From b77d4d55c689431675fdbb9aaf19cf26c32969c8 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 21 May 2026 17:44:55 +0200 Subject: [PATCH 1/6] Add new headers for CRR Cascaded Issue: CLDSRVCLT-14 --- models/backbeatRoutes/putMetadata.smithy | 10 ++++++++-- models/backbeatRoutes/putdata.smithy | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/models/backbeatRoutes/putMetadata.smithy b/models/backbeatRoutes/putMetadata.smithy index 2aa181d9..3d022b6d 100644 --- a/models/backbeatRoutes/putMetadata.smithy +++ b/models/backbeatRoutes/putMetadata.smithy @@ -33,12 +33,18 @@ structure PutMetadataInput { @httpHeader("X-Scal-Request-Uids") RequestUids: String, - + + @httpHeader("x-scal-micro-version-id") + MicroVersionId: String, + @httpPayload Body: Blob } structure PutMetadataOutput { /// Version ID of the stored metadata - versionId: String + versionId: String, + + @httpHeader("x-scal-cascade-loop-detected") + CascadeLoopDetected: Boolean } \ No newline at end of file diff --git a/models/backbeatRoutes/putdata.smithy b/models/backbeatRoutes/putdata.smithy index 45e45780..a16b45df 100644 --- a/models/backbeatRoutes/putdata.smithy +++ b/models/backbeatRoutes/putdata.smithy @@ -31,6 +31,9 @@ structure PutDataInput { @httpHeader("X-Scal-Request-Uids") RequestUids: String, + @httpHeader("x-scal-micro-version-id") + MicroVersionId: String, + @httpPayload @default("") Body: StreamingBlob @@ -45,7 +48,10 @@ structure PutDataOutput { @httpHeader("x-amz-server-side-encryption-customer-algorithm") SSECustomerAlgorithm: String, - + @httpHeader("x-amz-server-side-encryption-aws-kms-key-id") - SSEKMSKeyId: String + SSEKMSKeyId: String, + + @httpHeader("x-scal-cascade-loop-detected") + CascadeLoopDetected: Boolean } \ No newline at end of file From c333d5c8fe3bc8532f11e81389e3931a0c846437 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 21 May 2026 17:45:16 +0200 Subject: [PATCH 2/6] Add tests for CRR Cascaded Issue: CLDSRVCLT-14 --- tests/testApis.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/testApis.test.ts b/tests/testApis.test.ts index 947729f3..f777414b 100644 --- a/tests/testApis.test.ts +++ b/tests/testApis.test.ts @@ -16,7 +16,7 @@ import { createTestClient, testConfig } from './testSetup'; import { describeForMetadataBackend } from './testHelpers'; import assert from 'assert'; -describeForMetadataBackend('CloudServer Backbeat Routes API Tests', () => { +describe('CloudServer Backbeat Routes API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; let s3client: S3Client; @@ -37,7 +37,8 @@ describeForMetadataBackend('CloudServer Backbeat Routes API Tests', () => { CanonicalID: testConfig.canonicalID, ContentMD5: etag, Body: getData.Body, - VersioningRequired: true + VersioningRequired: true, + // MicroVersionId: '12345', }; const command2 = new PutDataCommand(putInput); From 4b85273c59f8340afa4a1940d8ba129afa310c35 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Thu, 21 May 2026 17:45:51 +0200 Subject: [PATCH 3/6] Bump version to 1.0.9 Issue: CLDSRVCLT-14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ec5a8a19..15670614 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserverclient", - "version": "1.0.8", + "version": "1.0.9", "engines": { "node": ">=20" }, From b7d908e66d96f75e1093b296bb208522e7d52845 Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Tue, 26 May 2026 18:25:46 +0200 Subject: [PATCH 4/6] further tests crr cascade, to be refined --- package.json | 3 +- tests/testApis.test.ts | 36 ++++- tests/testCrrCascaded.test.ts | 291 ++++++++++++++++++++++++++++++++++ yarn.lock | 5 + 4 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 tests/testCrrCascaded.test.ts diff --git a/package.json b/package.json index 15670614..3d0f2022 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,8 @@ "dependencies": { "@aws-sdk/client-s3": "^3.1009.0", "JSONStream": "^1.3.5", - "fast-xml-parser": "^5.5.7" + "fast-xml-parser": "^5.5.7", + "uuid": "11" }, "resolutions": { "flatted": "^3.4.2" diff --git a/tests/testApis.test.ts b/tests/testApis.test.ts index f777414b..d8989aa7 100644 --- a/tests/testApis.test.ts +++ b/tests/testApis.test.ts @@ -15,6 +15,7 @@ import { S3Client, GetObjectCommand as S3getCommand } from '@aws-sdk/client-s3'; import { createTestClient, testConfig } from './testSetup'; import { describeForMetadataBackend } from './testHelpers'; import assert from 'assert'; +import { v7 as uuidv7 } from 'uuid'; describe('CloudServer Backbeat Routes API Tests', () => { let backbeatRoutesClient: BackbeatRoutesClient; @@ -38,7 +39,6 @@ describe('CloudServer Backbeat Routes API Tests', () => { ContentMD5: etag, Body: getData.Body, VersioningRequired: true, - // MicroVersionId: '12345', }; const command2 = new PutDataCommand(putInput); @@ -51,6 +51,40 @@ describe('CloudServer Backbeat Routes API Tests', () => { assert.ok(locationAny[0].key !== undefined); }); + it('should test putData for CRR-Cascaded', async () => { + const command = new S3getCommand({ + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + }); + const getData = await s3client.send(command); + const etag = getData.ETag?.replace(/"/g, '') || ''; + let older = uuidv7(); + // sleep for 10ms + await new Promise(resolve => setTimeout(resolve, 1000)); + const putInput: PutDataCommandInput = { + Bucket: testConfig.bucketName, + Key: testConfig.objectKey, + CanonicalID: testConfig.canonicalID, + ContentMD5: etag, + Body: getData.Body, + VersioningRequired: true, + MicroVersionId: uuidv7(), + }; + + const command2 = new PutDataCommand(putInput); + addContentLengthMiddleware( + command2, + getData.ContentLength + ); + const data = await backbeatRoutesClient.send(command2); + console.log("AAAA data: ", data); + command2.input.MicroVersionId = older; + const data2 = await backbeatRoutesClient.send(command2); + console.log("AAAA data: ", data2); + // const locationAny: any = data.Location as any; + // assert.ok(locationAny[0].key !== undefined); + }); + it('should test GetSingleObject', async () => { const getInput: GetObjectInput = { Bucket: testConfig.bucketName, diff --git a/tests/testCrrCascaded.test.ts b/tests/testCrrCascaded.test.ts new file mode 100644 index 00000000..41b185bb --- /dev/null +++ b/tests/testCrrCascaded.test.ts @@ -0,0 +1,291 @@ +/** + * Functional tests for CRR Cascaded Replication (CLDSRV-897). + * + * Tests the loop-detection and stale-rejection behaviour added to the + * putData and putMetadata Backbeat routes via the x-scal-micro-version-id + * / x-scal-cascade-loop-detected headers. + * + * Requires a cloudserver running at http://localhost:8000 with any metadata + * backend (mem / file / scality). No CRR configuration is needed — the + * cascade-triggering tests (replicationInfo rewrite) are out of scope here + * and belong in an integration test with a fully configured stack. + */ + +import { + BackbeatRoutesClient, + PutDataCommand, + PutDataInput, + PutMetadataCommand, + PutMetadataInput, + GetMetadataCommand, + GetMetadataInput, +} from '../src/index'; +import { + S3Client, + GetObjectCommand, + PutObjectCommand, +} from '@aws-sdk/client-s3'; +import { createTestClient, testConfig } from './testSetup'; +import { addContentLengthMiddleware } from '../src/utils'; +import assert from 'assert'; +import { v7 as uuidv7 } from 'uuid'; + +// ─── helpers ───────────────────────────────────────────────────────────────── + +/** + * Returns a minimal valid object-metadata JSON body as a Uint8Array. + * Suitable for direct use as the PutMetadata Body field. + */ +function buildMetadataBody(overrides: Record = {}): Uint8Array { + const base: Record = { + 'content-length': testConfig.objectData.length, + 'content-type': 'text/plain', + 'last-modified': new Date().toISOString(), + 'etag': '"d41d8cd98f00b204e9800998ecf8427e"', + 'x-amz-version-id': 'null', + // Must match the canonical ID of accessKey1 (account "Bart") from cloudserver's + // conf/authdata.json — NOT testConfig.canonicalID which is a different account. + // If owner-id doesn't match the authenticated user, isObjAuthorized() denies + // the second putMetadata call (after the first one overwrites the stored metadata). + 'owner-id': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + 'owner-display-name': 'test', + 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', + 'replicationInfo': { + status: 'REPLICA', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + ...overrides, + }; + return new TextEncoder().encode(JSON.stringify(base)); +} + +/** + * Sends a PutMetadata request with a given microVersionId UUID. + * The UUID is set both in the x-scal-micro-version-id header (MicroVersionId + * field) and inside the metadata JSON body, so the stored object will carry + * the correct microVersionId for subsequent comparisons. + */ +async function putMetadataWithUUID( + client: BackbeatRoutesClient, + bucket: string, + key: string, + uuid: string, + bodyOverrides: Record = {}, +) { + const body = buildMetadataBody({ microVersionId: uuid, ...bodyOverrides }); + const input: PutMetadataInput = { + Bucket: bucket, + Key: key, + MicroVersionId: uuid, + Body: body, + }; + return client.send(new PutMetadataCommand(input)); +} + +/** + * Fetches a fresh object stream via S3 GetObject and immediately sends it as + * a PutData body. Always gets a fresh stream to avoid the "consumed stream" + * hang (a readable stream can only be piped once). + */ +async function putDataWithUUID( + client: BackbeatRoutesClient, + s3: S3Client, + bucket: string, + key: string, + uuid: string, +) { + const getResult = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + const etag = getResult.ETag?.replace(/"/g, '') || ''; + const input: PutDataInput = { + Bucket: bucket, + Key: key, + CanonicalID: testConfig.canonicalID, + ContentMD5: etag, + Body: getResult.Body, + VersioningRequired: true, + MicroVersionId: uuid, + }; + const cmd = new PutDataCommand(input); + addContentLengthMiddleware(cmd, getResult.ContentLength); + return client.send(cmd); +} + +/** + * Asserts that a call throws an error with httpStatusCode === expectedCode. + */ +async function assertThrowsHttpCode( + fn: () => Promise, + expectedCode: number, + message?: string, +) { + try { + await fn(); + assert.fail(`Expected HTTP ${expectedCode} but the call succeeded. ${message ?? ''}`); + } catch (err: any) { + if (err.message?.startsWith('Expected HTTP')) throw err; + assert.strictEqual( + err.$metadata?.httpStatusCode, + expectedCode, + `Expected HTTP ${expectedCode}, got ${err.$metadata?.httpStatusCode}. ${message ?? ''}`, + ); + } +} + +// ─── suite ─────────────────────────────────────────────────────────────────── + +describe('CRR Cascaded Replication — putData & putMetadata', () => { + let backbeatRoutesClient: BackbeatRoutesClient; + let s3client: S3Client; + + // Per-test object keys to avoid cross-test state pollution. + const keys = { + putMetadataLoop: `${testConfig.objectKey}-pm-loop`, + putMetadataStale: `${testConfig.objectKey}-pm-stale`, + putDataLoop: `${testConfig.objectKey}-pd-loop`, + putDataStale: `${testConfig.objectKey}-pd-stale`, + baseline: `${testConfig.objectKey}-baseline`, + }; + + beforeAll(async () => { + ({ backbeatRoutesClient, s3client } = createTestClient()); + + // Pre-create one S3 object per key so that objMd is never null + // when the route handler reads it. + await Promise.all( + Object.values(keys).map(key => + s3client.send(new PutObjectCommand({ + Bucket: testConfig.bucketName, + Key: key, + Body: testConfig.objectData, + })), + ), + ); + }); + + // ── putMetadata ─────────────────────────────────────────────────────────── + + describe('putMetadata', () => { + + it('returns CascadeLoopDetected=true when the same UUID is written twice', async () => { + const uuid = uuidv7(); + const bucket = testConfig.bucketName; + const key = keys.putMetadataLoop; + + // First write: no existing microVersionId yet → should succeed normally. + const first = await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); + assert.ok(!first.CascadeLoopDetected, 'First write should NOT be a loop'); + + // Second write with the same UUID → loop detected. + const second = await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); + assert.strictEqual(second.CascadeLoopDetected, true, 'Second write with same UUID should be loop-detected'); + assert.strictEqual(second.$metadata.httpStatusCode, 200); + }); + + it('returns 409 Conflict when the incoming UUID is older than the stored one', async () => { + const olderUUID = uuidv7(); + const newerUUID = uuidv7(); // generated after → lexicographically larger + const bucket = testConfig.bucketName; + const key = keys.putMetadataStale; + + // Establish a newer microVersionId in the stored metadata. + await putMetadataWithUUID(backbeatRoutesClient, bucket, key, newerUUID); + + // Attempt to write an older UUID → should be rejected as stale. + await assertThrowsHttpCode( + () => putMetadataWithUUID(backbeatRoutesClient, bucket, key, olderUUID), + 409, + 'Stale putMetadata (older UUID) should return 409', + ); + }); + + }); + + // ── putData ─────────────────────────────────────────────────────────────── + + describe('putData', () => { + + it('returns CascadeLoopDetected=true when the data UUID matches the stored metadata UUID', async () => { + const uuid = uuidv7(); + const bucket = testConfig.bucketName; + const key = keys.putDataLoop; + + // Commit the UUID into the object's stored metadata first. + await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); + + // putData with the same UUID: the server sees incoming == existing → loop. + const result = await putDataWithUUID(backbeatRoutesClient, s3client, bucket, key, uuid); + assert.strictEqual(result.CascadeLoopDetected, true, 'putData with same UUID as stored metadata should be loop-detected'); + assert.strictEqual(result.$metadata.httpStatusCode, 200); + }); + + it('returns 409 Conflict when the data UUID is older than the stored metadata UUID', async () => { + const olderUUID = uuidv7(); + const newerUUID = uuidv7(); + const bucket = testConfig.bucketName; + const key = keys.putDataStale; + + // Commit the newer UUID into stored metadata. + await putMetadataWithUUID(backbeatRoutesClient, bucket, key, newerUUID); + + // putData with the older UUID → stale → 409. + await assertThrowsHttpCode( + () => putDataWithUUID(backbeatRoutesClient, s3client, bucket, key, olderUUID), + 409, + 'Stale putData (older UUID) should return 409', + ); + }); + + }); + + // ── baseline (no MicroVersionId header) ────────────────────────────────── + + describe('baseline — no MicroVersionId header', () => { + + it('putData without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { + const bucket = testConfig.bucketName; + const key = keys.baseline; + + const getResult = await s3client.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + const etag = getResult.ETag?.replace(/"/g, '') || ''; + const input: PutDataInput = { + Bucket: bucket, + Key: key, + CanonicalID: testConfig.canonicalID, + ContentMD5: etag, + Body: getResult.Body, + VersioningRequired: true, + // MicroVersionId intentionally omitted + }; + const cmd = new PutDataCommand(input); + addContentLengthMiddleware(cmd, getResult.ContentLength); + const result = await backbeatRoutesClient.send(cmd); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.ok(!result.CascadeLoopDetected, 'CascadeLoopDetected should be falsy without MicroVersionId'); + }); + + it('putMetadata without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { + const bucket = testConfig.bucketName; + const key = keys.baseline; + const body = buildMetadataBody(); + const input: PutMetadataInput = { + Bucket: bucket, + Key: key, + Body: body, + // MicroVersionId intentionally omitted + }; + const result = await backbeatRoutesClient.send(new PutMetadataCommand(input)); + + assert.strictEqual(result.$metadata.httpStatusCode, 200); + assert.ok(!result.CascadeLoopDetected, 'CascadeLoopDetected should be falsy without MicroVersionId'); + }); + + }); + +}); diff --git a/yarn.lock b/yarn.lock index 62853f7b..1f713771 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4258,6 +4258,11 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +uuid@11: + version "11.1.1" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.1.tgz#f6d81d2e1c65d00762e5e29b16c5d2d995e208ad" + integrity sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ== + v8-to-istanbul@^9.0.1: version "9.3.0" resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz#b9572abfa62bd556c16d75fdebc1a411d5ff3175" From 5f6387884cb95b2b5160a1584001e0e0b821ed85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 27 May 2026 18:37:35 +0200 Subject: [PATCH 5/6] Add attachExpectContinueMiddleware helper for opt-in 100-continue Expose a per-command helper that wraps the AWS SDK Expect: 100-continue middleware so callers can opt commands like PutData into the handshake, letting the server reject invalid requests before the body is streamed. The threshold is configurable to skip the cost on small payloads. Issue: CLDSRVCLT-15 --- package.json | 1 + src/utils.ts | 43 ++++++ tests/testExpectContinue.test.ts | 240 +++++++++++++++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 tests/testExpectContinue.test.ts diff --git a/package.json b/package.json index 3d0f2022..69c4be2e 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1009.0", + "@aws-sdk/middleware-expect-continue": "^3.972.8", "JSONStream": "^1.3.5", "fast-xml-parser": "^5.5.7", "uuid": "11" diff --git a/src/utils.ts b/src/utils.ts index db1dfead..8812565a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,51 @@ +import { addExpectContinueMiddleware } from '@aws-sdk/middleware-expect-continue'; +import { MiddlewareStack, RequestHandler } from '@smithy/types'; import { XMLParser } from 'fast-xml-parser'; import { CloudserverBackbeatRoutesServiceException } from '../build/smithy/cloudserverBackbeatRoutes/typescript-codegen'; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type WithMiddlewareStack = { middlewareStack?: MiddlewareStack }; + +/** + * Attach the AWS SDK Expect: 100-continue middleware to a single command. + * + * Use this on commands whose target route honors 100-continue server-side. + * Pass the client's requestHandler so the underlying middleware can skip + * the header when running on FetchHttpHandler. + * + * @param command - The command to attach the middleware to. + * @param requestHandler - The client's requestHandler, used by the AWS SDK + * middleware to detect FetchHttpHandler and skip the header in that case. + * @param expectContinueHeader - Controls when the header is set: + * - `true` (default): always set the header on body-carrying requests. + * - `false`: never set the header (middleware no-op). + * - `number`: only set the header when the body's Content-Length is + * greater than or equal to this threshold (in bytes). Useful to skip + * the handshake cost on small payloads. + */ +export function attachExpectContinueMiddleware( + command: TCommand & WithMiddlewareStack, + requestHandler?: RequestHandler, + expectContinueHeader: boolean | number = true, +): TCommand { + if (!command.middlewareStack) { + throw new Error('Command does not have a middleware stack'); + } + + command.middlewareStack.add( + addExpectContinueMiddleware({ + runtime: 'node', + requestHandler, + expectContinueHeader, + }), + { step: 'build', name: 'expectContinue' }, + ); + + return command; +} + /** * Adds middleware to manually set the Content-Length header on a command. * diff --git a/tests/testExpectContinue.test.ts b/tests/testExpectContinue.test.ts new file mode 100644 index 00000000..e8c07161 --- /dev/null +++ b/tests/testExpectContinue.test.ts @@ -0,0 +1,240 @@ +import http, { Server } from 'http'; +import { AddressInfo } from 'net'; +import { promisify } from 'util'; +import { + BackbeatRoutesClient, + PutDataCommand, + GetObjectCommand, + attachExpectContinueMiddleware, +} from '../src/index'; + +jest.setTimeout(20000); + +let server: Server; +let client: BackbeatRoutesClient; +let sendContinue: boolean; +let earlyReject: boolean; +let unsolicitedContinue: boolean; +let continueSent: boolean; +let captured: { + method?: string; + headers: http.IncomingHttpHeaders; + body: Buffer; + bodyArrivedBeforeContinueSent: boolean; + headersReceivedAt?: number; + firstBodyChunkAt?: number; +}; + +describe('Expect: 100-continue middleware on PutDataCommand', () => { + beforeAll(async () => { + server = http.createServer(); + + const handle = (req: http.IncomingMessage, res: http.ServerResponse) => { + captured.method = req.method; + captured.headers = req.headers; + if (captured.headersReceivedAt === undefined) { + captured.headersReceivedAt = Date.now(); + } + if (unsolicitedContinue && !continueSent) { + res.writeContinue(); + continueSent = true; + } + const chunks: Buffer[] = []; + + req.on('data', chunk => { + if (captured.firstBodyChunkAt === undefined) { + captured.firstBodyChunkAt = Date.now(); + } + if (!continueSent) { + captured.bodyArrivedBeforeContinueSent = true; + } + chunks.push(chunk); + }); + + req.on('end', () => { + captured.body = Buffer.concat(chunks); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify([{ key: 'k', dataStoreName: 'd' }])); + }); + }; + + // Once a 'checkContinue' listener exists, Node stops auto-sending 100 Continue. + server.on('checkContinue', (req, res) => { + if (captured.headersReceivedAt === undefined) { + captured.headersReceivedAt = Date.now(); + } + if (earlyReject) { + captured.method = req.method; + captured.headers = req.headers; + req.on('data', chunk => { + captured.bodyArrivedBeforeContinueSent = true; + captured.body = Buffer.concat([captured.body, chunk]); + }); + res.writeHead(409); + res.end(); + return; + } + + if (sendContinue) { + res.writeContinue(); + continueSent = true; + } + + handle(req, res); + }); + server.on('request', handle); + + await promisify(server.listen).call(server, 0, '127.0.0.1'); + const { port } = server.address() as AddressInfo; + + client = new BackbeatRoutesClient({ + endpoint: `http://127.0.0.1:${port}`, + credentials: { accessKeyId: 'a', secretAccessKey: 'b' }, + region: 'us-east-1', + maxAttempts: 1, + }); + }); + + afterAll(async () => { + client.destroy(); + await promisify(server.close).call(server); + }); + + beforeEach(() => { + sendContinue = true; + earlyReject = false; + unsolicitedContinue = false; + continueSent = false; + captured = { headers: {}, body: Buffer.alloc(0), bodyArrivedBeforeContinueSent: false }; + }); + + const putData = (Body: Buffer) => client.send(attachExpectContinueMiddleware( + new PutDataCommand({ + Bucket: 'bucket', + Key: 'obj', + ContentMD5: 'x', + CanonicalID: 'c', + Body, + }), + client.config.requestHandler, + )); + + it('sets Expect and waits for 100 before streaming the body', async () => { + const body = Buffer.from('hello-world'); + await putData(body); + + expect(captured.method).toBe('PUT'); + expect(captured.headers.expect).toBe('100-continue'); + expect(captured.bodyArrivedBeforeContinueSent).toBe(false); + expect(captured.body.length).toBe(body.length); + }); + + it('does NOT set Expect on body-less commands (GetObject)', async () => { + await client + .send(new GetObjectCommand({ Bucket: 'bucket', Key: 'obj' })) + .catch(() => undefined); + expect(captured.headers.expect).toBeUndefined(); + }); + + it('does NOT set Expect on PutData without attachExpectContinueMiddleware', async () => { + await client.send(new PutDataCommand({ + Bucket: 'bucket', + Key: 'obj', + ContentMD5: 'x', + CanonicalID: 'c', + Body: Buffer.from('hello-world'), + })); + expect(captured.method).toBe('PUT'); + expect(captured.headers.expect).toBeUndefined(); + }); + + it('honors a numeric expectContinueHeader threshold (below threshold => no header)', async () => { + const body = Buffer.from('tiny'); + await client.send(attachExpectContinueMiddleware( + new PutDataCommand({ + Bucket: 'bucket', + Key: 'obj', + ContentMD5: 'x', + CanonicalID: 'c', + Body: body, + }), + client.config.requestHandler, + 1024, + )); + expect(captured.method).toBe('PUT'); + expect(captured.headers.expect).toBeUndefined(); + expect(captured.body.length).toBe(body.length); + }); + + it('honors a numeric expectContinueHeader threshold (at/above threshold => header set)', async () => { + const body = Buffer.alloc(1024, 'a'); + await client.send(attachExpectContinueMiddleware( + new PutDataCommand({ + Bucket: 'bucket', + Key: 'obj', + ContentMD5: 'x', + CanonicalID: 'c', + Body: body, + }), + client.config.requestHandler, + 1024, + )); + expect(captured.headers.expect).toBe('100-continue'); + expect(captured.body.length).toBe(body.length); + }); + + it('still uploads when the server sends an unsolicited 100-continue', async () => { + unsolicitedContinue = true; + const body = Buffer.from('hello-world'); + await client.send(new PutDataCommand({ + Bucket: 'bucket', + Key: 'obj', + ContentMD5: 'x', + CanonicalID: 'c', + Body: body, + })); + expect(captured.method).toBe('PUT'); + expect(captured.headers.expect).toBeUndefined(); + expect(captured.body.length).toBe(body.length); + }); + + it('still uploads if the server never sends 100-continue (falls back after timeout)', async () => { + sendContinue = false; + const body = Buffer.from('hello-world'); + await putData(body); + + expect(captured.headers.expect).toBe('100-continue'); + expect(captured.body.length).toBe(body.length); + }); + + it('waits ~6s before streaming the body when no 100-continue is received', async () => { + sendContinue = false; + const body = Buffer.from('hello-world'); + await putData(body); + + expect(captured.headersReceivedAt).toBeDefined(); + expect(captured.firstBodyChunkAt).toBeDefined(); + const waited = captured.firstBodyChunkAt! - captured.headersReceivedAt!; + expect(waited).toBeGreaterThanOrEqual(5500); + expect(waited).toBeLessThan(8000); + expect(captured.body.length).toBe(body.length); + }); + + + it('surfaces an early 4xx response without streaming the body', async () => { + earlyReject = true; + const body = Buffer.from('hello-world'); + + const err = await putData(body).then( + () => { throw new Error('expected request to fail'); }, + (e: Error & { $metadata?: { httpStatusCode?: number } }) => e, + ); + + expect(err.$metadata?.httpStatusCode).toBe(409); + expect(captured.method).toBe('PUT'); + expect(captured.headers.expect).toBe('100-continue'); + expect(captured.bodyArrivedBeforeContinueSent).toBe(false); + expect(captured.body.length).toBe(0); + }); + +}); From 12080a38522b9ad33440ec57115f47bdf6b7055c Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Wed, 27 May 2026 20:05:48 +0200 Subject: [PATCH 6/6] update test crr cascade --- tests/testCrrCascaded.test.ts | 162 +++++++++++++++++++++------------- 1 file changed, 102 insertions(+), 60 deletions(-) diff --git a/tests/testCrrCascaded.test.ts b/tests/testCrrCascaded.test.ts index 41b185bb..5efb079b 100644 --- a/tests/testCrrCascaded.test.ts +++ b/tests/testCrrCascaded.test.ts @@ -17,8 +17,6 @@ import { PutDataInput, PutMetadataCommand, PutMetadataInput, - GetMetadataCommand, - GetMetadataInput, } from '../src/index'; import { S3Client, @@ -26,15 +24,44 @@ import { PutObjectCommand, } from '@aws-sdk/client-s3'; import { createTestClient, testConfig } from './testSetup'; -import { addContentLengthMiddleware } from '../src/utils'; +import { addContentLengthMiddleware, attachExpectContinueMiddleware } from '../src/utils'; import assert from 'assert'; -import { v7 as uuidv7 } from 'uuid'; +import { Readable } from 'stream'; + +// ─── microVersionId test helpers ───────────────────────────────────────────── +// +// Uses Arsenal's VersionID module via a relative path to cloudserver's +// node_modules. TODO: replace with a proper devDependency once the package.json +// is updated to reference the Arsenal repo. + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const VersionIDUtils = require('../../cloudserver/node_modules/arsenal').versioning.VersionID; + +/** + * Generates a microVersionId pair for use in tests. + * raw — 27-char stored format (goes in the metadata body JSON) + * wire — 32-char base62 wire format (goes in the MicroVersionId header) + * + * The versionId format is reverse-chronological: a smaller raw value means a + * more recent write. Calling this twice in sequence produces a pair where the + * first call's raw value is *larger* (older) than the second call's. + */ +function generateTestMVId(): { raw: string; wire: string } { + // Pass empty instanceId so the raw value is always 14+6+7=27 chars + // regardless of S3_VERSION_ID_ENCODING_TYPE. encode() uses base62 for + // any 27-char input, giving the expected 32-char wire value. + const raw: string = VersionIDUtils.generateVersionId('', 'RG001'); + const wire: string = VersionIDUtils.encode(raw); + return { raw, wire }; +} // ─── helpers ───────────────────────────────────────────────────────────────── /** * Returns a minimal valid object-metadata JSON body as a Uint8Array. - * Suitable for direct use as the PutMetadata Body field. + * Sets owner-id to the canonical ID of accessKey1 (account "Bart") from + * cloudserver's conf/authdata.json so isObjAuthorized() passes for the + * authenticated account used by BackbeatRoutesClient. */ function buildMetadataBody(overrides: Record = {}): Uint8Array { const base: Record = { @@ -46,7 +73,7 @@ function buildMetadataBody(overrides: Record = {}): Uint8Array // Must match the canonical ID of accessKey1 (account "Bart") from cloudserver's // conf/authdata.json — NOT testConfig.canonicalID which is a different account. // If owner-id doesn't match the authenticated user, isObjAuthorized() denies - // the second putMetadata call (after the first one overwrites the stored metadata). + // subsequent putMetadata calls on the same key. 'owner-id': '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', 'owner-display-name': 'test', 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', @@ -66,53 +93,66 @@ function buildMetadataBody(overrides: Record = {}): Uint8Array } /** - * Sends a PutMetadata request with a given microVersionId UUID. - * The UUID is set both in the x-scal-micro-version-id header (MicroVersionId - * field) and inside the metadata JSON body, so the stored object will carry - * the correct microVersionId for subsequent comparisons. + * Sends a PutMetadata request with a given microVersionId pair. + * mvId.wire — sent as the x-scal-micro-version-id header (32-char base62) + * mvId.raw — stored inside the metadata JSON body (27-char raw) + * + * Builds a minimal valid metadata body directly — no GetMetadata call needed. + * The cascade check on the server uses objMd fetched from the metadata backend, + * not what's in the request body, so a minimal body is sufficient. */ -async function putMetadataWithUUID( +async function putMetadataWithMVId( client: BackbeatRoutesClient, bucket: string, key: string, - uuid: string, + mvId: { raw: string; wire: string }, bodyOverrides: Record = {}, ) { - const body = buildMetadataBody({ microVersionId: uuid, ...bodyOverrides }); - const input: PutMetadataInput = { + const body = buildMetadataBody({ microVersionId: mvId.raw, ...bodyOverrides }); + const result = await client.send(new PutMetadataCommand({ Bucket: bucket, Key: key, - MicroVersionId: uuid, + MicroVersionId: mvId.wire, Body: body, - }; - return client.send(new PutMetadataCommand(input)); + })); + return result; } /** - * Fetches a fresh object stream via S3 GetObject and immediately sends it as - * a PutData body. Always gets a fresh stream to avoid the "consumed stream" - * hang (a readable stream can only be piped once). + * Sends a PutData request with a given microVersionId pair. + * + * Uses testConfig.objectData directly as the body instead of fetching the + * object via GetObject. For loop and stale cases the server rejects from + * headers alone (Expect: 100-continue) before reading any body bytes, so the + * body content is irrelevant. + * + * Attaches the Expect: 100-continue middleware so the server can reject the + * request (loop or stale) BEFORE the body stream is sent. Without this, the + * server closes the connection early, the SDK gets ECONNRESET, and retries + * indefinitely with an already-consumed stream → test hangs. */ -async function putDataWithUUID( +async function putDataWithMVId( client: BackbeatRoutesClient, - s3: S3Client, bucket: string, key: string, - uuid: string, + mvId: { raw: string; wire: string }, ) { - const getResult = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - const etag = getResult.ETag?.replace(/"/g, '') || ''; + const data = testConfig.objectData; + const bodyBytes = typeof data === 'string' ? Buffer.from(data) : data; const input: PutDataInput = { Bucket: bucket, Key: key, CanonicalID: testConfig.canonicalID, - ContentMD5: etag, - Body: getResult.Body, + ContentMD5: '8c68b1ec59642e3994c995eccfee553b', // md5('iAmSomeData') + Body: Readable.from([bodyBytes]), VersioningRequired: true, - MicroVersionId: uuid, + MicroVersionId: mvId.wire, }; const cmd = new PutDataCommand(input); - addContentLengthMiddleware(cmd, getResult.ContentLength); + addContentLengthMiddleware(cmd, bodyBytes.length); + // Attach Expect: 100-continue so the server can reject loop/stale requests + // from headers alone, before any body bytes are sent. + attachExpectContinueMiddleware(cmd, client.config.requestHandler); return client.send(cmd); } @@ -172,35 +212,37 @@ describe('CRR Cascaded Replication — putData & putMetadata', () => { describe('putMetadata', () => { - it('returns CascadeLoopDetected=true when the same UUID is written twice', async () => { - const uuid = uuidv7(); + it('1111 returns CascadeLoopDetected=true when the same UUID is written twice', async () => { + const mvId = generateTestMVId(); const bucket = testConfig.bucketName; const key = keys.putMetadataLoop; // First write: no existing microVersionId yet → should succeed normally. - const first = await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); + const first = await putMetadataWithMVId(backbeatRoutesClient, bucket, key, mvId); assert.ok(!first.CascadeLoopDetected, 'First write should NOT be a loop'); - // Second write with the same UUID → loop detected. - const second = await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); - assert.strictEqual(second.CascadeLoopDetected, true, 'Second write with same UUID should be loop-detected'); + // Second write with the same mvId → loop detected. + const second = await putMetadataWithMVId(backbeatRoutesClient, bucket, key, mvId); + assert.strictEqual(second.CascadeLoopDetected, true, 'Second write with same mvId should be loop-detected'); assert.strictEqual(second.$metadata.httpStatusCode, 200); }); - it('returns 409 Conflict when the incoming UUID is older than the stored one', async () => { - const olderUUID = uuidv7(); - const newerUUID = uuidv7(); // generated after → lexicographically larger + it('2222 returns 409 Conflict when the incoming UUID is older than the stored one', async () => { + // Format is reverse-chronological: the ID generated first has a + // larger raw value and is therefore "older". + const olderMVId = generateTestMVId(); + const newerMVId = generateTestMVId(); // generated after → smaller raw value → newer const bucket = testConfig.bucketName; const key = keys.putMetadataStale; - // Establish a newer microVersionId in the stored metadata. - await putMetadataWithUUID(backbeatRoutesClient, bucket, key, newerUUID); + // Establish the newer microVersionId in the stored metadata. + await putMetadataWithMVId(backbeatRoutesClient, bucket, key, newerMVId); - // Attempt to write an older UUID → should be rejected as stale. + // Attempt to write the older one → should be rejected as stale. await assertThrowsHttpCode( - () => putMetadataWithUUID(backbeatRoutesClient, bucket, key, olderUUID), + () => putMetadataWithMVId(backbeatRoutesClient, bucket, key, olderMVId), 409, - 'Stale putMetadata (older UUID) should return 409', + 'Stale putMetadata (older mvId) should return 409', ); }); @@ -210,34 +252,34 @@ describe('CRR Cascaded Replication — putData & putMetadata', () => { describe('putData', () => { - it('returns CascadeLoopDetected=true when the data UUID matches the stored metadata UUID', async () => { - const uuid = uuidv7(); + it('3333 returns CascadeLoopDetected=true when the data UUID matches the stored metadata UUID', async () => { + const mvId = generateTestMVId(); const bucket = testConfig.bucketName; const key = keys.putDataLoop; - // Commit the UUID into the object's stored metadata first. - await putMetadataWithUUID(backbeatRoutesClient, bucket, key, uuid); + // Commit the mvId into the object's stored metadata first. + await putMetadataWithMVId(backbeatRoutesClient, bucket, key, mvId); - // putData with the same UUID: the server sees incoming == existing → loop. - const result = await putDataWithUUID(backbeatRoutesClient, s3client, bucket, key, uuid); - assert.strictEqual(result.CascadeLoopDetected, true, 'putData with same UUID as stored metadata should be loop-detected'); + // putData with the same mvId: server sees incoming == existing → loop. + const result = await putDataWithMVId(backbeatRoutesClient, bucket, key, mvId); + assert.strictEqual(result.CascadeLoopDetected, true, 'putData with same mvId as stored metadata should be loop-detected'); assert.strictEqual(result.$metadata.httpStatusCode, 200); }); - it('returns 409 Conflict when the data UUID is older than the stored metadata UUID', async () => { - const olderUUID = uuidv7(); - const newerUUID = uuidv7(); + it('4444 returns 409 Conflict when the data UUID is older than the stored metadata UUID', async () => { + const olderMVId = generateTestMVId(); + const newerMVId = generateTestMVId(); // generated after → smaller raw → newer const bucket = testConfig.bucketName; const key = keys.putDataStale; - // Commit the newer UUID into stored metadata. - await putMetadataWithUUID(backbeatRoutesClient, bucket, key, newerUUID); + // Commit the newer mvId into stored metadata. + await putMetadataWithMVId(backbeatRoutesClient, bucket, key, newerMVId); - // putData with the older UUID → stale → 409. + // putData with the older mvId → stale → 409. await assertThrowsHttpCode( - () => putDataWithUUID(backbeatRoutesClient, s3client, bucket, key, olderUUID), + () => putDataWithMVId(backbeatRoutesClient, bucket, key, olderMVId), 409, - 'Stale putData (older UUID) should return 409', + 'Stale putData (older mvId) should return 409', ); }); @@ -247,7 +289,7 @@ describe('CRR Cascaded Replication — putData & putMetadata', () => { describe('baseline — no MicroVersionId header', () => { - it('putData without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { + it('5555 putData without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { const bucket = testConfig.bucketName; const key = keys.baseline; @@ -270,7 +312,7 @@ describe('CRR Cascaded Replication — putData & putMetadata', () => { assert.ok(!result.CascadeLoopDetected, 'CascadeLoopDetected should be falsy without MicroVersionId'); }); - it('putMetadata without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { + it('6666 putMetadata without MicroVersionId succeeds and does not set CascadeLoopDetected', async () => { const bucket = testConfig.bucketName; const key = keys.baseline; const body = buildMetadataBody();