diff --git a/eslint.config.mjs b/eslint.config.mjs index fa498da2..eb59432b 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -67,6 +67,9 @@ export default defineConfig([ project: [ "frontend/tsconfig.json", "lambdas/*/tsconfig.json", + "tests/integration/tsconfig.json", + "tests/performance/tsconfig.json", + "tests/test-support/tsconfig.json", "tests/test-team/tsconfig.json", "utils/*/tsconfig.json", ], @@ -197,7 +200,7 @@ export default defineConfig([ }, }, { - files: ["**/utils/**", "tests/test-team/**", "lambdas/**/src/**"], + files: ["**/utils/**", "tests/test-team/**", "tests/performance/helpers/**", "lambdas/**/src/**"], rules: { "import-x/prefer-default-export": 0, }, @@ -211,19 +214,14 @@ export default defineConfig([ }, }, { - // Files inside helpers/ are loaded transitively by globalSetup/globalTeardown which run - // outside Jest's module system, so moduleNameMapper does not apply — relative imports only - files: ["tests/integration/helpers/**"], - rules: { - "no-relative-import-paths/no-relative-import-paths": 0, - }, - }, - { - // globalSetup/globalTeardown run outside Jest's module system so moduleNameMapper - // does not apply — relative imports are the only way to reach local helpers + // helpers/ files use relative imports between each other; test files import + // directly from local helpers using relative paths files: [ - "tests/integration/jest.global-setup.ts", - "tests/integration/jest.global-teardown.ts", + "tests/integration/helpers/**", + "tests/integration/*.test.ts", + "tests/performance/helpers/**", + "tests/performance/*.test.ts", + "tests/test-support/helpers/**", ], rules: { "no-relative-import-paths/no-relative-import-paths": 0, @@ -273,6 +271,12 @@ export default defineConfig([ "import-x/first": "off", }, }, + { + files: ["tests/performance/**/*.ts"], + rules: { + "no-console": "off", + }, + }, // misc rule overrides { diff --git a/infrastructure/terraform/components/callbacks/README.md b/infrastructure/terraform/components/callbacks/README.md index f5056ab5..68dcccec 100644 --- a/infrastructure/terraform/components/callbacks/README.md +++ b/infrastructure/terraform/components/callbacks/README.md @@ -20,6 +20,7 @@ | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [deploy\_mock\_webhook](#input\_deploy\_mock\_webhook) | Flag to deploy mock webhook lambda for integration testing (test/dev environments only) | `bool` | `false` | no | | [enable\_event\_anomaly\_detection](#input\_enable\_event\_anomaly\_detection) | Enable CloudWatch anomaly detection alarm for inbound event queue message reception | `bool` | `true` | no | +| [enable\_xray\_tracing](#input\_enable\_xray\_tracing) | Enable AWS X-Ray active tracing for Lambda functions | `bool` | `false` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | | [event\_anomaly\_band\_width](#input\_event\_anomaly\_band\_width) | The width of the anomaly detection band. Higher values (e.g. 4-6) reduce sensitivity and noise, lower values (e.g. 2-3) increase sensitivity. Recommended: 2-4. | `number` | `3` | no | | [event\_anomaly\_evaluation\_periods](#input\_event\_anomaly\_evaluation\_periods) | Number of evaluation periods for the anomaly alarm. Each period is defined by event\_anomaly\_period. | `number` | `2` | no | @@ -43,12 +44,12 @@ | Name | Source | Version | |------|--------|---------| -| [client\_config\_bucket](#module\_client\_config\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a | +| [client\_config\_bucket](#module\_client\_config\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-s3bucket.zip | n/a | | [client\_destination](#module\_client\_destination) | ../../modules/client-destination | n/a | -| [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-lambda.zip | n/a | -| [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-kms.zip | n/a | -| [mock\_webhook\_lambda](#module\_mock\_webhook\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-lambda.zip | n/a | -| [sqs\_inbound\_event](#module\_sqs\_inbound\_event) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a | +| [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | +| [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-kms.zip | n/a | +| [mock\_webhook\_lambda](#module\_mock\_webhook\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | +| [sqs\_inbound\_event](#module\_sqs\_inbound\_event) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip | n/a | ## Outputs | Name | Description | diff --git a/infrastructure/terraform/components/callbacks/module_kms.tf b/infrastructure/terraform/components/callbacks/module_kms.tf index 758c62aa..327b5641 100644 --- a/infrastructure/terraform/components/callbacks/module_kms.tf +++ b/infrastructure/terraform/components/callbacks/module_kms.tf @@ -1,5 +1,5 @@ module "kms" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-kms.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-kms.zip" aws_account_id = var.aws_account_id component = var.component diff --git a/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf b/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf index 9a6de177..2bea7016 100644 --- a/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf +++ b/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf @@ -1,6 +1,6 @@ module "mock_webhook_lambda" { count = var.deploy_mock_webhook ? 1 : 0 - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-lambda.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip" function_name = "mock-webhook" description = "Mock webhook endpoint for integration testing - logs received callbacks to CloudWatch" diff --git a/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf b/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf index 3c7e16f9..2e3080fe 100644 --- a/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf +++ b/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf @@ -1,5 +1,5 @@ module "sqs_inbound_event" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip" aws_account_id = var.aws_account_id component = var.component diff --git a/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf b/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf index e267118f..2a0c70a7 100644 --- a/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf +++ b/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf @@ -1,5 +1,5 @@ module "client_transform_filter_lambda" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-lambda.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip" function_name = "client-transform-filter" description = "Lambda function that transforms and filters events coming to through the eventpipe" @@ -30,6 +30,7 @@ module "client_transform_filter_lambda" { force_lambda_code_deploy = var.force_lambda_code_deploy enable_lambda_insights = false + enable_xray_tracing = var.enable_xray_tracing log_destination_arn = local.log_destination_arn log_subscription_role_arn = local.acct.log_subscription_role_arn diff --git a/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf b/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf index 58b016e4..50113222 100644 --- a/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf +++ b/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf @@ -1,5 +1,5 @@ module "client_config_bucket" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-s3bucket.zip" name = "subscription-config" diff --git a/infrastructure/terraform/components/callbacks/variables.tf b/infrastructure/terraform/components/callbacks/variables.tf index b82546b0..81eb56e1 100644 --- a/infrastructure/terraform/components/callbacks/variables.tf +++ b/infrastructure/terraform/components/callbacks/variables.tf @@ -169,6 +169,12 @@ variable "deploy_mock_webhook" { default = false } +variable "enable_xray_tracing" { + type = bool + description = "Enable AWS X-Ray active tracing for Lambda functions" + default = false +} + variable "message_root_uri" { type = string description = "The root URI used for constructing message links in callback payloads" diff --git a/infrastructure/terraform/modules/client-destination/README.md b/infrastructure/terraform/modules/client-destination/README.md index 1cbd4706..ca03e02c 100644 --- a/infrastructure/terraform/modules/client-destination/README.md +++ b/infrastructure/terraform/modules/client-destination/README.md @@ -28,7 +28,7 @@ No requirements. | Name | Source | Version | |------|--------|---------| -| [target\_dlq](#module\_target\_dlq) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a | +| [target\_dlq](#module\_target\_dlq) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip | n/a | ## Outputs No outputs. diff --git a/infrastructure/terraform/modules/client-destination/module_target_dlq.tf b/infrastructure/terraform/modules/client-destination/module_target_dlq.tf index 5a1457e5..e6ce0187 100644 --- a/infrastructure/terraform/modules/client-destination/module_target_dlq.tf +++ b/infrastructure/terraform/modules/client-destination/module_target_dlq.tf @@ -1,5 +1,5 @@ module "target_dlq" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip" aws_account_id = var.aws_account_id component = var.component diff --git a/package-lock.json b/package-lock.json index 1bad9dd3..45430ea6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,8 @@ "src/models", "lambdas/mock-webhook-lambda", "tests/integration", + "tests/performance", + "tests/test-support", "tools/client-subscriptions-management" ], "devDependencies": { @@ -473,7 +475,6 @@ "version": "3.1017.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1017.0.tgz", "integrity": "sha512-Y5FRcAo1lkeOMp6+q7bGSAP3NUdR61VLYzW9J+ksz1KhHLQfCQEzNaTzjwIJyEW2FjJ8w08b/tcScG0Fde0NiA==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -1011,7 +1012,6 @@ "version": "3.972.17", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.17.tgz", "integrity": "sha512-LnzPRRoDXGtlFV2G1p2rsY6fRKrbf6Pvvc21KliSLw3+NmQca2+Aa1QIMRbpQvZYedsSqkGYwxe+qvXwQ2uxDw==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.6", @@ -3021,6 +3021,10 @@ "resolved": "src/models", "link": true }, + "node_modules/@nhs-notify-client-callbacks/test-support": { + "resolved": "tests/test-support", + "link": true + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4974,7 +4978,6 @@ "version": "2.0.31", "resolved": "https://registry.npmjs.org/async-wait-until/-/async-wait-until-2.0.31.tgz", "integrity": "sha512-9VCfHvc4f36oT6sG5p16aKc9zojf3wF4FrjNDxU3Db51SJ1bQ5lWAWtQDDZPysTwSLKBDzNZ083qPkTIj6XnrA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.14.0", @@ -9725,6 +9728,10 @@ "resolved": "tests/integration", "link": true }, + "node_modules/nhs-notify-client-callbacks-performance-tests": { + "resolved": "tests/performance", + "link": true + }, "node_modules/nhs-notify-client-transform-filter-lambda": { "resolved": "lambdas/client-transform-filter-lambda", "link": true @@ -12893,6 +12900,7 @@ "@aws-sdk/client-sqs": "^3.990.0", "@nhs-notify-client-callbacks/logger": "*", "@nhs-notify-client-callbacks/models": "*", + "@nhs-notify-client-callbacks/test-support": "*", "async-wait-until": "^2.0.12" }, "devDependencies": { @@ -12904,6 +12912,37 @@ "typescript": "^5.8.2" } }, + "tests/performance": { + "name": "nhs-notify-client-callbacks-performance-tests", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.991.0", + "@aws-sdk/client-s3": "^3.821.0", + "@aws-sdk/client-sqs": "^3.990.0", + "@nhs-notify-client-callbacks/models": "*", + "@nhs-notify-client-callbacks/test-support": "*", + "async-wait-until": "^2.0.12" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "typescript": "^5.8.2" + } + }, + "tests/test-support": { + "name": "@nhs-notify-client-callbacks/test-support", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.991.0", + "@aws-sdk/client-s3": "^3.821.0", + "@aws-sdk/client-sqs": "^3.990.0" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "typescript": "^5.8.2" + } + }, "tools/client-subscriptions-management": { "version": "0.0.1", "dependencies": { diff --git a/package.json b/package.json index 68238b47..ec5117a3 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,8 @@ "src/models", "lambdas/mock-webhook-lambda", "tests/integration", + "tests/performance", + "tests/test-support", "tools/client-subscriptions-management" ] } diff --git a/scripts/config/sonar-scanner.properties b/scripts/config/sonar-scanner.properties index 71326c80..65325df0 100644 --- a/scripts/config/sonar-scanner.properties +++ b/scripts/config/sonar-scanner.properties @@ -5,5 +5,5 @@ sonar.qualitygate.wait=true sonar.sourceEncoding=UTF-8 sonar.terraform.provider.aws.version=5.54.1 sonar.cpd.exclusions=**.test.*, src/models/** -sonar.coverage.exclusions=tests/**, lambdas/**/src/__tests__/**, src/**/src/__tests__/**, src/models/**, scripts/**/src/__tests__/**, tools/**/src/__tests__/**, **/jest.config.* +sonar.coverage.exclusions=tests/test-support/**, tests/**, lambdas/**/src/__tests__/**, src/**/src/__tests__/**, src/models/**, scripts/**/src/__tests__/**, tools/**/src/__tests__/**, **/jest.config.* sonar.javascript.lcov.reportPaths=lcov.info diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt index 16c46e54..102dbaba 100644 --- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt +++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt @@ -1,6 +1,5 @@ ajv auditability -[A-Z]+s Bitwarden bot [Cc]onfig @@ -33,6 +32,7 @@ repo [Rr]unbook sed Syft +teardown Terraform toolchain Trufflehog diff --git a/src/logger/src/__tests__/index.test.ts b/src/logger/src/__tests__/index.test.ts index 3c4670f9..82314612 100644 --- a/src/logger/src/__tests__/index.test.ts +++ b/src/logger/src/__tests__/index.test.ts @@ -2,6 +2,7 @@ import pino from "pino"; import { LogContext, Logger, + REDACT_PATHS, extractCorrelationId, logLifecycleEvent, logger, @@ -33,6 +34,7 @@ const mockLoggerMethods = pino() as jest.Mocked>; type PinoConfig = { formatters: { level: (label: string) => { level: string } }; timestamp: () => string; + redact: string[]; }; const capturedPinoConfig = (pino as unknown as jest.Mock).mock .calls[0][0] as PinoConfig; @@ -297,6 +299,10 @@ describe("pino configuration", () => { const result = capturedPinoConfig.timestamp(); expect(result).toMatch(/^,"timestamp":"\d{4}-\d{2}-\d{2}T/); }); + + it("should configure pino with redact paths", () => { + expect(capturedPinoConfig.redact).toEqual(REDACT_PATHS); + }); }); describe("logLifecycleEvent", () => { diff --git a/src/logger/src/__tests__/redaction.test.ts b/src/logger/src/__tests__/redaction.test.ts new file mode 100644 index 00000000..6c95c982 --- /dev/null +++ b/src/logger/src/__tests__/redaction.test.ts @@ -0,0 +1,149 @@ +import pino from "pino"; +import { REDACT_PATHS } from ".."; + +function createTestLogger(): { + logger: pino.Logger; + getOutput: () => string[]; +} { + const lines: string[] = []; + const stream: pino.DestinationStream = { + write(msg: string) { + lines.push(msg); + }, + }; + const logger = pino( + { + level: "debug", + redact: REDACT_PATHS, + }, + stream, + ); + return { logger, getOutput: () => lines }; +} + +function parseLastLine(lines: string[]): Record { + return JSON.parse(lines.at(-1)!) as Record; +} + +describe("pino redaction behaviour", () => { + it("should redact messageReference from log output", () => { + const { getOutput, logger } = createTestLogger(); + + logger.info( + { messageReference: "patient-nhs-1234567890", messageId: "msg-001" }, + "Callback generated", + ); + + const entry = parseLastLine(getOutput()); + expect(entry.messageReference).toBe("[Redacted]"); + expect(entry.messageId).toBe("msg-001"); + }); + + it("should redact channelStatusDescription from log output", () => { + const { getOutput, logger } = createTestLogger(); + + logger.info( + { + channelStatusDescription: "Failed to deliver to patient address", + channelStatus: "failed", + }, + "Channel status", + ); + + const entry = parseLastLine(getOutput()); + expect(entry.channelStatusDescription).toBe("[Redacted]"); + expect(entry.channelStatus).toBe("failed"); + }); + + it("should redact messageStatusDescription from log output", () => { + const { getOutput, logger } = createTestLogger(); + + logger.info( + { + messageStatusDescription: "Delivery failed for recipient", + messageStatus: "failed", + }, + "Message status", + ); + + const entry = parseLastLine(getOutput()); + expect(entry.messageStatusDescription).toBe("[Redacted]"); + expect(entry.messageStatus).toBe("failed"); + }); + + it("should redact nested error HTTP internals from log output", () => { + const { getOutput, logger } = createTestLogger(); + + logger.error( + { + error: { + message: "Request failed", + config: { headers: { Authorization: "Bearer secret-token" } }, + request: { url: "https://example.com/api" }, + response: { status: 500, data: "Internal Server Error" }, + }, + }, + "HTTP error", + ); + + const entry = parseLastLine(getOutput()); + const errorObj = entry.error as Record; + expect(errorObj.config).toBe("[Redacted]"); + expect(errorObj.request).toBe("[Redacted]"); + expect(errorObj.response).toBe("[Redacted]"); + expect(errorObj.message).toBe("Request failed"); + }); + + it("should redact nested err HTTP internals from log output", () => { + const { getOutput, logger } = createTestLogger(); + + logger.error( + { + err: { + message: "Connection refused", + config: { baseURL: "https://api.example.com" }, + request: { method: "POST" }, + response: { data: "sensitive-response-body" }, + }, + }, + "Connection error", + ); + + const entry = parseLastLine(getOutput()); + const errObj = entry.err as Record; + expect(errObj.config).toBe("[Redacted]"); + expect(errObj.request).toBe("[Redacted]"); + expect(errObj.response).toBe("[Redacted]"); + expect(errObj.message).toBe("Connection refused"); + }); + + it("should not redact fields outside the redact list", () => { + const { getOutput, logger } = createTestLogger(); + + logger.info( + { + correlationId: "corr-safe", + clientId: "client-123", + eventType: "status-update", + messageId: "msg-456", + }, + "Normal log entry", + ); + + const entry = parseLastLine(getOutput()); + expect(entry.correlationId).toBe("corr-safe"); + expect(entry.clientId).toBe("client-123"); + expect(entry.eventType).toBe("status-update"); + expect(entry.messageId).toBe("msg-456"); + }); + + it("should handle absent redacted fields without error", () => { + const { getOutput, logger } = createTestLogger(); + + logger.info({ clientId: "client-789" }, "No sensitive fields"); + + const entry = parseLastLine(getOutput()); + expect(entry.clientId).toBe("client-789"); + expect(entry.messageReference).toBeUndefined(); + }); +}); diff --git a/src/logger/src/index.ts b/src/logger/src/index.ts index 7d5c3ede..087f195a 100644 --- a/src/logger/src/index.ts +++ b/src/logger/src/index.ts @@ -1,5 +1,17 @@ import pino from "pino"; +export const REDACT_PATHS = [ + "messageReference", + "channelStatusDescription", + "messageStatusDescription", + "error.config", + "error.request", + "error.response", + "err.config", + "err.request", + "err.response", +]; + export interface LogContext { correlationId?: string; clientId?: string; @@ -22,6 +34,7 @@ const basePinoLogger = pino( }, }, timestamp: () => `,"timestamp":"${new Date().toISOString()}"`, + redact: REDACT_PATHS, }, pino.destination({ sync: true }), ); diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index f0406004..298f4468 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -1,24 +1,26 @@ import { GetQueueAttributesCommand, SQSClient } from "@aws-sdk/client-sqs"; +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import type { MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; import { - awaitSignedCallbacksFromWebhookLogGroup, buildInboundEventQueueUrl, buildLambdaLogGroupName, buildMockClientDlqQueueUrl, - computeExpectedSignature, createCloudWatchLogsClient, - createMessageStatusPublishEvent, createSqsClient, - ensureInboundQueueIsEmpty, getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; +import { awaitSignedCallbacksFromWebhookLogGroup } from "./helpers/cloudwatch"; +import { createMessageStatusPublishEvent } from "./helpers/event-factories"; +import sendEventToDlqAndRedrive from "./helpers/redrive"; +import { computeExpectedSignature } from "./helpers/signature"; +import { + ensureInboundQueueIsEmpty, purgeQueues, - sendEventToDlqAndRedrive, sendSqsEvent, -} from "helpers"; -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +} from "./helpers/sqs"; describe("DLQ Redrive", () => { let sqsClient: SQSClient; diff --git a/tests/integration/event-bus-to-webhook.test.ts b/tests/integration/event-bus-to-webhook.test.ts index 0af0076c..b94ccf0b 100644 --- a/tests/integration/event-bus-to-webhook.test.ts +++ b/tests/integration/event-bus-to-webhook.test.ts @@ -1,29 +1,35 @@ import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import { type ChannelStatusData, type MessageStatusData, type StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; import { - awaitQueueMessage, - awaitQueueMessageByMessageId, buildInboundEventDlqQueueUrl, buildInboundEventQueueUrl, buildLambdaLogGroupName, buildMockClientDlqQueueUrl, - computeExpectedSignature, - createChannelStatusPublishEvent, createCloudWatchLogsClient, - createMessageStatusPublishEvent, createSqsClient, - ensureInboundQueueIsEmpty, getDeploymentDetails, - processChannelStatusEvent, - processMessageStatusEvent, +} from "@nhs-notify-client-callbacks/test-support/helpers"; +import { computeExpectedSignature } from "./helpers/signature"; +import { + createChannelStatusPublishEvent, + createMessageStatusPublishEvent, +} from "./helpers/event-factories"; +import { + awaitQueueMessage, + awaitQueueMessageByMessageId, + ensureInboundQueueIsEmpty, purgeQueues, sendSqsEvent, -} from "helpers"; -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +} from "./helpers/sqs"; +import { + processChannelStatusEvent, + processMessageStatusEvent, +} from "./helpers/status-events"; describe("SQS to Webhook Integration", () => { let sqsClient: SQSClient; diff --git a/tests/integration/helpers/index.ts b/tests/integration/helpers/index.ts deleted file mode 100644 index 7f021566..00000000 --- a/tests/integration/helpers/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from "./deployment"; -export * from "./clients"; -export * from "./sqs"; -export * from "./cloudwatch"; -export { default as sendEventToDlqAndRedrive } from "./redrive"; -export * from "./status-events"; -export * from "./event-factories"; -export * from "./signature"; diff --git a/tests/integration/helpers/sqs.ts b/tests/integration/helpers/sqs.ts index 49f70e78..636eacad 100644 --- a/tests/integration/helpers/sqs.ts +++ b/tests/integration/helpers/sqs.ts @@ -11,8 +11,6 @@ import { import { logger } from "@nhs-notify-client-callbacks/logger"; import { waitUntil } from "async-wait-until"; -import type { DeploymentDetails } from "./deployment"; - const QUEUE_WAIT_TIMEOUT_MS = 60_000; const POLL_INTERVAL_MS = 500; const SQS_MAX_NUMBER_OF_MESSAGES = 1; @@ -33,38 +31,6 @@ function buildReceiveMessageInput( }; } -function buildQueueUrl( - { accountId, component, environment, project, region }: DeploymentDetails, - name: string, - options?: { appendQueueSuffix?: boolean }, -): string { - const appendQueueSuffix = options?.appendQueueSuffix ?? true; - const queueName = appendQueueSuffix - ? `${project}-${environment}-${component}-${name}-queue` - : `${project}-${environment}-${component}-${name}`; - return `https://sqs.${region}.amazonaws.com/${accountId}/${queueName}`; -} - -export function buildInboundEventQueueUrl( - deploymentDetails: DeploymentDetails, -): string { - return buildQueueUrl(deploymentDetails, "inbound-event"); -} - -export function buildInboundEventDlqQueueUrl( - deploymentDetails: DeploymentDetails, -): string { - return buildQueueUrl(deploymentDetails, "inbound-event-dlq", { - appendQueueSuffix: false, - }); -} - -export function buildMockClientDlqQueueUrl( - deploymentDetails: DeploymentDetails, -): string { - return buildQueueUrl(deploymentDetails, "mock-client-dlq"); -} - export async function sendSqsEvent( client: SQSClient, queueUrl: string, diff --git a/tests/integration/infrastructure-exists.test.ts b/tests/integration/infrastructure-exists.test.ts index 14aba92a..37c88dc6 100644 --- a/tests/integration/infrastructure-exists.test.ts +++ b/tests/integration/infrastructure-exists.test.ts @@ -4,7 +4,7 @@ import { buildSubscriptionConfigBucketName, createS3Client, getDeploymentDetails, -} from "helpers"; +} from "@nhs-notify-client-callbacks/test-support/helpers"; describe("Infrastructure exists", () => { let s3Client: S3Client; diff --git a/tests/integration/jest.config.ts b/tests/integration/jest.config.ts index fd9a3fa4..390a96ca 100644 --- a/tests/integration/jest.config.ts +++ b/tests/integration/jest.config.ts @@ -9,9 +9,6 @@ export default { ...(nodeJestConfig.coveragePathIgnorePatterns ?? []), "/helpers/", ], - moduleNameMapper: { - "^helpers$": "/helpers/index", - }, setupFilesAfterEnv: [ ...(nodeJestConfig.setupFilesAfterEnv ?? []), "/jest.setup.ts", diff --git a/tests/integration/jest.global-setup.ts b/tests/integration/jest.global-setup.ts index 3f4d58bd..59d8e0b1 100644 --- a/tests/integration/jest.global-setup.ts +++ b/tests/integration/jest.global-setup.ts @@ -3,7 +3,7 @@ import { buildSubscriptionConfigBucketName, createS3Client, getDeploymentDetails, -} from "./helpers"; +} from "@nhs-notify-client-callbacks/test-support/helpers"; const mockClientSubscriptionKey = "client_subscriptions/mock-client.json"; diff --git a/tests/integration/jest.global-teardown.ts b/tests/integration/jest.global-teardown.ts index 192c1892..32fe7f9b 100644 --- a/tests/integration/jest.global-teardown.ts +++ b/tests/integration/jest.global-teardown.ts @@ -3,7 +3,7 @@ import { buildSubscriptionConfigBucketName, createS3Client, getDeploymentDetails, -} from "./helpers"; +} from "@nhs-notify-client-callbacks/test-support/helpers"; const mockClientSubscriptionKey = "client_subscriptions/mock-client.json"; diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 19a41013..c49a24d8 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -1,25 +1,29 @@ import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import type { MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; import { - awaitAllEmfMetricsInLogGroup, - awaitQueueMessageByMessageId, - awaitSignedCallbacksFromWebhookLogGroup, buildInboundEventDlqQueueUrl, buildInboundEventQueueUrl, buildLambdaLogGroupName, buildMockClientDlqQueueUrl, createCloudWatchLogsClient, - createMessageStatusPublishEvent, createSqsClient, - ensureInboundQueueIsEmpty, getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; +import { + awaitAllEmfMetricsInLogGroup, + awaitSignedCallbacksFromWebhookLogGroup, +} from "./helpers/cloudwatch"; +import { createMessageStatusPublishEvent } from "./helpers/event-factories"; +import { + awaitQueueMessageByMessageId, + ensureInboundQueueIsEmpty, purgeQueues, sendSqsEvent, -} from "helpers"; -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +} from "./helpers/sqs"; describe("Metrics", () => { let sqsClient: SQSClient; diff --git a/tests/integration/package.json b/tests/integration/package.json index 0e67fe21..0ff076c7 100644 --- a/tests/integration/package.json +++ b/tests/integration/package.json @@ -15,7 +15,8 @@ "@aws-sdk/client-cloudwatch-logs": "^3.991.0", "@aws-sdk/client-s3": "^3.821.0", "@aws-sdk/client-sqs": "^3.990.0", - "async-wait-until": "^2.0.12" + "async-wait-until": "^2.0.12", + "@nhs-notify-client-callbacks/test-support": "*" }, "devDependencies": { "@aws-sdk/client-sqs": "^3.990.0", diff --git a/tests/integration/tsconfig.json b/tests/integration/tsconfig.json index a5cc2b81..c0ab68dc 100644 --- a/tests/integration/tsconfig.json +++ b/tests/integration/tsconfig.json @@ -1,12 +1,7 @@ { "compilerOptions": { "baseUrl": ".", - "isolatedModules": true, - "paths": { - "helpers": [ - "./helpers/index" - ] - } + "isolatedModules": true }, "extends": "../../tsconfig.base.json", "include": [ diff --git a/tests/performance/README.md b/tests/performance/README.md new file mode 100644 index 00000000..beb9b209 --- /dev/null +++ b/tests/performance/README.md @@ -0,0 +1,34 @@ +# performance + +Load tests for the client-callbacks service. These tests run against a real deployed AWS environment — they are not unit tests and cannot run locally without a live stack. + +## Prerequisites + +- AWS credentials configured for the target environment +- The service deployed to the target environment + +## Environment Variables + +| Variable | Required | Default | Description | +| --- | --- | --- | --- | +| `ENVIRONMENT` | Yes | — | Target environment name (e.g. `dev`) | +| `AWS_ACCOUNT_ID` | Yes | — | AWS account ID for the target environment | +| `AWS_REGION` | No | `eu-west-2` | AWS region | +| `PROJECT` | No | `nhs` | Project name prefix used in resource naming | +| `COMPONENT` | No | `callbacks` | Component name used in resource naming | + +## Running + +From the repository root: + +```bash +ENVIRONMENT=dev AWS_ACCOUNT_ID=123456789012 npm run test:performance --workspace tests/performance +``` + +## What the Tests Do + +The global setup creates a test client subscription config in the S3 subscription config bucket. + +The load test sends ~3,000 events/s to the SQS inbound queue for 30 seconds, then reads CloudWatch Logs to assert that the p95 Lambda processing time is below 500ms. + +The global teardown removes the test client subscription config from S3. diff --git a/tests/performance/helpers/cloudwatch.ts b/tests/performance/helpers/cloudwatch.ts new file mode 100644 index 00000000..498523af --- /dev/null +++ b/tests/performance/helpers/cloudwatch.ts @@ -0,0 +1,156 @@ +import { + CloudWatchLogsClient, + FilterLogEventsCommand, + GetQueryResultsCommand, + StartQueryCommand, +} from "@aws-sdk/client-cloudwatch-logs"; +import { waitUntil } from "async-wait-until"; + +const POLL_INTERVAL_MS = 2000; +const COLLECT_TIMEOUT_MS = 120_000; + +type BatchCompletedLogEntry = { + processingTimeMs: number; + batchSize: number; + successful: number; + failed: number; + filtered: number; +}; + +export async function collectBatchProcessingTimes( + client: CloudWatchLogsClient, + logGroupName: string, + expectedCount: number, + startTime: number, +): Promise { + const collected: number[] = []; + + await waitUntil( + async () => { + const response = await client.send( + new FilterLogEventsCommand({ + logGroupName, + startTime, + filterPattern: '{ $.msg = "batch-processing-completed" }', + }), + ); + + for (const event of response.events ?? []) { + if (event.message) { + try { + const entry = JSON.parse(event.message) as BatchCompletedLogEntry; + if (typeof entry.processingTimeMs === "number") { + collected.push(entry.processingTimeMs); + } + } catch { + // skip unparseable entries + } + } + } + + return collected.length >= expectedCount; + }, + { timeout: COLLECT_TIMEOUT_MS, intervalBetweenAttempts: POLL_INTERVAL_MS }, + ); + + return collected; +} + +export function computePercentile( + samples: number[], + percentile: number, +): number { + if (samples.length === 0) { + throw new Error("Cannot compute percentile of empty array"); + } + + const sorted = [...samples].sort((a, b) => a - b); + const index = Math.ceil((percentile / 100) * sorted.length) - 1; + return sorted[Math.max(0, index)]; +} + +const INSIGHTS_QUERY_TIMEOUT_MS = 60_000; +const INSIGHTS_COLLECT_TIMEOUT_MS = 300_000; + +async function runInsightsQuery( + client: CloudWatchLogsClient, + logGroupName: string, + startTimeSec: number, + endTimeSec: number, + percentile: number, +): Promise<{ count: number; percentileMs: number } | null> { + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupName, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + 'filter msg = "batch-processing-completed"', + `| stats count(*) as eventCount, pct(processingTimeMs, ${percentile}) as p`, + ].join("\n"), + }), + ); + + if (!queryId) return null; + + const deadline = Date.now() + INSIGHTS_QUERY_TIMEOUT_MS; + + while (Date.now() < deadline) { + await new Promise((resolve) => { + setTimeout(resolve, 2000); + }); + + const response = await client.send(new GetQueryResultsCommand({ queryId })); + + if (response.status === "Failed" || response.status === "Cancelled") { + return null; + } + + if (response.status === "Complete") { + const row = response.results?.[0]; + if (!row) return null; + + return { + count: Number(row.find((f) => f.field === "eventCount")?.value ?? 0), + percentileMs: Number(row.find((f) => f.field === "p")?.value ?? 0), + }; + } + } + + return null; +} + +export async function waitForBatchProcessingPercentile( + client: CloudWatchLogsClient, + logGroupName: string, + testStartTime: number, + expectedCount: number, + percentile: number, +): Promise<{ count: number; percentileMs: number }> { + const startTimeSec = Math.floor(testStartTime / 1000); + let result = { count: 0, percentileMs: 0 }; + + await waitUntil( + async () => { + const endTimeSec = Math.floor((Date.now() + 60_000) / 1000); + const queryResult = await runInsightsQuery( + client, + logGroupName, + startTimeSec, + endTimeSec, + percentile, + ); + + if (!queryResult) return false; + + result = queryResult; + return result.count >= expectedCount; + }, + { + timeout: INSIGHTS_COLLECT_TIMEOUT_MS, + intervalBetweenAttempts: POLL_INTERVAL_MS, + }, + ); + + return result; +} diff --git a/tests/performance/helpers/deployment.ts b/tests/performance/helpers/deployment.ts new file mode 100644 index 00000000..5d6ee82e --- /dev/null +++ b/tests/performance/helpers/deployment.ts @@ -0,0 +1,10 @@ +import { + type DeploymentDetails, + buildLambdaLogGroupName, +} from "@nhs-notify-client-callbacks/test-support/helpers/deployment"; + +export function buildTransformFilterLambdaLogGroupName( + details: DeploymentDetails, +): string { + return buildLambdaLogGroupName(details, "client-transform-filter"); +} diff --git a/tests/performance/helpers/event-factories.ts b/tests/performance/helpers/event-factories.ts new file mode 100644 index 00000000..17cc98e1 --- /dev/null +++ b/tests/performance/helpers/event-factories.ts @@ -0,0 +1,81 @@ +import type { + ChannelStatusData, + MessageStatusData, + StatusPublishEvent, +} from "@nhs-notify-client-callbacks/models"; +import { EventTypes } from "@nhs-notify-client-callbacks/models"; + +export function createMessageStatusPublishEvent( + overrides?: Partial, +): StatusPublishEvent { + const messageId = overrides?.messageId ?? crypto.randomUUID(); + const messageReference = + overrides?.messageReference ?? `ref-${crypto.randomUUID()}`; + + const data: MessageStatusData = { + clientId: "perf-test-client", + messageId, + messageReference, + messageStatus: "DELIVERED", + channels: [{ type: "NHSAPP", channelStatus: "DELIVERED" }], + timestamp: new Date().toISOString(), + routingPlan: { + id: crypto.randomUUID(), + name: "perf-test-routing-plan", + version: "v1.0.0", + createdDate: new Date().toISOString(), + }, + ...overrides, + }; + + return { + specversion: "1.0", + id: crypto.randomUUID(), + source: "/nhs/england/notify/development/primary/data-plane/messaging", + subject: `customer/${crypto.randomUUID()}/message/${messageId}`, + type: EventTypes.MESSAGE_STATUS_PUBLISHED, + time: new Date().toISOString(), + datacontenttype: "application/json", + dataschema: + "https://notify.nhs.uk/schemas/message-status-published-v1.json", + traceparent: "00-4d678967f96e353c07a0a31c1849b500-07f83ba58dd8df70-01", + data, + }; +} + +export function createChannelStatusPublishEvent( + overrides?: Partial, +): StatusPublishEvent { + const messageId = overrides?.messageId ?? crypto.randomUUID(); + const messageReference = + overrides?.messageReference ?? `ref-${crypto.randomUUID()}`; + + const data: ChannelStatusData = { + clientId: "perf-test-client", + messageId, + messageReference, + channel: "NHSAPP", + channelStatus: "DELIVERED", + channelStatusDescription: "perf-test", + supplierStatus: "delivered", + cascadeType: "primary", + cascadeOrder: 0, + timestamp: new Date().toISOString(), + retryCount: 0, + ...overrides, + }; + + return { + specversion: "1.0", + id: crypto.randomUUID(), + source: "/nhs/england/notify/development/primary/data-plane/messaging", + subject: `customer/${crypto.randomUUID()}/message/${messageId}`, + type: EventTypes.CHANNEL_STATUS_PUBLISHED, + time: new Date().toISOString(), + datacontenttype: "application/json", + dataschema: + "https://notify.nhs.uk/schemas/channel-status-published-v1.json", + traceparent: "00-4d678967f96e353c07a0a31c1849b500-07f83ba58dd8df70-01", + data, + }; +} diff --git a/tests/performance/helpers/index.ts b/tests/performance/helpers/index.ts new file mode 100644 index 00000000..194022a3 --- /dev/null +++ b/tests/performance/helpers/index.ts @@ -0,0 +1,4 @@ +export * from "./cloudwatch"; +export * from "./deployment"; +export * from "./event-factories"; +export * from "./sqs"; diff --git a/tests/performance/helpers/sqs.ts b/tests/performance/helpers/sqs.ts new file mode 100644 index 00000000..e8d5b171 --- /dev/null +++ b/tests/performance/helpers/sqs.ts @@ -0,0 +1,72 @@ +import { + SQSClient, + SendMessageBatchCommand, + SendMessageCommand, +} from "@aws-sdk/client-sqs"; +import type { StatusPublishEvent } from "@nhs-notify-client-callbacks/models"; + +export async function sendSqsEvent( + client: SQSClient, + queueUrl: string, + event: StatusPublishEvent, +): Promise { + await client.send( + new SendMessageCommand({ + QueueUrl: queueUrl, + MessageBody: JSON.stringify(event), + }), + ); +} + +const SQS_MAX_BATCH_SIZE = 10; + +export async function sendSqsBatch( + client: SQSClient, + queueUrl: string, + events: StatusPublishEvent[], +): Promise { + await client.send( + new SendMessageBatchCommand({ + QueueUrl: queueUrl, + Entries: events.map((event, index) => ({ + Id: String(index), + MessageBody: JSON.stringify(event), + })), + }), + ); +} + +export async function generateSqsLoad( + client: SQSClient, + queueUrl: string, + targetEventsPerSecond: number, + durationSeconds: number, + eventFactory: () => StatusPublishEvent, +): Promise<{ sent: number; durationMs: number }> { + const batchesPerSecond = Math.ceil( + targetEventsPerSecond / SQS_MAX_BATCH_SIZE, + ); + const start = Date.now(); + let sent = 0; + + for (let second = 0; second < durationSeconds; second++) { + const waveStart = Date.now(); + + const results = await Promise.all( + Array.from({ length: batchesPerSecond }, () => { + const batch = Array.from({ length: SQS_MAX_BATCH_SIZE }, eventFactory); + return sendSqsBatch(client, queueUrl, batch).then(() => batch.length); + }), + ); + sent += results.reduce((sum, count) => sum + count, 0); + + const remaining = 1000 - (Date.now() - waveStart); + if (remaining > 0 && second < durationSeconds - 1) { + await new Promise((resolve) => { + setTimeout(resolve, remaining); + }); + } + } + + return { sent, durationMs: Date.now() - start }; +} diff --git a/tests/performance/jest.config.ts b/tests/performance/jest.config.ts new file mode 100644 index 00000000..bdc5714a --- /dev/null +++ b/tests/performance/jest.config.ts @@ -0,0 +1,16 @@ +import { nodeJestConfig } from "../../jest.config.base"; + +export default { + ...nodeJestConfig, + modulePaths: [""], + globalSetup: "/jest.global-setup.ts", + globalTeardown: "/jest.global-teardown.ts", + collectCoverage: false, + moduleNameMapper: { + "^helpers$": "/helpers/index", + }, + // Run performance tests serially to avoid queue contention + maxWorkers: 1, + // Force exit after tests complete — real AWS SDK clients keep connections alive + forceExit: true, +}; diff --git a/tests/performance/jest.global-setup.ts b/tests/performance/jest.global-setup.ts new file mode 100644 index 00000000..fb13302d --- /dev/null +++ b/tests/performance/jest.global-setup.ts @@ -0,0 +1,61 @@ +import { PutObjectCommand } from "@aws-sdk/client-s3"; +import { + buildSubscriptionConfigBucketName, + createS3Client, + getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; + +const PERF_CLIENT_SUBSCRIPTION_KEY = + "client_subscriptions/perf-test-client.json"; + +const perfClientSubscriptionBody = JSON.stringify({ + clientId: "perf-test-client", + subscriptions: [ + { + subscriptionId: "perf-test-client-message", + subscriptionType: "MessageStatus", + messageStatuses: ["DELIVERED"], + targetIds: ["445527ff-277b-43a4-a4b0-15eedbd71598"], + }, + { + subscriptionId: "perf-test-client-channel", + subscriptionType: "ChannelStatus", + channelStatuses: ["DELIVERED"], + channelType: "NHSAPP", + supplierStatuses: ["delivered"], + targetIds: ["445527ff-277b-43a4-a4b0-15eedbd71598"], + }, + ], + targets: [ + { + type: "API", + targetId: "445527ff-277b-43a4-a4b0-15eedbd71598", + invocationEndpoint: "https://perf-test.internal/webhook", + invocationMethod: "POST", + invocationRateLimit: 100, + apiKey: { + headerName: "x-api-key", + headerValue: "perf-test-api-key", + }, + }, + ], +}); + +export default async function globalSetup() { + const deploymentDetails = getDeploymentDetails(); + const bucketName = buildSubscriptionConfigBucketName(deploymentDetails); + const client = createS3Client(deploymentDetails); + + try { + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: PERF_CLIENT_SUBSCRIPTION_KEY, + ContentType: "application/json", + Body: perfClientSubscriptionBody, + }), + ); + } finally { + client.destroy(); + } +} diff --git a/tests/performance/jest.global-teardown.ts b/tests/performance/jest.global-teardown.ts new file mode 100644 index 00000000..3fca4ddb --- /dev/null +++ b/tests/performance/jest.global-teardown.ts @@ -0,0 +1,26 @@ +import { DeleteObjectCommand } from "@aws-sdk/client-s3"; +import { + buildSubscriptionConfigBucketName, + createS3Client, + getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; + +const PERF_CLIENT_SUBSCRIPTION_KEY = + "client_subscriptions/perf-test-client.json"; + +export default async function globalTeardown() { + const deploymentDetails = getDeploymentDetails(); + const bucketName = buildSubscriptionConfigBucketName(deploymentDetails); + const client = createS3Client(deploymentDetails); + + try { + await client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: PERF_CLIENT_SUBSCRIPTION_KEY, + }), + ); + } finally { + client.destroy(); + } +} diff --git a/tests/performance/lambda-throughput.test.ts b/tests/performance/lambda-throughput.test.ts new file mode 100644 index 00000000..5a543ab6 --- /dev/null +++ b/tests/performance/lambda-throughput.test.ts @@ -0,0 +1,76 @@ +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { SQSClient } from "@aws-sdk/client-sqs"; +import { + buildInboundEventQueueUrl, + createCloudWatchLogsClient, + createSqsClient, + getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; +import { + buildTransformFilterLambdaLogGroupName, + createMessageStatusPublishEvent, + generateSqsLoad, + waitForBatchProcessingPercentile, +} from "helpers"; + +const TARGET_EPS = 3000; +const LOAD_DURATION_SECONDS = 30; +const P95_LATENCY_THRESHOLD_MS = 500; + +describe("Lambda throughput and latency under load", () => { + let sqsClient: SQSClient; + let cloudWatchClient: CloudWatchLogsClient; + let inboundQueueUrl: string; + let lambdaLogGroupName: string; + + beforeAll(() => { + const deploymentDetails = getDeploymentDetails(); + + sqsClient = createSqsClient(deploymentDetails); + cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); + inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); + lambdaLogGroupName = + buildTransformFilterLambdaLogGroupName(deploymentDetails); + }); + + afterAll(() => { + sqsClient.destroy(); + cloudWatchClient.destroy(); + }); + + it(`should sustain ~${TARGET_EPS} events/s for ${LOAD_DURATION_SECONDS}s with p95 Lambda processing time below ${P95_LATENCY_THRESHOLD_MS}ms`, async () => { + const testStartTime = Date.now(); + + const { durationMs, sent } = await generateSqsLoad( + sqsClient, + inboundQueueUrl, + TARGET_EPS, + LOAD_DURATION_SECONDS, + createMessageStatusPublishEvent, + ); + + const achievedEps = Math.round(sent / (durationMs / 1000)); + console.log( + `Load generation: ${sent} events in ${durationMs}ms (${achievedEps} eps achieved)`, + ); + + // Accept ≥90% of sent events processed — accounts for any events routed to DLQ + // due to transient Lambda errors under concurrency pressure. + const minExpectedCount = Math.floor(sent * 0.9); + + const { count, percentileMs } = await waitForBatchProcessingPercentile( + cloudWatchClient, + lambdaLogGroupName, + testStartTime, + minExpectedCount, + 95, + ); + + console.log( + `Processing: ${count} events logged, p95 Lambda processing time: ${percentileMs}ms`, + ); + + expect(count).toBeGreaterThanOrEqual(minExpectedCount); + expect(percentileMs).toBeLessThan(P95_LATENCY_THRESHOLD_MS); + }, 600_000); +}); diff --git a/tests/performance/package.json b/tests/performance/package.json new file mode 100644 index 00000000..fc75b276 --- /dev/null +++ b/tests/performance/package.json @@ -0,0 +1,26 @@ +{ + "name": "nhs-notify-client-callbacks-performance-tests", + "version": "0.0.1", + "private": true, + "scripts": { + "test:performance": "jest", + "test:unit": "echo 'No unit tests in performance workspace - skipping'", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.991.0", + "@aws-sdk/client-s3": "^3.821.0", + "@aws-sdk/client-sqs": "^3.990.0", + "@nhs-notify-client-callbacks/models": "*", + "async-wait-until": "^2.0.12", + "@nhs-notify-client-callbacks/test-support": "*" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "typescript": "^5.8.2" + } +} diff --git a/tests/performance/tsconfig.json b/tests/performance/tsconfig.json new file mode 100644 index 00000000..a5cc2b81 --- /dev/null +++ b/tests/performance/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "isolatedModules": true, + "paths": { + "helpers": [ + "./helpers/index" + ] + } + }, + "extends": "../../tsconfig.base.json", + "include": [ + "**/*.ts", + "jest.config.ts" + ] +} diff --git a/tests/integration/helpers/clients.ts b/tests/test-support/helpers/clients.ts similarity index 100% rename from tests/integration/helpers/clients.ts rename to tests/test-support/helpers/clients.ts diff --git a/tests/integration/helpers/deployment.ts b/tests/test-support/helpers/deployment.ts similarity index 100% rename from tests/integration/helpers/deployment.ts rename to tests/test-support/helpers/deployment.ts diff --git a/tests/test-support/helpers/index.ts b/tests/test-support/helpers/index.ts new file mode 100644 index 00000000..c20c8f36 --- /dev/null +++ b/tests/test-support/helpers/index.ts @@ -0,0 +1,3 @@ +export * from "./clients"; +export * from "./deployment"; +export * from "./sqs"; diff --git a/tests/test-support/helpers/sqs.ts b/tests/test-support/helpers/sqs.ts new file mode 100644 index 00000000..8ac11d75 --- /dev/null +++ b/tests/test-support/helpers/sqs.ts @@ -0,0 +1,33 @@ +import type { DeploymentDetails } from "./deployment"; + +function buildQueueUrl( + { accountId, component, environment, project, region }: DeploymentDetails, + name: string, + options?: { appendQueueSuffix?: boolean }, +): string { + const appendQueueSuffix = options?.appendQueueSuffix ?? true; + const queueName = appendQueueSuffix + ? `${project}-${environment}-${component}-${name}-queue` + : `${project}-${environment}-${component}-${name}`; + return `https://sqs.${region}.amazonaws.com/${accountId}/${queueName}`; +} + +export function buildInboundEventQueueUrl( + deploymentDetails: DeploymentDetails, +): string { + return buildQueueUrl(deploymentDetails, "inbound-event"); +} + +export function buildInboundEventDlqQueueUrl( + deploymentDetails: DeploymentDetails, +): string { + return buildQueueUrl(deploymentDetails, "inbound-event-dlq", { + appendQueueSuffix: false, + }); +} + +export function buildMockClientDlqQueueUrl( + deploymentDetails: DeploymentDetails, +): string { + return buildQueueUrl(deploymentDetails, "mock-client-dlq"); +} diff --git a/tests/test-support/package.json b/tests/test-support/package.json new file mode 100644 index 00000000..0667dde9 --- /dev/null +++ b/tests/test-support/package.json @@ -0,0 +1,24 @@ +{ + "name": "@nhs-notify-client-callbacks/test-support", + "version": "0.0.1", + "private": true, + "exports": { + "./helpers": "./helpers/index.ts", + "./helpers/*": "./helpers/*.ts" + }, + "scripts": { + "test:unit": "echo 'No unit tests in test-support workspace - skipping'", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.991.0", + "@aws-sdk/client-s3": "^3.821.0", + "@aws-sdk/client-sqs": "^3.990.0" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "typescript": "^5.8.2" + } +} diff --git a/tests/test-support/tsconfig.json b/tests/test-support/tsconfig.json new file mode 100644 index 00000000..90eea1bd --- /dev/null +++ b/tests/test-support/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "isolatedModules": true + }, + "extends": "../../tsconfig.base.json", + "include": [ + "**/*.ts" + ] +} diff --git a/tools/client-subscriptions-management/client-subscriptions-management b/tools/client-subscriptions-management/client-subscriptions-management new file mode 100755 index 00000000..f8bdc80c --- /dev/null +++ b/tools/client-subscriptions-management/client-subscriptions-management @@ -0,0 +1,3 @@ +#!/bin/bash +cd $(dirname "${BASH_SOURCE[0]}") +npx tsx ./src/entrypoint/cli/index.ts "$@"