diff --git a/.editorconfig b/.editorconfig index fd5c3548..e9bc6fac 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,6 +14,9 @@ indent_size = unset [*.py] indent_size = 4 +[*.ts] +quote_type = single + [{Dockerfile,Dockerfile.}*] indent_size = 4 diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index ac8f25b4..dda24569 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -13,6 +13,8 @@ No requirements. | [apim\_auth\_token\_url](#input\_apim\_auth\_token\_url) | URL to generate an APIM auth token | `string` | `"https://int.api.service.nhs.uk/oauth2/token"` | no | | [apim\_base\_url](#input\_apim\_base\_url) | The URL used to send requests to PDM | `string` | `"https://int.api.service.nhs.uk"` | no | | [apim\_keygen\_schedule](#input\_apim\_keygen\_schedule) | Schedule to refresh key pairs if necessary | `string` | `"cron(0 14 * * ? *)"` | no | +| [athena\_query\_max\_polling\_attempts](#input\_athena\_query\_max\_polling\_attempts) | The number of times athena will be polled to check if a query is completed | `number` | `50` | no | +| [athena\_query\_polling\_time\_seconds](#input\_athena\_query\_polling\_time\_seconds) | The amount of time in seconds to wait between each athena poll | `number` | `15` | no | | [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes | | [aws\_account\_type](#input\_aws\_account\_type) | The AWS Account Type | `string` | n/a | yes | | [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"dl"` | no | @@ -65,6 +67,7 @@ No requirements. | [print\_sender](#module\_print\_sender) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [print\_status\_handler](#module\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [report\_event\_transformer](#module\_report\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | +| [report\_generator](#module\_report\_generator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-lambda.zip | n/a | | [report\_scheduler](#module\_report\_scheduler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [s3bucket\_cf\_logs](#module\_s3bucket\_cf\_logs) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-s3bucket.zip | n/a | | [s3bucket\_file\_quarantine](#module\_s3bucket\_file\_quarantine) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-s3bucket.zip | n/a | @@ -84,6 +87,7 @@ No requirements. | [sqs\_print\_analyser](#module\_sqs\_print\_analyser) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_print\_sender](#module\_sqs\_print\_sender) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_print\_status\_handler](#module\_sqs\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | +| [sqs\_report\_generator](#module\_sqs\_report\_generator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_scanner](#module\_sqs\_scanner) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | diff --git a/infrastructure/terraform/components/dl/cloudwatch_event_rule_generate_report.tf b/infrastructure/terraform/components/dl/cloudwatch_event_rule_generate_report.tf new file mode 100644 index 00000000..3d14b8d2 --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_event_rule_generate_report.tf @@ -0,0 +1,18 @@ +resource "aws_cloudwatch_event_rule" "generate_report" { + name = "${local.csi}-generate-report" + description = "Generate Report event rule" + event_bus_name = aws_cloudwatch_event_bus.main.name + event_pattern = jsonencode({ + "detail" : { + "type" : [ + "uk.nhs.notify.digital.letters.reporting.generate.report.v1" + ], + } + }) +} + +resource "aws_cloudwatch_event_target" "generate_report_report_generator" { + rule = aws_cloudwatch_event_rule.generate_report.name + arn = module.sqs_report_generator.sqs_queue_arn + event_bus_name = aws_cloudwatch_event_bus.main.name +} diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf index 81f851df..a2e56de4 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf @@ -44,6 +44,10 @@ resource "aws_glue_catalog_table" "event_record" { name = "type" type = "string" } + columns { + name = "letterstatus" + type = "string" + } } partition_keys { diff --git a/infrastructure/terraform/components/dl/lambda_event_source_mapping_report_generator.tf b/infrastructure/terraform/components/dl/lambda_event_source_mapping_report_generator.tf new file mode 100644 index 00000000..432181a6 --- /dev/null +++ b/infrastructure/terraform/components/dl/lambda_event_source_mapping_report_generator.tf @@ -0,0 +1,10 @@ +resource "aws_lambda_event_source_mapping" "report_generator" { + event_source_arn = module.sqs_report_generator.sqs_queue_arn + function_name = module.report_generator.function_name + batch_size = var.queue_batch_size + maximum_batching_window_in_seconds = var.queue_batch_window_seconds + + function_response_types = [ + "ReportBatchItemFailures" + ] +} diff --git a/infrastructure/terraform/components/dl/module_lambda_report_generator.tf b/infrastructure/terraform/components/dl/module_lambda_report_generator.tf new file mode 100644 index 00000000..49dcb551 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_lambda_report_generator.tf @@ -0,0 +1,156 @@ +module "report_generator" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-lambda.zip" + + function_name = "report-generator" + description = "A function to generate reports from an event" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.report_generator_lambda.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "report-generator/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 128 + timeout = 60 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.log_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + "ATHENA_WORKGROUP" = aws_athena_workgroup.reporting.name + "ATHENA_DATABASE" = aws_glue_catalog_database.reporting.name + "EVENT_PUBLISHER_EVENT_BUS_ARN" = aws_cloudwatch_event_bus.main.arn + "EVENT_PUBLISHER_DLQ_URL" = module.sqs_event_publisher_errors.sqs_queue_url + "MAX_POLL_LIMIT" = var.athena_query_max_polling_attempts + "REPORTING_BUCKET" = module.s3bucket_reporting.bucket + "REPORT_NAME" = "completed_communications" + "WAIT_FOR_IN_SECONDS" = var.athena_query_polling_time_seconds + } +} + +data "aws_iam_policy_document" "report_generator_lambda" { + statement { + sid = "AllowS3Get" + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:GetBucketLocation", + "s3:ListBucket" + ] + + resources = [ + "${module.s3bucket_reporting.arn}/*", + "${module.s3bucket_reporting.arn}" + ] + } + + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } + + statement { + sid = "AllowAthenaAccess" + effect = "Allow" + + actions = [ + "athena:StartQueryExecution", + "athena:GetQueryResults", + "athena:GetQueryExecution" + ] + + resources = [ + "arn:aws:athena:${var.region}:${var.aws_account_id}:workgroup/${aws_athena_workgroup.reporting.name}" + ] + } + + statement { + sid = "AllowGlueAccess" + effect = "Allow" + + actions = [ + "glue:GetTable", + "glue:GetDatabase", + "glue:GetPartition", + "glue:GetPartitions", + ] + + resources = [ + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + "arn:aws:glue:${var.region}:${var.aws_account_id}:database/${aws_glue_catalog_database.reporting.name}", + "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.reporting.name}/*" + ] + } + + statement { + sid = "SQSPermissionsReportGeneratorQueue" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ] + + resources = [ + module.sqs_report_generator.sqs_queue_arn, + ] + } + + statement { + sid = "PutEvents" + effect = "Allow" + + actions = [ + "events:PutEvents", + ] + + resources = [ + aws_cloudwatch_event_bus.main.arn, + ] + } + + statement { + sid = "SQSPermissionsEventPublisherDLQ" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch", + ] + + resources = [ + module.sqs_event_publisher_errors.sqs_queue_arn, + ] + } +} diff --git a/infrastructure/terraform/components/dl/module_sqs_report_generator.tf b/infrastructure/terraform/components/dl/module_sqs_report_generator.tf new file mode 100644 index 00000000..a494ee60 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_sqs_report_generator.tf @@ -0,0 +1,44 @@ +module "sqs_report_generator" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + name = "report-generator" + + sqs_kms_key_arn = module.kms.key_arn + + visibility_timeout_seconds = 60 + + create_dlq = true + + sqs_policy_overload = data.aws_iam_policy_document.sqs_report_generator.json +} + +data "aws_iam_policy_document" "sqs_report_generator" { + statement { + sid = "AllowEventBridgeToSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-report-generator-queue" + ] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.generate_report.arn] + } + } +} diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index 844617b2..feb61151 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -219,3 +219,15 @@ variable "metadata_refresh_schedule" { description = "Schedule for refreshing reporting metadata." default = "cron(10 6-22 * * ? *)" # 10 minutes past the hour, between 06:00 - 22:00 } + +variable "athena_query_max_polling_attempts" { + type = number + description = "The number of times athena will be polled to check if a query is completed" + default = 50 +} + +variable "athena_query_polling_time_seconds" { + type = number + description = "The amount of time in seconds to wait between each athena poll" + default = 15 +} diff --git a/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts b/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts index f65b566b..2565a77b 100644 --- a/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts +++ b/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts @@ -51,6 +51,7 @@ describe('Firehose Handler', () => { reasonText: digitalLettersEvent.data.reasonText, senderId: digitalLettersEvent.data.senderId, supplierId: digitalLettersEvent.data.supplierId, + letterStatus: digitalLettersEvent.data.status, time: digitalLettersEvent.time, type: digitalLettersEvent.type, }); diff --git a/lambdas/report-event-transformer/src/__tests__/test-data.ts b/lambdas/report-event-transformer/src/__tests__/test-data.ts index 31927313..e74dc86e 100644 --- a/lambdas/report-event-transformer/src/__tests__/test-data.ts +++ b/lambdas/report-event-transformer/src/__tests__/test-data.ts @@ -24,6 +24,7 @@ export const digitalLettersEvent = { reasonCode: 'FAILURE001', reasonText: 'Letter has too many pages', senderId: 'sender1', + status: 'DISPATCHED', supplierId: 'supplier1', }, } as DigitalLettersEvent; diff --git a/lambdas/report-event-transformer/src/apis/firehose-handler.ts b/lambdas/report-event-transformer/src/apis/firehose-handler.ts index 5b46340e..04273b04 100644 --- a/lambdas/report-event-transformer/src/apis/firehose-handler.ts +++ b/lambdas/report-event-transformer/src/apis/firehose-handler.ts @@ -62,6 +62,7 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { reasonCode, reasonText, senderId, + status, supplierId, } = validatedRecord.event.data; const { time, type } = validatedRecord.event; @@ -72,6 +73,7 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { pageCount, senderId, supplierId, + letterStatus: status, reasonCode, reasonText, time, diff --git a/lambdas/report-event-transformer/src/types/events.ts b/lambdas/report-event-transformer/src/types/events.ts index d88818f6..0410780c 100644 --- a/lambdas/report-event-transformer/src/types/events.ts +++ b/lambdas/report-event-transformer/src/types/events.ts @@ -8,6 +8,7 @@ export const $DigitalLettersEvent = z.object({ reasonText: z.string().optional(), senderId: z.string(), supplierId: z.string().optional(), + status: z.string().optional(), }), time: z.string(), type: z.string(), @@ -20,6 +21,7 @@ export type FlatDigitalLettersEvent = { pageCount?: number; senderId: string; supplierId?: string; + letterStatus?: string; reasonCode?: string; reasonText?: string; time: string; diff --git a/lambdas/report-generator/jest.config.ts b/lambdas/report-generator/jest.config.ts new file mode 100644 index 00000000..6d5f3fe9 --- /dev/null +++ b/lambdas/report-generator/jest.config.ts @@ -0,0 +1,14 @@ +import { baseJestConfig } from '../../jest.config.base'; + +const config = baseJestConfig; + +config.coverageThreshold = { + global: { + branches: 88, + functions: 100, + lines: 97, + statements: 97, + }, +}; + +export default config; diff --git a/lambdas/report-generator/package.json b/lambdas/report-generator/package.json new file mode 100644 index 00000000..34b50efc --- /dev/null +++ b/lambdas/report-generator/package.json @@ -0,0 +1,24 @@ +{ + "dependencies": { + "@aws-sdk/client-athena": "^3.984.0", + "digital-letters-events": "^0.0.1", + "utils": "^0.0.1" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "typescript": "^5.9.3" + }, + "name": "nhs-notify-digital-letters-report-generator", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts && cp -r src/queries dist/", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/report-generator/src/__tests__/apis/sqs-trigger-lambda.test.ts b/lambdas/report-generator/src/__tests__/apis/sqs-trigger-lambda.test.ts new file mode 100644 index 00000000..9e802812 --- /dev/null +++ b/lambdas/report-generator/src/__tests__/apis/sqs-trigger-lambda.test.ts @@ -0,0 +1,337 @@ +import { randomUUID } from 'node:crypto'; +import { GenerateReport } from 'digital-letters-events'; +import { EventPublisher, Logger } from 'utils'; +import type { SQSEvent, SQSRecord } from 'aws-lambda'; +import type { + ReportGenerator, + ReportGeneratorResult, +} from 'app/report-generator'; +import { createHandler } from 'apis/sqs-trigger-lambda'; + +jest.mock('node:crypto'); + +const mockUuid = '123e4567-e89b-12d3-a456-426614174000'; + +const createMockSQSRecord = ( + messageId: string, + event: Partial, +): SQSRecord => ({ + messageId, + receiptHandle: 'receipt-handle', + body: JSON.stringify({ + detail: { + id: event.id || mockUuid, + source: + event.source || + '/nhs/england/notify/development/primary/data-plane/digitalletters/reporting', + specversion: event.specversion || '1.0', + type: + event.type || + 'uk.nhs.notify.digital.letters.reporting.generate.report.v1', + dataschema: + event.dataschema || + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-reporting-generate-report-data.schema.json', + time: event.time || new Date().toISOString(), + recordedtime: event.recordedtime || new Date().toISOString(), + subject: event.subject || 'customer/5661de82-7453-44a1-9922-e0c98e5411c1', + traceparent: + event.traceparent || + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + severitynumber: event.severitynumber || 2, + data: event.data || { + senderId: 'fce3ddee-2aca-4b2e-90a8-ce4da3787792', + reportDate: '2025-01-01', + }, + }, + }), + attributes: { + ApproximateReceiveCount: '1', + SentTimestamp: '1234567890', + SenderId: 'sender-id', + ApproximateFirstReceiveTimestamp: '1234567890', + }, + messageAttributes: {}, + md5OfBody: 'md5', + eventSource: 'aws:sqs', + eventSourceARN: 'arn:aws:sqs:region:account:queue', + awsRegion: 'us-east-1', +}); + +const createMockSQSEvent = (records: SQSRecord[]): SQSEvent => ({ + Records: records, +}); + +describe('sqs-trigger-lambda', () => { + let mockReportGenerator: jest.Mocked; + let mockEventPublisher: jest.Mocked; + let mockLogger: jest.Mocked; + + beforeEach(() => { + jest.clearAllMocks(); + jest.mocked(randomUUID).mockReturnValue(mockUuid); + + mockReportGenerator = { + generate: jest.fn(), + } as unknown as jest.Mocked; + + mockEventPublisher = { + sendEvents: jest.fn().mockResolvedValue([]), + } as unknown as jest.Mocked; + + mockLogger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as unknown as jest.Mocked; + }); + + describe('createHandler', () => { + it('should process valid SQS records successfully', async () => { + const reportUri = 's3://bucket/report.csv'; + const mockResult: ReportGeneratorResult = { + outcome: 'generated', + reportUri, + }; + mockReportGenerator.generate.mockResolvedValue(mockResult); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + const response = await handler(sqsEvent); + + expect(response.batchItemFailures).toEqual([]); + expect(mockReportGenerator.generate).toHaveBeenCalledTimes(1); + expect(mockEventPublisher.sendEvents).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + expect.objectContaining({ + description: 'Processed SQS Event.', + retrieved: 1, + generated: 1, + failed: 0, + }), + ); + }); + + it('should handle invalid JSON in SQS record body', async () => { + const sqsEvent: SQSEvent = { + Records: [ + { + ...createMockSQSRecord('msg-1', {}), + body: 'invalid-json', + }, + ], + }; + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + const response = await handler(sqsEvent); + + expect(response.batchItemFailures).toEqual([{ itemIdentifier: 'msg-1' }]); + expect(mockLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ + description: 'Error parsing SQS record', + }), + ); + expect(mockReportGenerator.generate).not.toHaveBeenCalled(); + }); + + it('should handle validation failure for event schema', async () => { + const record = createMockSQSRecord('msg-1', { + type: 'invalid-type' as any, + data: {} as any, + }); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + const response = await handler(sqsEvent); + + expect(response.batchItemFailures).toEqual([{ itemIdentifier: 'msg-1' }]); + expect(mockLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ + description: 'Error parsing queue entry', + }), + ); + expect(mockReportGenerator.generate).not.toHaveBeenCalled(); + }); + + it('should add to batch failures when report generation fails', async () => { + const mockResult: ReportGeneratorResult = { outcome: 'failed' }; + mockReportGenerator.generate.mockResolvedValue(mockResult); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + const response = await handler(sqsEvent); + + expect(response.batchItemFailures).toEqual([{ itemIdentifier: 'msg-1' }]); + expect(mockLogger.info).toHaveBeenCalledWith( + expect.objectContaining({ + failed: 1, + generated: 0, + }), + ); + }); + + it('should handle exceptions during report generation', async () => { + const error = new Error('Generation error'); + mockReportGenerator.generate.mockRejectedValue(error); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + const response = await handler(sqsEvent); + + expect(response.batchItemFailures).toEqual([{ itemIdentifier: 'msg-1' }]); + expect(mockLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ + err: error, + description: 'Error during SQS trigger handler', + }), + ); + }); + + it('should not publish events when there are no successful items', async () => { + mockReportGenerator.generate.mockResolvedValue({ outcome: 'failed' }); + + const record = createMockSQSRecord('msg-1', { + data: { senderId: 'sender-123' } as any, + }); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + await handler(sqsEvent); + + expect(mockEventPublisher.sendEvents).not.toHaveBeenCalled(); + }); + + it('should generate correct ReportGenerated events', async () => { + const reportUri = 's3://bucket/report.csv'; + mockReportGenerator.generate.mockResolvedValue({ + outcome: 'generated', + reportUri, + }); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + await handler(sqsEvent); + + expect(mockEventPublisher.sendEvents).toHaveBeenCalledWith( + [ + expect.objectContaining({ + id: mockUuid, + type: 'uk.nhs.notify.digital.letters.reporting.report.generated.v1', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-reporting-report-generated-data.schema.json', + data: { + senderId: 'fce3ddee-2aca-4b2e-90a8-ce4da3787792', + reportUri, + }, + }), + ], + expect.any(Function), + ); + }); + + it('should log a warning if some events fail to publish', async () => { + const reportUri = 's3://bucket/report.csv'; + mockReportGenerator.generate.mockResolvedValue({ + outcome: 'generated', + reportUri, + }); + mockEventPublisher.sendEvents.mockResolvedValue([ + { + id: 'event-1', + source: 'event-source', + type: 'event-type', + }, + ]); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + await handler(sqsEvent); + + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + description: 'Some successful events failed to publish', + failedCount: 1, + totalAttempted: 1, + }), + ); + }); + + it('should log a warning if publishing events throws an exception', async () => { + const reportUri = 's3://bucket/report.csv'; + const error = new Error('Publish error'); + mockReportGenerator.generate.mockResolvedValue({ + outcome: 'generated', + reportUri, + }); + mockEventPublisher.sendEvents.mockRejectedValue(error); + + const record = createMockSQSRecord('msg-1', {}); + const sqsEvent = createMockSQSEvent([record]); + + const handler = createHandler({ + reportGenerator: mockReportGenerator, + eventPublisher: mockEventPublisher, + logger: mockLogger, + }); + + await handler(sqsEvent); + + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + err: error, + description: 'Failed to send successful events to EventBridge', + eventCount: 1, + }), + ); + }); + }); +}); diff --git a/lambdas/report-generator/src/__tests__/app/report-generator.test.ts b/lambdas/report-generator/src/__tests__/app/report-generator.test.ts new file mode 100644 index 00000000..57d63a17 --- /dev/null +++ b/lambdas/report-generator/src/__tests__/app/report-generator.test.ts @@ -0,0 +1,121 @@ +import { IReportService, Logger } from 'utils'; +import { GenerateReport } from 'digital-letters-events'; +import fs from 'node:fs'; +import { ReportGenerator } from 'app/report-generator'; + +jest.mock('node:fs'); + +describe('ReportGenerator', () => { + let mockLogger: jest.Mocked; + let mockReportService: jest.Mocked; + let reportGenerator: ReportGenerator; + const reportName = 'completed_communications'; + + beforeEach(() => { + mockLogger = { + info: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + debug: jest.fn(), + } as unknown as jest.Mocked; + + mockReportService = { + generateReport: jest.fn(), + } as jest.Mocked; + + reportGenerator = new ReportGenerator( + mockLogger, + mockReportService, + reportName, + ); + + jest.clearAllMocks(); + }); + + describe('generate', () => { + const mockEvent: GenerateReport = { + data: { + senderId: 'sender-123', + reportDate: '2025-01-15', + }, + specversion: '1.0', + type: 'uk.nhs.notify.digital.letters.reporting.generate.report.v1', + source: 'test', + id: 'test-id', + time: '2025-01-15T10:00:00Z', + datacontenttype: 'application/json', + subject: 'customer/5661de82-7453-44a1-9922-e0c98e5411c1', + traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + recordedtime: '2025-12-15T10:00:00Z', + severitynumber: 2, + }; + + const mockQuery = + 'SELECT * FROM reports WHERE date = $1 AND sender_id = $2'; + + beforeEach(() => { + (fs.readFileSync as jest.Mock).mockReturnValue(mockQuery); + }); + + it('should successfully generate a report', async () => { + const expectedLocation = + 's3://bucket/reports/sender-123/completed_communications/completed_communications_sender-123_2025-01-15.csv'; + mockReportService.generateReport.mockResolvedValue(expectedLocation); + + const result = await reportGenerator.generate(mockEvent); + + expect(fs.readFileSync).toHaveBeenCalledWith( + '/var/task/queries/report.sql', + 'utf8', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + 'Generating report for sender sender-123 and date 2025-01-15', + ); + expect(mockReportService.generateReport).toHaveBeenCalledWith( + mockQuery, + ["'2025-01-15'", "'sender-123'"], + 'transactional-reports/sender-123/completed_communications/completed_communications_2025-01-15.csv', + ); + expect(result).toEqual({ + outcome: 'generated', + reportUri: expectedLocation, + }); + }); + + it('should return failed outcome when report service throws error', async () => { + const error = new Error('Database connection failed'); + mockReportService.generateReport.mockRejectedValue(error); + + const result = await reportGenerator.generate(mockEvent); + + expect(mockLogger.error).toHaveBeenCalledWith({ + err: error, + description: 'Error generating report', + senderId: 'sender-123', + reportDate: '2025-01-15', + }); + expect(result).toEqual({ + outcome: 'failed', + }); + }); + + it('should return failed outcome when file read throws error', async () => { + const error = new Error('File not found'); + (fs.readFileSync as jest.Mock).mockImplementation(() => { + throw error; + }); + + const result = await reportGenerator.generate(mockEvent); + + expect(mockLogger.error).toHaveBeenCalledWith({ + err: error, + description: 'Error generating report', + senderId: 'sender-123', + reportDate: '2025-01-15', + }); + expect(result).toEqual({ + outcome: 'failed', + }); + }); + }); +}); diff --git a/lambdas/report-generator/src/__tests__/container.test.ts b/lambdas/report-generator/src/__tests__/container.test.ts new file mode 100644 index 00000000..637177e2 --- /dev/null +++ b/lambdas/report-generator/src/__tests__/container.test.ts @@ -0,0 +1,33 @@ +import { createContainer } from 'container'; + +jest.mock('infra/config', () => ({ + loadConfig: jest.fn(() => ({ + athenaDatabase: 'test-database', + athenaWorkgroup: 'test-workgroup', + eventPublisherDlqUrl: 'test-url', + eventPublisherEventBusArn: 'test-arn', + maxPollLimit: 10, + reportName: 'test-report', + reportingBucket: 'test-bucket', + waitForInSeconds: 5, + })), +})); + +jest.mock('utils', () => ({ + ...jest.requireActual('utils'), + AthenaRepository: jest.fn(() => ({})), + ReportService: jest.fn(() => ({})), + createStorageRepository: jest.fn(() => ({})), + s3Client: {}, + eventBridgeClient: {}, + EventPublisher: jest.fn(() => ({})), + logger: {}, + sqsClient: {}, +})); + +describe('container', () => { + it('should create container', () => { + const container = createContainer(); + expect(container).toBeDefined(); + }); +}); diff --git a/lambdas/report-generator/src/__tests__/index.test.ts b/lambdas/report-generator/src/__tests__/index.test.ts new file mode 100644 index 00000000..6241b09f --- /dev/null +++ b/lambdas/report-generator/src/__tests__/index.test.ts @@ -0,0 +1,11 @@ +import { handler } from 'index'; + +jest.mock('container', () => ({ + createContainer: jest.fn(() => ({})), +})); + +describe('index', () => { + it('should export handler', () => { + expect(handler).toBeDefined(); + }); +}); diff --git a/lambdas/report-generator/src/__tests__/infra/config.test.ts b/lambdas/report-generator/src/__tests__/infra/config.test.ts new file mode 100644 index 00000000..2902c80f --- /dev/null +++ b/lambdas/report-generator/src/__tests__/infra/config.test.ts @@ -0,0 +1,15 @@ +import { loadConfig } from 'infra/config'; + +jest.mock('utils', () => ({ + defaultConfigReader: { + getValue: jest.fn(), + getInt: jest.fn(), + }, +})); + +describe('config', () => { + it('should load config', () => { + const config = loadConfig(); + expect(config).toBeDefined(); + }); +}); diff --git a/lambdas/report-generator/src/apis/sqs-trigger-lambda.ts b/lambdas/report-generator/src/apis/sqs-trigger-lambda.ts new file mode 100644 index 00000000..2861b6c9 --- /dev/null +++ b/lambdas/report-generator/src/apis/sqs-trigger-lambda.ts @@ -0,0 +1,204 @@ +import type { + SQSBatchItemFailure, + SQSBatchResponse, + SQSEvent, +} from 'aws-lambda'; +import { randomUUID } from 'node:crypto'; +import type { + ReportGenerator, + ReportGeneratorOutcome, + ReportGeneratorResult, +} from 'app/report-generator'; +import generateReportValidator from 'digital-letters-events/GenerateReport.js'; +import reportGeneratedValidator from 'digital-letters-events/ReportGenerated.js'; +import { GenerateReport, ReportGenerated } from 'digital-letters-events'; +import { EventPublisher, Logger } from 'utils'; + +interface ProcessingResult { + result: ReportGeneratorResult; + item: GenerateReport; +} + +interface CreateHandlerDependencies { + reportGenerator: ReportGenerator; + eventPublisher: EventPublisher; + logger: Logger; +} + +interface ValidatedRecord { + messageId: string; + event: GenerateReport; +} + +function validateRecord( + { body, messageId }: { body: string; messageId: string }, + logger: Logger, +): ValidatedRecord | null { + try { + const sqsEventBody = JSON.parse(body); + const sqsEventDetail = sqsEventBody.detail; + + const isEventValid = generateReportValidator(sqsEventDetail); + if (!isEventValid) { + logger.error({ + err: generateReportValidator.errors, + description: 'Error parsing queue entry', + }); + return null; + } + + return { messageId, event: sqsEventDetail }; + } catch (error) { + logger.error({ + err: error, + description: 'Error parsing SQS record', + }); + return null; + } +} + +async function processRecord( + { event, messageId }: ValidatedRecord, + reportGenerator: ReportGenerator, + logger: Logger, + batchItemFailures: SQSBatchItemFailure[], +): Promise { + try { + const result = await reportGenerator.generate(event); + + if (result.outcome === 'failed') { + batchItemFailures.push({ itemIdentifier: messageId }); + } + + return { result, item: event }; + } catch (error) { + logger.error({ + err: error, + description: 'Error during SQS trigger handler', + }); + batchItemFailures.push({ itemIdentifier: messageId }); + return { result: { outcome: 'failed' }, item: event }; + } +} + +interface CategorizedResults { + processed: Record; + successfulItems: { event: GenerateReport; reportUri: string }[]; +} + +function categorizeResults( + results: PromiseSettledResult[], + logger: Logger, +): CategorizedResults { + const processed: Record = { + retrieved: results.length, + generated: 0, + failed: 0, + }; + + const successfulItems: { + event: GenerateReport; + reportUri: string; + }[] = []; + + for (const result of results) { + if (result.status === 'fulfilled') { + const { item, result: itemResult } = result.value; + processed[itemResult.outcome] += 1; + + if (itemResult.outcome === 'generated') { + successfulItems.push({ + event: item, + reportUri: itemResult.reportUri, + }); + } + } else { + logger.error({ err: result.reason }); + processed.failed += 1; + } + } + + return { processed, successfulItems }; +} + +async function publishSuccessfulEvents( + successfulItems: { event: GenerateReport; reportUri: string }[], + eventPublisher: EventPublisher, + logger: Logger, +): Promise { + if (successfulItems.length === 0) return; + + try { + const reportGeneratedEvents: ReportGenerated[] = successfulItems.map( + ({ event, reportUri }) => ({ + ...event, + id: randomUUID(), + time: new Date().toISOString(), + recordedtime: new Date().toISOString(), + type: 'uk.nhs.notify.digital.letters.reporting.report.generated.v1', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-reporting-report-generated-data.schema.json', + data: { + senderId: event.data.senderId, + reportUri, + }, + }), + ); + + const submittedFailedEvents = + await eventPublisher.sendEvents( + reportGeneratedEvents, + reportGeneratedValidator, + ); + if (submittedFailedEvents.length > 0) { + logger.warn({ + description: 'Some successful events failed to publish', + failedCount: submittedFailedEvents.length, + totalAttempted: successfulItems.length, + }); + } + } catch (error) { + logger.warn({ + err: error, + description: 'Failed to send successful events to EventBridge', + eventCount: successfulItems.length, + }); + } +} + +export const createHandler = ({ + eventPublisher, + logger, + reportGenerator, +}: CreateHandlerDependencies) => + async function handler(sqsEvent: SQSEvent): Promise { + const batchItemFailures: SQSBatchItemFailure[] = []; + + const validatedRecords: ValidatedRecord[] = []; + for (const record of sqsEvent.Records) { + const validated = validateRecord(record, logger); + if (validated) { + validatedRecords.push(validated); + } else { + batchItemFailures.push({ itemIdentifier: record.messageId }); + } + } + + const promises = validatedRecords.map((record) => + processRecord(record, reportGenerator, logger, batchItemFailures), + ); + + const results = await Promise.allSettled(promises); + const { processed, successfulItems } = categorizeResults(results, logger); + + await publishSuccessfulEvents(successfulItems, eventPublisher, logger); + + logger.info({ + description: 'Processed SQS Event.', + ...processed, + }); + + return { batchItemFailures }; + }; + +export default createHandler; diff --git a/lambdas/report-generator/src/app/report-generator.ts b/lambdas/report-generator/src/app/report-generator.ts new file mode 100644 index 00000000..61b998f2 --- /dev/null +++ b/lambdas/report-generator/src/app/report-generator.ts @@ -0,0 +1,51 @@ +import { IReportService, Logger } from 'utils'; +import { GenerateReport } from 'digital-letters-events'; +import fs from 'node:fs'; + +export type ReportGeneratorOutcome = 'generated' | 'failed'; + +export type ReportGeneratorResult = + | { + outcome: 'generated'; + reportUri: string; + } + | { + outcome: 'failed'; + }; + +export class ReportGenerator { + constructor( + private readonly logger: Logger, + private readonly reportService: IReportService, + private readonly reportName: string, + ) {} + + async generate(event: GenerateReport): Promise { + const { reportDate, senderId } = event.data; + + try { + const query = fs.readFileSync('/var/task/queries/report.sql', 'utf8'); + const reportFilePath = `transactional-reports/${senderId}/${this.reportName}/${this.reportName}_${reportDate}.csv`; + + this.logger.info( + `Generating report for sender ${senderId} and date ${reportDate}`, + ); + + const location = await this.reportService.generateReport( + query, + [`'${reportDate}'`, `'${senderId}'`], + reportFilePath, + ); + + return { outcome: 'generated', reportUri: location }; + } catch (error) { + this.logger.error({ + err: error, + description: 'Error generating report', + senderId, + reportDate, + }); + return { outcome: 'failed' }; + } + } +} diff --git a/lambdas/report-generator/src/container.ts b/lambdas/report-generator/src/container.ts new file mode 100644 index 00000000..8aaef6dd --- /dev/null +++ b/lambdas/report-generator/src/container.ts @@ -0,0 +1,72 @@ +import { + AthenaRepository, + EventPublisher, + ReportService, + createStorageRepository, + eventBridgeClient, + logger, + region, + s3Client, + sqsClient, +} from 'utils'; +import { loadConfig } from 'infra/config'; +import { ReportGenerator } from 'app/report-generator'; +import { AthenaClient } from '@aws-sdk/client-athena'; + +export const createContainer = () => { + const { + athenaDatabase, + athenaWorkgroup, + eventPublisherDlqUrl, + eventPublisherEventBusArn, + maxPollLimit, + reportName, + reportingBucket, + waitForInSeconds, + } = loadConfig(); + + const athenaClient = new AthenaClient({ + region: region(), + }); + + const dataRepository = new AthenaRepository(athenaClient, { + athenaWorkgroup, + athenaDatabase, + }); + + const storageRepository = createStorageRepository({ + s3Client, + reportingBucketName: reportingBucket, + logger, + }); + + const reportService = new ReportService( + dataRepository, + storageRepository, + maxPollLimit, + waitForInSeconds, + logger, + ); + + const reportGenerator = new ReportGenerator( + logger, + reportService, + reportName, + ); + + const eventPublisher = new EventPublisher({ + eventBusArn: eventPublisherEventBusArn, + dlqUrl: eventPublisherDlqUrl, + logger, + sqsClient, + eventBridgeClient, + }); + + return { + reportGenerator, + eventPublisher, + logger, + }; +}; + +export default createContainer; diff --git a/lambdas/report-generator/src/index.ts b/lambdas/report-generator/src/index.ts new file mode 100644 index 00000000..65f60969 --- /dev/null +++ b/lambdas/report-generator/src/index.ts @@ -0,0 +1,6 @@ +import { createHandler } from 'apis/sqs-trigger-lambda'; +import { createContainer } from 'container'; + +export const handler = createHandler(createContainer()); + +export default handler; diff --git a/lambdas/report-generator/src/infra/config.ts b/lambdas/report-generator/src/infra/config.ts new file mode 100644 index 00000000..b93c0d4e --- /dev/null +++ b/lambdas/report-generator/src/infra/config.ts @@ -0,0 +1,29 @@ +import { defaultConfigReader } from 'utils'; + +export type ReportGeneratorConfig = { + athenaWorkgroup: string; + athenaDatabase: string; + eventPublisherEventBusArn: string; + eventPublisherDlqUrl: string; + maxPollLimit: number; + reportingBucket: string; + reportName: string; + waitForInSeconds: number; +}; + +export function loadConfig(): ReportGeneratorConfig { + return { + athenaWorkgroup: defaultConfigReader.getValue('ATHENA_WORKGROUP'), + athenaDatabase: defaultConfigReader.getValue('ATHENA_DATABASE'), + eventPublisherEventBusArn: defaultConfigReader.getValue( + 'EVENT_PUBLISHER_EVENT_BUS_ARN', + ), + eventPublisherDlqUrl: defaultConfigReader.getValue( + 'EVENT_PUBLISHER_DLQ_URL', + ), + maxPollLimit: defaultConfigReader.getInt('MAX_POLL_LIMIT'), + reportingBucket: defaultConfigReader.getValue('REPORTING_BUCKET'), + reportName: defaultConfigReader.getValue('REPORT_NAME'), + waitForInSeconds: defaultConfigReader.getInt('WAIT_FOR_IN_SECONDS'), + }; +} diff --git a/lambdas/report-generator/src/queries/report.sql b/lambdas/report-generator/src/queries/report.sql new file mode 100644 index 00000000..f9a63703 --- /dev/null +++ b/lambdas/report-generator/src/queries/report.sql @@ -0,0 +1,55 @@ +WITH vars AS ( + SELECT CAST(? AS DATE) AS dt, + ? AS senderid +), +"translated_events" AS ( + SELECT e.messagereference, + e.time, + CASE + WHEN e.type LIKE '%.item.dequeued.%' + OR e.type LIKE '%.item.removed.%' THEN 'Digital' + WHEN e.type LIKE '%.print.letter.transitioned.%' THEN 'Print' ELSE NULL + END as communicationtype, + CASE + WHEN e.type LIKE '%.item.dequeued.%' THEN 'Unread' + WHEN e.type LIKE '%.item.removed.%' THEN 'Read' + WHEN e.letterstatus = 'RETURNED' THEN 'Returned' + WHEN e.letterstatus = 'FAILED' THEN 'Failed' + WHEN e.letterstatus = 'DISPATCHED' THEN 'Dispatched' + WHEN e.letterstatus = 'REJECTED' THEN 'Rejected' ELSE NULL + END as status + FROM event_record e + CROSS JOIN vars v + WHERE e.senderid = v.senderid + AND e.__year = year(v.dt) + AND e.__month = month(v.dt) + AND e.__day = day(v.dt) +), +"ordered_events" AS ( + SELECT ROW_NUMBER() OVER ( + PARTITION BY te.messagereference, te.communicationtype + ORDER BY te.time DESC, + CASE + -- Digital Priority Order + WHEN te.status = 'Read' THEN 2 + WHEN te.status = 'Unread' THEN 1 + -- Print Priority Order + WHEN te.status = 'Returned' THEN 4 + WHEN te.status = 'Failed' THEN 3 + WHEN te.status = 'Dispatched' THEN 2 + WHEN te.status = 'Rejected' THEN 1 ELSE 0 + END DESC + ) AS "row_number", + te.messagereference, + te.time, + te.communicationtype, + te.status + FROM "translated_events" AS te + where te.status IS NOT NULL +) +SELECT oe.messagereference as "Message Reference", + oe.time as "Time", + oe.communicationtype as "Communication Type", + oe.status as "Status" +FROM "ordered_events" AS oe +WHERE oe.row_number = 1 diff --git a/lambdas/report-generator/tsconfig.json b/lambdas/report-generator/tsconfig.json new file mode 100644 index 00000000..f7bcaa1f --- /dev/null +++ b/lambdas/report-generator/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "baseUrl": "./src/", + "isolatedModules": true + }, + "extends": "@tsconfig/node22/tsconfig.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/lambdas/ttl-poll-lambda/src/__tests__/infra/dynamo-repository.test.ts b/lambdas/ttl-poll-lambda/src/__tests__/infra/dynamo-repository.test.ts index 5f4d579f..b978f78a 100644 --- a/lambdas/ttl-poll-lambda/src/__tests__/infra/dynamo-repository.test.ts +++ b/lambdas/ttl-poll-lambda/src/__tests__/infra/dynamo-repository.test.ts @@ -4,7 +4,7 @@ import { QueryCommand, } from '@aws-sdk/lib-dynamodb'; import { mockClient } from 'aws-sdk-client-mock'; -import { Logger, deleteDynamoBatch, dynamoDocumentClient } from 'utils'; +import { Logger, deleteDynamoBatch } from 'utils'; import { mock, mockFn } from 'jest-mock-extended'; import { DynamoRepository } from 'infra/dynamo-repository'; import 'aws-sdk-client-mock-jest'; @@ -20,7 +20,7 @@ const mockDynamoDeleteBatch = mockFn(); const dynamoRepository = new DynamoRepository( mockTableName, - dynamoDocumentClient, + mockDynamoClient as any, logger, mockDynamoDeleteBatch, ); diff --git a/package-lock.json b/package-lock.json index 6b76b351..87df4ac6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "lambdas/report-scheduler", "lambdas/report-event-transformer", "lambdas/move-scanned-files-lambda", + "lambdas/report-generator", "utils/utils", "utils/sender-management", "src/cloudevents", @@ -3533,6 +3534,313 @@ "dev": true, "license": "MIT" }, + "lambdas/report-generator": { + "name": "nhs-notify-digital-letters-report-generator", + "version": "0.0.1", + "dependencies": { + "@aws-sdk/client-athena": "^3.984.0", + "digital-letters-events": "^0.0.1", + "utils": "^0.0.1" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "typescript": "^5.9.3" + } + }, + "lambdas/report-generator/node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "lambdas/report-generator/node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/@sinclair/typebox": { + "version": "0.27.10", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", + "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", + "dev": true, + "license": "MIT" + }, + "lambdas/report-generator/node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "lambdas/report-generator/node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "lambdas/report-generator/node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "lambdas/report-generator/node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "lambdas/report-generator/node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "lambdas/report-generator/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, "lambdas/report-scheduler": { "name": "nhs-notify-digital-letters-report-scheduler-lambda", "version": "0.0.1", @@ -4905,6 +5213,72 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-athena": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.989.0.tgz", + "integrity": "sha512-nehYJYXNjazNa8oe/GjKNyNJ96FzMDQHAhaq3Em9icUWo7ndKHTGGKFHoPo2TORayP7yxqVwQ1TSGgEBLOHwpw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.9", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.989.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.23.0", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.10", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-athena/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.981.0", "license": "Apache-2.0", @@ -4959,6 +5333,7 @@ "node_modules/@aws-sdk/client-dynamodb": { "version": "3.981.0", "license": "Apache-2.0", + "peer": true, "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", @@ -5007,128 +5382,130 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-dynamodb": { - "version": "3.980.0", + "node_modules/@aws-sdk/client-eventbridge": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-eventbridge/-/client-eventbridge-3.989.0.tgz", + "integrity": "sha512-0NYU5ifF509g3YHYF5lSUxEVyBHn5Y7dCp7f/LzEP4rLy4HPM7Amg8dEpk1voyXC9bvTPDI9uJwpC6Tfxgz9Qg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-node": "^3.972.4", - "@aws-sdk/dynamodb-codec": "^3.972.5", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.3", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/signature-v4-multi-region": "3.989.0", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", "@smithy/util-utf8": "^4.2.0", - "@smithy/util-waiter": "^4.2.8", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "node_modules/@aws-sdk/client-eventbridge/node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.989.0.tgz", + "integrity": "sha512-rVhR/BUZdnru7tLlxWD+uzoKB1LAs2L0pcoh6rYgIYuCtQflnsC6Ud0SpfqIsOapBSBKXdoW73IITFf+XFMdCQ==", "license": "Apache-2.0", "dependencies": { + "@aws-sdk/middleware-sdk-s3": "^3.972.9", "@aws-sdk/types": "^3.973.1", + "@smithy/protocol-http": "^5.3.8", + "@smithy/signature-v4": "^5.3.8", "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.972.5", + "node_modules/@aws-sdk/client-eventbridge/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@smithy/core": "^3.22.0", - "@smithy/smithy-client": "^4.11.1", + "@aws-sdk/types": "^3.973.1", "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "3.980.0" } }, - "node_modules/@aws-sdk/client-eventbridge": { - "version": "3.981.0", + "node_modules/@aws-sdk/client-firehose": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-firehose/-/client-firehose-3.989.0.tgz", + "integrity": "sha512-y5zUH7XHHiZPbdSm/f98g0yBIF3+F51yzmPvbrSRHNpzmDf74haszBfEOjBKGOat3oZJngzw8XsZM72BXSfKJQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-node": "^3.972.4", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/signature-v4-multi-region": "3.981.0", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.981.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5139,25 +5516,43 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-firehose/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-lambda": { - "version": "3.981.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.989.0.tgz", + "integrity": "sha512-poODSUZ7QbpP6tUXkuJkW8UUAiOfuATV4jzS/FV2SXhd+y388x7o8cOYcwLR2hcCfMrfDqGGVAkYMcjebQyVQw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-node": "^3.972.4", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.981.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/eventstream-serde-browser": "^4.2.8", "@smithy/eventstream-serde-config-resolver": "^4.3.8", "@smithy/eventstream-serde-node": "^4.2.8", @@ -5165,25 +5560,25 @@ "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.12", "@smithy/util-utf8": "^4.2.0", "@smithy/util-waiter": "^4.2.8", "tslib": "^2.6.2" @@ -5192,10 +5587,25 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-lambda/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.981.0", "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", @@ -5258,45 +5668,47 @@ } }, "node_modules/@aws-sdk/client-sqs": { - "version": "3.981.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.989.0.tgz", + "integrity": "sha512-6c0AdFqpdUiTOhLWeSgDtlZ1PMGfPkSQn11mSOvBKdEUJDRO9whTx/cAQNGiJYJybQA2HoL/4+mdlfTpcyxjzA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-node": "^3.972.4", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-sdk-sqs": "^3.972.5", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-sdk-sqs": "^3.972.7", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.981.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/md5-js": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5307,44 +5719,62 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-ssm": { - "version": "3.981.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ssm/-/client-ssm-3.989.0.tgz", + "integrity": "sha512-RaOXWdwUb8YyDazc6b79X2hZhMyzWSUXEUCSpS0dWOGlu51CPlTSPKJq3iNIKzz2Vf5XkaeM4lUN5QhCfMJBQw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-node": "^3.972.4", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.981.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5356,43 +5786,61 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-ssm/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-sso": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.989.0.tgz", + "integrity": "sha512-3sC+J1ru5VFXLgt9KZmXto0M7mnV5RkS6FNGwRMK3XrojSjHso9DLOWjbnXhbNv4motH8vu53L1HK2VC1+Nj5w==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5404,7 +5852,9 @@ } }, "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5418,17 +5868,19 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.5", + "version": "3.973.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.9.tgz", + "integrity": "sha512-cyUOfJSizn8da7XrBEFBf4UMI4A6JQNX6ZFcKtYmh/CrwfzsDcabv3k/z0bNwQ3pX5aeq5sg/8Bs/ASiL0bJaA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", - "@aws-sdk/xml-builder": "^3.972.2", - "@smithy/core": "^3.22.0", + "@aws-sdk/xml-builder": "^3.972.4", + "@smithy/core": "^3.23.0", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-base64": "^4.3.0", "@smithy/util-middleware": "^4.2.8", @@ -5451,10 +5903,12 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.7.tgz", + "integrity": "sha512-r8kBtglvLjGxBT87l6Lqkh9fL8yJJ6O4CYQPjKlj3AkCuL4/4784x3rxxXWw9LTKXOo114VB6mjxAuy5pI7XIg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/types": "^4.12.0", @@ -5465,18 +5919,20 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.5", + "version": "3.972.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.9.tgz", + "integrity": "sha512-40caFblEg/TPrp9EpvyMxp4xlJ5TuTI+A8H6g8FhHn2hfH2PObFAPLF9d5AljK/G69E1YtTklkuQeAwPlV3w8Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.12", "tslib": "^2.6.2" }, "engines": { @@ -5484,17 +5940,19 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.7.tgz", + "integrity": "sha512-zeYKrMwM5bCkHFho/x3+1OL0vcZQ0OhTR7k35tLq74+GP5ieV3juHXTZfa2LVE0Bg75cHIIerpX0gomVOhzo/w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-login": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-env": "^3.972.7", + "@aws-sdk/credential-provider-http": "^3.972.9", + "@aws-sdk/credential-provider-login": "^3.972.7", + "@aws-sdk/credential-provider-process": "^3.972.7", + "@aws-sdk/credential-provider-sso": "^3.972.7", + "@aws-sdk/credential-provider-web-identity": "^3.972.7", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5507,11 +5965,13 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.7.tgz", + "integrity": "sha512-Q103cLU6OjAllYjX7+V+PKQw654jjvZUkD+lbUUiFbqut6gR5zwl1DrelvJPM5hnzIty7BCaxaRB3KMuz3M/ug==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", @@ -5524,15 +5984,17 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.4", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.8.tgz", + "integrity": "sha512-AaDVOT7iNJyLjc3j91VlucPZ4J8Bw+eu9sllRDugJqhHWYyR3Iyp2huBUW8A3+DfHoh70sxGkY92cThAicSzlQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-ini": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", + "@aws-sdk/credential-provider-env": "^3.972.7", + "@aws-sdk/credential-provider-http": "^3.972.9", + "@aws-sdk/credential-provider-ini": "^3.972.7", + "@aws-sdk/credential-provider-process": "^3.972.7", + "@aws-sdk/credential-provider-sso": "^3.972.7", + "@aws-sdk/credential-provider-web-identity": "^3.972.7", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5545,10 +6007,12 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.7.tgz", + "integrity": "sha512-hxMo1V3ujWWrQSONxQJAElnjredkRpB6p8SDjnvRq70IwYY38R/CZSys0IbhRPxdgWZ5j12yDRk2OXhxw4Gj3g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5560,12 +6024,14 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.7.tgz", + "integrity": "sha512-ZGKBOHEj8Ap15jhG2XMncQmKLTqA++2DVU2eZfLu3T/pkwDyhCp5eZv5c/acFxbZcA/6mtxke+vzO/n+aeHs4A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.980.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/token-providers": "3.980.0", + "@aws-sdk/client-sso": "3.989.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/token-providers": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5577,11 +6043,13 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.7.tgz", + "integrity": "sha512-AbYupBIoSJoVMlbMqBhNvPhqj+CdGtzW7Uk4ZIMBm2br18pc3rkG1VaKVFV85H87QCvLHEnni1idJjaX1wOmIw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5592,6 +6060,23 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.972.10", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.972.10.tgz", + "integrity": "sha512-Q41dpb2kqx5FEq/qtTHXB5ocHVLaFzdKTogKHEZyylV0Wt+uJwhAPMzJ46+dL/4K20ZPGvkNRK3cOqp8Su/CIw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.973.9", + "@smithy/core": "^3.23.0", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/util-base64": "^4.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/endpoint-cache": { "version": "3.972.2", "license": "Apache-2.0", @@ -5621,25 +6106,6 @@ "@aws-sdk/client-dynamodb": "3.981.0" } }, - "node_modules/@aws-sdk/lib-storage": { - "version": "3.981.0", - "license": "Apache-2.0", - "dependencies": { - "@smithy/abort-controller": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/smithy-client": "^4.11.1", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-s3": "3.981.0" - } - }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { "version": "3.972.3", "license": "Apache-2.0", @@ -5685,13 +6151,15 @@ } }, "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.972.7.tgz", + "integrity": "sha512-YU/5rpz8k2mwFGi2M0px9ChOQZY7Bbow5knB2WLRVPqDM/cG8T5zj55UaWS1qcaFpE7vCX9a9/kvYBlKGcD+KA==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/crc64-nvme": "3.972.0", "@aws-sdk/types": "^3.973.1", "@smithy/is-array-buffer": "^4.2.0", @@ -5699,7 +6167,7 @@ "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.12", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, @@ -5759,21 +6227,23 @@ } }, "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.5", + "version": "3.972.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.9.tgz", + "integrity": "sha512-F4Ak2HM7te/o3izFTqg/jUTBLjavpaJ5iynKM6aLMwNddXbwAZQ1VbIG8RFUHBo7fBHj2eeN2FNLtIFT4ejWYQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@aws-sdk/util-arn-parser": "^3.972.2", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/node-config-provider": "^4.3.8", "@smithy/protocol-http": "^5.3.8", "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-config-provider": "^4.2.0", "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.12", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" }, @@ -5782,11 +6252,13 @@ } }, "node_modules/@aws-sdk/middleware-sdk-sqs": { - "version": "3.972.5", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.7.tgz", + "integrity": "sha512-DcJLYE4sRjgUyb2SupQGaRgBYc+j89N9nXeMT0PwwVvaBGmKqcxa7PFvz0kBnQrBckPWlfrPyyyMwOeT5BEp6Q==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-hex-encoding": "^4.2.0", "@smithy/util-utf8": "^4.2.0", @@ -5809,13 +6281,15 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.5", + "version": "3.972.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.9.tgz", + "integrity": "sha512-1g1B7yf7KzessB0mKNiV9gAHEwbM662xgU+VE4LxyGe6kVGZ8LqYsngjhE+Stna09CJ7Pxkjr6Uq1OtbGwJJJg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", - "@smithy/core": "^3.22.0", + "@aws-sdk/util-endpoints": "3.989.0", + "@smithy/core": "^3.23.0", "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" @@ -5825,7 +6299,9 @@ } }, "node_modules/@aws-sdk/middleware-user-agent/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5839,42 +6315,44 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.989.0.tgz", + "integrity": "sha512-Dbk2HMPU3mb6RrSRzgf0WCaWSbgtZG258maCpuN2/ONcAQNpOTw99V5fU5CA1qVK6Vkm4Fwj2cnOnw7wbGVlOw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5886,7 +6364,9 @@ } }, "node_modules/@aws-sdk/nested-clients/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5929,11 +6409,13 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.989.0.tgz", + "integrity": "sha512-OdBByMv+OjOZoekrk4THPFpLuND5aIQbDHCGh3n2rvifAbm31+6e0OLhxSeCF1UMPm+nKq12bXYYEoCIx5SQBg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -6013,10 +6495,12 @@ } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.972.7.tgz", + "integrity": "sha512-oyhv+FjrgHjP+F16cmsrJzNP4qaRJzkV1n9Lvv4uyh3kLqo3rIe9NSBSBa35f2TedczfG2dD+kaQhHBB47D6Og==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/types": "^3.973.1", "@smithy/node-config-provider": "^4.3.8", "@smithy/types": "^4.12.0", @@ -6035,7 +6519,9 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.3", + "version": "3.972.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.4.tgz", + "integrity": "sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==", "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.12.0", @@ -8614,7 +9100,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.22.1", + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.0.tgz", + "integrity": "sha512-Yq4UPVoQICM9zHnByLmG8632t2M0+yap4T7ANVw482J0W7HW0pOuxwVmeOwzJqX2Q89fkXz0Vybz55Wj2Xzrsg==", "license": "Apache-2.0", "dependencies": { "@smithy/middleware-serde": "^4.2.9", @@ -8623,7 +9111,7 @@ "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.11", + "@smithy/util-stream": "^4.5.12", "@smithy/util-utf8": "^4.2.0", "@smithy/uuid": "^1.1.0", "tslib": "^2.6.2" @@ -8804,10 +9292,12 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.13", + "version": "4.4.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.14.tgz", + "integrity": "sha512-FUFNE5KVeaY6U/GL0nzAAHkaCHzXLZcY1EhtQnsAqhD8Du13oPKtMB9/0WK4/LK6a/T5OZ24wPoSShff5iI6Ag==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.22.1", + "@smithy/core": "^3.23.0", "@smithy/middleware-serde": "^4.2.9", "@smithy/node-config-provider": "^4.3.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -8821,13 +9311,15 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.30", + "version": "4.4.31", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.31.tgz", + "integrity": "sha512-RXBzLpMkIrxBPe4C8OmEOHvS8aH9RUuCOH++Acb5jZDEblxDjyg6un72X9IcbrGTJoiUwmI7hLypNfuDACypbg==", "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.8", "@smithy/protocol-http": "^5.3.8", "@smithy/service-error-classification": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -8875,7 +9367,9 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.4.9", + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.10.tgz", + "integrity": "sha512-u4YeUwOWRZaHbWaebvrs3UhwQwj+2VNmcVCwXcYTvPIuVyM7Ex1ftAj+fdbG/P4AkBwLq/+SKn+ydOI4ZJE9PA==", "license": "Apache-2.0", "dependencies": { "@smithy/abort-controller": "^4.2.8", @@ -8972,15 +9466,17 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.11.2", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.11.3.tgz", + "integrity": "sha512-Q7kY5sDau8OoE6Y9zJoRGgje8P4/UY0WzH8R2ok0PDh+iJ+ZnEKowhjEqYafVcubkbYxQVaqwm3iufktzhprGg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.22.1", - "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/core": "^3.23.0", + "@smithy/middleware-endpoint": "^4.4.14", "@smithy/middleware-stack": "^4.2.8", "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.11", + "@smithy/util-stream": "^4.5.12", "tslib": "^2.6.2" }, "engines": { @@ -9063,11 +9559,13 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.29", + "version": "4.3.30", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.30.tgz", + "integrity": "sha512-cMni0uVU27zxOiU8TuC8pQLC1pYeZ/xEMxvchSK/ILwleRd1ugobOcIRr5vXtcRqKd4aBLWlpeBoDPJJ91LQng==", "license": "Apache-2.0", "dependencies": { "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" }, @@ -9076,14 +9574,16 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.32", + "version": "4.2.33", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.33.tgz", + "integrity": "sha512-LEb2aq5F4oZUSzWBG7S53d4UytZSkOEJPXcBq/xbG2/TmK9EW5naUZ8lKu1BEyWMzdHIzEVN16M3k8oxDq+DJA==", "license": "Apache-2.0", "dependencies": { "@smithy/config-resolver": "^4.4.6", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" }, @@ -9137,11 +9637,13 @@ } }, "node_modules/@smithy/util-stream": { - "version": "4.5.11", + "version": "4.5.12", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.12.tgz", + "integrity": "sha512-D8tgkrmhAX/UNeCZbqbEO3uqyghUnEmmoO9YEvRuwxjlkKKUE7FOgCJnqpTlQPe9MApdWPky58mNQQHbnCzoNg==", "license": "Apache-2.0", "dependencies": { "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.9", + "@smithy/node-http-handler": "^4.4.10", "@smithy/types": "^4.12.0", "@smithy/util-base64": "^4.3.0", "@smithy/util-buffer-from": "^4.2.0", @@ -10930,6 +11432,12 @@ "node": ">=18" } }, + "node_modules/csv-parse": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz", + "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==", + "license": "MIT" + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "dev": true, @@ -12610,6 +13118,8 @@ }, "node_modules/fast-xml-parser": { "version": "5.3.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.4.tgz", + "integrity": "sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==", "funding": [ { "type": "github", @@ -17631,6 +18141,10 @@ "resolved": "lambdas/report-event-transformer", "link": true }, + "node_modules/nhs-notify-digital-letters-report-generator": { + "resolved": "lambdas/report-generator", + "link": true + }, "node_modules/nhs-notify-digital-letters-report-scheduler-lambda": { "resolved": "lambdas/report-scheduler", "link": true @@ -19217,6 +19731,8 @@ }, "node_modules/strnum": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", "funding": [ { "type": "github", @@ -21003,8 +21519,10 @@ "name": "nhs-notify-digital-letters-integration-tests", "version": "0.0.1", "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0", @@ -21013,6 +21531,7 @@ "@faker-js/faker": "^9.6.0", "@nhsdigital/nhs-notify-event-schemas-supplier-api": "1.0.6", "@playwright/test": "^1.51.1", + "csv-parse": "^6.1.0", "digital-letters-events": "^0.0.1", "sender-management": "^0.0.1", "utils": "^0.0.1", @@ -21313,14 +21832,15 @@ "utils/utils": { "version": "0.0.1", "dependencies": { - "@aws-sdk/client-dynamodb": "^3.914.0", - "@aws-sdk/client-eventbridge": "^3.918.0", - "@aws-sdk/client-lambda": "^3.914.0", - "@aws-sdk/client-s3": "^3.914.0", - "@aws-sdk/client-sqs": "^3.914.0", - "@aws-sdk/client-ssm": "^3.914.0", - "@aws-sdk/lib-dynamodb": "^3.914.0", - "@aws-sdk/lib-storage": "^3.914.0", + "@aws-sdk/client-athena": "^3.984.0", + "@aws-sdk/client-dynamodb": "^3.984.0", + "@aws-sdk/client-eventbridge": "^3.984.0", + "@aws-sdk/client-lambda": "^3.984.0", + "@aws-sdk/client-s3": "^3.984.0", + "@aws-sdk/client-sqs": "^3.984.0", + "@aws-sdk/client-ssm": "^3.984.0", + "@aws-sdk/lib-dynamodb": "^3.984.0", + "@aws-sdk/lib-storage": "^3.984.0", "async-mutex": "^0.4.0", "axios": "^1.13.5", "date-fns": "^4.1.0", @@ -21342,6 +21862,216 @@ "typescript": "^5.8.2" } }, + "utils/utils/node_modules/@aws-sdk/client-dynamodb": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.989.0.tgz", + "integrity": "sha512-CzZFyQIKjlXWYE2rdMWfOFPF0PbFUehrjBKLjz7kJLb5XZwICrQudBA8/j/EqlxNtw5EknjxXrS3oNBpEwSe2Q==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", + "@aws-sdk/dynamodb-codec": "^3.972.10", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.3", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.9", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.989.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.23.0", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.10", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "@smithy/util-waiter": "^4.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "utils/utils/node_modules/@aws-sdk/client-s3": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.989.0.tgz", + "integrity": "sha512-ccz2miIetWAgrJYmKCpSnRjF8jew7DPstl54nufhfPMtM1MLxD2z55eSk1eJj3Umhu4CioNN1aY1ILT7fwlSiw==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", + "@aws-sdk/middleware-bucket-endpoint": "^3.972.3", + "@aws-sdk/middleware-expect-continue": "^3.972.3", + "@aws-sdk/middleware-flexible-checksums": "^3.972.7", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-location-constraint": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-sdk-s3": "^3.972.9", + "@aws-sdk/middleware-ssec": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.9", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/signature-v4-multi-region": "3.989.0", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.989.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.23.0", + "@smithy/eventstream-serde-browser": "^4.2.8", + "@smithy/eventstream-serde-config-resolver": "^4.3.8", + "@smithy/eventstream-serde-node": "^4.2.8", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-blob-browser": "^4.2.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/hash-stream-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/md5-js": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.10", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-stream": "^4.5.12", + "@smithy/util-utf8": "^4.2.0", + "@smithy/util-waiter": "^4.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "utils/utils/node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.989.0.tgz", + "integrity": "sha512-EfbnesyLsoy8Xu/jJwU0+r58Gs129aF9Uehm1B6RwLSnhaXSH6+2R4AA5PRAtdRfPVD1ewSq4bk4rqCE9iBC0Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/util-dynamodb": "3.989.0", + "@smithy/core": "^3.23.0", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.989.0" + } + }, + "utils/utils/node_modules/@aws-sdk/lib-storage": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.989.0.tgz", + "integrity": "sha512-8pJXMJ7MT5At/5ANFC68IbhfG8hNe0/ISsbtdVopgQEsiZEAHr0HDNoPcyoRnc3RTzjykz7Q95uf/Lpz3PQNmA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/abort-controller": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/smithy-client": "^4.11.3", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.989.0" + } + }, + "utils/utils/node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.989.0.tgz", + "integrity": "sha512-rVhR/BUZdnru7tLlxWD+uzoKB1LAs2L0pcoh6rYgIYuCtQflnsC6Ud0SpfqIsOapBSBKXdoW73IITFf+XFMdCQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/middleware-sdk-s3": "^3.972.9", + "@aws-sdk/types": "^3.973.1", + "@smithy/protocol-http": "^5.3.8", + "@smithy/signature-v4": "^5.3.8", + "@smithy/types": "^4.12.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "utils/utils/node_modules/@aws-sdk/util-dynamodb": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.989.0.tgz", + "integrity": "sha512-k3dtJG2OxJ6PAr3XfNDbzLaLPnN5iwO45Ikwla7RrToFw5pzRJ4ugj40ZPBEF2JyO5xmrXJppSA0Wo56ci86yQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.989.0" + } + }, + "utils/utils/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "utils/utils/node_modules/@jest/core": { "version": "29.7.0", "dev": true, diff --git a/package.json b/package.json index 3bceba37..be1c38a5 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "lambdas/report-scheduler", "lambdas/report-event-transformer", "lambdas/move-scanned-files-lambda", + "lambdas/report-generator", "utils/utils", "utils/sender-management", "src/cloudevents", diff --git a/project.code-workspace b/project.code-workspace index 5edb866b..c97f7ea5 100644 --- a/project.code-workspace +++ b/project.code-workspace @@ -89,6 +89,7 @@ { "name": "python-schema-generator", "rootPath": "src/python-schema-generator" }, { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" }, { "name": "report-event-transformer", "rootPath": "lambdas/report-event-transformer" }, + { "name": "report-generator", "rootPath": "lambdas/report-generator" }, { "name": "sender-management", "rootPath": "utils/sender-management" }, { "name": "ttl-create-lambda", "rootPath": "lambdas/ttl-create-lambda/" }, { "name": "ttl-handle-expiry-lambda", "rootPath": "lambdas/ttl-handle-expiry-lambda" }, diff --git a/tests/playwright/config/component/component.config.ts b/tests/playwright/config/component/component.config.ts index 340e7bb3..87ca74fa 100644 --- a/tests/playwright/config/component/component.config.ts +++ b/tests/playwright/config/component/component.config.ts @@ -13,10 +13,19 @@ export default defineConfig({ name: 'senders:setup', testMatch: 'senders.setup.ts', }, + { + name: 'firehose:setup', + testMatch: 'firehose.setup.ts', + teardown: 'firehose:teardown', + }, + { + name: 'firehose:teardown', + testMatch: 'firehose.teardown.ts', + }, { name: 'component', testMatch: '*.component.spec.ts', - dependencies: ['senders:setup'], + dependencies: ['senders:setup', 'firehose:setup'], teardown: 'component:teardown', }, { diff --git a/tests/playwright/config/component/firehose.setup.ts b/tests/playwright/config/component/firehose.setup.ts new file mode 100644 index 00000000..f7ecf537 --- /dev/null +++ b/tests/playwright/config/component/firehose.setup.ts @@ -0,0 +1,21 @@ +import { test as setup } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +setup('Reduce Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/config/component/firehose.teardown.ts b/tests/playwright/config/component/firehose.teardown.ts new file mode 100644 index 00000000..11844458 --- /dev/null +++ b/tests/playwright/config/component/firehose.teardown.ts @@ -0,0 +1,21 @@ +import { test as teardown } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +teardown('Restore Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index 9c98f58b..0fa41764 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -42,10 +42,16 @@ export const EVENT_BUS_LOG_GROUP_NAME = `/aws/vendedlogs/events/event-bus/${CSI} // DynamoDB export const TTL_TABLE_NAME = `${CSI}-ttl`; +// Glue +export const GLUE_DATABASE_NAME = `${CSI}-reporting`; +export const GLUE_TABLE_NAME = 'event_record'; + // S3 export const LETTERS_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-letters`; export const NON_PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-non-pii-data`; export const PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-pii-data`; +export const REPORTING_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-reporting`; +export const REPORTING_S3_FIREHOSE_OUTPUT_KEY_PREFIX = `kinesis-firehose-output/reporting/parquet/${GLUE_TABLE_NAME}`; export const FILE_SAFE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-safe`; export const UNSCANNED_FILES_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-main-acct-digi-unscanned-files`; export const FILE_QUARANTINE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-quarantine`; @@ -61,3 +67,13 @@ export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pr export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; export const PRINT_SENDER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-sender`; export const MOVE_SCANNED_FILES_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-move-scanned-files`; + +// Data Firehose +export const FIREHOSE_STREAM_NAME = `${CSI}-to-s3-reporting`; +export const TERRAFORM_DESTINATION_BUFFER_INTERVAL = 300; +export const TERRAFORM_PROCESSOR_BUFFER_INTERVAL = 301; +export const MINIMUM_DESTINATION_BUFFER_INTERVAL = 60; +export const MINIMUM_PROCESSOR_BUFFER_INTERVAL = 0; + +// Athena +export const ATHENA_WORKGROUP_NAME = CSI; diff --git a/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts new file mode 100644 index 00000000..3e8344b8 --- /dev/null +++ b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts @@ -0,0 +1,223 @@ +import { expect, test } from '@playwright/test'; +import { + ATHENA_WORKGROUP_NAME, + ENV, + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, + REPORTING_S3_BUCKET_NAME, + REPORTING_S3_FIREHOSE_OUTPUT_KEY_PREFIX, +} from 'constants/backend-constants'; +import { + GenerateReport, + ItemRemoved, + MESHInboxMessageDownloaded, +} from 'digital-letters-events'; +import generateReportValidator from 'digital-letters-events/GenerateReport.js'; +import itemRemovedValidator from 'digital-letters-events/ItemRemoved.js'; +import messageDownloadedValidator from 'digital-letters-events/MESHInboxMessageDownloaded.js'; +import { + QueryExecutionState, + getQueryState, + triggerTableMetadataRefresh, +} from 'helpers/athena-helpers'; +import { getLogsFromCloudwatch } from 'helpers/cloudwatch-helpers'; +import eventPublisher from 'helpers/event-bus-helpers'; +import expectToPassEventually from 'helpers/expectations'; +import { downloadFromS3, existsInS3 } from 'helpers/s3-helpers'; +import { v4 as uuidv4 } from 'uuid'; +import { parse } from 'csv-parse/sync'; + +test.describe('Digital Letters - Report Generator', () => { + test('should generate a report containing the expected statuses', async () => { + // We need to wait for events to make their way from EventBridge -> Firehose -> S3 -> Glue + test.setTimeout(700_000); + + // Use a random sender ID, so we can be sure that if there are files with this prefix + // in S3 they've been created by this test. + const senderId = `report-generator-test-${uuidv4()}`; + console.log(`Using senderId: ${senderId}`); + + // Communication type should be Digital, and the status should be Read + const itemRemovedEventId = uuidv4(); + const itemRemovedEventTime = new Date().toISOString(); + await eventPublisher.sendEvents( + [ + { + id: itemRemovedEventId, + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/queue', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.queue.item.removed.v1', + time: itemRemovedEventTime, + recordedtime: itemRemovedEventTime, + severitynumber: 2, + traceparent: + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-queue-item-removed-data.schema.json', + severitytext: 'INFO', + data: { + messageReference: 'component-test-itemRemoved', + senderId, + messageUri: `https://example.com/ttl/resource/${itemRemovedEventId}`, + }, + }, + ], + itemRemovedValidator, + ); + + // TODO: Send a ItemDequeued event - communication type should be Digital, and the status should be Unread + // TODO: Send a PrintLetterTransitioned event, with status REJECTED - communication type should be Print, and the status should be Rejected + // TODO: Send a PrintLetterTransitioned event, with status DISPATCHED - communication type should be Print, and the status should be Dispatched + // TODO: Send a PrintLetterTransitioned event, with status FAILED - communication type should be Print, and the status should be Failed + // TODO: Send a PrintLetterTransitioned event, with status RETURNED - communication type should be Print, and the status should be Returned + + // Send a MESHInboxMessageDownloaded event (to prove it isn't included in the report) + const downloadedEventId = uuidv4(); + const downloadedEventTime = new Date().toISOString(); + await eventPublisher.sendEvents( + [ + { + id: downloadedEventId, + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/mesh', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.mesh.inbox.message.downloaded.v1', + time: downloadedEventTime, + recordedtime: downloadedEventTime, + severitynumber: 2, + traceparent: + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-downloaded-data.schema.json', + severitytext: 'INFO', + data: { + meshMessageId: '12345', + messageUri: `https://example.com/ttl/resource/${downloadedEventId}`, + messageReference: 'component-test-messageDownloaded', + senderId, + }, + }, + ], + messageDownloadedValidator, + ); + + await expectToPassEventually( + async () => { + console.log( + 'Checking for events in S3 with prefix:', + `${REPORTING_S3_FIREHOSE_OUTPUT_KEY_PREFIX}/senderid=${senderId}`, + ); + const eventsHaveBeenWrittenToS3 = await existsInS3( + REPORTING_S3_BUCKET_NAME, + `${REPORTING_S3_FIREHOSE_OUTPUT_KEY_PREFIX}/senderid=${senderId}`, + ); + + expect(eventsHaveBeenWrittenToS3).toBeTruthy(); + }, + 300_000, + 10, + ); + + // Trigger a metadata refresh for the Glue table, which will cause it to pick up any new files in S3 + const refreshQueryExecutionId = await triggerTableMetadataRefresh( + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, + ATHENA_WORKGROUP_NAME, + ); + + await expectToPassEventually(async () => { + console.log( + 'Waiting for Glue table metadata refresh to complete, query execution ID:', + refreshQueryExecutionId, + ); + const refreshQueryState = await getQueryState(refreshQueryExecutionId); + + expect(refreshQueryState).toEqual(QueryExecutionState.SUCCEEDED); + }); + + const generateReportEventId = uuidv4(); + const generateReportEventTime = new Date().toISOString(); + const reportDate = new Date().toISOString().split('T')[0]; + await eventPublisher.sendEvents( + [ + { + id: generateReportEventId, + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/reporting', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.reporting.generate.report.v1', + time: generateReportEventTime, + recordedtime: generateReportEventTime, + severitynumber: 2, + traceparent: + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-reporting-generate-report-data.schema.json', + severitytext: 'INFO', + data: { + senderId, + reportDate, + }, + }, + ], + generateReportValidator, + ); + + const reportKey = `transactional-reports/${senderId}/completed_communications/completed_communications_${reportDate}.csv`; + + await expectToPassEventually( + async () => { + console.log(`Looking for report with prefix: ${reportKey}`); + + const reportHasBeenWrittenToS3 = await existsInS3( + REPORTING_S3_BUCKET_NAME, + reportKey, + ); + + expect(reportHasBeenWrittenToS3).toBeTruthy(); + }, + 300_000, + 10, + ); + + const report = await downloadFromS3(REPORTING_S3_BUCKET_NAME, reportKey); + + console.log('Received report:', report.body); + + const reportRows = parse(report.body, { columns: true }); + expect(reportRows).toEqual([ + { + 'Message Reference': 'component-test-itemRemoved', + Time: itemRemovedEventTime, + 'Communication Type': 'Digital', + Status: 'Read', + }, + ]); + + // Verify ReportGenerated event published + await expectToPassEventually(async () => { + const eventLogEntry = await getLogsFromCloudwatch( + `/aws/vendedlogs/events/event-bus/nhs-${ENV}-dl`, + [ + '$.message_type = "EVENT_RECEIPT"', + '$.details.detail_type = "uk.nhs.notify.digital.letters.reporting.report.generated.v1"', + `$.details.event_detail = "*\\"senderId\\":\\"${senderId}\\"*"`, + ], + ); + + expect(eventLogEntry.length).toEqual(1); + }); + }); + + // TODO: Add a test that proves the priority of events is applied as expected? +}); diff --git a/tests/playwright/helpers/athena-helpers.ts b/tests/playwright/helpers/athena-helpers.ts new file mode 100644 index 00000000..2d15ff24 --- /dev/null +++ b/tests/playwright/helpers/athena-helpers.ts @@ -0,0 +1,61 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + QueryExecutionState, + StartQueryExecutionCommand, +} from '@aws-sdk/client-athena'; + +export { QueryExecutionState } from '@aws-sdk/client-athena'; + +const client = new AthenaClient(); + +/** + * Triggers a metadata refresh for an Athena table using the MSCK REPAIR TABLE command. + * + * This will cause any new files in S3 to be picked up. + * + * @param database - The name of the Athena database + * @param tableName - The name of the table to repair + * @param workgroup - The Athena workgroup to run the query in + * @returns The query execution ID + */ +export async function triggerTableMetadataRefresh( + database: string, + tableName: string, + workgroup: string, +): Promise { + const command = new StartQueryExecutionCommand({ + QueryString: `MSCK REPAIR TABLE ${tableName};`, + QueryExecutionContext: { + Database: database, + Catalog: 'AwsDataCatalog', + }, + WorkGroup: workgroup, + }); + + const response = await client.send(command); + + if (!response.QueryExecutionId) { + throw new Error( + 'Failed to start query execution - no query execution ID returned', + ); + } + + return response.QueryExecutionId; +} + +export async function getQueryState( + queryExecutionId: string, +): Promise { + const queryExecutionInfo = await client.send( + new GetQueryExecutionCommand({ + QueryExecutionId: queryExecutionId, + }), + ); + + if (!queryExecutionInfo.QueryExecution?.Status?.State) { + throw new Error('Failed to get query execution state'); + } + + return queryExecutionInfo.QueryExecution?.Status?.State; +} diff --git a/tests/playwright/helpers/data-firehose-helpers.ts b/tests/playwright/helpers/data-firehose-helpers.ts new file mode 100644 index 00000000..384fef24 --- /dev/null +++ b/tests/playwright/helpers/data-firehose-helpers.ts @@ -0,0 +1,116 @@ +import { + DescribeDeliveryStreamCommand, + DescribeDeliveryStreamCommandOutput, + ExtendedS3DestinationUpdate, + FirehoseClient, + UpdateDestinationCommand, +} from '@aws-sdk/client-firehose'; +import { FIREHOSE_STREAM_NAME, REGION } from 'constants/backend-constants'; + +export async function alterFirehoseBufferIntervals(bufferIntervalConfig: { + expected: { + destination: number; + processor: number; + }; + update: { + destination: number; + processor: number; + }; +}) { + const client = new FirehoseClient({ region: REGION }); + + const deliveryStreamDetails: DescribeDeliveryStreamCommandOutput = + await client.send( + new DescribeDeliveryStreamCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + }), + ); + + const destinations = + deliveryStreamDetails.DeliveryStreamDescription?.Destinations ?? []; + if (destinations.length !== 1) { + throw new Error('expected a single delivery destination'); + } + + const destination = destinations[0]; + + const currentDestinationBufferInterval = + destination.ExtendedS3DestinationDescription?.BufferingHints + ?.IntervalInSeconds; + + if ( + currentDestinationBufferInterval !== + bufferIntervalConfig.expected.destination + ) { + throw new Error( + `Expected destination buffer size to be ${bufferIntervalConfig.expected.destination} - got ${currentDestinationBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const processors = + destination.ExtendedS3DestinationDescription?.ProcessingConfiguration + ?.Processors; + + if (processors?.length !== 1) { + throw new Error('Expected one processor to be configured'); + } + + const processor = processors[0]; + + const currentProcessorBufferInterval = processor.Parameters?.find( + (p) => p.ParameterName === 'BufferIntervalInSeconds', + )?.ParameterValue; + + const otherParams = + processor.Parameters?.filter( + (p) => p.ParameterName !== 'BufferIntervalInSeconds', + ) ?? []; + + if ( + currentProcessorBufferInterval !== + bufferIntervalConfig.expected.processor.toString() + ) { + throw new Error( + `Expected processor buffer size to be ${bufferIntervalConfig.expected.processor} - got ${currentProcessorBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const destinationId = destination.DestinationId; + + if (!destinationId) { + throw new Error('Destination ID not found'); + } + + const updatedDestinationConfig: ExtendedS3DestinationUpdate = { + ...destination.ExtendedS3DestinationDescription, + BufferingHints: { + ...destination.ExtendedS3DestinationDescription?.BufferingHints, + IntervalInSeconds: bufferIntervalConfig.update.destination, + }, + ProcessingConfiguration: { + ...destination.ExtendedS3DestinationDescription?.ProcessingConfiguration, + Processors: [ + { + ...processor, + Parameters: [ + ...otherParams, + { + ParameterName: 'BufferIntervalInSeconds', + ParameterValue: bufferIntervalConfig.update.processor.toString(), + }, + ], + }, + ], + }, + }; + + await client.send( + new UpdateDestinationCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + DestinationId: destinationId, + CurrentDeliveryStreamVersionId: + deliveryStreamDetails.DeliveryStreamDescription?.VersionId, + ExtendedS3DestinationUpdate: updatedDestinationConfig, + }), + ); +} diff --git a/tests/playwright/helpers/expectations.ts b/tests/playwright/helpers/expectations.ts index e680b43a..2f171fa7 100644 --- a/tests/playwright/helpers/expectations.ts +++ b/tests/playwright/helpers/expectations.ts @@ -37,10 +37,10 @@ test.afterEach(async () => { async function expectToPassEventually( expectationFunction: () => Promise, timeout = 30, + delay = 1, ): Promise { const invocationToken = Symbol('invocationToken'); const startTime = Date.now(); - const delay = 1; for (;;) { try { diff --git a/tests/playwright/helpers/s3-helpers.ts b/tests/playwright/helpers/s3-helpers.ts index 108e90c5..b2d6eb0a 100644 --- a/tests/playwright/helpers/s3-helpers.ts +++ b/tests/playwright/helpers/s3-helpers.ts @@ -61,4 +61,12 @@ async function downloadFromS3( }; } -export { downloadFromS3, uploadToS3 }; +async function existsInS3(bucket: string, keyPrefix: string): Promise { + const objects = await s3.send( + new ListObjectsV2Command({ Bucket: bucket, Prefix: keyPrefix }), + ); + + return (objects.Contents?.length ?? 0) > 0; +} + +export { downloadFromS3, existsInS3, uploadToS3 }; diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 156975af..41967755 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -1,7 +1,9 @@ { "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0", @@ -10,6 +12,7 @@ "@faker-js/faker": "^9.6.0", "@nhsdigital/nhs-notify-event-schemas-supplier-api": "1.0.6", "@playwright/test": "^1.51.1", + "csv-parse": "^6.1.0", "digital-letters-events": "^0.0.1", "sender-management": "^0.0.1", "utils": "^0.0.1", diff --git a/utils/utils/package.json b/utils/utils/package.json index 7f873bcc..780d0109 100644 --- a/utils/utils/package.json +++ b/utils/utils/package.json @@ -1,13 +1,14 @@ { "dependencies": { - "@aws-sdk/client-dynamodb": "^3.914.0", - "@aws-sdk/client-eventbridge": "^3.918.0", - "@aws-sdk/client-lambda": "^3.914.0", - "@aws-sdk/client-s3": "^3.914.0", - "@aws-sdk/client-sqs": "^3.914.0", - "@aws-sdk/client-ssm": "^3.914.0", - "@aws-sdk/lib-dynamodb": "^3.914.0", - "@aws-sdk/lib-storage": "^3.914.0", + "@aws-sdk/client-athena": "^3.984.0", + "@aws-sdk/client-dynamodb": "^3.984.0", + "@aws-sdk/client-eventbridge": "^3.984.0", + "@aws-sdk/client-lambda": "^3.984.0", + "@aws-sdk/client-s3": "^3.984.0", + "@aws-sdk/client-sqs": "^3.984.0", + "@aws-sdk/client-ssm": "^3.984.0", + "@aws-sdk/lib-dynamodb": "^3.984.0", + "@aws-sdk/lib-storage": "^3.984.0", "async-mutex": "^0.4.0", "axios": "^1.13.5", "date-fns": "^4.1.0", diff --git a/utils/utils/src/__tests__/reporting/data-repository.test.ts b/utils/utils/src/__tests__/reporting/data-repository.test.ts new file mode 100644 index 00000000..16563430 --- /dev/null +++ b/utils/utils/src/__tests__/reporting/data-repository.test.ts @@ -0,0 +1,117 @@ +import { AthenaClient } from '@aws-sdk/client-athena'; +import { mockClient } from 'aws-sdk-client-mock'; +import { AthenaRepository } from '../../reporting/data-repository'; + +const athenaClientMock = mockClient(AthenaClient); + +describe('AthenaRepository', () => { + let repository: AthenaRepository; + const mockConfig = { + athenaWorkgroup: 'test-workgroup', + athenaDatabase: 'test-database', + }; + + beforeEach(() => { + athenaClientMock.reset(); + repository = new AthenaRepository(new AthenaClient({}), mockConfig); + }); + + describe('startQuery', () => { + it('should start query execution and return query execution ID', async () => { + const mockQueryExecutionId = 'query-123'; + athenaClientMock.onAnyCommand().resolves({ + QueryExecutionId: mockQueryExecutionId, + }); + + const result = await repository.startQuery('SELECT * FROM table', [ + 'param1', + 'param2', + ]); + + expect(result).toBe(mockQueryExecutionId); + }); + + it('should send correct parameters to Athena client', async () => { + const query = 'SELECT * FROM table WHERE id = ?'; + const executionParameters = ['123']; + + athenaClientMock.onAnyCommand().resolves({ + QueryExecutionId: 'query-456', + }); + + await repository.startQuery(query, executionParameters); + + const calls = athenaClientMock.commandCalls( + Object.getPrototypeOf(athenaClientMock.calls()[0].args[0]).constructor, + ); + expect(calls[0].args[0].input).toEqual({ + QueryString: query, + WorkGroup: 'test-workgroup', + QueryExecutionContext: { Database: 'test-database' }, + ExecutionParameters: executionParameters, + }); + }); + + it('should handle empty execution parameters', async () => { + athenaClientMock.onAnyCommand().resolves({ + QueryExecutionId: 'query-789', + }); + + const result = await repository.startQuery('SELECT 1', []); + + expect(result).toBe('query-789'); + }); + + it('should propagate Athena client errors', async () => { + const mockError = new Error('Athena service error'); + athenaClientMock.onAnyCommand().rejects(mockError); + + await expect( + repository.startQuery('SELECT * FROM table', []), + ).rejects.toThrow('Athena service error'); + }); + }); + + describe('getQueryStatus', () => { + it('should return query execution state', async () => { + athenaClientMock.onAnyCommand().resolves({ + QueryExecution: { + Status: { + State: 'SUCCEEDED', + }, + }, + }); + + const result = await repository.getQueryStatus('query-123'); + + expect(result).toBe('SUCCEEDED'); + }); + + it('should return undefined when QueryExecution is missing', async () => { + athenaClientMock.onAnyCommand().resolves({}); + + const result = await repository.getQueryStatus('query-999'); + + expect(result).toBeUndefined(); + }); + + it('should return undefined when Status is missing', async () => { + athenaClientMock.onAnyCommand().resolves({ + QueryExecution: {}, + }); + + const result = await repository.getQueryStatus('query-888'); + + expect(result).toBeUndefined(); + }); + + it('should propagate Athena client errors', async () => { + const mockError = new Error('Query not found'); + athenaClientMock.onAnyCommand().rejects(mockError); + + await expect(repository.getQueryStatus('invalid-query')).rejects.toThrow( + 'Query not found', + ); + }); + }); +}); diff --git a/utils/utils/src/__tests__/reporting/report-service.test.ts b/utils/utils/src/__tests__/reporting/report-service.test.ts new file mode 100644 index 00000000..170c5559 --- /dev/null +++ b/utils/utils/src/__tests__/reporting/report-service.test.ts @@ -0,0 +1,227 @@ +import { Logger } from '../../logger'; +import { ReportService } from '../../reporting/report-service'; +import { IDataRepository } from '../../reporting/data-repository'; +import { IStorageRepository } from '../../reporting/storage-repository'; +import { sleep } from '../../util-retry/sleep'; + +jest.mock('../../util-retry/sleep'); + +describe('ReportService', () => { + let mockDataRepository: jest.Mocked; + let mockStorageRepository: jest.Mocked; + let mockLogger: jest.Mocked; + let reportService: ReportService; + + const defaultMaxPollLimit = 10; + const defaultWaitForInSeconds = 1; + + beforeEach(() => { + mockDataRepository = { + startQuery: jest.fn(), + getQueryStatus: jest.fn(), + } as jest.Mocked; + + mockStorageRepository = { + publishReport: jest.fn(), + } as jest.Mocked; + + mockLogger = { + child: jest.fn().mockReturnThis(), + info: jest.fn(), + error: jest.fn(), + } as unknown as jest.Mocked; + + reportService = new ReportService( + mockDataRepository, + mockStorageRepository, + defaultMaxPollLimit, + defaultWaitForInSeconds, + mockLogger, + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('generateReport', () => { + const query = 'SELECT * FROM test_table'; + const executionParameters = ['param1', 'param2']; + const reportFilePath = 's3://bucket/report.csv'; + const queryExecutionId = 'test-execution-id-123'; + + it('should generate report successfully when query succeeds', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus.mockResolvedValue('SUCCEEDED'); + mockStorageRepository.publishReport.mockResolvedValue(reportFilePath); + + const result = await reportService.generateReport( + query, + executionParameters, + reportFilePath, + ); + + expect(mockDataRepository.startQuery).toHaveBeenCalledWith( + query, + executionParameters, + ); + expect(mockDataRepository.getQueryStatus).toHaveBeenCalledWith( + queryExecutionId, + ); + expect(mockStorageRepository.publishReport).toHaveBeenCalledWith( + queryExecutionId, + reportFilePath, + ); + expect(result).toBe(reportFilePath); + expect(mockLogger.child).toHaveBeenCalledWith({ queryExecutionId }); + expect(mockLogger.info).toHaveBeenCalledWith('Athena query started.'); + expect(mockLogger.info).toHaveBeenCalledWith('Athena query finished.'); + }); + + it('should throw error when query fails', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus.mockResolvedValue('FAILED'); + + await expect( + reportService.generateReport( + query, + executionParameters, + reportFilePath, + ), + ).rejects.toThrow('Failed to generate report. Query status: FAILED'); + + expect(mockStorageRepository.publishReport).not.toHaveBeenCalled(); + }); + + it('should throw error when query is cancelled', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus.mockResolvedValue('CANCELLED'); + + await expect( + reportService.generateReport( + query, + executionParameters, + reportFilePath, + ), + ).rejects.toThrow('Failed to generate report. Query status: CANCELLED'); + }); + + it('should poll until query succeeds', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus + .mockResolvedValueOnce('QUEUED') + .mockResolvedValueOnce('RUNNING') + .mockResolvedValueOnce('RUNNING') + .mockResolvedValueOnce('SUCCEEDED'); + mockStorageRepository.publishReport.mockResolvedValue(reportFilePath); + + await reportService.generateReport( + query, + executionParameters, + reportFilePath, + ); + + expect(mockDataRepository.getQueryStatus).toHaveBeenCalledTimes(4); + expect(sleep).toHaveBeenCalledTimes(4); + expect(sleep).toHaveBeenCalledWith(defaultWaitForInSeconds); + }); + + it('should throw error when max poll limit is reached', async () => { + const shortPollLimit = 3; + const shortReportService = new ReportService( + mockDataRepository, + mockStorageRepository, + shortPollLimit, + defaultWaitForInSeconds, + mockLogger, + ); + + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus.mockResolvedValue('RUNNING'); + + await expect( + shortReportService.generateReport( + query, + executionParameters, + reportFilePath, + ), + ).rejects.toThrow('Failed to generate report. Query status: RUNNING'); + + expect(mockDataRepository.getQueryStatus).toHaveBeenCalledTimes( + shortPollLimit, + ); + }); + + it('should handle UNKNOWN status and continue polling', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus + .mockResolvedValueOnce('QUEUED') + .mockResolvedValueOnce('UNKNOWN') + .mockResolvedValueOnce('SUCCEEDED'); + mockStorageRepository.publishReport.mockResolvedValue(reportFilePath); + + await reportService.generateReport( + query, + executionParameters, + reportFilePath, + ); + + expect(mockDataRepository.getQueryStatus).toHaveBeenCalledTimes(3); + }); + + it('should respect custom wait time between polls', async () => { + const customWaitTime = 5; + const customReportService = new ReportService( + mockDataRepository, + mockStorageRepository, + defaultMaxPollLimit, + customWaitTime, + mockLogger, + ); + + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus + .mockResolvedValueOnce('RUNNING') + .mockResolvedValueOnce('SUCCEEDED'); + mockStorageRepository.publishReport.mockResolvedValue(reportFilePath); + + await customReportService.generateReport( + query, + executionParameters, + reportFilePath, + ); + + expect(sleep).toHaveBeenCalledWith(customWaitTime); + }); + + it('should throw an error if the query is not started successfully', async () => { + // eslint-disable-next-line unicorn/no-useless-undefined -- We want to explicitly set the return value. + mockDataRepository.startQuery.mockResolvedValue(undefined); + + await expect( + reportService.generateReport( + query, + executionParameters, + reportFilePath, + ), + ).rejects.toThrow('failed to obtain a query executionId from Athena'); + }); + + it('should continue polling if getQueryStatus returns undefined', async () => { + mockDataRepository.startQuery.mockResolvedValue(queryExecutionId); + mockDataRepository.getQueryStatus + // eslint-disable-next-line unicorn/no-useless-undefined -- We want to explicitly set the return value. + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce('SUCCEEDED'); + mockStorageRepository.publishReport.mockResolvedValue(reportFilePath); + + await reportService.generateReport( + query, + executionParameters, + reportFilePath, + ); + + expect(mockDataRepository.getQueryStatus).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/utils/utils/src/__tests__/reporting/storage-repository.test.ts b/utils/utils/src/__tests__/reporting/storage-repository.test.ts new file mode 100644 index 00000000..4fb65e96 --- /dev/null +++ b/utils/utils/src/__tests__/reporting/storage-repository.test.ts @@ -0,0 +1,76 @@ +import { CopyObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import { createStorageRepository } from '../../reporting/storage-repository'; +import { Logger } from '../../logger'; + +const s3Mock = mockClient(S3Client); + +describe('StorageRepository', () => { + const mockLogger = { + debug: jest.fn(), + error: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + } as unknown as jest.Mocked; + const reportingBucketName = 'test-reporting-bucket'; + let storageRepository: ReturnType; + + beforeEach(() => { + s3Mock.reset(); + storageRepository = createStorageRepository({ + s3Client: new S3Client({}), + reportingBucketName, + logger: mockLogger, + }); + }); + + describe('publishReport', () => { + it('should copy report from athena-output to target path', async () => { + const reportQueryId = 'query-123'; + const reportFilePath = 'reports/2024/report.csv'; + + s3Mock.on(CopyObjectCommand).resolves({}); + + const result = await storageRepository.publishReport( + reportQueryId, + reportFilePath, + ); + + expect(result).toBe(`s3://${reportingBucketName}/${reportFilePath}`); + expect(s3Mock.calls()).toHaveLength(1); + + const copyCommand = s3Mock.call(0).args[0].input; + expect(copyCommand).toEqual({ + CopySource: `${reportingBucketName}/athena-output/${reportQueryId}.csv`, + Bucket: reportingBucketName, + Key: reportFilePath, + }); + }); + + it('should throw error when S3 copy fails', async () => { + const reportQueryId = 'query-456'; + const reportFilePath = 'reports/2024/failed-report.csv'; + const s3Error = new Error('S3 CopyObject failed'); + + s3Mock.on(CopyObjectCommand).rejects(s3Error); + + await expect( + storageRepository.publishReport(reportQueryId, reportFilePath), + ).rejects.toThrow('S3 CopyObject failed'); + }); + + it('should construct correct S3 URIs for nested paths', async () => { + const reportQueryId = 'query-789'; + const reportFilePath = 'reports/2024/01/daily/report.csv'; + + s3Mock.on(CopyObjectCommand).resolves({}); + + const result = await storageRepository.publishReport( + reportQueryId, + reportFilePath, + ); + + expect(result).toBe(`s3://${reportingBucketName}/${reportFilePath}`); + }); + }); +}); diff --git a/utils/utils/src/index.ts b/utils/utils/src/index.ts index 8da5a845..4b9333bc 100644 --- a/utils/utils/src/index.ts +++ b/utils/utils/src/index.ts @@ -15,3 +15,4 @@ export * from './event-bridge-utils'; export * from './key-generation-utils'; export * from './schema-utils'; export * from './pdm-client'; +export * from './reporting'; diff --git a/utils/utils/src/reporting/data-repository.ts b/utils/utils/src/reporting/data-repository.ts new file mode 100644 index 00000000..da7bbad5 --- /dev/null +++ b/utils/utils/src/reporting/data-repository.ts @@ -0,0 +1,71 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + StartQueryExecutionCommand, +} from '@aws-sdk/client-athena'; +import type { Logger } from '../logger'; + +export type DataRepositoryDependencies = { + athenaClient: AthenaClient; + config: Record; + logger: Logger; +}; + +export type IDataRepository = { + startQuery( + query: string, + executionParameters: string[], + ): Promise; + getQueryStatus(reportQueryId: string): Promise; +}; + +export class AthenaRepository implements IDataRepository { + readonly athenaClient: AthenaClient; + + readonly workGroup: string; + + readonly database: string; + + constructor(athenaClient: AthenaClient, config: Record) { + this.athenaClient = athenaClient; + this.workGroup = config.athenaWorkgroup; + this.database = config.athenaDatabase; + } + + /** + * Asynchronously starts a query execution in Athena. + * + * @param {string} query - The query string to execute. + * @return {Promise} - The ID of the query execution. + */ + async startQuery(query: string, executionParameters: string[]) { + const executionCommand = new StartQueryExecutionCommand({ + QueryString: query, + WorkGroup: this.workGroup, + QueryExecutionContext: { Database: this.database }, + ExecutionParameters: executionParameters, + }); + + const { QueryExecutionId } = await this.athenaClient.send(executionCommand); + + return QueryExecutionId; + } + + /** + * Retrieves the status of a query execution. + * + * @param {string} reportQueryId - The ID of the query execution. + * @return {State} The state of the query execution. + */ + async getQueryStatus(reportQueryId: string) { + const getQueryExecutionCommand = new GetQueryExecutionCommand({ + QueryExecutionId: reportQueryId, + }); + + const { QueryExecution } = await this.athenaClient.send( + getQueryExecutionCommand, + ); + + return QueryExecution?.Status?.State; + } +} diff --git a/utils/utils/src/reporting/index.ts b/utils/utils/src/reporting/index.ts new file mode 100644 index 00000000..c89877d1 --- /dev/null +++ b/utils/utils/src/reporting/index.ts @@ -0,0 +1,3 @@ +export * from './data-repository'; +export * from './report-service'; +export * from './storage-repository'; diff --git a/utils/utils/src/reporting/report-service.ts b/utils/utils/src/reporting/report-service.ts new file mode 100644 index 00000000..c4bd5393 --- /dev/null +++ b/utils/utils/src/reporting/report-service.ts @@ -0,0 +1,96 @@ +import type { Logger } from '../logger'; +import { sleep } from '../util-retry/sleep'; +import { IDataRepository } from './data-repository'; +import { IStorageRepository } from './storage-repository'; + +export interface IReportService { + generateReport( + query: string, + executionParameters: string[], + reportFilePath: string, + ): Promise; +} + +export class ReportService implements IReportService { + readonly dataRepository: IDataRepository; + + readonly storageRepository: IStorageRepository; + + readonly maxPollLimit: number; + + readonly waitForInSeconds: number; + + readonly logger: Logger; + + constructor( + dataRepository: IDataRepository, + storageRepository: IStorageRepository, + maxPollLimit: number, + waitForInSeconds: number, + logger: Logger, + ) { + this.dataRepository = dataRepository; + this.storageRepository = storageRepository; + this.maxPollLimit = maxPollLimit; + this.waitForInSeconds = waitForInSeconds; + this.logger = logger; + } + + async generateReport( + query: string, + executionParameters: string[], + reportFilePath: string, + ): Promise { + const queryExecutionId = await this.dataRepository.startQuery( + query, + executionParameters, + ); + + if (!queryExecutionId) { + throw new Error('failed to obtain a query executionId from Athena'); + } + + const logger = this.logger.child({ queryExecutionId }); + + logger.info('Athena query started.'); + + const status = await this.poll( + queryExecutionId, + this.maxPollLimit, + this.waitForInSeconds, + ); + + if (status !== 'SUCCEEDED') { + throw new Error(`Failed to generate report. Query status: ${status}`); + } + + logger.info('Athena query finished.'); + + return this.storageRepository.publishReport( + queryExecutionId, + reportFilePath, + ); + } + + private async poll( + queryId: string, + maxPollLimit: number, + waitForInSeconds: number, + ) { + let count = 0; + let status = 'QUEUED'; + + while ( + count < maxPollLimit && + ['QUEUED', 'RUNNING', 'UNKNOWN'].includes(status) + ) { + status = (await this.dataRepository.getQueryStatus(queryId)) || 'UNKNOWN'; + + count += 1; + + await sleep(waitForInSeconds); + } + + return status; + } +} diff --git a/utils/utils/src/reporting/storage-repository.ts b/utils/utils/src/reporting/storage-repository.ts new file mode 100644 index 00000000..5a9444e6 --- /dev/null +++ b/utils/utils/src/reporting/storage-repository.ts @@ -0,0 +1,39 @@ +import { CopyObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import type { Logger } from '../logger'; + +export type IStorageRepository = { + publishReport: ( + reportQueryId: string, + reportFilePath: string, + ) => Promise; +}; + +type StorageRepositoryDependencies = { + s3Client: S3Client; + reportingBucketName: string; + logger: Logger; +}; + +export const createStorageRepository = ({ + logger, + reportingBucketName, + s3Client, +}: StorageRepositoryDependencies): IStorageRepository => ({ + async publishReport(reportQueryId: string, reportFilePath: string) { + logger.debug( + `Publishing report data to ${reportFilePath} for query ${reportQueryId}`, + ); + + const copyObjectCommand = new CopyObjectCommand({ + CopySource: `${reportingBucketName}/athena-output/${reportQueryId}.csv`, + Bucket: reportingBucketName, + Key: reportFilePath, + }); + + await s3Client.send(copyObjectCommand); + + logger.info(`Report stored at ${reportingBucketName}/${reportFilePath}.`); + + return `s3://${reportingBucketName}/${reportFilePath}`; + }, +});