From fcc6fdf49b05a889bcc44db2e3535b1a33765fba Mon Sep 17 00:00:00 2001 From: Ian Hodges Date: Mon, 26 Jan 2026 12:15:33 +0000 Subject: [PATCH 01/14] CCM-13295 add basic status reporter component --- .../terraform/components/dl/README.md | 2 + .../cloudwatch_event_rule_status_recorder.tf | 19 +++ ...da_event_source_mapping_status_recorder.tf | 10 ++ .../dl/module_lambda_status_recorder.tf | 72 ++++++++++++ .../dl/module_sqs_status_recorder.tf | 42 +++++++ lambdas/status-recorder/jest.config.ts | 5 + lambdas/status-recorder/package.json | 25 ++++ .../src/__tests__/apis/sqs-handler.test.ts | 100 ++++++++++++++++ .../src/__tests__/container.test.ts | 18 +++ .../src/__tests__/index.test.ts | 15 +++ .../src/__tests__/infra/config.test.ts | 15 +++ .../src/__tests__/test-data.ts | 61 ++++++++++ .../status-recorder/src/apis/sqs-handler.ts | 110 ++++++++++++++++++ lambdas/status-recorder/src/container.ts | 11 ++ lambdas/status-recorder/src/index.ts | 6 + lambdas/status-recorder/src/infra/config.ts | 11 ++ lambdas/status-recorder/src/types/events.ts | 25 ++++ lambdas/status-recorder/tsconfig.json | 11 ++ package-lock.json | 75 ++++++++++++ package.json | 1 + 20 files changed, 634 insertions(+) create mode 100644 infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf create mode 100644 infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf create mode 100644 infrastructure/terraform/components/dl/module_lambda_status_recorder.tf create mode 100644 infrastructure/terraform/components/dl/module_sqs_status_recorder.tf create mode 100644 lambdas/status-recorder/jest.config.ts create mode 100644 lambdas/status-recorder/package.json create mode 100644 lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts create mode 100644 lambdas/status-recorder/src/__tests__/container.test.ts create mode 100644 lambdas/status-recorder/src/__tests__/index.test.ts create mode 100644 lambdas/status-recorder/src/__tests__/infra/config.test.ts create mode 100644 lambdas/status-recorder/src/__tests__/test-data.ts create mode 100644 lambdas/status-recorder/src/apis/sqs-handler.ts create mode 100644 lambdas/status-recorder/src/container.ts create mode 100644 lambdas/status-recorder/src/index.ts create mode 100644 lambdas/status-recorder/src/infra/config.ts create mode 100644 lambdas/status-recorder/src/types/events.ts create mode 100644 lambdas/status-recorder/tsconfig.json diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index f811d47a..1ac5c9cf 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -62,8 +62,10 @@ No requirements. | [sqs\_pdm\_poll](#module\_sqs\_pdm\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_pdm\_uploader](#module\_sqs\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_print\_status\_handler](#module\_sqs\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | +| [sqs\_status\_recorder](#module\_sqs\_status\_recorder) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | +| [status\_recorder](#module\_status\_recorder) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_create](#module\_ttl\_create) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_handle\_expiry](#module\_ttl\_handle\_expiry) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_poll](#module\_ttl\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | diff --git a/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf b/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf new file mode 100644 index 00000000..b7323dfd --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf @@ -0,0 +1,19 @@ +resource "aws_cloudwatch_event_rule" "status_recorder" { + name = "${local.csi}-status-recorder" + description = "Status recorder event rule" + event_bus_name = aws_cloudwatch_event_bus.main.name + + event_pattern = jsonencode({ + "detail" : { + "type" : [{ + "prefix" : "uk.nhs.notify.digital.letters." + }] + } + }) +} + +resource "aws_cloudwatch_event_target" "status_recorder" { + rule = aws_cloudwatch_event_rule.status_recorder.name + arn = module.sqs_status_recorder.sqs_queue_arn + event_bus_name = aws_cloudwatch_event_bus.main.name +} diff --git a/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf b/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf new file mode 100644 index 00000000..44efff2f --- /dev/null +++ b/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf @@ -0,0 +1,10 @@ +resource "aws_lambda_event_source_mapping" "status_recorder" { + event_source_arn = module.sqs_status_recorder.sqs_queue_arn + function_name = module.status_recorder.function_name + batch_size = var.queue_batch_size + maximum_batching_window_in_seconds = var.queue_batch_window_seconds + + function_response_types = [ + "ReportBatchItemFailures" + ] +} diff --git a/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf b/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf new file mode 100644 index 00000000..ee8b4251 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf @@ -0,0 +1,72 @@ +module "status_recorder" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip" + + function_name = "status-recorder" + description = "A function for processing all digital letter events" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.status_recorder.json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "status-recorder/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 128 + timeout = 60 + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + + log_destination_arn = local.log_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + "ATHENA_ARN" = "Some Value" + } +} + +data "aws_iam_policy_document" "status_recorder" { + statement { + sid = "SQSPermissionsDLQs" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch", + ] + + resources = [ + module.sqs_event_publisher_errors.sqs_queue_arn, + ] + } + + statement { + sid = "SQSPermissionsStatusRecorderQueue" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + ] + + resources = [ + module.sqs_status_recorder.sqs_queue_arn, + ] + } +} diff --git a/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf b/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf new file mode 100644 index 00000000..e8220d48 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf @@ -0,0 +1,42 @@ +module "sqs_status_recorder" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + name = "status-recorder" + sqs_kms_key_arn = module.kms.key_arn + visibility_timeout_seconds = 60 + delay_seconds = 5 + create_dlq = true + max_receive_count = 1 + sqs_policy_overload = data.aws_iam_policy_document.sqs_status_recorder.json +} + +data "aws_iam_policy_document" "sqs_status_recorder" { + statement { + sid = "AllowEventBridgeToSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage" + ] + + resources = [ + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-status-recorder-queue" + ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [ aws_cloudwatch_event_rule.status_recorder.arn ] + } + } +} diff --git a/lambdas/status-recorder/jest.config.ts b/lambdas/status-recorder/jest.config.ts new file mode 100644 index 00000000..c02601ae --- /dev/null +++ b/lambdas/status-recorder/jest.config.ts @@ -0,0 +1,5 @@ +import { baseJestConfig } from '../../jest.config.base'; + +const config = baseJestConfig; + +export default config; diff --git a/lambdas/status-recorder/package.json b/lambdas/status-recorder/package.json new file mode 100644 index 00000000..d463e25e --- /dev/null +++ b/lambdas/status-recorder/package.json @@ -0,0 +1,25 @@ +{ + "dependencies": { + "aws-lambda": "^1.0.7", + "utils": "^0.0.1", + "zod": "^4.1.12" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "jest-mock-extended": "^3.0.7", + "typescript": "^5.9.3" + }, + "name": "nhs-notify-digital-letters-status-recorder", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts b/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts new file mode 100644 index 00000000..bd7d196a --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts @@ -0,0 +1,100 @@ +import { mock } from 'jest-mock-extended'; +import { createHandler } from 'apis/sqs-handler'; +import { Logger } from 'utils'; +import { digitalLettersEvent, recordEvent } from '__tests__/test-data'; + +const logger = mock(); + +const handler = createHandler({ + athenaArn: 'some value', + logger, +}); + +describe('SQS Handler', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('status', () => { + it('should process valid digital letters event and log report event', async () => { + const response = await handler(recordEvent([digitalLettersEvent])); + + expect(logger.info).toHaveBeenCalledWith( + 'Received SQS Event of 1 record(s)', + ); + + expect(logger.info).toHaveBeenCalledWith({ + description: 'Following report events will be sent to some value', + validEvents: [ + { + messageReference: digitalLettersEvent.data.messageReference, + senderId: digitalLettersEvent.data.senderId, + pageCount: digitalLettersEvent.data.pageCount, + supplierId: digitalLettersEvent.data.supplierId, + time: digitalLettersEvent.time, + type: digitalLettersEvent.type, + }, + ], + }); + + expect(logger.info).toHaveBeenCalledWith( + '1 of 1 records processed successfully', + ); + + expect(response).toEqual({ batchItemFailures: [] }); + }); + }); + + describe('errors', () => { + it('should return failed SQS records to the queue if an error occurs while parsing them', async () => { + const event = recordEvent([digitalLettersEvent]); + event.Records[0].body = 'not-json'; + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: new SyntaxError( + `Unexpected token 'o', "not-json" is not valid JSON`, + ), + description: 'Error parsing SQS record', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + + it('should return failed items to the queue if an invalid event is received', async () => { + const invalidEvent = { + ...digitalLettersEvent, + type: 123, // Invalid: should be string + }; + const event = recordEvent([invalidEvent as any]); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: expect.objectContaining({ + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ['type'], + }), + ]), + }), + description: 'Error parsing queue item', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result).toEqual({ + batchItemFailures: [{ itemIdentifier: '1' }], + }); + }); + }); +}); diff --git a/lambdas/status-recorder/src/__tests__/container.test.ts b/lambdas/status-recorder/src/__tests__/container.test.ts new file mode 100644 index 00000000..007ff523 --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/container.test.ts @@ -0,0 +1,18 @@ +import { createContainer } from 'container'; + +jest.mock('infra/config', () => ({ + loadConfig: jest.fn(() => ({ + athenaArn: 'some value', + })), +})); + +jest.mock('utils', () => ({ + logger: {}, +})); + +describe('container', () => { + it('should create container', () => { + const container = createContainer(); + expect(container).toBeDefined(); + }); +}); diff --git a/lambdas/status-recorder/src/__tests__/index.test.ts b/lambdas/status-recorder/src/__tests__/index.test.ts new file mode 100644 index 00000000..b5465321 --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/index.test.ts @@ -0,0 +1,15 @@ +import { handler } from 'index'; + +jest.mock('apis/sqs-handler', () => ({ + createHandler: jest.fn(() => jest.fn()), +})); + +jest.mock('container', () => ({ + createContainer: jest.fn(() => ({})), +})); + +describe('index', () => { + it('should export handler', () => { + expect(handler).toBeDefined(); + }); +}); diff --git a/lambdas/status-recorder/src/__tests__/infra/config.test.ts b/lambdas/status-recorder/src/__tests__/infra/config.test.ts new file mode 100644 index 00000000..2902c80f --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/infra/config.test.ts @@ -0,0 +1,15 @@ +import { loadConfig } from 'infra/config'; + +jest.mock('utils', () => ({ + defaultConfigReader: { + getValue: jest.fn(), + getInt: jest.fn(), + }, +})); + +describe('config', () => { + it('should load config', () => { + const config = loadConfig(); + expect(config).toBeDefined(); + }); +}); diff --git a/lambdas/status-recorder/src/__tests__/test-data.ts b/lambdas/status-recorder/src/__tests__/test-data.ts new file mode 100644 index 00000000..4ead7ba8 --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/test-data.ts @@ -0,0 +1,61 @@ +import { SQSEvent, SQSRecord } from 'aws-lambda'; +import { DigitalLettersEvent } from 'types/events'; + +const baseEvent = { + id: '550e8400-e29b-41d4-a716-446655440001', + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/pdm', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.pdm.resource.submitted.v1', + time: '2023-06-20T12:00:00Z', + recordedtime: '2023-06-20T12:00:00.250Z', + severitynumber: 2, + traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10/digital-letter-base-data.schema.json', + severitytext: 'INFO', + data: { + resourceId: 'a2bcbb42-ab7e-42b6-88d6-74f8d3ca4a09', + messageReference: 'ref1', + senderId: 'sender1', + }, +}; + +export const digitalLettersEvent = { + ...baseEvent, + type: 'uk.nhs.notify.digital.letters.pdm.resource.submitted.v1', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-submitted-data.schema.json', +} as DigitalLettersEvent; + +const busEvent = { + version: '0', + id: 'ab07d406-0797-e919-ff9b-3ad9c5498114', +}; + +const sqsRecord = { + messageId: '1', + receiptHandle: 'abc', + attributes: { + ApproximateReceiveCount: '1', + SentTimestamp: '2025-07-03T14:23:30Z', + SenderId: 'sender-id', + ApproximateFirstReceiveTimestamp: '2025-07-03T14:23:30Z', + }, + messageAttributes: {}, + md5OfBody: '', + eventSource: 'aws:sqs', + eventSourceARN: '', + awsRegion: '', +} as SQSRecord; + +export const recordEvent = (events: DigitalLettersEvent[]): SQSEvent => ({ + Records: events.map((event, i) => ({ + ...sqsRecord, + messageId: String(i + 1), + body: JSON.stringify({ ...busEvent, detail: event }), + })), +}); diff --git a/lambdas/status-recorder/src/apis/sqs-handler.ts b/lambdas/status-recorder/src/apis/sqs-handler.ts new file mode 100644 index 00000000..d8432082 --- /dev/null +++ b/lambdas/status-recorder/src/apis/sqs-handler.ts @@ -0,0 +1,110 @@ +import type { + SQSBatchItemFailure, + SQSBatchResponse, + SQSEvent, +} from 'aws-lambda'; +import { + $DigitalLettersEvent, + DigitalLettersEvent, + ReportEvent, +} from 'types/events'; +import { Logger } from 'utils'; + +export interface HandlerDependencies { + athenaArn: string; + logger: Logger; +} + +type ValidatedRecord = { + messageId: string; + event: DigitalLettersEvent; +}; + +function validateRecord( + { body, messageId }: { body: string; messageId: string }, + logger: Logger, +): ValidatedRecord | null { + try { + const sqsEventBody = JSON.parse(body); + const sqsEventDetail = sqsEventBody.detail; + + const { + data: item, + error: parseError, + success: parseSuccess, + } = $DigitalLettersEvent.safeParse(sqsEventDetail); + + if (!parseSuccess) { + logger.warn({ + err: parseError, + description: 'Error parsing queue item', + }); + + return null; + } + + return { messageId, event: item }; + } catch (error) { + logger.warn({ + err: error, + description: 'Error parsing SQS record', + }); + + return null; + } +} + +function generateReportEvent(event: DigitalLettersEvent): ReportEvent { + const { + messageReference, + pageCount, + senderId: senderID, + supplierId, + } = event.data; + const { time, type } = event; + + return { + messageReference, + senderId: senderID, + pageCount, + supplierId, + time, + type, + }; +} + +export const createHandler = ({ athenaArn, logger }: HandlerDependencies) => + async function handler(sqsEvent: SQSEvent): Promise { + const receivedItemCount = sqsEvent.Records.length; + const batchItemFailures: SQSBatchItemFailure[] = []; + const validatedRecords: ValidatedRecord[] = []; + const validEvents: ReportEvent[] = []; + + logger.info(`Received SQS Event of ${receivedItemCount} record(s)`); + + for (const record of sqsEvent.Records) { + const validated = validateRecord(record, logger); + if (validated) { + validatedRecords.push(validated); + } else { + batchItemFailures.push({ itemIdentifier: record.messageId }); + } + } + + for (const validatedRecord of validatedRecords) { + const { event } = validatedRecord; + validEvents.push(generateReportEvent(event)); + } + + logger.info({ + description: `Following report events will be sent to ${athenaArn}`, + validEvents, + }); + + const processedItemCount = receivedItemCount - batchItemFailures.length; + logger.info( + `${processedItemCount} of ${receivedItemCount} records processed successfully`, + ); + + return { batchItemFailures }; + }; diff --git a/lambdas/status-recorder/src/container.ts b/lambdas/status-recorder/src/container.ts new file mode 100644 index 00000000..b7bee1e1 --- /dev/null +++ b/lambdas/status-recorder/src/container.ts @@ -0,0 +1,11 @@ +import { HandlerDependencies } from 'apis/sqs-handler'; +import { loadConfig } from 'infra/config'; +import { logger } from 'utils'; + +export const createContainer = (): HandlerDependencies => { + const { athenaArn } = loadConfig(); + + return { athenaArn, logger }; +}; + +export default createContainer; diff --git a/lambdas/status-recorder/src/index.ts b/lambdas/status-recorder/src/index.ts new file mode 100644 index 00000000..f25a8086 --- /dev/null +++ b/lambdas/status-recorder/src/index.ts @@ -0,0 +1,6 @@ +import { createHandler } from 'apis/sqs-handler'; +import { createContainer } from 'container'; + +export const handler = createHandler(createContainer()); + +export default handler; diff --git a/lambdas/status-recorder/src/infra/config.ts b/lambdas/status-recorder/src/infra/config.ts new file mode 100644 index 00000000..5a3e87f3 --- /dev/null +++ b/lambdas/status-recorder/src/infra/config.ts @@ -0,0 +1,11 @@ +import { defaultConfigReader } from 'utils'; + +export type Config = { + athenaArn: string; +}; + +export function loadConfig(): Config { + return { + athenaArn: defaultConfigReader.getValue('ATHENA_ARN'), + }; +} diff --git a/lambdas/status-recorder/src/types/events.ts b/lambdas/status-recorder/src/types/events.ts new file mode 100644 index 00000000..f7e2b0ad --- /dev/null +++ b/lambdas/status-recorder/src/types/events.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +export const $DigitalLettersEvent = z.object({ + data: z.object({ + messageReference: z.string(), + senderId: z.string(), + pageCount: z.number().optional(), + supplierId: z.string().optional(), + }), + time: z.string(), + type: z.string(), +}); + +export const $ReportEvent = z.object({ + messageReference: z.string(), + senderId: z.string(), + pageCount: z.number().optional(), + supplierId: z.string().optional(), + time: z.string(), + type: z.string(), +}); + +export type DigitalLettersEvent = z.infer; + +export type ReportEvent = z.infer; diff --git a/lambdas/status-recorder/tsconfig.json b/lambdas/status-recorder/tsconfig.json new file mode 100644 index 00000000..f7bcaa1f --- /dev/null +++ b/lambdas/status-recorder/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "baseUrl": "./src/", + "isolatedModules": true + }, + "extends": "@tsconfig/node22/tsconfig.json", + "include": [ + "src/**/*", + "jest.config.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index 8bb82526..e0036e16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "lambdas/pdm-uploader-lambda", "lambdas/core-notifier-lambda", "lambdas/print-status-handler", + "lambdas/status-recorder", "utils/utils", "utils/sender-management", "src/cloudevents", @@ -579,6 +580,76 @@ "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0" } }, + "lambdas/status-recorder": { + "name": "nhs-notify-digital-letters-status-recorder", + "version": "0.0.1", + "dependencies": { + "aws-lambda": "^1.0.7", + "utils": "^0.0.1", + "zod": "^4.1.12" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.2", + "@types/aws-lambda": "^8.10.155", + "@types/jest": "^29.5.14", + "jest": "^29.7.0", + "jest-mock-extended": "^3.0.7", + "typescript": "^5.9.3" + } + }, + "lambdas/status-recorder/node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "lambdas/status-recorder/node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "lambdas/status-recorder/node_modules/jest-mock-extended": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/jest-mock-extended/-/jest-mock-extended-3.0.7.tgz", + "integrity": "sha512-7lsKdLFcW9B9l5NzZ66S/yTQ9k8rFtnwYdCNuRU/81fqDWicNDVhitTSPnrGmNeNm0xyw0JHexEOShrIKRCIRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ts-essentials": "^10.0.0" + }, + "peerDependencies": { + "jest": "^24.0.0 || ^25.0.0 || ^26.0.0 || ^27.0.0 || ^28.0.0 || ^29.0.0", + "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "lambdas/ttl-create-lambda": { "name": "nhs-notify-digital-letters-ttl-create-lambda", "version": "0.0.1", @@ -12474,6 +12545,10 @@ "resolved": "lambdas/print-status-handler", "link": true }, + "node_modules/nhs-notify-digital-letters-status-recorder": { + "resolved": "lambdas/status-recorder", + "link": true + }, "node_modules/nhs-notify-digital-letters-ttl-create-lambda": { "resolved": "lambdas/ttl-create-lambda", "link": true diff --git a/package.json b/package.json index 125f8fb2..7e6fc28f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "lambdas/pdm-uploader-lambda", "lambdas/core-notifier-lambda", "lambdas/print-status-handler", + "lambdas/status-recorder", "utils/utils", "utils/sender-management", "src/cloudevents", From 064e205d93aa7c53b50d9befedd7e2b2df759f30 Mon Sep 17 00:00:00 2001 From: Ian Hodges Date: Mon, 26 Jan 2026 12:28:48 +0000 Subject: [PATCH 02/14] CCM-13295 remove docker example --- .../examples/python/.tool-versions.example | 2 - scripts/docker/examples/python/Dockerfile | 33 ------------ .../examples/python/Dockerfile.effective | 54 ------------------- scripts/docker/examples/python/VERSION | 1 - .../examples/python/assets/hello_world/app.py | 12 ----- .../assets/hello_world/requirements.txt | 12 ----- .../docker/examples/python/tests/goss.yaml | 8 --- 7 files changed, 122 deletions(-) delete mode 100644 scripts/docker/examples/python/.tool-versions.example delete mode 100644 scripts/docker/examples/python/Dockerfile delete mode 100644 scripts/docker/examples/python/Dockerfile.effective delete mode 100644 scripts/docker/examples/python/VERSION delete mode 100644 scripts/docker/examples/python/assets/hello_world/app.py delete mode 100644 scripts/docker/examples/python/assets/hello_world/requirements.txt delete mode 100644 scripts/docker/examples/python/tests/goss.yaml diff --git a/scripts/docker/examples/python/.tool-versions.example b/scripts/docker/examples/python/.tool-versions.example deleted file mode 100644 index 92093116..00000000 --- a/scripts/docker/examples/python/.tool-versions.example +++ /dev/null @@ -1,2 +0,0 @@ -# python, SEE: https://hub.docker.com/_/python/tags -# docker/python 3.11.4-alpine3.18@sha256:0135ae6442d1269379860b361760ad2cf6ab7c403d21935a8015b48d5bf78a86 diff --git a/scripts/docker/examples/python/Dockerfile b/scripts/docker/examples/python/Dockerfile deleted file mode 100644 index d0780aa4..00000000 --- a/scripts/docker/examples/python/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# `*:latest` will be replaced with a corresponding version stored in the '.tool-versions' file -# hadolint ignore=DL3007 -FROM python:latest as base - -# === Builder ================================================================== - -FROM base AS builder -COPY ./assets/hello_world/requirements.txt /requirements.txt -WORKDIR /packages -RUN set -eux; \ - \ - # Install dependencies - pip install \ - --requirement /requirements.txt \ - --prefix=/packages \ - --no-warn-script-location \ - --no-cache-dir - -# === Runtime ================================================================== - -FROM base -ENV \ - LANG="C.UTF-8" \ - LC_ALL="C.UTF-8" \ - PYTHONDONTWRITEBYTECODE="1" \ - PYTHONUNBUFFERED="1" \ - TZ="UTC" -COPY --from=builder /packages /usr/local -COPY ./assets/hello_world /hello_world -WORKDIR /hello_world -USER nobody -CMD [ "python", "app.py" ] -EXPOSE 8000 diff --git a/scripts/docker/examples/python/Dockerfile.effective b/scripts/docker/examples/python/Dockerfile.effective deleted file mode 100644 index 3f1ea6b0..00000000 --- a/scripts/docker/examples/python/Dockerfile.effective +++ /dev/null @@ -1,54 +0,0 @@ -# `*:latest` will be replaced with a corresponding version stored in the '.tool-versions' file -FROM python:3.11.4-alpine3.18@sha256:0135ae6442d1269379860b361760ad2cf6ab7c403d21935a8015b48d5bf78a86 as base - -# === Builder ================================================================== - -FROM base AS builder -COPY ./assets/hello_world/requirements.txt /requirements.txt -WORKDIR /packages -RUN set -eux; \ - \ - # Install dependencies - pip install \ - --requirement /requirements.txt \ - --prefix=/packages \ - --no-warn-script-location \ - --no-cache-dir - -# === Runtime ================================================================== - -FROM base -ENV \ - LANG="C.UTF-8" \ - LC_ALL="C.UTF-8" \ - PYTHONDONTWRITEBYTECODE="1" \ - PYTHONUNBUFFERED="1" \ - TZ="UTC" -COPY --from=builder /packages /usr/local -COPY ./assets/hello_world /hello_world -WORKDIR /hello_world -USER nobody -CMD [ "python", "app.py" ] -EXPOSE 8000 - -# === Metadata ================================================================= - -ARG IMAGE -ARG TITLE -ARG DESCRIPTION -ARG LICENCE -ARG GIT_URL -ARG GIT_BRANCH -ARG GIT_COMMIT_HASH -ARG BUILD_DATE -ARG BUILD_VERSION -LABEL \ - org.opencontainers.image.base.name=$IMAGE \ - org.opencontainers.image.title="$TITLE" \ - org.opencontainers.image.description="$DESCRIPTION" \ - org.opencontainers.image.licenses="$LICENCE" \ - org.opencontainers.image.url=$GIT_URL \ - org.opencontainers.image.ref.name=$GIT_BRANCH \ - org.opencontainers.image.revision=$GIT_COMMIT_HASH \ - org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.version=$BUILD_VERSION diff --git a/scripts/docker/examples/python/VERSION b/scripts/docker/examples/python/VERSION deleted file mode 100644 index 8acdd82b..00000000 --- a/scripts/docker/examples/python/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.1 diff --git a/scripts/docker/examples/python/assets/hello_world/app.py b/scripts/docker/examples/python/assets/hello_world/app.py deleted file mode 100644 index 4844e89c..00000000 --- a/scripts/docker/examples/python/assets/hello_world/app.py +++ /dev/null @@ -1,12 +0,0 @@ -from flask import Flask -from flask_wtf.csrf import CSRFProtect - -app = Flask(__name__) -csrf = CSRFProtect() -csrf.init_app(app) - -@app.route("/") -def index(): - return "Hello World!" - -app.run(host='0.0.0.0', port=8000) diff --git a/scripts/docker/examples/python/assets/hello_world/requirements.txt b/scripts/docker/examples/python/assets/hello_world/requirements.txt deleted file mode 100644 index 1921a428..00000000 --- a/scripts/docker/examples/python/assets/hello_world/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -blinker==1.6.2 -click==8.1.7 -Flask-WTF==1.2.0 -Flask==2.3.3 -itsdangerous==2.1.2 -Jinja2==3.1.6 -MarkupSafe==2.1.3 -pip==25.3 -setuptools==78.1.1 -Werkzeug==3.1.4 -wheel==0.41.1 -WTForms==3.0.1 diff --git a/scripts/docker/examples/python/tests/goss.yaml b/scripts/docker/examples/python/tests/goss.yaml deleted file mode 100644 index 589db37b..00000000 --- a/scripts/docker/examples/python/tests/goss.yaml +++ /dev/null @@ -1,8 +0,0 @@ -package: - python: - installed: true - -command: - pip list | grep -i flask: - exit-status: 0 - timeout: 60000 From e6ff328916cca07529eb60f0f7a78602a273c191 Mon Sep 17 00:00:00 2001 From: Ian Hodges Date: Thu, 29 Jan 2026 11:24:25 +0000 Subject: [PATCH 03/14] CCM-13295 pivot to using a core-like process --- .../terraform/components/dl/README.md | 4 + .../dl/athena_workgroup_reporting.tf | 19 ++ ...cument_eventbridge_firehose_assume_role.tf | 40 +++ .../cloudwatch_event_rule_status_recorder.tf | 3 +- .../dl/cloudwatch_log_group_kinesis_logs.tf | 9 + .../dl/glue_catalog_database_reporting.tf | 4 + .../dl/glue_catalog_table_transactions.tf | 78 ++++++ .../dl/glue_crawler_transactions.tf | 97 ++++++++ ...irehose_delivery_stream_to_s3_reporting.tf | 228 ++++++++++++++++++ .../terraform/components/dl/locals.tf | 6 + .../dl/s3_bucket_policy_reporting.tf | 28 +++ .../components/dl/s3_bucket_reporting.tf | 68 ++++++ .../terraform/components/dl/variables.tf | 30 ++- .../__tests__/apis/firehose-handler.test.ts | 112 +++++++++ .../src/__tests__/apis/sqs-handler.test.ts | 100 -------- .../src/__tests__/container.test.ts | 6 - .../src/__tests__/index.test.ts | 2 +- .../src/__tests__/infra/config.test.ts | 15 -- .../src/__tests__/test-data.ts | 18 +- .../src/apis/firehose-handler.ts | 117 +++++++++ .../status-recorder/src/apis/sqs-handler.ts | 110 --------- lambdas/status-recorder/src/container.ts | 7 +- lambdas/status-recorder/src/index.ts | 2 +- lambdas/status-recorder/src/infra/config.ts | 11 - lambdas/status-recorder/src/types/events.ts | 20 +- 25 files changed, 881 insertions(+), 253 deletions(-) create mode 100644 infrastructure/terraform/components/dl/athena_workgroup_reporting.tf create mode 100644 infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf create mode 100644 infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf create mode 100644 infrastructure/terraform/components/dl/glue_catalog_database_reporting.tf create mode 100644 infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf create mode 100644 infrastructure/terraform/components/dl/glue_crawler_transactions.tf create mode 100644 infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf create mode 100644 infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf create mode 100644 infrastructure/terraform/components/dl/s3_bucket_reporting.tf create mode 100644 lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts delete mode 100644 lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts delete mode 100644 lambdas/status-recorder/src/__tests__/infra/config.test.ts create mode 100644 lambdas/status-recorder/src/apis/firehose-handler.ts delete mode 100644 lambdas/status-recorder/src/apis/sqs-handler.ts delete mode 100644 lambdas/status-recorder/src/infra/config.ts diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index 1ac5c9cf..e5c11c15 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -17,6 +17,7 @@ No requirements. | [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"dl"` | no | | [core\_notify\_url](#input\_core\_notify\_url) | The URL used to send requests to Notify | `string` | `"https://sandbox.api.service.nhs.uk"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | +| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | | [enable\_dynamodb\_delete\_protection](#input\_enable\_dynamodb\_delete\_protection) | Enable DynamoDB Delete Protection on all Tables | `bool` | `true` | no | | [enable\_mock\_mesh](#input\_enable\_mock\_mesh) | Enable mock mesh access (dev only). Grants lambda permission to read mock-mesh prefix in non-pii bucket. | `bool` | `false` | no | | [enable\_pdm\_mock](#input\_enable\_pdm\_mock) | Flag indicating whether to deploy PDM mock API (should be false in production environments) | `bool` | `true` | no | @@ -26,15 +27,18 @@ No requirements. | [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes | | [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no | | [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no | +| [log\_retention\_days](#input\_log\_retention\_days) | How many days to retain Cloudwatch logs | `number` | `180` | no | | [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no | | [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"rate(5 minutes)"` | no | | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | | [pdm\_mock\_access\_token](#input\_pdm\_mock\_access\_token) | Mock access token for PDM API authentication (used in local/dev environments) | `string` | `"mock-pdm-token"` | no | | [pdm\_use\_non\_mock\_token](#input\_pdm\_use\_non\_mock\_token) | Whether to use the shared APIM access token from SSM (/component/environment/apim/access\_token) instead of the mock token | `bool` | `false` | no | +| [pii\_data\_retention\_policy\_days](#input\_pii\_data\_retention\_policy\_days) | The number of days for data retention policy for PII | `number` | `534` | no | | [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | | [queue\_batch\_size](#input\_queue\_batch\_size) | maximum number of queue items to process | `number` | `10` | no | | [queue\_batch\_window\_seconds](#input\_queue\_batch\_window\_seconds) | maximum time in seconds between processing events | `number` | `1` | no | | [region](#input\_region) | The AWS Region | `string` | n/a | yes | +| [s3\_enable\_force\_destroy](#input\_s3\_enable\_force\_destroy) | Allow force destroy of buckets and contents via Terraform - DO NOT ENABLE IN PRODUCTION | `bool` | `false` | no | | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Shared Infra Account ID (numeric) | `string` | n/a | yes | | [ttl\_poll\_schedule](#input\_ttl\_poll\_schedule) | Schedule to poll for any overdue TTL records | `string` | `"rate(10 minutes)"` | no | ## Modules diff --git a/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf b/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf new file mode 100644 index 00000000..5a9f47a9 --- /dev/null +++ b/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf @@ -0,0 +1,19 @@ +resource "aws_athena_workgroup" "reporting" { + name = local.csi + description = "Athena Workgroup for ${var.environment}" + force_destroy = true + + configuration { + enforce_workgroup_configuration = true + + result_configuration { + expected_bucket_owner = var.aws_account_id + output_location = "s3://${aws_s3_bucket.reporting.bucket}/athena-output/" + + encryption_configuration { + encryption_option = "SSE_KMS" + kms_key_arn = module.kms.key_arn + } + } + } +} diff --git a/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf b/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf new file mode 100644 index 00000000..7c7a4504 --- /dev/null +++ b/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf @@ -0,0 +1,40 @@ +# IAM role for EventBridge to write to Kinesis Firehose +data "aws_iam_policy_document" "eventbridge_firehose_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "eventbridge_firehose" { + name = "${local.csi}-eventbridge-firehose" + description = "Role for EventBridge to write to Kinesis Firehose" + assume_role_policy = data.aws_iam_policy_document.eventbridge_firehose_assume_role.json +} + +data "aws_iam_policy_document" "eventbridge_firehose_policy" { + statement { + effect = "Allow" + + actions = [ + "firehose:PutRecord", + "firehose:PutRecordBatch" + ] + + resources = [ + aws_kinesis_firehose_delivery_stream.to_s3_reporting.arn + ] + } +} + +resource "aws_iam_role_policy" "eventbridge_firehose" { + name = "${local.csi}-eventbridge-firehose" + role = aws_iam_role.eventbridge_firehose.id + policy = data.aws_iam_policy_document.eventbridge_firehose_policy.json +} diff --git a/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf b/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf index b7323dfd..56ddd590 100644 --- a/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf +++ b/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf @@ -14,6 +14,7 @@ resource "aws_cloudwatch_event_rule" "status_recorder" { resource "aws_cloudwatch_event_target" "status_recorder" { rule = aws_cloudwatch_event_rule.status_recorder.name - arn = module.sqs_status_recorder.sqs_queue_arn + arn = aws_kinesis_firehose_delivery_stream.to_s3_reporting.arn + role_arn = aws_iam_role.eventbridge_firehose.arn event_bus_name = aws_cloudwatch_event_bus.main.name } diff --git a/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf b/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf new file mode 100644 index 00000000..0adc3684 --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf @@ -0,0 +1,9 @@ +resource "aws_cloudwatch_log_group" "kinesis_logs" { + name = "/aws/kinesisfirehose/${local.csi}-to-s3-reporting" + retention_in_days = var.log_retention_days +} + +resource "aws_cloudwatch_log_stream" "reporting_kinesis_logs" { + name = "${local.csi}reportingKinesisLogs" + log_group_name = aws_cloudwatch_log_group.kinesis_logs.name +} diff --git a/infrastructure/terraform/components/dl/glue_catalog_database_reporting.tf b/infrastructure/terraform/components/dl/glue_catalog_database_reporting.tf new file mode 100644 index 00000000..34b7c8b0 --- /dev/null +++ b/infrastructure/terraform/components/dl/glue_catalog_database_reporting.tf @@ -0,0 +1,4 @@ +resource "aws_glue_catalog_database" "reporting" { + name = "${local.csi}-reporting" + description = "Reporting database for ${var.environment}" +} diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf new file mode 100644 index 00000000..28bfd067 --- /dev/null +++ b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf @@ -0,0 +1,78 @@ +resource "aws_glue_catalog_table" "transactions" { + name = "transaction_history" + description = "transaction history for ${var.environment}" + database_name = aws_glue_catalog_database.reporting.name + + table_type = "EXTERNAL_TABLE" + + storage_descriptor { + location = "s3://${aws_s3_bucket.reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" + + input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" + + ser_de_info { + serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + } + + # additional columns must be added at the end of the list + columns { + name = "messagereference" + type = "string" + } + columns { + name = "pagecount" + type = "int" + } + columns { + name = "supplierid" + type = "string" + } + columns { + name = "time" + type = "string" + } + columns { + name = "type" + type = "string" + } + } + + partition_keys { + name = "senderid" + type = "string" + } + + partition_keys { + name = "__year" + type = "int" + } + partition_keys { + name = "__month" + type = "int" + } + partition_keys { + name = "__day" + type = "int" + } + + parameters = { + EXTERNAL = "TRUE" + "parquet.compression" = "SNAPPY" + } +} + +resource "aws_glue_partition_index" "transaction_data" { + database_name = aws_glue_catalog_database.reporting.name + table_name = aws_glue_catalog_table.transactions.name + + partition_index { + index_name = "data" + keys = ["senderid", "__year", "__month", "__day"] + } + + timeouts { + create = "60m" + delete = "60m" + } +} diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf new file mode 100644 index 00000000..b27df175 --- /dev/null +++ b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf @@ -0,0 +1,97 @@ +resource "aws_glue_crawler" "transactions" { + name = "${local.csi}-transactions-crawler" + database_name = aws_glue_catalog_database.reporting.name + role = aws_iam_role.glue_crawler.arn + table_prefix = "" + + schedule = "cron(0 * * * ? *)" # Run every hour + + s3_target { + path = "s3://${aws_s3_bucket.reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" + } + + schema_change_policy { + delete_behavior = "LOG" + update_behavior = "LOG" + } + + recrawl_policy { + recrawl_behavior = "CRAWL_NEW_FOLDERS_ONLY" + } + + configuration = jsonencode({ + Version = 1.0 + Grouping = { + TableGroupingPolicy = "CombineCompatibleSchemas" + } + CrawlerOutput = { + Partitions = { + AddOrUpdateBehavior = "InheritFromTable" + } + Tables = { + AddOrUpdateBehavior = "MergeNewColumns" + } + } + }) + + depends_on = [aws_glue_catalog_table.transactions] +} + +resource "aws_iam_role" "glue_crawler" { + name = "${local.csi}-glue-crawler" + description = "Role for Glue Crawler to access S3 and Glue Catalog" + assume_role_policy = data.aws_iam_policy_document.glue_crawler_assume_role.json +} + +data "aws_iam_policy_document" "glue_crawler_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role_policy_attachment" "glue_crawler_service" { + role = aws_iam_role.glue_crawler.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" +} + +data "aws_iam_policy_document" "glue_crawler_policy" { + statement { + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + + resources = [ + aws_s3_bucket.reporting.arn, + "${aws_s3_bucket.reporting.arn}/${local.firehose_output_path_prefix}/*" + ] + } + + statement { + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + ] + + resources = [ + module.kms.key_arn + ] + } +} + +resource "aws_iam_role_policy" "glue_crawler" { + name = "${local.csi}-glue-crawler" + role = aws_iam_role.glue_crawler.id + policy = data.aws_iam_policy_document.glue_crawler_policy.json +} diff --git a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf new file mode 100644 index 00000000..71041494 --- /dev/null +++ b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf @@ -0,0 +1,228 @@ +resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { + name = "${local.csi}-to-s3-reporting" + + destination = "extended_s3" + + extended_s3_configuration { + role_arn = aws_iam_role.firehose_role.arn + bucket_arn = aws_s3_bucket.reporting.arn + + prefix = "${local.firehose_output_path_prefix}/reporting/parquet/transaction_history/senderid=!{partitionKeyFromLambda:senderId}/__year=!{partitionKeyFromLambda:year}/__month=!{partitionKeyFromLambda:month}/__day=!{partitionKeyFromLambda:day}/" + error_output_prefix = "${local.firehose_output_path_prefix}/errors/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/!{firehose:error-output-type}/" + + buffering_size = 128 + buffering_interval = 300 + + dynamic_partitioning_configuration { + enabled = true + } + + processing_configuration { + enabled = "true" + + processors { + type = "Lambda" + + parameters { + parameter_name = "LambdaArn" + parameter_value = "${module.status_recorder.function_arn}:$LATEST" + } + parameters { + parameter_name = "RoleArn" + parameter_value = aws_iam_role.firehose_role.arn + } + parameters { + parameter_name = "BufferSizeInMBs" + parameter_value = 1 + } + parameters { + parameter_name = "BufferIntervalInSeconds" + parameter_value = 301 + } + } + } + + data_format_conversion_configuration { + input_format_configuration { + deserializer { + open_x_json_ser_de {} + } + } + + output_format_configuration { + serializer { + parquet_ser_de {} + } + } + + schema_configuration { + database_name = aws_glue_catalog_table.transactions.database_name + role_arn = aws_iam_role.firehose_role.arn + table_name = aws_glue_catalog_table.transactions.name + } + } + + cloudwatch_logging_options { + enabled = true + log_group_name = aws_cloudwatch_log_group.kinesis_logs.name + log_stream_name = aws_cloudwatch_log_stream.reporting_kinesis_logs.name + } + } + +} + +resource "aws_iam_role" "firehose_role" { + name = "${local.csi}-firehose" + description = "Firehose Role" + assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json +} + +data "aws_iam_policy_document" "firehose_assume_role" { + statement { + sid = "FirehoseAssumeRole" + effect = "Allow" + + principals { + type = "Service" + identifiers = [ + "firehose.amazonaws.com" + ] + } + + actions = [ + "sts:AssumeRole" + ] + } +} + +resource "aws_iam_policy" "firehose_policy" { + name = "${local.csi}-firehose" + description = "Firehose Policy" + path = "/" + policy = data.aws_iam_policy_document.firehose_policy.json +} + +resource "aws_iam_role_policy_attachment" "firehose" { + role = aws_iam_role.firehose_role.name + policy_arn = aws_iam_policy.firehose_policy.arn +} + +data "aws_iam_policy_document" "firehose_policy" { + version = "2012-10-17" + + statement { + actions = [ + "logs:PutLogEvents", + ] + + resources = [ + aws_cloudwatch_log_group.kinesis_logs.arn, + aws_cloudwatch_log_stream.reporting_kinesis_logs.arn + ] + + effect = "Allow" + } + + statement { + actions = [ + "lambda:InvokeFunction", + "lambda:GetFunctionConfiguration", + ] + + resources = [ + "${module.status_recorder.function_arn}:$LATEST", + ] + } + + statement { + sid = "AllowSSE" + effect = "Allow" + actions = [ + "kms:DescribeKey", + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*", + ] + + resources = [ + module.kms.key_arn + ] + } + + statement { + sid = "DestinationS3Access" + effect = "Allow" + + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject", + ] + + resources = [ + aws_s3_bucket.reporting.arn, + "${aws_s3_bucket.reporting.arn}/${local.firehose_output_path_prefix}/*" + ] + } + + statement { + sid = "EncryptTargetData" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey" + ] + resources = [ + module.kms.key_arn + ] + + condition { + test = "StringLike" + variable = "kms:EncryptionContext:aws:s3:arn" + values = [ + "${aws_s3_bucket.reporting.arn}" + ] + } + + condition { + test = "StringEquals" + variable = "kms:ViaService" + values = [ + "s3.${var.region}.amazonaws.com" + ] + } + } + + statement { + sid = "AllowListDataStream" + effect = "Allow" + actions = [ + "kinesis:ListStreams" + ] + + resources = [ + "*" + ] + } + + statement { + sid = "AllowGlueTableAccess" + effect = "Allow" + actions = [ + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions" + ] + + resources = [ + aws_glue_catalog_table.transactions.arn, + aws_glue_catalog_database.reporting.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog" + ] + } +} diff --git a/infrastructure/terraform/components/dl/locals.tf b/infrastructure/terraform/components/dl/locals.tf index 9298b97b..5434187c 100644 --- a/infrastructure/terraform/components/dl/locals.tf +++ b/infrastructure/terraform/components/dl/locals.tf @@ -11,4 +11,10 @@ locals { root_domain_id = local.acct.route53_zone_ids["digital-letters"] ttl_shard_count = 3 deploy_pdm_mock = var.enable_pdm_mock + firehose_output_path_prefix = "kinesis-firehose-output" + this_account = var.aws_account_id + pii_retention_config = { + current_days = var.pii_data_retention_policy_days, + non_current_days = 14 + } } diff --git a/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf b/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf new file mode 100644 index 00000000..62c9f056 --- /dev/null +++ b/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf @@ -0,0 +1,28 @@ +resource "aws_s3_bucket_policy" "reporting" { + bucket = aws_s3_bucket.reporting.id + policy = data.aws_iam_policy_document.reporting.json +} + +data "aws_iam_policy_document" "reporting" { + statement { + effect = "Deny" + actions = ["s3:*"] + resources = [ + aws_s3_bucket.reporting.arn, + "${aws_s3_bucket.reporting.arn}/*", + ] + + principals { + type = "AWS" + identifiers = ["*"] + } + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = [ + false + ] + } + } +} diff --git a/infrastructure/terraform/components/dl/s3_bucket_reporting.tf b/infrastructure/terraform/components/dl/s3_bucket_reporting.tf new file mode 100644 index 00000000..ecda410f --- /dev/null +++ b/infrastructure/terraform/components/dl/s3_bucket_reporting.tf @@ -0,0 +1,68 @@ +resource "aws_s3_bucket" "reporting" { + bucket = "${local.csi_global}-reporting" + force_destroy = var.s3_enable_force_destroy + + tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups }) +} + +resource "aws_s3_bucket_ownership_controls" "reporting" { + bucket = aws_s3_bucket.reporting.id + + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "reporting" { + bucket = aws_s3_bucket.reporting.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = module.kms.key_arn + } + bucket_key_enabled = true + } +} + +resource "aws_s3_bucket_versioning" "reporting" { + bucket = aws_s3_bucket.reporting.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_public_access_block" "reporting" { + depends_on = [ + aws_s3_bucket_policy.reporting + ] + + bucket = aws_s3_bucket.reporting.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_lifecycle_configuration" "reporting" { + bucket = aws_s3_bucket.reporting.id + expected_bucket_owner = local.this_account + + rule { + id = "reporting" + status = "Enabled" + + filter { + } + + expiration { + days = local.pii_retention_config.current_days + } + + noncurrent_version_expiration { + noncurrent_days = local.pii_retention_config.non_current_days + } + } +} diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index be79f015..59f6d27f 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -140,7 +140,6 @@ variable "apim_base_url" { default = "https://int.api.service.nhs.uk" } - variable "core_notify_url" { type = string description = "The URL used to send requests to Notify" @@ -176,3 +175,32 @@ variable "enable_pdm_mock" { description = "Flag indicating whether to deploy PDM mock API (should be false in production environments)" default = true } + +variable "s3_enable_force_destroy" { + type = bool + description = "Allow force destroy of buckets and contents via Terraform - DO NOT ENABLE IN PRODUCTION" + default = false + + validation { + condition = !(var.s3_enable_force_destroy && var.environment == "prod") + error_message = "s3_enable_force_destroy must not be set to true when environment is 'prod'." + } +} + +variable "enable_backups" { + type = bool + description = "Enable backups" + default = false +} + +variable "pii_data_retention_policy_days" { + type = number + description = "The number of days for data retention policy for PII" + default = 534 +} + +variable "log_retention_days" { + type = number + description = "How many days to retain Cloudwatch logs" + default = 180 +} diff --git a/lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts b/lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts new file mode 100644 index 00000000..10942086 --- /dev/null +++ b/lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts @@ -0,0 +1,112 @@ +import { mock } from 'jest-mock-extended'; +import { createHandler } from 'apis/firehose-handler'; +import { Logger } from 'utils'; +import { digitalLettersEvent, firehoseEvent } from '__tests__/test-data'; + +const logger = mock(); + +const handler = createHandler({ + logger, +}); + +describe('Firehose Handler', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('status', () => { + it('should process valid digital letters event and return report event', async () => { + const response = await handler(firehoseEvent([digitalLettersEvent])); + + expect(logger.info).toHaveBeenCalledWith( + 'Received Firehose Event of 1 record(s)', + ); + + expect(logger.info).toHaveBeenCalledWith( + '1 of 1 records processed successfully', + ); + + expect(response.records).toHaveLength(1); + expect(response.records[0]).toMatchObject({ + recordId: '1', + result: 'Ok', + metadata: { + partitionKeys: { + year: '2023', + month: '6', + day: '20', + senderId: 'sender1', + }, + }, + }); + + // Verify the data is base64 encoded JSON + const decodedData = JSON.parse( + Buffer.from(response.records[0].data!, 'base64').toString('utf8'), + ); + expect(decodedData).toEqual({ + messageReference: digitalLettersEvent.data.messageReference, + senderId: digitalLettersEvent.data.senderId, + pageCount: digitalLettersEvent.data.pageCount, + supplierId: digitalLettersEvent.data.supplierId, + time: digitalLettersEvent.time, + type: digitalLettersEvent.type, + }); + }); + }); + + describe('errors', () => { + it('should return ProcessingFailed for records with invalid JSON', async () => { + const event = firehoseEvent([digitalLettersEvent]); + event.records[0].data = Buffer.from('not-json').toString('base64'); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: expect.any(SyntaxError), + description: 'Error parsing firehose record', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result.records).toHaveLength(1); + expect(result.records[0]).toMatchObject({ + recordId: '1', + result: 'ProcessingFailed', + }); + }); + + it('should return ProcessingFailed for records with invalid event schema', async () => { + const invalidEvent = { + ...digitalLettersEvent, + type: 123, // Invalid: should be string + }; + const event = firehoseEvent([invalidEvent as any]); + + const result = await handler(event); + + expect(logger.warn).toHaveBeenCalledWith({ + err: expect.objectContaining({ + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ['type'], + }), + ]), + }), + description: 'Error parsing firehose item', + }); + + expect(logger.info).toHaveBeenCalledWith( + '0 of 1 records processed successfully', + ); + + expect(result.records).toHaveLength(1); + expect(result.records[0]).toMatchObject({ + recordId: '1', + result: 'ProcessingFailed', + }); + }); + }); +}); diff --git a/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts b/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts deleted file mode 100644 index bd7d196a..00000000 --- a/lambdas/status-recorder/src/__tests__/apis/sqs-handler.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { mock } from 'jest-mock-extended'; -import { createHandler } from 'apis/sqs-handler'; -import { Logger } from 'utils'; -import { digitalLettersEvent, recordEvent } from '__tests__/test-data'; - -const logger = mock(); - -const handler = createHandler({ - athenaArn: 'some value', - logger, -}); - -describe('SQS Handler', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('status', () => { - it('should process valid digital letters event and log report event', async () => { - const response = await handler(recordEvent([digitalLettersEvent])); - - expect(logger.info).toHaveBeenCalledWith( - 'Received SQS Event of 1 record(s)', - ); - - expect(logger.info).toHaveBeenCalledWith({ - description: 'Following report events will be sent to some value', - validEvents: [ - { - messageReference: digitalLettersEvent.data.messageReference, - senderId: digitalLettersEvent.data.senderId, - pageCount: digitalLettersEvent.data.pageCount, - supplierId: digitalLettersEvent.data.supplierId, - time: digitalLettersEvent.time, - type: digitalLettersEvent.type, - }, - ], - }); - - expect(logger.info).toHaveBeenCalledWith( - '1 of 1 records processed successfully', - ); - - expect(response).toEqual({ batchItemFailures: [] }); - }); - }); - - describe('errors', () => { - it('should return failed SQS records to the queue if an error occurs while parsing them', async () => { - const event = recordEvent([digitalLettersEvent]); - event.Records[0].body = 'not-json'; - - const result = await handler(event); - - expect(logger.warn).toHaveBeenCalledWith({ - err: new SyntaxError( - `Unexpected token 'o', "not-json" is not valid JSON`, - ), - description: 'Error parsing SQS record', - }); - - expect(logger.info).toHaveBeenCalledWith( - '0 of 1 records processed successfully', - ); - - expect(result).toEqual({ - batchItemFailures: [{ itemIdentifier: '1' }], - }); - }); - - it('should return failed items to the queue if an invalid event is received', async () => { - const invalidEvent = { - ...digitalLettersEvent, - type: 123, // Invalid: should be string - }; - const event = recordEvent([invalidEvent as any]); - - const result = await handler(event); - - expect(logger.warn).toHaveBeenCalledWith({ - err: expect.objectContaining({ - issues: expect.arrayContaining([ - expect.objectContaining({ - path: ['type'], - }), - ]), - }), - description: 'Error parsing queue item', - }); - - expect(logger.info).toHaveBeenCalledWith( - '0 of 1 records processed successfully', - ); - - expect(result).toEqual({ - batchItemFailures: [{ itemIdentifier: '1' }], - }); - }); - }); -}); diff --git a/lambdas/status-recorder/src/__tests__/container.test.ts b/lambdas/status-recorder/src/__tests__/container.test.ts index 007ff523..1859a92a 100644 --- a/lambdas/status-recorder/src/__tests__/container.test.ts +++ b/lambdas/status-recorder/src/__tests__/container.test.ts @@ -1,11 +1,5 @@ import { createContainer } from 'container'; -jest.mock('infra/config', () => ({ - loadConfig: jest.fn(() => ({ - athenaArn: 'some value', - })), -})); - jest.mock('utils', () => ({ logger: {}, })); diff --git a/lambdas/status-recorder/src/__tests__/index.test.ts b/lambdas/status-recorder/src/__tests__/index.test.ts index b5465321..c1ecd810 100644 --- a/lambdas/status-recorder/src/__tests__/index.test.ts +++ b/lambdas/status-recorder/src/__tests__/index.test.ts @@ -1,6 +1,6 @@ import { handler } from 'index'; -jest.mock('apis/sqs-handler', () => ({ +jest.mock('apis/firehose-handler', () => ({ createHandler: jest.fn(() => jest.fn()), })); diff --git a/lambdas/status-recorder/src/__tests__/infra/config.test.ts b/lambdas/status-recorder/src/__tests__/infra/config.test.ts deleted file mode 100644 index 2902c80f..00000000 --- a/lambdas/status-recorder/src/__tests__/infra/config.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { loadConfig } from 'infra/config'; - -jest.mock('utils', () => ({ - defaultConfigReader: { - getValue: jest.fn(), - getInt: jest.fn(), - }, -})); - -describe('config', () => { - it('should load config', () => { - const config = loadConfig(); - expect(config).toBeDefined(); - }); -}); diff --git a/lambdas/status-recorder/src/__tests__/test-data.ts b/lambdas/status-recorder/src/__tests__/test-data.ts index 4ead7ba8..950a731d 100644 --- a/lambdas/status-recorder/src/__tests__/test-data.ts +++ b/lambdas/status-recorder/src/__tests__/test-data.ts @@ -1,4 +1,4 @@ -import { SQSEvent, SQSRecord } from 'aws-lambda'; +import { FirehoseTransformationEvent, SQSEvent, SQSRecord } from 'aws-lambda'; import { DigitalLettersEvent } from 'types/events'; const baseEvent = { @@ -59,3 +59,19 @@ export const recordEvent = (events: DigitalLettersEvent[]): SQSEvent => ({ body: JSON.stringify({ ...busEvent, detail: event }), })), }); + +export const firehoseEvent = ( + events: DigitalLettersEvent[], +): FirehoseTransformationEvent => ({ + invocationId: 'test-invocation-id', + deliveryStreamArn: + 'arn:aws:firehose:eu-west-2:123456789012:deliverystream/test', + region: 'eu-west-2', + records: events.map((event, i) => ({ + recordId: String(i + 1), + approximateArrivalTimestamp: Date.now(), + data: Buffer.from(JSON.stringify({ ...busEvent, detail: event })).toString( + 'base64', + ), + })), +}); diff --git a/lambdas/status-recorder/src/apis/firehose-handler.ts b/lambdas/status-recorder/src/apis/firehose-handler.ts new file mode 100644 index 00000000..18bfdc13 --- /dev/null +++ b/lambdas/status-recorder/src/apis/firehose-handler.ts @@ -0,0 +1,117 @@ +import type { + FirehoseTransformationEvent, + FirehoseTransformationEventRecord, + FirehoseTransformationResult, + FirehoseTransformationResultRecord, +} from 'aws-lambda'; +import { + $DigitalLettersEvent, + DigitalLettersEvent, + FlatDigitalLettersEvent, + ReportEvent, +} from 'types/events'; +import { Logger } from 'utils'; + +export interface HandlerDependencies { + logger: Logger; +} + +type ValidatedRecord = { + recordId: string; + event: DigitalLettersEvent; +}; + +function validateRecord( + { data, recordId }: FirehoseTransformationEventRecord, + logger: Logger, +): ValidatedRecord | null { + try { + const eventBody = JSON.parse(Buffer.from(data, 'base64').toString('utf8')); + const eventDetail = eventBody.detail; + + const { + data: item, + error: parseError, + success: parseSuccess, + } = $DigitalLettersEvent.safeParse(eventDetail); + + if (!parseSuccess) { + logger.warn({ + err: parseError, + description: 'Error parsing firehose item', + }); + + return null; + } + + return { recordId, event: item }; + } catch (error) { + logger.warn({ + err: error, + description: 'Error parsing firehose record', + }); + + return null; + } +} + +function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { + const { messageReference, pageCount, senderId, supplierId } = + validatedRecord.event.data; + const { time, type } = validatedRecord.event; + + const flattenedEvent: FlatDigitalLettersEvent = { + messageReference, + senderId, + pageCount, + supplierId, + time, + type, + }; + + return { + recordId: validatedRecord.recordId, + data: Buffer.from(JSON.stringify(flattenedEvent)).toString('base64'), + result: 'Ok', + metadata: { + partitionKeys: { + year: new Date(flattenedEvent.time).getUTCFullYear().toString(), + month: (new Date(flattenedEvent.time).getUTCMonth() + 1).toString(), + day: new Date(flattenedEvent.time).getUTCDate().toString(), + senderId: flattenedEvent.senderId, + }, + }, + }; +} + +export const createHandler = ({ logger }: HandlerDependencies) => + async function handler( + firehoseTransformationEvent: FirehoseTransformationEvent, + ): Promise { + const receivedItemCount = firehoseTransformationEvent.records.length; + const failedEvents: FirehoseTransformationResultRecord[] = []; + const validatedRecords: ValidatedRecord[] = []; + const validEvents: ReportEvent[] = []; + + logger.info(`Received Firehose Event of ${receivedItemCount} record(s)`); + + for (const record of firehoseTransformationEvent.records) { + const validated = validateRecord(record, logger); + if (validated) { + validatedRecords.push(validated); + } else { + failedEvents.push({ ...record, result: 'ProcessingFailed' }); + } + } + + for (const validatedRecord of validatedRecords) { + validEvents.push(generateReportEvent(validatedRecord)); + } + + const processedItemCount = receivedItemCount - failedEvents.length; + logger.info( + `${processedItemCount} of ${receivedItemCount} records processed successfully`, + ); + + return { records: [...validEvents, ...failedEvents] }; + }; diff --git a/lambdas/status-recorder/src/apis/sqs-handler.ts b/lambdas/status-recorder/src/apis/sqs-handler.ts deleted file mode 100644 index d8432082..00000000 --- a/lambdas/status-recorder/src/apis/sqs-handler.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { - SQSBatchItemFailure, - SQSBatchResponse, - SQSEvent, -} from 'aws-lambda'; -import { - $DigitalLettersEvent, - DigitalLettersEvent, - ReportEvent, -} from 'types/events'; -import { Logger } from 'utils'; - -export interface HandlerDependencies { - athenaArn: string; - logger: Logger; -} - -type ValidatedRecord = { - messageId: string; - event: DigitalLettersEvent; -}; - -function validateRecord( - { body, messageId }: { body: string; messageId: string }, - logger: Logger, -): ValidatedRecord | null { - try { - const sqsEventBody = JSON.parse(body); - const sqsEventDetail = sqsEventBody.detail; - - const { - data: item, - error: parseError, - success: parseSuccess, - } = $DigitalLettersEvent.safeParse(sqsEventDetail); - - if (!parseSuccess) { - logger.warn({ - err: parseError, - description: 'Error parsing queue item', - }); - - return null; - } - - return { messageId, event: item }; - } catch (error) { - logger.warn({ - err: error, - description: 'Error parsing SQS record', - }); - - return null; - } -} - -function generateReportEvent(event: DigitalLettersEvent): ReportEvent { - const { - messageReference, - pageCount, - senderId: senderID, - supplierId, - } = event.data; - const { time, type } = event; - - return { - messageReference, - senderId: senderID, - pageCount, - supplierId, - time, - type, - }; -} - -export const createHandler = ({ athenaArn, logger }: HandlerDependencies) => - async function handler(sqsEvent: SQSEvent): Promise { - const receivedItemCount = sqsEvent.Records.length; - const batchItemFailures: SQSBatchItemFailure[] = []; - const validatedRecords: ValidatedRecord[] = []; - const validEvents: ReportEvent[] = []; - - logger.info(`Received SQS Event of ${receivedItemCount} record(s)`); - - for (const record of sqsEvent.Records) { - const validated = validateRecord(record, logger); - if (validated) { - validatedRecords.push(validated); - } else { - batchItemFailures.push({ itemIdentifier: record.messageId }); - } - } - - for (const validatedRecord of validatedRecords) { - const { event } = validatedRecord; - validEvents.push(generateReportEvent(event)); - } - - logger.info({ - description: `Following report events will be sent to ${athenaArn}`, - validEvents, - }); - - const processedItemCount = receivedItemCount - batchItemFailures.length; - logger.info( - `${processedItemCount} of ${receivedItemCount} records processed successfully`, - ); - - return { batchItemFailures }; - }; diff --git a/lambdas/status-recorder/src/container.ts b/lambdas/status-recorder/src/container.ts index b7bee1e1..431346a9 100644 --- a/lambdas/status-recorder/src/container.ts +++ b/lambdas/status-recorder/src/container.ts @@ -1,11 +1,8 @@ -import { HandlerDependencies } from 'apis/sqs-handler'; -import { loadConfig } from 'infra/config'; +import { HandlerDependencies } from 'apis/firehose-handler'; import { logger } from 'utils'; export const createContainer = (): HandlerDependencies => { - const { athenaArn } = loadConfig(); - - return { athenaArn, logger }; + return { logger }; }; export default createContainer; diff --git a/lambdas/status-recorder/src/index.ts b/lambdas/status-recorder/src/index.ts index f25a8086..0794289c 100644 --- a/lambdas/status-recorder/src/index.ts +++ b/lambdas/status-recorder/src/index.ts @@ -1,4 +1,4 @@ -import { createHandler } from 'apis/sqs-handler'; +import { createHandler } from 'apis/firehose-handler'; import { createContainer } from 'container'; export const handler = createHandler(createContainer()); diff --git a/lambdas/status-recorder/src/infra/config.ts b/lambdas/status-recorder/src/infra/config.ts deleted file mode 100644 index 5a3e87f3..00000000 --- a/lambdas/status-recorder/src/infra/config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { defaultConfigReader } from 'utils'; - -export type Config = { - athenaArn: string; -}; - -export function loadConfig(): Config { - return { - athenaArn: defaultConfigReader.getValue('ATHENA_ARN'), - }; -} diff --git a/lambdas/status-recorder/src/types/events.ts b/lambdas/status-recorder/src/types/events.ts index f7e2b0ad..d8438ce3 100644 --- a/lambdas/status-recorder/src/types/events.ts +++ b/lambdas/status-recorder/src/types/events.ts @@ -11,7 +11,7 @@ export const $DigitalLettersEvent = z.object({ type: z.string(), }); -export const $ReportEvent = z.object({ +export const $FlatDigitalLettersEvent = z.object({ messageReference: z.string(), senderId: z.string(), pageCount: z.number().optional(), @@ -20,6 +20,24 @@ export const $ReportEvent = z.object({ type: z.string(), }); +// Custom type similar to FirehoseTransformationResultRecord from aws-lambda, +// but with strict metadata typing for dynamic partitioning keys +export const $ReportEvent = z.object({ + recordId: z.string(), + data: z.string(), + result: z.enum(['Ok', 'Dropped', 'ProcessingFailed']), + metadata: z.object({ + partitionKeys: z.object({ + year: z.string(), + month: z.string(), + day: z.string(), + senderId: z.string(), + }), + }), +}); + export type DigitalLettersEvent = z.infer; +export type FlatDigitalLettersEvent = z.infer; + export type ReportEvent = z.infer; From 1a793162f5dd8b5298badda2b43f3d62ce6bcf0b Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Thu, 29 Jan 2026 17:45:02 +0000 Subject: [PATCH 04/14] CCM-13295: Configure Glue crawler to use a table target --- .../terraform/components/dl/glue_crawler_transactions.tf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf index b27df175..7ad5ee1f 100644 --- a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf @@ -2,12 +2,12 @@ resource "aws_glue_crawler" "transactions" { name = "${local.csi}-transactions-crawler" database_name = aws_glue_catalog_database.reporting.name role = aws_iam_role.glue_crawler.arn - table_prefix = "" schedule = "cron(0 * * * ? *)" # Run every hour - s3_target { - path = "s3://${aws_s3_bucket.reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" + catalog_target { + database_name = aws_glue_catalog_database.reporting.name + tables = [aws_glue_catalog_table.transactions.name] } schema_change_policy { @@ -33,8 +33,6 @@ resource "aws_glue_crawler" "transactions" { } } }) - - depends_on = [aws_glue_catalog_table.transactions] } resource "aws_iam_role" "glue_crawler" { From 94a3361c6f9bf56d3ded5db0cd268fb2fed031cc Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Thu, 29 Jan 2026 19:52:31 +0000 Subject: [PATCH 05/14] CCM-13295: Configure Glue crawler to crawl everything --- .../terraform/components/dl/glue_crawler_transactions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf index 7ad5ee1f..b0a02c93 100644 --- a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf @@ -16,7 +16,7 @@ resource "aws_glue_crawler" "transactions" { } recrawl_policy { - recrawl_behavior = "CRAWL_NEW_FOLDERS_ONLY" + recrawl_behavior = "CRAWL_EVERYTHING" } configuration = jsonencode({ From 4ed046541fbb8366752bf9d7374c59095ef4381c Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:25:43 +0000 Subject: [PATCH 06/14] CCM-13295: Update Glue table config so crawler works --- .../terraform/components/dl/glue_catalog_table_transactions.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf index 28bfd067..eaf22934 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf @@ -59,6 +59,8 @@ resource "aws_glue_catalog_table" "transactions" { parameters = { EXTERNAL = "TRUE" "parquet.compression" = "SNAPPY" + compressionType = "none" + classification = "parquet" } } From c8496937f459213d8d87eeb4f10858368a80af7a Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:48:45 +0000 Subject: [PATCH 07/14] CCM-13295: Update workspace config to include new Node applications in Jest config --- project.code-workspace | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/project.code-workspace b/project.code-workspace index b7407a2a..9ee010ae 100644 --- a/project.code-workspace +++ b/project.code-workspace @@ -80,15 +80,22 @@ "terminal.integrated.scrollback": 10000, "shellcheck.run": "onSave", "jest.virtualFolders": [ + { "name": "core-notifier-lambda", "rootPath": "lambdas/core-notifier-lambda" }, { "name": "key-generation", "rootPath": "lambdas/key-generation" }, - { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" }, + { "name": "pdm-mock-lambda", "rootPath": "lambdas/pdm-mock-lambda" }, + { "name": "pdm-poll-lambda", "rootPath": "lambdas/pdm-poll-lambda" }, + { "name": "pdm-uploader-lambda", "rootPath": "lambdas/pdm-uploader-lambda" }, + { "name": "print-status-handler", "rootPath": "lambdas/print-status-handler" }, { "name": "python-schema-generator", "rootPath": "src/python-schema-generator" }, + { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" }, + { "name": "sender-management", "rootPath": "utils/sender-management" }, + { "name": "status-recorder", "rootPath": "lambdas/status-recorder" }, { "name": "ttl-create-lambda", "rootPath": "lambdas/ttl-create-lambda/" }, { "name": "ttl-handle-expiry-lambda", "rootPath": "lambdas/ttl-handle-expiry-lambda" }, { "name": "ttl-poll-lambda", "rootPath": "lambdas/ttl-poll-lambda" }, - { "name": "sender-management", "rootPath": "utils/sender-management" }, { "name": "utils", "rootPath": "utils/utils" }, ], + "testing.defaultGutterClickAction": "runWithCoverage", }, "extensions": { "recommendations": [ From 9d34c1d17c484a16a503c05a59f90b7b0ec4b930 Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Fri, 30 Jan 2026 14:56:03 +0000 Subject: [PATCH 08/14] CCM-13295: Tidy TF --- .../terraform/components/dl/README.md | 3 -- ...tf => cloudwatch_event_rule_all_events.tf} | 10 ++--- .../dl/cloudwatch_log_group_kinesis_logs.tf | 2 +- ...da_event_source_mapping_status_recorder.tf | 10 ----- .../terraform/components/dl/locals.tf | 1 - .../dl/module_lambda_status_recorder.tf | 40 ------------------ .../dl/module_sqs_status_recorder.tf | 42 ------------------- .../components/dl/s3_bucket_reporting.tf | 4 +- .../terraform/components/dl/variables.tf | 22 +++------- 9 files changed, 13 insertions(+), 121 deletions(-) rename infrastructure/terraform/components/dl/{cloudwatch_event_rule_status_recorder.tf => cloudwatch_event_rule_all_events.tf} (57%) delete mode 100644 infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf delete mode 100644 infrastructure/terraform/components/dl/module_sqs_status_recorder.tf diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index e5c11c15..3f6bd86c 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -27,7 +27,6 @@ No requirements. | [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes | | [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no | | [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no | -| [log\_retention\_days](#input\_log\_retention\_days) | How many days to retain Cloudwatch logs | `number` | `180` | no | | [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no | | [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"rate(5 minutes)"` | no | | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | @@ -38,7 +37,6 @@ No requirements. | [queue\_batch\_size](#input\_queue\_batch\_size) | maximum number of queue items to process | `number` | `10` | no | | [queue\_batch\_window\_seconds](#input\_queue\_batch\_window\_seconds) | maximum time in seconds between processing events | `number` | `1` | no | | [region](#input\_region) | The AWS Region | `string` | n/a | yes | -| [s3\_enable\_force\_destroy](#input\_s3\_enable\_force\_destroy) | Allow force destroy of buckets and contents via Terraform - DO NOT ENABLE IN PRODUCTION | `bool` | `false` | no | | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Shared Infra Account ID (numeric) | `string` | n/a | yes | | [ttl\_poll\_schedule](#input\_ttl\_poll\_schedule) | Schedule to poll for any overdue TTL records | `string` | `"rate(10 minutes)"` | no | ## Modules @@ -66,7 +64,6 @@ No requirements. | [sqs\_pdm\_poll](#module\_sqs\_pdm\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_pdm\_uploader](#module\_sqs\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_print\_status\_handler](#module\_sqs\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | -| [sqs\_status\_recorder](#module\_sqs\_status\_recorder) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a | | [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [status\_recorder](#module\_status\_recorder) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | diff --git a/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf b/infrastructure/terraform/components/dl/cloudwatch_event_rule_all_events.tf similarity index 57% rename from infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf rename to infrastructure/terraform/components/dl/cloudwatch_event_rule_all_events.tf index 56ddd590..f27e48de 100644 --- a/infrastructure/terraform/components/dl/cloudwatch_event_rule_status_recorder.tf +++ b/infrastructure/terraform/components/dl/cloudwatch_event_rule_all_events.tf @@ -1,6 +1,6 @@ -resource "aws_cloudwatch_event_rule" "status_recorder" { - name = "${local.csi}-status-recorder" - description = "Status recorder event rule" +resource "aws_cloudwatch_event_rule" "all_events" { + name = "${local.csi}-all-events" + description = "Event rule to match all Digital Letters events" event_bus_name = aws_cloudwatch_event_bus.main.name event_pattern = jsonencode({ @@ -12,8 +12,8 @@ resource "aws_cloudwatch_event_rule" "status_recorder" { }) } -resource "aws_cloudwatch_event_target" "status_recorder" { - rule = aws_cloudwatch_event_rule.status_recorder.name +resource "aws_cloudwatch_event_target" "reporting_firehose" { + rule = aws_cloudwatch_event_rule.all_events.name arn = aws_kinesis_firehose_delivery_stream.to_s3_reporting.arn role_arn = aws_iam_role.eventbridge_firehose.arn event_bus_name = aws_cloudwatch_event_bus.main.name diff --git a/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf b/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf index 0adc3684..4133622f 100644 --- a/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf +++ b/infrastructure/terraform/components/dl/cloudwatch_log_group_kinesis_logs.tf @@ -1,6 +1,6 @@ resource "aws_cloudwatch_log_group" "kinesis_logs" { name = "/aws/kinesisfirehose/${local.csi}-to-s3-reporting" - retention_in_days = var.log_retention_days + retention_in_days = var.log_retention_in_days } resource "aws_cloudwatch_log_stream" "reporting_kinesis_logs" { diff --git a/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf b/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf deleted file mode 100644 index 44efff2f..00000000 --- a/infrastructure/terraform/components/dl/lambda_event_source_mapping_status_recorder.tf +++ /dev/null @@ -1,10 +0,0 @@ -resource "aws_lambda_event_source_mapping" "status_recorder" { - event_source_arn = module.sqs_status_recorder.sqs_queue_arn - function_name = module.status_recorder.function_name - batch_size = var.queue_batch_size - maximum_batching_window_in_seconds = var.queue_batch_window_seconds - - function_response_types = [ - "ReportBatchItemFailures" - ] -} diff --git a/infrastructure/terraform/components/dl/locals.tf b/infrastructure/terraform/components/dl/locals.tf index 5434187c..013e2298 100644 --- a/infrastructure/terraform/components/dl/locals.tf +++ b/infrastructure/terraform/components/dl/locals.tf @@ -12,7 +12,6 @@ locals { ttl_shard_count = 3 deploy_pdm_mock = var.enable_pdm_mock firehose_output_path_prefix = "kinesis-firehose-output" - this_account = var.aws_account_id pii_retention_config = { current_days = var.pii_data_retention_policy_days, non_current_days = 14 diff --git a/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf b/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf index ee8b4251..aaa31156 100644 --- a/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf +++ b/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf @@ -14,10 +14,6 @@ module "status_recorder" { log_retention_in_days = var.log_retention_in_days kms_key_arn = module.kms.key_arn - iam_policy_document = { - body = data.aws_iam_policy_document.status_recorder.json - } - function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] function_code_base_path = local.aws_lambda_functions_dir_path function_code_dir = "status-recorder/dist" @@ -33,40 +29,4 @@ module "status_recorder" { log_destination_arn = local.log_destination_arn log_subscription_role_arn = local.acct.log_subscription_role_arn - - lambda_env_vars = { - "ATHENA_ARN" = "Some Value" - } -} - -data "aws_iam_policy_document" "status_recorder" { - statement { - sid = "SQSPermissionsDLQs" - effect = "Allow" - - actions = [ - "sqs:SendMessage", - "sqs:SendMessageBatch", - ] - - resources = [ - module.sqs_event_publisher_errors.sqs_queue_arn, - ] - } - - statement { - sid = "SQSPermissionsStatusRecorderQueue" - effect = "Allow" - - actions = [ - "sqs:ReceiveMessage", - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - ] - - resources = [ - module.sqs_status_recorder.sqs_queue_arn, - ] - } } diff --git a/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf b/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf deleted file mode 100644 index e8220d48..00000000 --- a/infrastructure/terraform/components/dl/module_sqs_status_recorder.tf +++ /dev/null @@ -1,42 +0,0 @@ -module "sqs_status_recorder" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip" - - aws_account_id = var.aws_account_id - component = local.component - environment = var.environment - project = var.project - region = var.region - name = "status-recorder" - sqs_kms_key_arn = module.kms.key_arn - visibility_timeout_seconds = 60 - delay_seconds = 5 - create_dlq = true - max_receive_count = 1 - sqs_policy_overload = data.aws_iam_policy_document.sqs_status_recorder.json -} - -data "aws_iam_policy_document" "sqs_status_recorder" { - statement { - sid = "AllowEventBridgeToSendMessage" - effect = "Allow" - - principals { - type = "Service" - identifiers = ["events.amazonaws.com"] - } - - actions = [ - "sqs:SendMessage" - ] - - resources = [ - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-status-recorder-queue" - ] - - condition { - test = "ArnLike" - variable = "aws:SourceArn" - values = [ aws_cloudwatch_event_rule.status_recorder.arn ] - } - } -} diff --git a/infrastructure/terraform/components/dl/s3_bucket_reporting.tf b/infrastructure/terraform/components/dl/s3_bucket_reporting.tf index ecda410f..6383d8c3 100644 --- a/infrastructure/terraform/components/dl/s3_bucket_reporting.tf +++ b/infrastructure/terraform/components/dl/s3_bucket_reporting.tf @@ -1,6 +1,6 @@ resource "aws_s3_bucket" "reporting" { bucket = "${local.csi_global}-reporting" - force_destroy = var.s3_enable_force_destroy + force_destroy = var.force_destroy tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups }) } @@ -48,7 +48,7 @@ resource "aws_s3_bucket_public_access_block" "reporting" { resource "aws_s3_bucket_lifecycle_configuration" "reporting" { bucket = aws_s3_bucket.reporting.id - expected_bucket_owner = local.this_account + expected_bucket_owner = var.aws_account_id rule { id = "reporting" diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index 59f6d27f..1a964e11 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -168,6 +168,11 @@ variable "force_destroy" { type = bool description = "Flag to force deletion of S3 buckets" default = false + + validation { + condition = !(var.force_destroy && var.environment == "prod") + error_message = "force_destroy must not be set to true when environment is 'prod'." + } } variable "enable_pdm_mock" { @@ -176,17 +181,6 @@ variable "enable_pdm_mock" { default = true } -variable "s3_enable_force_destroy" { - type = bool - description = "Allow force destroy of buckets and contents via Terraform - DO NOT ENABLE IN PRODUCTION" - default = false - - validation { - condition = !(var.s3_enable_force_destroy && var.environment == "prod") - error_message = "s3_enable_force_destroy must not be set to true when environment is 'prod'." - } -} - variable "enable_backups" { type = bool description = "Enable backups" @@ -198,9 +192,3 @@ variable "pii_data_retention_policy_days" { description = "The number of days for data retention policy for PII" default = 534 } - -variable "log_retention_days" { - type = number - description = "How many days to retain Cloudwatch logs" - default = 180 -} From ce2c54da423fb6b1737ec5793a0a8ee5776f705a Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Fri, 30 Jan 2026 15:06:04 +0000 Subject: [PATCH 09/14] CCM-13295: Tidy lambda code --- .../src/__tests__/test-data.ts | 41 +++------------ .../src/apis/firehose-handler.ts | 16 +++--- lambdas/status-recorder/src/types/events.ts | 50 +++++++++---------- 3 files changed, 37 insertions(+), 70 deletions(-) diff --git a/lambdas/status-recorder/src/__tests__/test-data.ts b/lambdas/status-recorder/src/__tests__/test-data.ts index 950a731d..e8043885 100644 --- a/lambdas/status-recorder/src/__tests__/test-data.ts +++ b/lambdas/status-recorder/src/__tests__/test-data.ts @@ -1,4 +1,4 @@ -import { FirehoseTransformationEvent, SQSEvent, SQSRecord } from 'aws-lambda'; +import { FirehoseTransformationEvent } from 'aws-lambda'; import { DigitalLettersEvent } from 'types/events'; const baseEvent = { @@ -31,35 +31,6 @@ export const digitalLettersEvent = { 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-submitted-data.schema.json', } as DigitalLettersEvent; -const busEvent = { - version: '0', - id: 'ab07d406-0797-e919-ff9b-3ad9c5498114', -}; - -const sqsRecord = { - messageId: '1', - receiptHandle: 'abc', - attributes: { - ApproximateReceiveCount: '1', - SentTimestamp: '2025-07-03T14:23:30Z', - SenderId: 'sender-id', - ApproximateFirstReceiveTimestamp: '2025-07-03T14:23:30Z', - }, - messageAttributes: {}, - md5OfBody: '', - eventSource: 'aws:sqs', - eventSourceARN: '', - awsRegion: '', -} as SQSRecord; - -export const recordEvent = (events: DigitalLettersEvent[]): SQSEvent => ({ - Records: events.map((event, i) => ({ - ...sqsRecord, - messageId: String(i + 1), - body: JSON.stringify({ ...busEvent, detail: event }), - })), -}); - export const firehoseEvent = ( events: DigitalLettersEvent[], ): FirehoseTransformationEvent => ({ @@ -70,8 +41,12 @@ export const firehoseEvent = ( records: events.map((event, i) => ({ recordId: String(i + 1), approximateArrivalTimestamp: Date.now(), - data: Buffer.from(JSON.stringify({ ...busEvent, detail: event })).toString( - 'base64', - ), + data: Buffer.from( + JSON.stringify({ + version: '0', + id: 'ab07d406-0797-e919-ff9b-3ad9c5498114', + detail: event, + }), + ).toString('base64'), })), }); diff --git a/lambdas/status-recorder/src/apis/firehose-handler.ts b/lambdas/status-recorder/src/apis/firehose-handler.ts index 18bfdc13..171003a2 100644 --- a/lambdas/status-recorder/src/apis/firehose-handler.ts +++ b/lambdas/status-recorder/src/apis/firehose-handler.ts @@ -59,6 +59,7 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { const { messageReference, pageCount, senderId, supplierId } = validatedRecord.event.data; const { time, type } = validatedRecord.event; + const eventTime = new Date(time); const flattenedEvent: FlatDigitalLettersEvent = { messageReference, @@ -75,10 +76,10 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { result: 'Ok', metadata: { partitionKeys: { - year: new Date(flattenedEvent.time).getUTCFullYear().toString(), - month: (new Date(flattenedEvent.time).getUTCMonth() + 1).toString(), - day: new Date(flattenedEvent.time).getUTCDate().toString(), - senderId: flattenedEvent.senderId, + year: eventTime.getUTCFullYear().toString(), + month: (eventTime.getUTCMonth() + 1).toString(), + day: eventTime.getUTCDate().toString(), + senderId, }, }, }; @@ -90,7 +91,6 @@ export const createHandler = ({ logger }: HandlerDependencies) => ): Promise { const receivedItemCount = firehoseTransformationEvent.records.length; const failedEvents: FirehoseTransformationResultRecord[] = []; - const validatedRecords: ValidatedRecord[] = []; const validEvents: ReportEvent[] = []; logger.info(`Received Firehose Event of ${receivedItemCount} record(s)`); @@ -98,16 +98,12 @@ export const createHandler = ({ logger }: HandlerDependencies) => for (const record of firehoseTransformationEvent.records) { const validated = validateRecord(record, logger); if (validated) { - validatedRecords.push(validated); + validEvents.push(generateReportEvent(validated)); } else { failedEvents.push({ ...record, result: 'ProcessingFailed' }); } } - for (const validatedRecord of validatedRecords) { - validEvents.push(generateReportEvent(validatedRecord)); - } - const processedItemCount = receivedItemCount - failedEvents.length; logger.info( `${processedItemCount} of ${receivedItemCount} records processed successfully`, diff --git a/lambdas/status-recorder/src/types/events.ts b/lambdas/status-recorder/src/types/events.ts index d8438ce3..c018f0f2 100644 --- a/lambdas/status-recorder/src/types/events.ts +++ b/lambdas/status-recorder/src/types/events.ts @@ -11,33 +11,29 @@ export const $DigitalLettersEvent = z.object({ type: z.string(), }); -export const $FlatDigitalLettersEvent = z.object({ - messageReference: z.string(), - senderId: z.string(), - pageCount: z.number().optional(), - supplierId: z.string().optional(), - time: z.string(), - type: z.string(), -}); - -// Custom type similar to FirehoseTransformationResultRecord from aws-lambda, -// but with strict metadata typing for dynamic partitioning keys -export const $ReportEvent = z.object({ - recordId: z.string(), - data: z.string(), - result: z.enum(['Ok', 'Dropped', 'ProcessingFailed']), - metadata: z.object({ - partitionKeys: z.object({ - year: z.string(), - month: z.string(), - day: z.string(), - senderId: z.string(), - }), - }), -}); - export type DigitalLettersEvent = z.infer; -export type FlatDigitalLettersEvent = z.infer; +export type FlatDigitalLettersEvent = { + messageReference: string; + senderId: string; + pageCount?: number; + supplierId?: string; + time: string; + type: string; +}; -export type ReportEvent = z.infer; +// Custom type similar to FirehoseTransformationResultRecord from aws-lambda, +// but with strict metadata typing for dynamic partitioning keys +export type ReportEvent = { + recordId: string; + data: string; + result: 'Ok' | 'Dropped' | 'ProcessingFailed'; + metadata: { + partitionKeys: { + year: string; + month: string; + day: string; + senderId: string; + }; + }; +}; From d889b1c056e89308d00fdeae2134aca90673c610 Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Mon, 2 Feb 2026 10:35:19 +0000 Subject: [PATCH 10/14] CCM-13295: Fix TF formatting --- ...am_policy_document_eventbridge_firehose_assume_role.tf | 6 +++--- .../terraform/components/dl/glue_crawler_transactions.tf | 8 ++++---- .../kinesis_firehose_delivery_stream_to_s3_reporting.tf | 8 ++++---- infrastructure/terraform/components/dl/variables.tf | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf b/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf index 7c7a4504..296905a7 100644 --- a/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf +++ b/infrastructure/terraform/components/dl/aws_iam_policy_document_eventbridge_firehose_assume_role.tf @@ -13,9 +13,9 @@ data "aws_iam_policy_document" "eventbridge_firehose_assume_role" { } resource "aws_iam_role" "eventbridge_firehose" { - name = "${local.csi}-eventbridge-firehose" - description = "Role for EventBridge to write to Kinesis Firehose" - assume_role_policy = data.aws_iam_policy_document.eventbridge_firehose_assume_role.json + name = "${local.csi}-eventbridge-firehose" + description = "Role for EventBridge to write to Kinesis Firehose" + assume_role_policy = data.aws_iam_policy_document.eventbridge_firehose_assume_role.json } data "aws_iam_policy_document" "eventbridge_firehose_policy" { diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf index b0a02c93..80a67a44 100644 --- a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf @@ -7,7 +7,7 @@ resource "aws_glue_crawler" "transactions" { catalog_target { database_name = aws_glue_catalog_database.reporting.name - tables = [aws_glue_catalog_table.transactions.name] + tables = [aws_glue_catalog_table.transactions.name] } schema_change_policy { @@ -36,9 +36,9 @@ resource "aws_glue_crawler" "transactions" { } resource "aws_iam_role" "glue_crawler" { - name = "${local.csi}-glue-crawler" - description = "Role for Glue Crawler to access S3 and Glue Catalog" - assume_role_policy = data.aws_iam_policy_document.glue_crawler_assume_role.json + name = "${local.csi}-glue-crawler" + description = "Role for Glue Crawler to access S3 and Glue Catalog" + assume_role_policy = data.aws_iam_policy_document.glue_crawler_assume_role.json } data "aws_iam_policy_document" "glue_crawler_assume_role" { diff --git a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf index 71041494..de732a2a 100644 --- a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf +++ b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf @@ -1,5 +1,5 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { - name = "${local.csi}-to-s3-reporting" + name = "${local.csi}-to-s3-reporting" destination = "extended_s3" @@ -72,9 +72,9 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { } resource "aws_iam_role" "firehose_role" { - name = "${local.csi}-firehose" - description = "Firehose Role" - assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json + name = "${local.csi}-firehose" + description = "Firehose Role" + assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json } data "aws_iam_policy_document" "firehose_assume_role" { diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index 1a964e11..7530e7c8 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -89,7 +89,7 @@ variable "parent_acct_environment" { variable "mesh_poll_schedule" { type = string description = "Schedule to poll MESH for messages" - default = "rate(5 minutes)" # Every 5 minutes + default = "rate(5 minutes)" # Every 5 minutes } variable "enable_mock_mesh" { From 7b72563d44e403b98ca545e7dc11cef29249c397 Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Mon, 2 Feb 2026 14:40:29 +0000 Subject: [PATCH 11/14] CCM-13295: Use shared module for reporting bucket --- .../terraform/components/dl/README.md | 2 +- .../dl/athena_workgroup_reporting.tf | 2 +- .../dl/glue_catalog_table_transactions.tf | 2 +- .../dl/glue_crawler_transactions.tf | 4 +- ...irehose_delivery_stream_to_s3_reporting.tf | 8 +- .../dl/module_s3bucket_reporting.tf | 75 +++++++++++++++++++ .../dl/s3_bucket_policy_reporting.tf | 28 ------- .../components/dl/s3_bucket_reporting.tf | 68 ----------------- .../terraform/components/dl/variables.tf | 6 -- 9 files changed, 84 insertions(+), 111 deletions(-) create mode 100644 infrastructure/terraform/components/dl/module_s3bucket_reporting.tf delete mode 100644 infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf delete mode 100644 infrastructure/terraform/components/dl/s3_bucket_reporting.tf diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index 3f6bd86c..ac141144 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -17,7 +17,6 @@ No requirements. | [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"dl"` | no | | [core\_notify\_url](#input\_core\_notify\_url) | The URL used to send requests to Notify | `string` | `"https://sandbox.api.service.nhs.uk"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | -| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | | [enable\_dynamodb\_delete\_protection](#input\_enable\_dynamodb\_delete\_protection) | Enable DynamoDB Delete Protection on all Tables | `bool` | `true` | no | | [enable\_mock\_mesh](#input\_enable\_mock\_mesh) | Enable mock mesh access (dev only). Grants lambda permission to read mock-mesh prefix in non-pii bucket. | `bool` | `false` | no | | [enable\_pdm\_mock](#input\_enable\_pdm\_mock) | Flag indicating whether to deploy PDM mock API (should be false in production environments) | `bool` | `true` | no | @@ -57,6 +56,7 @@ No requirements. | [s3bucket\_letters](#module\_s3bucket\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_non\_pii\_data](#module\_s3bucket\_non\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_pii\_data](#module\_s3bucket\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | +| [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_static\_assets](#module\_s3bucket\_static\_assets) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [sqs\_core\_notifier](#module\_sqs\_core\_notifier) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_event\_publisher\_errors](#module\_sqs\_event\_publisher\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | diff --git a/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf b/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf index 5a9f47a9..6529f19f 100644 --- a/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf +++ b/infrastructure/terraform/components/dl/athena_workgroup_reporting.tf @@ -8,7 +8,7 @@ resource "aws_athena_workgroup" "reporting" { result_configuration { expected_bucket_owner = var.aws_account_id - output_location = "s3://${aws_s3_bucket.reporting.bucket}/athena-output/" + output_location = "s3://${module.s3bucket_reporting.bucket}/athena-output/" encryption_configuration { encryption_option = "SSE_KMS" diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf index eaf22934..5336e0f2 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf @@ -6,7 +6,7 @@ resource "aws_glue_catalog_table" "transactions" { table_type = "EXTERNAL_TABLE" storage_descriptor { - location = "s3://${aws_s3_bucket.reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" + location = "s3://${module.s3bucket_reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf index 80a67a44..ac736ff3 100644 --- a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf @@ -69,8 +69,8 @@ data "aws_iam_policy_document" "glue_crawler_policy" { ] resources = [ - aws_s3_bucket.reporting.arn, - "${aws_s3_bucket.reporting.arn}/${local.firehose_output_path_prefix}/*" + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/${local.firehose_output_path_prefix}/*" ] } diff --git a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf index de732a2a..b13d48fc 100644 --- a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf +++ b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf @@ -5,7 +5,7 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { extended_s3_configuration { role_arn = aws_iam_role.firehose_role.arn - bucket_arn = aws_s3_bucket.reporting.arn + bucket_arn = module.s3bucket_reporting.arn prefix = "${local.firehose_output_path_prefix}/reporting/parquet/transaction_history/senderid=!{partitionKeyFromLambda:senderId}/__year=!{partitionKeyFromLambda:year}/__month=!{partitionKeyFromLambda:month}/__day=!{partitionKeyFromLambda:day}/" error_output_prefix = "${local.firehose_output_path_prefix}/errors/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/!{firehose:error-output-type}/" @@ -164,8 +164,8 @@ data "aws_iam_policy_document" "firehose_policy" { ] resources = [ - aws_s3_bucket.reporting.arn, - "${aws_s3_bucket.reporting.arn}/${local.firehose_output_path_prefix}/*" + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/${local.firehose_output_path_prefix}/*" ] } @@ -185,7 +185,7 @@ data "aws_iam_policy_document" "firehose_policy" { test = "StringLike" variable = "kms:EncryptionContext:aws:s3:arn" values = [ - "${aws_s3_bucket.reporting.arn}" + "${module.s3bucket_reporting.arn}" ] } diff --git a/infrastructure/terraform/components/dl/module_s3bucket_reporting.tf b/infrastructure/terraform/components/dl/module_s3bucket_reporting.tf new file mode 100644 index 00000000..739f7c99 --- /dev/null +++ b/infrastructure/terraform/components/dl/module_s3bucket_reporting.tf @@ -0,0 +1,75 @@ +module "s3bucket_reporting" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip" + + name = "reporting" + + aws_account_id = var.aws_account_id + region = var.region + project = var.project + environment = var.environment + component = local.component + + kms_key_arn = module.kms.key_arn + + policy_documents = [data.aws_iam_policy_document.s3bucket_reporting.json] + + force_destroy = var.force_destroy + + lifecycle_rules = [ + { + prefix = "" + enabled = true + + expiration = { + days = local.pii_retention_config.current_days + } + + noncurrent_version_expiration = { + noncurrent_days = local.pii_retention_config.non_current_days + } + } + ] +} + +data "aws_iam_policy_document" "s3bucket_reporting" { + statement { + sid = "AllowManagedAccountsToList" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + module.s3bucket_reporting.arn, + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } + + statement { + sid = "AllowManagedAccountsToGet" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + + resources = [ + "${module.s3bucket_reporting.arn}/*", + ] + + principals { + type = "AWS" + identifiers = [ + "arn:aws:iam::${var.aws_account_id}:root" + ] + } + } +} diff --git a/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf b/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf deleted file mode 100644 index 62c9f056..00000000 --- a/infrastructure/terraform/components/dl/s3_bucket_policy_reporting.tf +++ /dev/null @@ -1,28 +0,0 @@ -resource "aws_s3_bucket_policy" "reporting" { - bucket = aws_s3_bucket.reporting.id - policy = data.aws_iam_policy_document.reporting.json -} - -data "aws_iam_policy_document" "reporting" { - statement { - effect = "Deny" - actions = ["s3:*"] - resources = [ - aws_s3_bucket.reporting.arn, - "${aws_s3_bucket.reporting.arn}/*", - ] - - principals { - type = "AWS" - identifiers = ["*"] - } - - condition { - test = "Bool" - variable = "aws:SecureTransport" - values = [ - false - ] - } - } -} diff --git a/infrastructure/terraform/components/dl/s3_bucket_reporting.tf b/infrastructure/terraform/components/dl/s3_bucket_reporting.tf deleted file mode 100644 index 6383d8c3..00000000 --- a/infrastructure/terraform/components/dl/s3_bucket_reporting.tf +++ /dev/null @@ -1,68 +0,0 @@ -resource "aws_s3_bucket" "reporting" { - bucket = "${local.csi_global}-reporting" - force_destroy = var.force_destroy - - tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups }) -} - -resource "aws_s3_bucket_ownership_controls" "reporting" { - bucket = aws_s3_bucket.reporting.id - - rule { - object_ownership = "BucketOwnerPreferred" - } -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "reporting" { - bucket = aws_s3_bucket.reporting.id - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "aws:kms" - kms_master_key_id = module.kms.key_arn - } - bucket_key_enabled = true - } -} - -resource "aws_s3_bucket_versioning" "reporting" { - bucket = aws_s3_bucket.reporting.id - - versioning_configuration { - status = "Enabled" - } -} - -resource "aws_s3_bucket_public_access_block" "reporting" { - depends_on = [ - aws_s3_bucket_policy.reporting - ] - - bucket = aws_s3_bucket.reporting.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_lifecycle_configuration" "reporting" { - bucket = aws_s3_bucket.reporting.id - expected_bucket_owner = var.aws_account_id - - rule { - id = "reporting" - status = "Enabled" - - filter { - } - - expiration { - days = local.pii_retention_config.current_days - } - - noncurrent_version_expiration { - noncurrent_days = local.pii_retention_config.non_current_days - } - } -} diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index 7530e7c8..fb9b496a 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -181,12 +181,6 @@ variable "enable_pdm_mock" { default = true } -variable "enable_backups" { - type = bool - description = "Enable backups" - default = false -} - variable "pii_data_retention_policy_days" { type = number description = "The number of days for data retention policy for PII" From 2d23385512fd7f26306067ad51d689b2e6ee07cc Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Mon, 2 Feb 2026 15:01:55 +0000 Subject: [PATCH 12/14] CCM-13295: Rename transaction_history table to event_record --- ....tf => glue_catalog_table_event_record.tf} | 12 +-- .../dl/glue_crawler_transactions.tf | 95 ------------------- ...irehose_delivery_stream_to_s3_reporting.tf | 8 +- 3 files changed, 10 insertions(+), 105 deletions(-) rename infrastructure/terraform/components/dl/{glue_catalog_table_transactions.tf => glue_catalog_table_event_record.tf} (81%) delete mode 100644 infrastructure/terraform/components/dl/glue_crawler_transactions.tf diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf similarity index 81% rename from infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf rename to infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf index 5336e0f2..bf8993d8 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_transactions.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf @@ -1,12 +1,12 @@ -resource "aws_glue_catalog_table" "transactions" { - name = "transaction_history" - description = "transaction history for ${var.environment}" +resource "aws_glue_catalog_table" "event_record" { + name = "event_record" + description = "Event records for ${var.environment}" database_name = aws_glue_catalog_database.reporting.name table_type = "EXTERNAL_TABLE" storage_descriptor { - location = "s3://${module.s3bucket_reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/transaction_history" + location = "s3://${module.s3bucket_reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/event_staging" input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat" @@ -64,9 +64,9 @@ resource "aws_glue_catalog_table" "transactions" { } } -resource "aws_glue_partition_index" "transaction_data" { +resource "aws_glue_partition_index" "event_record" { database_name = aws_glue_catalog_database.reporting.name - table_name = aws_glue_catalog_table.transactions.name + table_name = aws_glue_catalog_table.event_record.name partition_index { index_name = "data" diff --git a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf b/infrastructure/terraform/components/dl/glue_crawler_transactions.tf deleted file mode 100644 index ac736ff3..00000000 --- a/infrastructure/terraform/components/dl/glue_crawler_transactions.tf +++ /dev/null @@ -1,95 +0,0 @@ -resource "aws_glue_crawler" "transactions" { - name = "${local.csi}-transactions-crawler" - database_name = aws_glue_catalog_database.reporting.name - role = aws_iam_role.glue_crawler.arn - - schedule = "cron(0 * * * ? *)" # Run every hour - - catalog_target { - database_name = aws_glue_catalog_database.reporting.name - tables = [aws_glue_catalog_table.transactions.name] - } - - schema_change_policy { - delete_behavior = "LOG" - update_behavior = "LOG" - } - - recrawl_policy { - recrawl_behavior = "CRAWL_EVERYTHING" - } - - configuration = jsonencode({ - Version = 1.0 - Grouping = { - TableGroupingPolicy = "CombineCompatibleSchemas" - } - CrawlerOutput = { - Partitions = { - AddOrUpdateBehavior = "InheritFromTable" - } - Tables = { - AddOrUpdateBehavior = "MergeNewColumns" - } - } - }) -} - -resource "aws_iam_role" "glue_crawler" { - name = "${local.csi}-glue-crawler" - description = "Role for Glue Crawler to access S3 and Glue Catalog" - assume_role_policy = data.aws_iam_policy_document.glue_crawler_assume_role.json -} - -data "aws_iam_policy_document" "glue_crawler_assume_role" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["glue.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -resource "aws_iam_role_policy_attachment" "glue_crawler_service" { - role = aws_iam_role.glue_crawler.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" -} - -data "aws_iam_policy_document" "glue_crawler_policy" { - statement { - effect = "Allow" - - actions = [ - "s3:GetObject", - "s3:ListBucket", - ] - - resources = [ - module.s3bucket_reporting.arn, - "${module.s3bucket_reporting.arn}/${local.firehose_output_path_prefix}/*" - ] - } - - statement { - effect = "Allow" - - actions = [ - "kms:Decrypt", - "kms:DescribeKey", - ] - - resources = [ - module.kms.key_arn - ] - } -} - -resource "aws_iam_role_policy" "glue_crawler" { - name = "${local.csi}-glue-crawler" - role = aws_iam_role.glue_crawler.id - policy = data.aws_iam_policy_document.glue_crawler_policy.json -} diff --git a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf index b13d48fc..9ba03d13 100644 --- a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf +++ b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf @@ -7,7 +7,7 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { role_arn = aws_iam_role.firehose_role.arn bucket_arn = module.s3bucket_reporting.arn - prefix = "${local.firehose_output_path_prefix}/reporting/parquet/transaction_history/senderid=!{partitionKeyFromLambda:senderId}/__year=!{partitionKeyFromLambda:year}/__month=!{partitionKeyFromLambda:month}/__day=!{partitionKeyFromLambda:day}/" + prefix = "${local.firehose_output_path_prefix}/reporting/parquet/${aws_glue_catalog_table.event_record.name}/senderid=!{partitionKeyFromLambda:senderId}/__year=!{partitionKeyFromLambda:year}/__month=!{partitionKeyFromLambda:month}/__day=!{partitionKeyFromLambda:day}/" error_output_prefix = "${local.firehose_output_path_prefix}/errors/!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/!{firehose:error-output-type}/" buffering_size = 128 @@ -56,9 +56,9 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { } schema_configuration { - database_name = aws_glue_catalog_table.transactions.database_name + database_name = aws_glue_catalog_database.reporting.name role_arn = aws_iam_role.firehose_role.arn - table_name = aws_glue_catalog_table.transactions.name + table_name = aws_glue_catalog_table.event_record.name } } @@ -220,7 +220,7 @@ data "aws_iam_policy_document" "firehose_policy" { ] resources = [ - aws_glue_catalog_table.transactions.arn, + aws_glue_catalog_table.event_record.arn, aws_glue_catalog_database.reporting.arn, "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog" ] From cda4aa9c48891b60d444132478962d79f787b33b Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Mon, 2 Feb 2026 17:06:27 +0000 Subject: [PATCH 13/14] CCM-13295: Rename status_recorder lambda to report_event_transformer --- .../terraform/components/dl/README.md | 2 +- ...irehose_delivery_stream_to_s3_reporting.tf | 4 ++-- ...module_lambda_report_event_transformer.tf} | 8 +++---- .../jest.config.ts | 0 .../package.json | 2 +- .../__tests__/apis/firehose-handler.test.ts | 0 .../src/__tests__/container.test.ts | 0 .../src/__tests__/index.test.ts | 0 .../src/__tests__/test-data.ts | 0 .../src/apis/firehose-handler.ts | 0 .../src/container.ts | 0 .../src/index.ts | 0 .../src/types/events.ts | 0 .../tsconfig.json | 0 package-lock.json | 24 +++++++++---------- package.json | 2 +- project.code-workspace | 2 +- 17 files changed, 22 insertions(+), 22 deletions(-) rename infrastructure/terraform/components/dl/{module_lambda_status_recorder.tf => module_lambda_report_event_transformer.tf} (82%) rename lambdas/{status-recorder => report-event-transformer}/jest.config.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/package.json (91%) rename lambdas/{status-recorder => report-event-transformer}/src/__tests__/apis/firehose-handler.test.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/__tests__/container.test.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/__tests__/index.test.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/__tests__/test-data.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/apis/firehose-handler.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/container.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/index.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/src/types/events.ts (100%) rename lambdas/{status-recorder => report-event-transformer}/tsconfig.json (100%) diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index eb9d2064..d15e7b28 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -54,6 +54,7 @@ No requirements. | [pdm\_uploader](#module\_pdm\_uploader) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [print\_analyser](#module\_print\_analyser) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [print\_status\_handler](#module\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | +| [report\_event\_transformer](#module\_report\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [s3bucket\_cf\_logs](#module\_s3bucket\_cf\_logs) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_file\_safe](#module\_s3bucket\_file\_safe) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | | [s3bucket\_letters](#module\_s3bucket\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a | @@ -71,7 +72,6 @@ No requirements. | [sqs\_scanner](#module\_sqs\_scanner) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | | [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a | -| [status\_recorder](#module\_status\_recorder) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_create](#module\_ttl\_create) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_handle\_expiry](#module\_ttl\_handle\_expiry) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [ttl\_poll](#module\_ttl\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | diff --git a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf index 9ba03d13..86efee7f 100644 --- a/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf +++ b/infrastructure/terraform/components/dl/kinesis_firehose_delivery_stream_to_s3_reporting.tf @@ -25,7 +25,7 @@ resource "aws_kinesis_firehose_delivery_stream" "to_s3_reporting" { parameters { parameter_name = "LambdaArn" - parameter_value = "${module.status_recorder.function_arn}:$LATEST" + parameter_value = "${module.report_event_transformer.function_arn}:$LATEST" } parameters { parameter_name = "RoleArn" @@ -130,7 +130,7 @@ data "aws_iam_policy_document" "firehose_policy" { ] resources = [ - "${module.status_recorder.function_arn}:$LATEST", + "${module.report_event_transformer.function_arn}:$LATEST", ] } diff --git a/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf b/infrastructure/terraform/components/dl/module_lambda_report_event_transformer.tf similarity index 82% rename from infrastructure/terraform/components/dl/module_lambda_status_recorder.tf rename to infrastructure/terraform/components/dl/module_lambda_report_event_transformer.tf index aaa31156..21121525 100644 --- a/infrastructure/terraform/components/dl/module_lambda_status_recorder.tf +++ b/infrastructure/terraform/components/dl/module_lambda_report_event_transformer.tf @@ -1,8 +1,8 @@ -module "status_recorder" { +module "report_event_transformer" { source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip" - function_name = "status-recorder" - description = "A function for processing all digital letter events" + function_name = "report-event-transformer" + description = "A function for transforming all digital letter events" aws_account_id = var.aws_account_id component = local.component @@ -16,7 +16,7 @@ module "status_recorder" { function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] function_code_base_path = local.aws_lambda_functions_dir_path - function_code_dir = "status-recorder/dist" + function_code_dir = "report-event-transformer/dist" function_include_common = true handler_function_name = "handler" runtime = "nodejs22.x" diff --git a/lambdas/status-recorder/jest.config.ts b/lambdas/report-event-transformer/jest.config.ts similarity index 100% rename from lambdas/status-recorder/jest.config.ts rename to lambdas/report-event-transformer/jest.config.ts diff --git a/lambdas/status-recorder/package.json b/lambdas/report-event-transformer/package.json similarity index 91% rename from lambdas/status-recorder/package.json rename to lambdas/report-event-transformer/package.json index d463e25e..a6d9f388 100644 --- a/lambdas/status-recorder/package.json +++ b/lambdas/report-event-transformer/package.json @@ -12,7 +12,7 @@ "jest-mock-extended": "^3.0.7", "typescript": "^5.9.3" }, - "name": "nhs-notify-digital-letters-status-recorder", + "name": "nhs-notify-digital-letters-report-event-transformer", "private": true, "scripts": { "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts", diff --git a/lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts b/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts similarity index 100% rename from lambdas/status-recorder/src/__tests__/apis/firehose-handler.test.ts rename to lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts diff --git a/lambdas/status-recorder/src/__tests__/container.test.ts b/lambdas/report-event-transformer/src/__tests__/container.test.ts similarity index 100% rename from lambdas/status-recorder/src/__tests__/container.test.ts rename to lambdas/report-event-transformer/src/__tests__/container.test.ts diff --git a/lambdas/status-recorder/src/__tests__/index.test.ts b/lambdas/report-event-transformer/src/__tests__/index.test.ts similarity index 100% rename from lambdas/status-recorder/src/__tests__/index.test.ts rename to lambdas/report-event-transformer/src/__tests__/index.test.ts diff --git a/lambdas/status-recorder/src/__tests__/test-data.ts b/lambdas/report-event-transformer/src/__tests__/test-data.ts similarity index 100% rename from lambdas/status-recorder/src/__tests__/test-data.ts rename to lambdas/report-event-transformer/src/__tests__/test-data.ts diff --git a/lambdas/status-recorder/src/apis/firehose-handler.ts b/lambdas/report-event-transformer/src/apis/firehose-handler.ts similarity index 100% rename from lambdas/status-recorder/src/apis/firehose-handler.ts rename to lambdas/report-event-transformer/src/apis/firehose-handler.ts diff --git a/lambdas/status-recorder/src/container.ts b/lambdas/report-event-transformer/src/container.ts similarity index 100% rename from lambdas/status-recorder/src/container.ts rename to lambdas/report-event-transformer/src/container.ts diff --git a/lambdas/status-recorder/src/index.ts b/lambdas/report-event-transformer/src/index.ts similarity index 100% rename from lambdas/status-recorder/src/index.ts rename to lambdas/report-event-transformer/src/index.ts diff --git a/lambdas/status-recorder/src/types/events.ts b/lambdas/report-event-transformer/src/types/events.ts similarity index 100% rename from lambdas/status-recorder/src/types/events.ts rename to lambdas/report-event-transformer/src/types/events.ts diff --git a/lambdas/status-recorder/tsconfig.json b/lambdas/report-event-transformer/tsconfig.json similarity index 100% rename from lambdas/status-recorder/tsconfig.json rename to lambdas/report-event-transformer/tsconfig.json diff --git a/package-lock.json b/package-lock.json index 613c7384..32fa2c07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "lambdas/core-notifier-lambda", "lambdas/print-status-handler", "lambdas/print-analyser", - "lambdas/status-recorder", + "lambdas/report-event-transformer", "utils/utils", "utils/sender-management", "src/cloudevents", @@ -724,8 +724,8 @@ "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0" } }, - "lambdas/status-recorder": { - "name": "nhs-notify-digital-letters-status-recorder", + "lambdas/report-event-transformer": { + "name": "nhs-notify-digital-letters-report-event-transformer", "version": "0.0.1", "dependencies": { "aws-lambda": "^1.0.7", @@ -741,7 +741,7 @@ "typescript": "^5.9.3" } }, - "lambdas/status-recorder/node_modules/@types/jest": { + "lambdas/report-event-transformer/node_modules/@types/jest": { "version": "29.5.14", "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", @@ -752,7 +752,7 @@ "pretty-format": "^29.0.0" } }, - "lambdas/status-recorder/node_modules/jest": { + "lambdas/report-event-transformer/node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", @@ -780,7 +780,7 @@ } } }, - "lambdas/status-recorder/node_modules/jest-mock-extended": { + "lambdas/report-event-transformer/node_modules/jest-mock-extended": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/jest-mock-extended/-/jest-mock-extended-3.0.7.tgz", "integrity": "sha512-7lsKdLFcW9B9l5NzZ66S/yTQ9k8rFtnwYdCNuRU/81fqDWicNDVhitTSPnrGmNeNm0xyw0JHexEOShrIKRCIRQ==", @@ -12719,16 +12719,16 @@ "resolved": "lambdas/pdm-uploader-lambda", "link": true }, - "node_modules/nhs-notify-digital-letters-print-status-handler": { - "resolved": "lambdas/print-status-handler", - "link": true - }, "node_modules/nhs-notify-digital-letters-print-analyser": { "resolved": "lambdas/print-analyser", "link": true }, - "node_modules/nhs-notify-digital-letters-status-recorder": { - "resolved": "lambdas/status-recorder", + "node_modules/nhs-notify-digital-letters-print-status-handler": { + "resolved": "lambdas/print-status-handler", + "link": true + }, + "node_modules/nhs-notify-digital-letters-report-event-transformer": { + "resolved": "lambdas/report-event-transformer", "link": true }, "node_modules/nhs-notify-digital-letters-ttl-create-lambda": { diff --git a/package.json b/package.json index d6229b9f..269e8cd5 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "lambdas/core-notifier-lambda", "lambdas/print-status-handler", "lambdas/print-analyser", - "lambdas/status-recorder", + "lambdas/report-event-transformer", "utils/utils", "utils/sender-management", "src/cloudevents", diff --git a/project.code-workspace b/project.code-workspace index 9ee010ae..16295b33 100644 --- a/project.code-workspace +++ b/project.code-workspace @@ -88,8 +88,8 @@ { "name": "print-status-handler", "rootPath": "lambdas/print-status-handler" }, { "name": "python-schema-generator", "rootPath": "src/python-schema-generator" }, { "name": "refresh-apim-access-token", "rootPath": "lambdas/refresh-apim-access-token" }, + { "name": "report-event-transformer", "rootPath": "lambdas/report-event-transformer" }, { "name": "sender-management", "rootPath": "utils/sender-management" }, - { "name": "status-recorder", "rootPath": "lambdas/status-recorder" }, { "name": "ttl-create-lambda", "rootPath": "lambdas/ttl-create-lambda/" }, { "name": "ttl-handle-expiry-lambda", "rootPath": "lambdas/ttl-handle-expiry-lambda" }, { "name": "ttl-poll-lambda", "rootPath": "lambdas/ttl-poll-lambda" }, From c80e87810bdaf5d17129efa24d4b7367390b7c92 Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Mon, 2 Feb 2026 17:26:12 +0000 Subject: [PATCH 14/14] CCM-13295: Fix table storage reference --- .../terraform/components/dl/glue_catalog_table_event_record.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf index bf8993d8..f03c9586 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf @@ -6,7 +6,7 @@ resource "aws_glue_catalog_table" "event_record" { table_type = "EXTERNAL_TABLE" storage_descriptor { - location = "s3://${module.s3bucket_reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/event_staging" + location = "s3://${module.s3bucket_reporting.bucket}/${local.firehose_output_path_prefix}/reporting/parquet/event_record" input_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat" output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"