CCM-13304: Generate reports

.editorconfig

@@ -14,6 +14,9 @@ indent_size = unset
 [*.py]
 indent_size = 4
 
+[*.ts]
+quote_type = single
+
 [{Dockerfile,Dockerfile.}*]
 indent_size = 4
 

infrastructure/terraform/components/dl/README.md

@@ -13,6 +13,8 @@ No requirements.
 | <a name="input_apim_auth_token_url"></a> [apim\_auth\_token\_url](#input\_apim\_auth\_token\_url) | URL to generate an APIM auth token | `string` | `"https://int.api.service.nhs.uk/oauth2/token"` | no |
 | <a name="input_apim_base_url"></a> [apim\_base\_url](#input\_apim\_base\_url) | The URL used to send requests to PDM | `string` | `"https://int.api.service.nhs.uk"` | no |
 | <a name="input_apim_keygen_schedule"></a> [apim\_keygen\_schedule](#input\_apim\_keygen\_schedule) | Schedule to refresh key pairs if necessary | `string` | `"cron(0 14 * * ? *)"` | no |
+| <a name="input_athena_query_max_polling_attempts"></a> [athena\_query\_max\_polling\_attempts](#input\_athena\_query\_max\_polling\_attempts) | The number of times athena will be polled to check if a query is completed | `number` | `50` | no |
+| <a name="input_athena_query_polling_time_seconds"></a> [athena\_query\_polling\_time\_seconds](#input\_athena\_query\_polling\_time\_seconds) | The amount of time in seconds to wait between each athena poll | `number` | `15` | no |
 | <a name="input_aws_account_id"></a> [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
 | <a name="input_aws_account_type"></a> [aws\_account\_type](#input\_aws\_account\_type) | The AWS Account Type | `string` | n/a | yes |
 | <a name="input_component"></a> [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"dl"` | no |
@@ -65,6 +67,7 @@ No requirements.
 | <a name="module_print_sender"></a> [print\_sender](#module\_print\_sender) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_print_status_handler"></a> [print\_status\_handler](#module\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_report_event_transformer"></a> [report\_event\_transformer](#module\_report\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
+| <a name="module_report_generator"></a> [report\_generator](#module\_report\_generator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-lambda.zip | n/a |
 | <a name="module_report_scheduler"></a> [report\_scheduler](#module\_report\_scheduler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_s3bucket_cf_logs"></a> [s3bucket\_cf\_logs](#module\_s3bucket\_cf\_logs) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_file_quarantine"></a> [s3bucket\_file\_quarantine](#module\_s3bucket\_file\_quarantine) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-s3bucket.zip | n/a |
@@ -84,6 +87,7 @@ No requirements.
 | <a name="module_sqs_print_analyser"></a> [sqs\_print\_analyser](#module\_sqs\_print\_analyser) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
 | <a name="module_sqs_print_sender"></a> [sqs\_print\_sender](#module\_sqs\_print\_sender) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
 | <a name="module_sqs_print_status_handler"></a> [sqs\_print\_status\_handler](#module\_sqs\_print\_status\_handler) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
+| <a name="module_sqs_report_generator"></a> [sqs\_report\_generator](#module\_sqs\_report\_generator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
 | <a name="module_sqs_scanner"></a> [sqs\_scanner](#module\_sqs\_scanner) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
 | <a name="module_sqs_ttl"></a> [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |
 | <a name="module_sqs_ttl_handle_expiry_errors"></a> [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip | n/a |

infrastructure/terraform/components/dl/cloudwatch_event_rule_generate_report.tf

@@ -0,0 +1,18 @@
+resource "aws_cloudwatch_event_rule" "generate_report" {
+  name           = "${local.csi}-generate-report"
+  description    = "Generate Report event rule"
+  event_bus_name = aws_cloudwatch_event_bus.main.name
+  event_pattern = jsonencode({
+    "detail" : {
+      "type" : [
+        "uk.nhs.notify.digital.letters.reporting.generate.report.v1"
+      ],
+    }
+  })
+}
+
+resource "aws_cloudwatch_event_target" "generate_report_report_generator" {
+  rule           = aws_cloudwatch_event_rule.generate_report.name
+  arn            = module.sqs_report_generator.sqs_queue_arn
+  event_bus_name = aws_cloudwatch_event_bus.main.name
+}

infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf

@@ -44,6 +44,10 @@ resource "aws_glue_catalog_table" "event_record" {
       name = "type"
       type = "string"
     }
+    columns {
+      name = "letterstatus"
+      type = "string"
+    }
   }
 
   partition_keys {

infrastructure/terraform/components/dl/lambda_event_source_mapping_report_generator.tf

@@ -0,0 +1,10 @@
+resource "aws_lambda_event_source_mapping" "report_generator" {
+  event_source_arn                   = module.sqs_report_generator.sqs_queue_arn
+  function_name                      = module.report_generator.function_name
+  batch_size                         = var.queue_batch_size
+  maximum_batching_window_in_seconds = var.queue_batch_window_seconds
+
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+}

infrastructure/terraform/components/dl/module_lambda_report_generator.tf

@@ -0,0 +1,156 @@
+module "report_generator" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-lambda.zip"
+
+  function_name = "report-generator"
+  description   = "A function to generate reports from an event"
+
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  group          = var.group
+
+  log_retention_in_days = var.log_retention_in_days
+  kms_key_arn           = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.report_generator_lambda.json
+  }
+
+  function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
+  function_code_base_path = local.aws_lambda_functions_dir_path
+  function_code_dir       = "report-generator/dist"
+  function_include_common = true
+  handler_function_name   = "handler"
+  runtime                 = "nodejs22.x"
+  memory                  = 128
+  timeout                 = 60
+  log_level               = var.log_level
+
+  force_lambda_code_deploy = var.force_lambda_code_deploy
+  enable_lambda_insights   = false
+
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
+  lambda_env_vars = {
+    "ATHENA_WORKGROUP"              = aws_athena_workgroup.reporting.name
+    "ATHENA_DATABASE"               = aws_glue_catalog_database.reporting.name
+    "EVENT_PUBLISHER_EVENT_BUS_ARN" = aws_cloudwatch_event_bus.main.arn
+    "EVENT_PUBLISHER_DLQ_URL"       = module.sqs_event_publisher_errors.sqs_queue_url
+    "MAX_POLL_LIMIT"                = var.athena_query_max_polling_attempts
+    "REPORTING_BUCKET"              = module.s3bucket_reporting.bucket
+    "REPORT_NAME"                   = "completed_communications"
+    "WAIT_FOR_IN_SECONDS"           = var.athena_query_polling_time_seconds
+  }
+}
+
+data "aws_iam_policy_document" "report_generator_lambda" {
+  statement {
+    sid    = "AllowS3Get"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject",
+      "s3:GetBucketLocation",
+      "s3:ListBucket"
+    ]
+
+    resources = [
+      "${module.s3bucket_reporting.arn}/*",
+      "${module.s3bucket_reporting.arn}"
+    ]
+  }
+
+  statement {
+    sid    = "KMSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowAthenaAccess"
+    effect = "Allow"
+
+    actions = [
+      "athena:StartQueryExecution",
+      "athena:GetQueryResults",
+      "athena:GetQueryExecution"
+    ]
+
+    resources = [
+      "arn:aws:athena:${var.region}:${var.aws_account_id}:workgroup/${aws_athena_workgroup.reporting.name}"
+    ]
+  }
+
+  statement {
+    sid    = "AllowGlueAccess"
+    effect = "Allow"
+
+    actions = [
+      "glue:GetTable",
+      "glue:GetDatabase",
+      "glue:GetPartition",
+      "glue:GetPartitions",
+    ]
+
+    resources = [
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog",
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:database/${aws_glue_catalog_database.reporting.name}",
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.reporting.name}/*"
+    ]
+  }
+
+  statement {
+    sid    = "SQSPermissionsReportGeneratorQueue"
+    effect = "Allow"
+
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:GetQueueUrl",
+    ]
+
+    resources = [
+      module.sqs_report_generator.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "PutEvents"
+    effect = "Allow"
+
+    actions = [
+      "events:PutEvents",
+    ]
+
+    resources = [
+      aws_cloudwatch_event_bus.main.arn,
+    ]
+  }
+
+  statement {
+    sid    = "SQSPermissionsEventPublisherDLQ"
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+      "sqs:SendMessageBatch",
+    ]
+
+    resources = [
+      module.sqs_event_publisher_errors.sqs_queue_arn,
+    ]
+  }
+}

infrastructure/terraform/components/dl/module_sqs_report_generator.tf

@@ -0,0 +1,44 @@
+module "sqs_report_generator" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.30/terraform-sqs.zip"
+
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  name           = "report-generator"
+
+  sqs_kms_key_arn = module.kms.key_arn
+
+  visibility_timeout_seconds = 60
+
+  create_dlq = true
+
+  sqs_policy_overload = data.aws_iam_policy_document.sqs_report_generator.json
+}
+
+data "aws_iam_policy_document" "sqs_report_generator" {
+  statement {
+    sid    = "AllowEventBridgeToSendMessage"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["events.amazonaws.com"]
+    }
+
+    actions = [
+      "sqs:SendMessage"
+    ]
+
+    resources = [
+      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-report-generator-queue"
+    ]
+
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.generate_report.arn]
+    }
+  }
+}

infrastructure/terraform/components/dl/variables.tf

@@ -219,3 +219,15 @@ variable "metadata_refresh_schedule" {
   description = "Schedule for refreshing reporting metadata."
   default     = "cron(10 6-22 * * ? *)" # 10 minutes past the hour, between 06:00 - 22:00
 }
+
+variable "athena_query_max_polling_attempts" {
+  type        = number
+  description = "The number of times athena will be polled to check if a query is completed"
+  default     = 50
+}
+
+variable "athena_query_polling_time_seconds" {
+  type        = number
+  description = "The amount of time in seconds to wait between each athena poll"
+  default     = 15
+}

lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts

@@ -51,6 +51,7 @@ describe('Firehose Handler', () => {
         reasonText: digitalLettersEvent.data.reasonText,
         senderId: digitalLettersEvent.data.senderId,
         supplierId: digitalLettersEvent.data.supplierId,
+        letterStatus: digitalLettersEvent.data.status,
         time: digitalLettersEvent.time,
         type: digitalLettersEvent.type,
       });

lambdas/report-event-transformer/src/__tests__/test-data.ts

@@ -24,6 +24,7 @@ export const digitalLettersEvent = {
     reasonCode: 'FAILURE001',
     reasonText: 'Letter has too many pages',
     senderId: 'sender1',
+    status: 'DISPATCHED',
     supplierId: 'supplier1',
   },
 } as DigitalLettersEvent;

lambdas/report-event-transformer/src/apis/firehose-handler.ts

@@ -62,6 +62,7 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent {
     reasonCode,
     reasonText,
     senderId,
+    status,
     supplierId,
   } = validatedRecord.event.data;
   const { time, type } = validatedRecord.event;
@@ -72,6 +73,7 @@ function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent {
     pageCount,
     senderId,
     supplierId,
+    letterStatus: status,
     reasonCode,
     reasonText,
     time,

lambdas/report-event-transformer/src/types/events.ts

@@ -8,6 +8,7 @@ export const $DigitalLettersEvent = z.object({
     reasonText: z.string().optional(),
     senderId: z.string(),
     supplierId: z.string().optional(),
+    status: z.string().optional(),
   }),
   time: z.string(),
   type: z.string(),
@@ -20,6 +21,7 @@ export type FlatDigitalLettersEvent = {
   pageCount?: number;
   senderId: string;
   supplierId?: string;
+  letterStatus?: string;
   reasonCode?: string;
   reasonText?: string;
   time: string;

lambdas/report-generator/jest.config.ts

@@ -0,0 +1,14 @@
+import { baseJestConfig } from '../../jest.config.base';
+
+const config = baseJestConfig;
+
+config.coverageThreshold = {
+  global: {
+    branches: 88,
+    functions: 100,
+    lines: 97,
+    statements: 97,
+  },
+};
+
+export default config;

lambdas/report-generator/package.json

@@ -0,0 +1,24 @@
+{
+  "dependencies": {
+    "@aws-sdk/client-athena": "^3.984.0",
+    "digital-letters-events": "^0.0.1",
+    "utils": "^0.0.1"
+  },
+  "devDependencies": {
+    "@tsconfig/node22": "^22.0.2",
+    "@types/aws-lambda": "^8.10.155",
+    "@types/jest": "^29.5.14",
+    "jest": "^29.7.0",
+    "typescript": "^5.9.3"
+  },
+  "name": "nhs-notify-digital-letters-report-generator",
+  "private": true,
+  "scripts": {
+    "lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts && cp -r src/queries dist/",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "test:unit": "jest",
+    "typecheck": "tsc --noEmit"
+  },
+  "version": "0.0.1"
+}