CCM-14974: Additional failure events in CSV reports

infrastructure/terraform/components/dl/data/failure_codes.csv

@@ -0,0 +1,5 @@
+code,description
+DL_PDMV_001,Letter rejected by PDM
+DL_PDMV_002,Timeout waiting for letter storage
+DL_CLIV_003,Attachment contains a virus
+DL_INTE_001,Request rejected by Core API

infrastructure/terraform/components/dl/glue_catalog_table_failure_code_lookup.tf

@@ -0,0 +1,39 @@
+resource "aws_glue_catalog_table" "failure_code_lookup" {
+  name          = "failure_code_lookup"
+  description   = "Lookup table for failure code descriptions"
+  database_name = aws_glue_catalog_database.reporting.name
+
+  table_type = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    location = "s3://${module.s3bucket_reporting.bucket}/reference-data/failure_codes/"
+
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name                  = "csv"
+      serialization_library = "org.apache.hadoop.hive.serde2.OpenCSVSerde"
+
+      parameters = {
+        "separatorChar"          = ","
+        "skip.header.line.count" = "1"
+      }
+    }
+
+    columns {
+      name = "code"
+      type = "string"
+    }
+
+    columns {
+      name = "description"
+      type = "string"
+    }
+  }
+
+  parameters = {
+    EXTERNAL       = "TRUE"
+    classification = "csv"
+  }
+}

infrastructure/terraform/components/dl/s3_object_failure_codes.tf

@@ -0,0 +1,17 @@
+# Auto-generated CSV containing failure code definitions
+# Source: src/digital-letters-events/failure-codes.ts
+# Build: make build / make generate (runs generate-dependencies)
+resource "aws_s3_object" "failure_codes" {
+  bucket       = module.s3bucket_reporting.bucket
+  key          = "reference-data/failure_codes/failure_codes.csv"
+  source       = "${path.module}/data/failure_codes.csv"
+  content_type = "text/csv"
+  etag         = filemd5("${path.module}/data/failure_codes.csv")
+
+  tags = merge(
+    local.default_tags,
+    {
+      Name = "${local.csi}-failure-codes-csv"
+    }
+  )
+}

infrastructure/terraform/components/dl/scripts/sql/reports/daily_report.sql

@@ -7,19 +7,35 @@ WITH vars AS (
         e.time,
         CASE
             WHEN e.type LIKE '%.item.dequeued.%'
-            OR e.type LIKE '%.queue.digital.letter.read.%' THEN 'Digital'
-            WHEN e.type LIKE '%.print.letter.transitioned.%' THEN 'Print' ELSE NULL
+            OR e.type LIKE '%.queue.digital.letter.read.%'
+            OR e.type LIKE '%.pdm.resource.submission.rejected.%'
+            OR e.type LIKE '%.pdm.resource.retries.exceeded.%'
+            OR e.type LIKE '%.messages.request.rejected.%' THEN 'Digital'
+            WHEN e.type LIKE '%.print.letter.transitioned.%'
+            OR e.type LIKE '%.print.file.quarantined.%' THEN 'Print' ELSE NULL
         END as communicationtype,
         CASE
             WHEN e.type LIKE '%.item.dequeued.%' THEN 'Unread'
             WHEN e.type LIKE '%.queue.digital.letter.read.%' THEN 'Read'
+            WHEN e.type LIKE '%.pdm.resource.submission.rejected.%' THEN 'Failed'
+            WHEN e.type LIKE '%.pdm.resource.retries.exceeded.%' THEN 'Failed'
+            WHEN e.type LIKE '%.messages.request.rejected.%' THEN 'Failed'
+            WHEN e.type LIKE '%.print.file.quarantined.%' THEN 'Failed'
             WHEN e.letterstatus = 'RETURNED' THEN 'Returned'
             WHEN e.letterstatus = 'FAILED' THEN 'Failed'
             WHEN e.letterstatus = 'DISPATCHED' THEN 'Dispatched'
             WHEN e.letterstatus = 'REJECTED' THEN 'Rejected' ELSE NULL
-        END as status
+        END as status,
+        e.reasoncode,
+        COALESCE(
+            CASE WHEN e.type LIKE '%.messages.request.rejected.%' THEN e.reasontext END,
+            fcl.description,
+            e.reasontext,
+            e.reasoncode
+        ) as reasontext
     FROM event_record e
         CROSS JOIN vars v
+        LEFT JOIN failure_code_lookup fcl ON e.reasoncode = fcl.code
     WHERE e.senderid = v.senderid
         AND e.__year = year(v.dt)
         AND e.__month = month(v.dt)
@@ -31,25 +47,31 @@ WITH vars AS (
             ORDER BY te.time DESC,
                 CASE
                     -- Digital Priority Order
+                    WHEN te.communicationtype = 'Digital' AND te.status = 'Failed' THEN 3
                     WHEN te.status = 'Read' THEN 2
                     WHEN te.status = 'Unread' THEN 1
                     -- Print Priority Order
                     WHEN te.status = 'Returned' THEN 4
-                    WHEN te.status = 'Failed' THEN 3
+                    WHEN te.communicationtype = 'Print' AND te.status = 'Failed' THEN 3
                     WHEN te.status = 'Dispatched' THEN 2
                     WHEN te.status = 'Rejected' THEN 1 ELSE 0
                 END DESC
         ) AS "row_number",
         te.messagereference,
         te.time,
         te.communicationtype,
-        te.status
+        te.status,
+        te.reasoncode,
+        te.reasontext
     FROM "translated_events" AS te
-    where te.status IS NOT NULL
+    WHERE te.status IS NOT NULL
+        AND te.communicationtype IS NOT NULL
 )
 SELECT oe.messagereference as "Message Reference",
     oe.time as "Time",
     oe.communicationtype as "Communication Type",
-    oe.status as "Status"
+    oe.status as "Status",
+    oe.reasoncode as "Reason Code",
+    oe.reasontext as "Reason"
 FROM "ordered_events" AS oe
 WHERE oe.row_number = 1

lambdas/core-notifier-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -256,11 +256,13 @@ describe('createHandler', () => {
       const handler = createHandler(dependencies);
       const { messageId } = sqsEvent.Records[0];
       const errorCode = 'VALIDATION_ERROR';
+      const failureReason = 'Request validation failed';
       const correlationId = 'corr-123';
       const error = new RequestNotifyError(
         new Error('Validation failed'),
         correlationId,
         errorCode,
+        failureReason,
       );
       // Add messageReference property dynamically to trigger the terminal error path
       (error as any).messageReference = messageReference;

lambdas/core-notifier-lambda/src/__tests__/app/notify-api-client.test.ts

@@ -211,8 +211,11 @@ describe('sendRequest', () => {
           mockRequest1.data.attributes.messageReference,
         ),
       ).rejects.toMatchObject({
-        errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
+        cause: error,
         correlationId: 'request-item-id_request-item-plan-id',
+        errorCode: 'CM_MISSING_ROUTING_PLAN_TEMPLATE',
+        failureReason:
+          'The templates required to use the routing plan were not found.',
       });
     },
   );

lambdas/core-notifier-lambda/src/__tests__/domain/mapper.test.ts

@@ -209,13 +209,15 @@ describe('mapper', () => {
   describe('mapPdmEventToMessageRequestRejected', () => {
     it('correctly maps PDM event to MessageRequestRejected', () => {
       const failureCode = 'INVALID_NHS_NUMBER';
+      const failureReason = 'NHS number is not valid';
       const mockDate = new Date('2024-01-15T12:00:00Z');
       jest.spyOn(globalThis, 'Date').mockImplementation(() => mockDate as any);
 
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result).toEqual({
@@ -232,40 +234,64 @@ describe('mapper', () => {
           failureCode: 'INVALID_NHS_NUMBER',
           messageUri:
             'https://www.nhsapp.service.nhs.uk/digital-letters?letterid=resource-789',
+          reasonCode: 'DL_INTE_001',
+          reasonText: 'NHS number is not valid',
         },
       });
 
       expect(mockRandomUUID).toHaveBeenCalled();
     });
 
+    it('includes reasonCode and reasonText for reporting', () => {
+      const failureCode = 'CM_DUPLICATE_REQUEST';
+      const failureReason = 'This request has already been received';
+      const result = mapPdmEventToMessageRequestRejected(
+        mockPdmEvent,
+        mockSender,
+        failureCode,
+        failureReason,
+      );
+
+      expect(result.data.reasonCode).toBe('DL_INTE_001');
+      expect(result.data.reasonText).toBe(
+        'This request has already been received',
+      );
+    });
+
     it('generates new UUID for event', () => {
       const failureCode = 'VALIDATION_ERROR';
+      const failureReason = 'Request validation failed';
       mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(mockRandomUUID).toHaveBeenCalledTimes(1);
     });
 
     it('includes failureCode in data', () => {
       const failureCode = 'ROUTING_FAILED';
+      const failureReason = 'Unable to route message';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.data.failureCode).toBe('ROUTING_FAILED');
     });
 
     it('includes messageUri with resource ID', () => {
       const failureCode = 'TIMEOUT';
+      const failureReason = 'Request timed out';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.data.messageUri).toBe(
@@ -275,32 +301,38 @@ describe('mapper', () => {
 
     it('uses sender senderId in data', () => {
       const failureCode = 'UNKNOWN_ERROR';
+      const failureReason = 'An unknown error occurred';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.data.senderId).toBe('test-sender-id');
     });
 
     it('uses messageReference from PDM event', () => {
       const failureCode = 'DUPLICATE_REQUEST';
+      const failureReason = 'Duplicate request detected';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.data.messageReference).toBe('msg-ref-123');
     });
 
     it('sets correct event type', () => {
       const failureCode = 'SYSTEM_ERROR';
+      const failureReason = 'System error occurred';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.type).toBe(
@@ -310,10 +342,12 @@ describe('mapper', () => {
 
     it('sets correct dataschema', () => {
       const failureCode = 'CONFIG_ERROR';
+      const failureReason = 'Configuration error';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.dataschema).toBe(
@@ -323,10 +357,12 @@ describe('mapper', () => {
 
     it('preserves CloudEvents properties from PDM event', () => {
       const failureCode = 'NETWORK_ERROR';
+      const failureReason = 'Network connection failed';
       const result = mapPdmEventToMessageRequestRejected(
         mockPdmEvent,
         mockSender,
         failureCode,
+        failureReason,
       );
 
       expect(result.specversion).toBe('1.0');

lambdas/core-notifier-lambda/src/apis/sqs-handler.ts

@@ -125,6 +125,7 @@ async function processSqsRecord(
         incoming,
         sender,
         error.errorCode,
+        error.failureReason,
       );
     } else {
       // this might be a transient error so we notify the queue to retry

lambdas/core-notifier-lambda/src/app/notify-api-client.ts

@@ -109,6 +109,7 @@ export class NotifyClient implements INotifyClient {
             error,
             correlationId,
             errorBody?.errors[0].code,
+            errorBody?.errors[0].detail,
           );
         }
       }

lambdas/core-notifier-lambda/src/domain/mapper.ts

@@ -8,6 +8,8 @@ import {
 } from 'digital-letters-events';
 import type { SingleMessageRequest } from 'domain/request';
 
+const CORE_API_FAILURE_CODE = 'DL_INTE_001';
+
 const DIGITAL_LETTER_URL =
   'https://www.nhsapp.service.nhs.uk/digital-letters?letterid=';
 
@@ -99,6 +101,7 @@ export function mapPdmEventToMessageRequestRejected(
   pdmResourceAvailable: PDMResourceAvailable,
   sender: Sender,
   notifyFailureCode: string,
+  failureReason: string,
 ): MessageRequestRejected {
   const { data } = pdmResourceAvailable;
   const { messageReference } = data;
@@ -117,6 +120,8 @@ export function mapPdmEventToMessageRequestRejected(
       senderId: sender.senderId,
       failureCode: notifyFailureCode,
       messageUri: `${DIGITAL_LETTER_URL}${data.resourceId}`,
+      reasonCode: CORE_API_FAILURE_CODE,
+      reasonText: failureReason,
     },
   };
 }

lambdas/core-notifier-lambda/src/domain/request-notify-error.ts

@@ -9,11 +9,19 @@ export class RequestNotifyError extends Error {
 
   readonly errorCode: string;
 
-  constructor(cause: Error, correlationId: string, errorCode: string) {
+  readonly failureReason: string;
+
+  constructor(
+    cause: Error,
+    correlationId: string,
+    errorCode: string,
+    failureReason: string,
+  ) {
     super('Error received from Core Notify API');
 
     this.cause = cause;
     this.correlationId = correlationId;
     this.errorCode = errorCode;
+    this.failureReason = failureReason;
   }
 }

lambdas/core-notifier-lambda/src/domain/request.ts

@@ -46,6 +46,7 @@ export type SingleMessageErrorResponse = {
     {
       id: string;
       code: string;
+      detail: string;
     },
   ];
 };

lambdas/move-scanned-files-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -132,6 +132,7 @@ describe('sqs-handler', () => {
           senderId: 'sender-id-2',
           letterUri: 'https://bucket/key2',
           createdAt: '2024-01-01T00:00:00Z',
+          reasonCode: 'DL_CLIV_003',
         },
         subject: 'test-subject',
         traceparent: 'test-traceparent',

lambdas/move-scanned-files-lambda/src/__tests__/domain/mapper.test.ts

@@ -106,6 +106,7 @@ describe('mapper', () => {
           senderId,
           letterUri,
           createdAt,
+          reasonCode: 'DL_CLIV_003',
         },
         recordedtime: '2024-01-15T10:30:00.000Z',
         severitynumber: 2,