CCM-12939 event audit pipeline

.github/workflows/manual-proxy-environment-deploy.yaml

@@ -18,6 +18,10 @@ on:
         required: false
         default: false
         type: boolean
+      nodejs_version:
+        description: "Node.js version, set by the CI/CD pipeline workflow"
+        required: true
+        type: string
 
 permissions:
   contents: read
@@ -36,11 +40,10 @@ jobs:
           node-version: 22
 
       - name: Npm install
-        working-directory: .
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: npm ci
-        shell: bash
+        uses: ./.github/actions/node-install
+        with:
+          node-version: ${{ inputs.nodejs_version }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV"
         id: pr_exists

infrastructure/terraform/components/api/README.md

@@ -18,8 +18,10 @@ No requirements.
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
 | <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
 | <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
-| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
-| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
+| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
+| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
+| <a name="input_enable_firehose_raw_message_delivery"></a> [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no |
+| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_eventpub_control_plane_bus_arn"></a> [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
 | <a name="input_eventpub_data_plane_bus_arn"></a> [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no |

infrastructure/terraform/components/api/glue_catalog_database_supplier.tf

@@ -0,0 +1,4 @@
+resource "aws_glue_catalog_database" "supplier" {
+  name        = "${local.csi}-supplier"
+  description = "Glue catalog database for Suppliers API"
+}

infrastructure/terraform/components/api/glue_catalog_table_events.tf

@@ -0,0 +1,86 @@
+resource "aws_glue_catalog_table" "events" {
+  name          = "${local.csi}-events_history"
+  database_name = aws_glue_catalog_database.supplier.name
+
+  table_type = "EXTERNAL_TABLE"
+
+  parameters = {
+    classification = "json"
+  }
+
+  storage_descriptor {
+    location      = "s3://${aws_s3_bucket.event_reporting.bucket}/events/"
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    columns {
+      name = "type"
+      type = "string"
+    }
+
+    columns {
+      name = "messageid"
+      type = "string"
+    }
+
+    columns {
+      name = "topicarn"
+      type = "string"
+    }
+
+    columns {
+      name = "message"
+      type = "string"
+    }
+
+    columns {
+      name = "timestamp"
+      type = "string"
+    }
+
+    columns {
+      name = "unsubscribeurl"
+      type = "string"
+    }
+
+    columns {
+      name = "change"
+      type = "double"
+    }
+
+    columns {
+      name = "price"
+      type = "double"
+    }
+
+    columns {
+      name = "ticker_symbol"
+      type = "string"
+    }
+
+    columns {
+      name = "sector"
+      type = "string"
+    }
+
+    columns {
+      name = "partition_0"
+      type = "string"
+    }
+
+    columns {
+      name = "partition_1"
+      type = "string"
+    }
+
+    columns {
+      name = "partition_2"
+      type = "string"
+    }
+
+    columns {
+      name = "partition_3"
+      type = "string"
+    }
+  }
+}

infrastructure/terraform/components/api/glue_crawler_event_crawler.tf

@@ -0,0 +1,24 @@
+resource "aws_glue_crawler" "event_crawler" {
+  count  = local.event_cache_bucket_name != null ? 1 : 0
+  name          = "event-crawler-${aws_glue_catalog_table.events.name}"
+  database_name = aws_glue_catalog_database.supplier.name
+  role          = aws_iam_role.glue_role.arn
+
+  table_prefix = ""
+  s3_target {
+    path = "s3://${local.csi_global}-eventcache/"
+  }
+
+  s3_target {
+    path = "s3://${local.csi_global}-eventsubeventcache/"
+  }
+  recrawl_policy {
+    recrawl_behavior = "CRAWL_EVERYTHING"
+  }
+
+  schema_change_policy {
+    delete_behavior = "LOG"
+    update_behavior = "UPDATE_IN_DATABASE"
+  }
+
+}

infrastructure/terraform/components/api/iam_role_glue.tf

@@ -0,0 +1,105 @@
+resource "aws_iam_role" "glue_role" {
+  name               = "${local.csi}-glue-role"
+  assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
+}
+
+data "aws_iam_policy_document" "glue_assume_role" {
+  statement {
+    sid    = "AllowGlueServiceAssumeRole"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["glue.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "glue_service_policy" {
+  name        = "${local.csi}-glue-service-policy"
+  description = "Policy for ${local.csi} Glue Service Role"
+  policy      = data.aws_iam_policy_document.glue_service_policy.json
+}
+
+data "aws_iam_policy_document" "glue_service_policy" {
+  statement {
+    sid    = "AllowGlueLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    resources = ["arn:aws:logs:*:*:*"]
+  }
+
+  statement {
+    sid    = "AllowListBucketAndGetLocation"
+    effect = "Allow"
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache",
+      "arn:aws:s3:::${local.csi_global}-eventsubeventcache"
+    ]
+  }
+  statement {
+    sid    = "AllowS3Access"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache/*",
+      "arn:aws:s3:::${local.csi_global}-eventsubeventcache/*"
+    ]
+  }
+  statement {
+    sid    = "GlueCatalogAccess"
+    effect = "Allow"
+    actions = [
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:CreateTable",
+      "glue:UpdateTable",
+      "glue:CreatePartition",
+      "glue:BatchCreatePartition",
+      "glue:GetPartition",
+      "glue:BatchGetPartition",
+      "glue:UpdatePartition"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    sid    = "S3TempAndGlueETL"
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject"
+    ]
+    resources = [
+      "arn:aws:s3:::aws-glue-*",
+      "arn:aws:s3:::aws-glue-*/*"
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
+  role       = aws_iam_role.glue_role.name
+  policy_arn = aws_iam_policy.glue_service_policy.arn
+}

infrastructure/terraform/components/api/locals.tf

@@ -31,4 +31,6 @@ locals {
 
   core_pdf_bucket_arn        = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"
   core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3"
+
+  event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
 }

infrastructure/terraform/components/api/module_lambda_letter_status_update.tf

@@ -82,4 +82,17 @@ data "aws_iam_policy_document" "letter_status_update" {
       module.letter_status_updates_queue.sqs_queue_arn
     ]
   }
+
+  statement {
+    sid    = "AllowSNSPublish"
+    effect = "Allow"
+
+    actions = [
+      "sns:Publish"
+    ]
+
+    resources = [
+      module.eventsub.sns_topic.arn
+    ]
+  }
 }

infrastructure/terraform/components/api/modules_eventsub.tf

@@ -12,6 +12,8 @@ module "eventsub" {
 
   default_tags = local.default_tags
 
+  glue_role_arn = aws_iam_role.glue_role.arn
+
   kms_key_arn           = module.kms.key_arn
   log_retention_in_days = var.log_retention_in_days
   log_level             = "INFO"
@@ -22,7 +24,7 @@ module "eventsub" {
   sns_success_logging_sample_percent = var.sns_success_logging_sample_percent
 
   event_cache_expiry_days = 30
-  enable_event_cache                 = var.enable_event_cache
+  enable_event_cache      = var.enable_event_cache
 
   shared_infra_account_id = var.shared_infra_account_id
 }

infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf

@@ -0,0 +1,51 @@
+resource "aws_s3_bucket_policy" "eventcache" {
+  count  = local.event_cache_bucket_name != null ? 1 : 0
+  bucket = local.event_cache_bucket_name
+  policy = data.aws_iam_policy_document.eventcache[0].json
+
+  depends_on = [module.eventpub]
+}
+
+data "aws_iam_policy_document" "eventcache" {
+  count = local.event_cache_bucket_name != null ? 1 : 0
+  statement {
+    sid    = "AllowGlueListBucketAndGetLocation"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache"
+    ]
+  }
+
+  # Object-level permissions: Get/Put/Delete objects
+  statement {
+    sid    = "AllowGlueObjectAccess"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache/*"
+    ]
+  }
+}

infrastructure/terraform/components/api/s3_event_reporting.tf

@@ -0,0 +1,19 @@
+resource "aws_s3_bucket" "event_reporting" {
+  bucket = "${local.csi_global}-event-reporting"
+
+  tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups })
+}
+resource "aws_s3_bucket_ownership_controls" "event_reporting" {
+  bucket = aws_s3_bucket.event_reporting.id
+
+  rule {
+    object_ownership = "BucketOwnerPreferred"
+  }
+}
+resource "aws_s3_bucket_versioning" "event_reporting" {
+  bucket = aws_s3_bucket.event_reporting.id
+
+  versioning_configuration {
+    status = "Enabled"
+  }
+}