diff --git a/.github/workflows/manual-proxy-environment-deploy.yaml b/.github/workflows/manual-proxy-environment-deploy.yaml
index d5e502309..63f65ccf4 100644
--- a/.github/workflows/manual-proxy-environment-deploy.yaml
+++ b/.github/workflows/manual-proxy-environment-deploy.yaml
@@ -18,6 +18,10 @@ on:
required: false
default: false
type: boolean
+ nodejs_version:
+ description: "Node.js version, set by the CI/CD pipeline workflow"
+ required: true
+ type: string
permissions:
contents: read
@@ -36,11 +40,10 @@ jobs:
node-version: 22
- name: Npm install
- working-directory: .
- env:
- NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: npm ci
- shell: bash
+ uses: ./.github/actions/node-install
+ with:
+ node-version: ${{ inputs.nodejs_version }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV"
id: pr_exists
diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md
index 01fe4c4be..93f30d0c0 100644
--- a/infrastructure/terraform/components/api/README.md
+++ b/infrastructure/terraform/components/api/README.md
@@ -18,8 +18,10 @@ No requirements.
| [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
| [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
| [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
-| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
-| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
+| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
+| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
+| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no |
+| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
| [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
| [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
| [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no |
diff --git a/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf
new file mode 100644
index 000000000..ae64cab23
--- /dev/null
+++ b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf
@@ -0,0 +1,4 @@
+resource "aws_glue_catalog_database" "supplier" {
+ name = "${local.csi}-supplier"
+ description = "Glue catalog database for Suppliers API"
+}
diff --git a/infrastructure/terraform/components/api/glue_catalog_table_events.tf b/infrastructure/terraform/components/api/glue_catalog_table_events.tf
new file mode 100644
index 000000000..a5fc161fc
--- /dev/null
+++ b/infrastructure/terraform/components/api/glue_catalog_table_events.tf
@@ -0,0 +1,86 @@
+resource "aws_glue_catalog_table" "events" {
+ name = "${local.csi}-events_history"
+ database_name = aws_glue_catalog_database.supplier.name
+
+ table_type = "EXTERNAL_TABLE"
+
+ parameters = {
+ classification = "json"
+ }
+
+ storage_descriptor {
+ location = "s3://${aws_s3_bucket.event_reporting.bucket}/events/"
+ input_format = "org.apache.hadoop.mapred.TextInputFormat"
+ output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+ columns {
+ name = "type"
+ type = "string"
+ }
+
+ columns {
+ name = "messageid"
+ type = "string"
+ }
+
+ columns {
+ name = "topicarn"
+ type = "string"
+ }
+
+ columns {
+ name = "message"
+ type = "string"
+ }
+
+ columns {
+ name = "timestamp"
+ type = "string"
+ }
+
+ columns {
+ name = "unsubscribeurl"
+ type = "string"
+ }
+
+ columns {
+ name = "change"
+ type = "double"
+ }
+
+ columns {
+ name = "price"
+ type = "double"
+ }
+
+ columns {
+ name = "ticker_symbol"
+ type = "string"
+ }
+
+ columns {
+ name = "sector"
+ type = "string"
+ }
+
+ columns {
+ name = "partition_0"
+ type = "string"
+ }
+
+ columns {
+ name = "partition_1"
+ type = "string"
+ }
+
+ columns {
+ name = "partition_2"
+ type = "string"
+ }
+
+ columns {
+ name = "partition_3"
+ type = "string"
+ }
+ }
+}
diff --git a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf
new file mode 100644
index 000000000..00c74066e
--- /dev/null
+++ b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf
@@ -0,0 +1,24 @@
+resource "aws_glue_crawler" "event_crawler" {
+ count = local.event_cache_bucket_name != null ? 1 : 0
+ name = "event-crawler-${aws_glue_catalog_table.events.name}"
+ database_name = aws_glue_catalog_database.supplier.name
+ role = aws_iam_role.glue_role.arn
+
+ table_prefix = ""
+ s3_target {
+ path = "s3://${local.csi_global}-eventcache/"
+ }
+
+ s3_target {
+ path = "s3://${local.csi_global}-eventsubeventcache/"
+ }
+ recrawl_policy {
+ recrawl_behavior = "CRAWL_EVERYTHING"
+ }
+
+ schema_change_policy {
+ delete_behavior = "LOG"
+ update_behavior = "UPDATE_IN_DATABASE"
+ }
+
+}
diff --git a/infrastructure/terraform/components/api/iam_role_glue.tf b/infrastructure/terraform/components/api/iam_role_glue.tf
new file mode 100644
index 000000000..d69215458
--- /dev/null
+++ b/infrastructure/terraform/components/api/iam_role_glue.tf
@@ -0,0 +1,105 @@
+resource "aws_iam_role" "glue_role" {
+ name = "${local.csi}-glue-role"
+ assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
+}
+
+data "aws_iam_policy_document" "glue_assume_role" {
+ statement {
+ sid = "AllowGlueServiceAssumeRole"
+ effect = "Allow"
+
+ principals {
+ type = "Service"
+ identifiers = ["glue.amazonaws.com"]
+ }
+
+ actions = [
+ "sts:AssumeRole",
+ ]
+ }
+}
+
+resource "aws_iam_policy" "glue_service_policy" {
+ name = "${local.csi}-glue-service-policy"
+ description = "Policy for ${local.csi} Glue Service Role"
+ policy = data.aws_iam_policy_document.glue_service_policy.json
+}
+
+data "aws_iam_policy_document" "glue_service_policy" {
+ statement {
+ sid = "AllowGlueLogging"
+ effect = "Allow"
+
+ actions = [
+ "logs:CreateLogGroup",
+ "logs:CreateLogStream",
+ "logs:PutLogEvents"
+ ]
+ resources = ["arn:aws:logs:*:*:*"]
+ }
+
+ statement {
+ sid = "AllowListBucketAndGetLocation"
+ effect = "Allow"
+
+ actions = [
+ "s3:ListBucket",
+ "s3:GetBucketLocation"
+ ]
+
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventcache",
+ "arn:aws:s3:::${local.csi_global}-eventsubeventcache"
+ ]
+ }
+ statement {
+ sid = "AllowS3Access"
+ effect = "Allow"
+
+ actions = [
+ "s3:GetObject",
+ "s3:GetObjectVersion",
+ "s3:PutObject",
+ "s3:DeleteObject"
+ ]
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventcache/*",
+ "arn:aws:s3:::${local.csi_global}-eventsubeventcache/*"
+ ]
+ }
+ statement {
+ sid = "GlueCatalogAccess"
+ effect = "Allow"
+ actions = [
+ "glue:GetDatabase",
+ "glue:GetDatabases",
+ "glue:GetTable",
+ "glue:GetTables",
+ "glue:CreateTable",
+ "glue:UpdateTable",
+ "glue:CreatePartition",
+ "glue:BatchCreatePartition",
+ "glue:GetPartition",
+ "glue:BatchGetPartition",
+ "glue:UpdatePartition"
+ ]
+ resources = ["*"]
+ }
+ statement {
+ sid = "S3TempAndGlueETL"
+ effect = "Allow"
+ actions = [
+ "s3:PutObject",
+ "s3:GetObject"
+ ]
+ resources = [
+ "arn:aws:s3:::aws-glue-*",
+ "arn:aws:s3:::aws-glue-*/*"
+ ]
+ }
+}
+
+resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
+ role = aws_iam_role.glue_role.name
+ policy_arn = aws_iam_policy.glue_service_policy.arn
+}
diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf
index 683156a05..2444e51a1 100644
--- a/infrastructure/terraform/components/api/locals.tf
+++ b/infrastructure/terraform/components/api/locals.tf
@@ -31,4 +31,6 @@ locals {
core_pdf_bucket_arn = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"
core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3"
+
+ event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
}
diff --git a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf
index 59393bd29..bd953b76f 100644
--- a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf
+++ b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf
@@ -82,4 +82,17 @@ data "aws_iam_policy_document" "letter_status_update" {
module.letter_status_updates_queue.sqs_queue_arn
]
}
+
+ statement {
+ sid = "AllowSNSPublish"
+ effect = "Allow"
+
+ actions = [
+ "sns:Publish"
+ ]
+
+ resources = [
+ module.eventsub.sns_topic.arn
+ ]
+ }
}
diff --git a/infrastructure/terraform/components/api/modules_eventsub.tf b/infrastructure/terraform/components/api/modules_eventsub.tf
index c55be96c0..c97b5a908 100644
--- a/infrastructure/terraform/components/api/modules_eventsub.tf
+++ b/infrastructure/terraform/components/api/modules_eventsub.tf
@@ -12,6 +12,8 @@ module "eventsub" {
default_tags = local.default_tags
+ glue_role_arn = aws_iam_role.glue_role.arn
+
kms_key_arn = module.kms.key_arn
log_retention_in_days = var.log_retention_in_days
log_level = "INFO"
@@ -22,7 +24,7 @@ module "eventsub" {
sns_success_logging_sample_percent = var.sns_success_logging_sample_percent
event_cache_expiry_days = 30
- enable_event_cache = var.enable_event_cache
+ enable_event_cache = var.enable_event_cache
shared_infra_account_id = var.shared_infra_account_id
}
diff --git a/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf
new file mode 100644
index 000000000..788a2581c
--- /dev/null
+++ b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf
@@ -0,0 +1,51 @@
+resource "aws_s3_bucket_policy" "eventcache" {
+ count = local.event_cache_bucket_name != null ? 1 : 0
+ bucket = local.event_cache_bucket_name
+ policy = data.aws_iam_policy_document.eventcache[0].json
+
+ depends_on = [module.eventpub]
+}
+
+data "aws_iam_policy_document" "eventcache" {
+ count = local.event_cache_bucket_name != null ? 1 : 0
+ statement {
+ sid = "AllowGlueListBucketAndGetLocation"
+ effect = "Allow"
+
+ principals {
+ type = "AWS"
+ identifiers = [aws_iam_role.glue_role.arn]
+ }
+
+ actions = [
+ "s3:ListBucket",
+ "s3:GetBucketLocation"
+ ]
+
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventcache"
+ ]
+ }
+
+ # Object-level permissions: Get/Put/Delete objects
+ statement {
+ sid = "AllowGlueObjectAccess"
+ effect = "Allow"
+
+ principals {
+ type = "AWS"
+ identifiers = [aws_iam_role.glue_role.arn]
+ }
+
+ actions = [
+ "s3:GetObject",
+ "s3:GetObjectVersion",
+ "s3:PutObject",
+ "s3:DeleteObject"
+ ]
+
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventcache/*"
+ ]
+ }
+}
diff --git a/infrastructure/terraform/components/api/s3_event_reporting.tf b/infrastructure/terraform/components/api/s3_event_reporting.tf
new file mode 100644
index 000000000..e61602d0c
--- /dev/null
+++ b/infrastructure/terraform/components/api/s3_event_reporting.tf
@@ -0,0 +1,19 @@
+resource "aws_s3_bucket" "event_reporting" {
+ bucket = "${local.csi_global}-event-reporting"
+
+ tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups })
+}
+resource "aws_s3_bucket_ownership_controls" "event_reporting" {
+ bucket = aws_s3_bucket.event_reporting.id
+
+ rule {
+ object_ownership = "BucketOwnerPreferred"
+ }
+}
+resource "aws_s3_bucket_versioning" "event_reporting" {
+ bucket = aws_s3_bucket.event_reporting.id
+
+ versioning_configuration {
+ status = "Enabled"
+ }
+}
diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf
index 47928a960..268de2b9d 100644
--- a/infrastructure/terraform/components/api/variables.tf
+++ b/infrastructure/terraform/components/api/variables.tf
@@ -163,17 +163,23 @@ variable "core_environment" {
}
+variable "enable_backups" {
+ type = bool
+ description = "Enable backups"
+ default = false
+}
+
# Event Pub/Sub cache settings
variable "enable_event_cache" {
type = bool
description = "Enable caching of events to an S3 bucket"
- default = false
+ default = true
}
variable "enable_sns_delivery_logging" {
type = bool
description = "Enable SNS Delivery Failure Notifications"
- default = false
+ default = true
}
variable "sns_success_logging_sample_percent" {
@@ -187,3 +193,9 @@ variable "enable_api_data_trace" {
description = "Enable API Gateway data trace logging"
default = false
}
+
+variable "enable_firehose_raw_message_delivery" {
+ type = bool
+ description = "Enables raw message delivery on firehose subscription"
+ default = true
+}
diff --git a/infrastructure/terraform/modules/eventsub/README.md b/infrastructure/terraform/modules/eventsub/README.md
index 859c5fd5b..5d297d1de 100644
--- a/infrastructure/terraform/modules/eventsub/README.md
+++ b/infrastructure/terraform/modules/eventsub/README.md
@@ -14,13 +14,14 @@
| [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
| [component](#input\_component) | The name of the terraformscaffold component calling this module | `string` | n/a | yes |
| [default\_tags](#input\_default\_tags) | Default tag map for application to all taggable resources in the module | `map(string)` | `{}` | no |
-| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
-| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `false` | no |
-| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
+| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
+| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no |
+| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
| [environment](#input\_environment) | The name of the terraformscaffold environment the module is called for | `string` | n/a | yes |
| [event\_cache\_buffer\_interval](#input\_event\_cache\_buffer\_interval) | The buffer interval for data firehose | `number` | `500` | no |
| [event\_cache\_expiry\_days](#input\_event\_cache\_expiry\_days) | s3 archiving expiry in days | `number` | `30` | no |
| [force\_destroy](#input\_force\_destroy) | When enabled will force destroy event-cache S3 bucket | `bool` | `false` | no |
+| [glue\_role\_arn](#input\_glue\_role\_arn) | ARN of the Glue execution role from the parent | `string` | n/a | yes |
| [group](#input\_group) | The name of the tfscaffold group | `string` | `null` | no |
| [kms\_key\_arn](#input\_kms\_key\_arn) | KMS key arn to use for this function | `string` | n/a | yes |
| [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"WARN"` | no |
diff --git a/infrastructure/terraform/modules/eventsub/locals.tf b/infrastructure/terraform/modules/eventsub/locals.tf
index e421035ac..1141f727b 100644
--- a/infrastructure/terraform/modules/eventsub/locals.tf
+++ b/infrastructure/terraform/modules/eventsub/locals.tf
@@ -12,6 +12,18 @@ locals {
"_",
"",
)
+ csi_global = replace(
+ format(
+ "%s-%s-%s-%s-%s",
+ var.project,
+ var.aws_account_id,
+ var.region,
+ var.environment,
+ var.component,
+ ),
+ "_",
+ "",
+ )
default_tags = merge(
var.default_tags,
{
diff --git a/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf
new file mode 100644
index 000000000..a5e32026e
--- /dev/null
+++ b/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf
@@ -0,0 +1,48 @@
+resource "aws_s3_bucket_policy" "eventcache" {
+ bucket = module.s3bucket_event_cache[0].bucket
+ policy = data.aws_iam_policy_document.eventcache.json
+ count = var.enable_event_cache ? 1 : 0
+}
+
+data "aws_iam_policy_document" "eventcache" {
+ statement {
+ sid = "AllowGlueListBucketAndGetLocation"
+ effect = "Allow"
+
+ principals {
+ type = "AWS"
+ identifiers = [var.glue_role_arn]
+ }
+
+ actions = [
+ "s3:ListBucket",
+ "s3:GetBucketLocation"
+ ]
+
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventsubeventcache"
+ ]
+ }
+
+ # Object-level permissions: Get/Put/Delete objects
+ statement {
+ sid = "AllowGlueObjectAccess"
+ effect = "Allow"
+
+ principals {
+ type = "AWS"
+ identifiers = [var.glue_role_arn]
+ }
+
+ actions = [
+ "s3:GetObject",
+ "s3:GetObjectVersion",
+ "s3:PutObject",
+ "s3:DeleteObject"
+ ]
+
+ resources = [
+ "arn:aws:s3:::${local.csi_global}-eventsubeventcache/*"
+ ]
+ }
+}
diff --git a/infrastructure/terraform/modules/eventsub/variables.tf b/infrastructure/terraform/modules/eventsub/variables.tf
index 4b73d4523..7594a366e 100644
--- a/infrastructure/terraform/modules/eventsub/variables.tf
+++ b/infrastructure/terraform/modules/eventsub/variables.tf
@@ -70,7 +70,7 @@ variable "event_cache_buffer_interval" {
variable "enable_sns_delivery_logging" {
type = bool
description = "Enable SNS Delivery Failure Notifications"
- default = false
+ default = true
}
variable "sns_success_logging_sample_percent" {
@@ -94,13 +94,13 @@ variable "event_cache_expiry_days" {
variable "enable_event_cache" {
type = bool
description = "Enable caching of events to an S3 bucket"
- default = false
+ default = true
}
variable "enable_firehose_raw_message_delivery" {
type = bool
description = "Enables raw message delivery on firehose subscription"
- default = false
+ default = true
}
variable "force_destroy" {
@@ -114,3 +114,8 @@ variable "shared_infra_account_id" {
description = "The AWS Account ID of the shared infrastructure account"
default = "000000000000"
}
+
+variable "glue_role_arn" {
+ type = string
+ description = "ARN of the Glue execution role from the parent"
+}
diff --git a/lambdas/letter-updates-transformer/src/letter-updates-transformer.ts b/lambdas/letter-updates-transformer/src/letter-updates-transformer.ts
index c53151c07..2389b232b 100644
--- a/lambdas/letter-updates-transformer/src/letter-updates-transformer.ts
+++ b/lambdas/letter-updates-transformer/src/letter-updates-transformer.ts
@@ -126,6 +126,7 @@ function buildMessage(
): PublishBatchRequestEntry {
return {
Id: `${event.id}-${index}`,
+ Subject: "LetterEvent",
Message: JSON.stringify(event),
};
}
diff --git a/lambdas/mi-updates-transformer/src/mi-updates-transformer.ts b/lambdas/mi-updates-transformer/src/mi-updates-transformer.ts
index 107b616fa..0908c7530 100644
--- a/lambdas/mi-updates-transformer/src/mi-updates-transformer.ts
+++ b/lambdas/mi-updates-transformer/src/mi-updates-transformer.ts
@@ -29,6 +29,7 @@ function buildMessage(
): PublishBatchRequestEntry {
const message = {
Id: event.id,
+ Subject: "MISubmittedEvent",
Message: JSON.stringify(event),
};
deps.logger.info({ description: "Built message", message });