From 06eee290107f96d832912470b4202dca73196f45 Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Fri, 16 Jan 2026 14:27:04 +0000 Subject: [PATCH 1/4] VED-901: Update extended attributes destination prefix. --- lambdas/filenameprocessor/src/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/filenameprocessor/src/constants.py b/lambdas/filenameprocessor/src/constants.py index dbe2be774..4dda64113 100644 --- a/lambdas/filenameprocessor/src/constants.py +++ b/lambdas/filenameprocessor/src/constants.py @@ -21,7 +21,7 @@ # Currently only COVID extended attributes files are supported, might be extended in future for other vaccine types EXTENDED_ATTRIBUTES_VACC_TYPE = "COVID" -DPS_DESTINATION_PREFIX = "dps_destination" +DPS_DESTINATION_PREFIX = "generic/EXTENDED_ATTRIBUTES_DAILY_1" EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX = "extended-attributes-archive" VALID_EA_VERSIONS = ["V1_5"] ERROR_TYPE_TO_STATUS_CODE_MAP = { From 07be76a74568cb3d4fb8dd1e4e9c79668f7dcb74 Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Fri, 16 Jan 2026 15:27:31 +0000 Subject: [PATCH 2/4] VED-901: Update tests. --- lambdas/filenameprocessor/tests/test_lambda_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lambdas/filenameprocessor/tests/test_lambda_handler.py b/lambdas/filenameprocessor/tests/test_lambda_handler.py index 9ee44bd79..ff8f14c02 100644 --- a/lambdas/filenameprocessor/tests/test_lambda_handler.py +++ b/lambdas/filenameprocessor/tests/test_lambda_handler.py @@ -288,8 +288,8 @@ def test_lambda_handler_extended_attributes_success(self, mock_get_redis_client) archived_obj = s3_client.get_object(Bucket=BucketNames.SOURCE, Key=archived_key) self.assertIsNotNone(archived_obj) - # Also verify file copied to DPS destination bucket under dps_destination/ - dps_key = f"dps_destination/{test_cases[0].file_key}" + # Also verify file copied to DPS destination bucket under generic/EXTENDED_ATTRIBUTES_DAILY_1/ + dps_key = f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{test_cases[0].file_key}" copied_obj = s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=dps_key) self.assertIsNotNone(copied_obj) @@ -463,7 +463,7 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi # Ensure processed path hit by checking archive move in source bucket s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{csv_key}") # And verify copy to DPS destination - s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{csv_key}") + s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{csv_key}") # .DAT accepted dat_key = MockFileDetails.extended_attributes_file.file_key[:-3] + "dat" @@ -474,7 +474,7 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi ): lambda_handler(self.make_event([self.make_record(dat_key)]), None) s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{dat_key}") - s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"dps_destination/{dat_key}") + s3_client.get_object(Bucket=BucketNames.DPS_DESTINATION, Key=f"generic/EXTENDED_ATTRIBUTES_DAILY_1/{dat_key}") # Invalid extension fails bad_ext_key = csv_key[:-3] + "txt" From 68afab45d9f621b35afbb0e51874382bd9b210ab Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Tue, 27 Jan 2026 14:55:47 +0000 Subject: [PATCH 3/4] VED-901: Upload EA files directly to the S3 submission upload bucket. --- .../environments/prod/blue/variables.tfvars | 3 ++- .../environments/prod/green/variables.tfvars | 3 ++- .../instance/file_name_processor.tf | 20 +++++-------------- infrastructure/instance/variables.tf | 12 ++++++++--- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/infrastructure/instance/environments/prod/blue/variables.tfvars b/infrastructure/instance/environments/prod/blue/variables.tfvars index 1d94e22f0..2e66b76a2 100644 --- a/infrastructure/instance/environments/prod/blue/variables.tfvars +++ b/infrastructure/instance/environments/prod/blue/variables.tfvars @@ -8,4 +8,5 @@ error_alarm_notifications_enabled = true mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false -dspp_kms_key_alias = "nhsd-dspp-core-prod-extended-attributes-gdp-key" +dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" +dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/infrastructure/instance/environments/prod/green/variables.tfvars b/infrastructure/instance/environments/prod/green/variables.tfvars index 4f270563f..2e66b76a2 100644 --- a/infrastructure/instance/environments/prod/green/variables.tfvars +++ b/infrastructure/instance/environments/prod/green/variables.tfvars @@ -8,4 +8,5 @@ error_alarm_notifications_enabled = true mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false -dspp_kms_key_alias = "nhsd-dspp-core-prod-extended-attributes-gdp-key" \ No newline at end of file +dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" +dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf index 4950d2422..29f5cf73b 100644 --- a/infrastructure/instance/file_name_processor.tf +++ b/infrastructure/instance/file_name_processor.tf @@ -3,18 +3,8 @@ locals { filename_lambda_dir = abspath("${path.root}/../../lambdas/filenameprocessor") filename_lambda_files = fileset(local.filename_lambda_dir, "**") filename_lambda_dir_sha = sha1(join("", [for f in local.filename_lambda_files : filesha1("${local.filename_lambda_dir}/${f}")])) - dps_bucket_name_for_extended_attribute = ( - var.environment == "prod" - ? "nhsd-dspp-core-prod-extended-attributes-gdp" - : "nhsd-dspp-core-ref-extended-attributes-gdp" - ) - dps_bucket_arn_for_extended_attribute = [ - "arn:aws:s3:::${local.dps_bucket_name_for_extended_attribute}/*" - ] } - - resource "aws_ecr_repository" "file_name_processor_lambda_repository" { image_scanning_configuration { scan_on_push = true @@ -178,7 +168,7 @@ resource "aws_iam_policy" "filenameprocessor_lambda_exec_policy" { "Action" : [ "s3:PutObject" ], - "Resource" : local.dps_bucket_arn_for_extended_attribute + "Resource" : ["arn:aws:s3:::${var.dspp_submission_s3_bucket_name}/*"] } ] }) @@ -264,14 +254,14 @@ resource "aws_iam_policy" "filenameprocessor_dps_extended_attribute_kms_policy" { Effect = "Allow", Action = [ - "kms:Decrypt", + "kms:Encrypt", "kms:GenerateDataKey", - "kms:DescribeKey" + "kms:DescribeKey", ], Resource = "arn:aws:kms:eu-west-2:${var.dspp_core_account_id}:key/*", "Condition" = { "ForAnyValue:StringEquals" = { - "kms:ResourceAliases" = "alias/${var.dspp_kms_key_alias}" + "kms:ResourceAliases" = "alias/${var.dspp_submission_kms_key_alias}" } } } @@ -329,7 +319,7 @@ resource "aws_lambda_function" "file_processor_lambda" { DPS_ACCOUNT_ID = var.dspp_core_account_id SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket - DPS_BUCKET_NAME = local.dps_bucket_name_for_extended_attribute + DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name QUEUE_URL = aws_sqs_queue.batch_file_created.url REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf index 88a2a502b..a2c935dd6 100644 --- a/infrastructure/instance/variables.tf +++ b/infrastructure/instance/variables.tf @@ -10,10 +10,16 @@ variable "csoc_account_id" { default = "693466633220" } -variable "dspp_kms_key_alias" { - description = "Alias name of the DPS KMS key allowed for SSE-KMS encryption" +variable "dspp_submission_s3_bucket_name" { + description = "Name of the DSPP (DPS) S3 bucket where extended attributes files should be submitted" type = string - default = "nhsd-dspp-core-ref-extended-attributes-gdp-key" + default = "nhsd-dspp-core-ref-s3-submission-upload" +} + +variable "dspp_submission_kms_key_alias" { + description = "Alias of the DSPP (DPS) KMS key required to encrypt extended attributes files" + type = string + default = "nhsd-dspp-core-ref-s3-submission-upload-key" } variable "create_mesh_processor" { From 0227aab6f2a4e59789e22aef5b17b6d25b4294a5 Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Tue, 27 Jan 2026 14:57:59 +0000 Subject: [PATCH 4/4] VED-901: Lint tfvars files. --- .../instance/environments/prod/blue/variables.tfvars | 4 ++-- .../instance/environments/prod/green/variables.tfvars | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/infrastructure/instance/environments/prod/blue/variables.tfvars b/infrastructure/instance/environments/prod/blue/variables.tfvars index 2e66b76a2..60c42bf83 100644 --- a/infrastructure/instance/environments/prod/blue/variables.tfvars +++ b/infrastructure/instance/environments/prod/blue/variables.tfvars @@ -8,5 +8,5 @@ error_alarm_notifications_enabled = true mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false -dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" -dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" +dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" +dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/infrastructure/instance/environments/prod/green/variables.tfvars b/infrastructure/instance/environments/prod/green/variables.tfvars index 2e66b76a2..60c42bf83 100644 --- a/infrastructure/instance/environments/prod/green/variables.tfvars +++ b/infrastructure/instance/environments/prod/green/variables.tfvars @@ -8,5 +8,5 @@ error_alarm_notifications_enabled = true mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false -dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" -dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" +dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" +dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/package.json b/package.json index 1efa4e255..b044633de 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "poetry -P quality_checks run ruff check --fix", "poetry -P quality_checks run ruff format" ], - "*.tf": "terraform fmt", + "*.{tf,tfvars}": "terraform fmt", "immunisation-fhir-api.{yaml,json}": "redocly lint --skip-rule=security-defined" } }