CCM-15541: Gateway Cluster Deployment

.gitleaksignore

@@ -1,6 +1,8 @@
 # SEE: https://github.com/gitleaks/gitleaks/blob/master/README.md#gitleaksignore
 
-9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/README.md:ipv4:16
-9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
+39565cc5ab1245e4e6a6368c19fd0aa9a187733a:infrastructure/terraform/components/reporting/README.md:ipv4:16
+39565cc5ab1245e4e6a6368c19fd0aa9a187733a:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
 ca243cb73d3804a14f3eeefa8073c96802420c52:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
 ca243cb73d3804a14f3eeefa8073c96802420c52:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
+d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43

docs/diagrams/reporting.drawio

@@ -157,4 +157,4 @@
             </root>
         </mxGraphModel>
     </diagram>
-</mxfile>
+</mxfile>

infrastructure/terraform/bin/test_mandatory_tfvars.sh

@@ -81,6 +81,3 @@ for tfvars_file in ./etc/env_eu-west-2_*; do
     fi
   done
 done
-
-
-

infrastructure/terraform/components/powerbi-gateway/.tool-versions

@@ -0,0 +1 @@
+terraform 1.10.1

infrastructure/terraform/components/powerbi-gateway/README.md

@@ -0,0 +1,44 @@
+<!-- BEGIN_TF_DOCS -->
+<!-- markdownlint-disable -->
+<!-- vale off -->
+
+## Requirements
+
+No requirements.
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_account_ids"></a> [account\_ids](#input\_account\_ids) | All AWS Account IDs for this project | `map(string)` | `{}` | no |
+| <a name="input_account_name"></a> [account\_name](#input\_account\_name) | The name of the AWS Account to deploy into (see globals.tfvars) | `string` | n/a | yes |
+| <a name="input_athena_driver_url"></a> [athena\_driver\_url](#input\_athena\_driver\_url) | Amazon Athena ODBC MSI download URL for PowerBI gateway bootstrap | `string` | `"https://downloads.athena.us-east-1.amazonaws.com/drivers/ODBC/v2.1.0.0/Windows/AmazonAthenaODBC-2.1.0.0.msi"` | no |
+| <a name="input_aws_account_id"></a> [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
+| <a name="input_component"></a> [component](#input\_component) | The name of the component | `string` | `"powerbi-gateway"` | no |
+| <a name="input_core_account_id"></a> [core\_account\_id](#input\_core\_account\_id) | The core account that contains the corresponding Glue Catalog | `string` | `1234567890` | no |
+| <a name="input_core_account_ids"></a> [core\_account\_ids](#input\_core\_account\_ids) | List of all corresponding core account id's that exist in the Non-Prod domain | `list(string)` | `[]` | no |
+| <a name="input_default_kms_deletion_window_in_days"></a> [default\_kms\_deletion\_window\_in\_days](#input\_default\_kms\_deletion\_window\_in\_days) | Default number of days to set KMS key deletion window | `number` | `14` | no |
+| <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
+| <a name="input_enable_powerbi_gateway"></a> [enable\_powerbi\_gateway](#input\_enable\_powerbi\_gateway) | Deploy EC2 instance for PowerBI On-Premises Gateway | `bool` | `true` | no |
+| <a name="input_enable_spot"></a> [enable\_spot](#input\_enable\_spot) | run Power BI On-Premises Gateway as spot instances | `bool` | `false` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | The name of the environment | `string` | n/a | yes |
+| <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | `"n/a"` | no |
+| <a name="input_instance_type"></a> [instance\_type](#input\_instance\_type) | The EC2 instance type. | `string` | `"t3.medium"` | no |
+| <a name="input_module"></a> [module](#input\_module) | The variable encapsulating the name of this module | `string` | `"n/a"` | no |
+| <a name="input_powerbi_gateway_instance_count"></a> [powerbi\_gateway\_instance\_count](#input\_powerbi\_gateway\_instance\_count) | Number of standalone Power BI On-Premises Gateway instances created directly from the launch template. | `number` | `2` | no |
+| <a name="input_private_subnet_cidrs"></a> [private\_subnet\_cidrs](#input\_private\_subnet\_cidrs) | List of CIDR blocks for private subnets. | `list(string)` | `[]` | no |
+| <a name="input_project"></a> [project](#input\_project) | The name of the Project we are bootstrapping tfscaffold for | `string` | n/a | yes |
+| <a name="input_public_subnet_cidrs"></a> [public\_subnet\_cidrs](#input\_public\_subnet\_cidrs) | List of CIDR blocks for public subnets. | `list(string)` | `[]` | no |
+| <a name="input_region"></a> [region](#input\_region) | The AWS Region | `string` | n/a | yes |
+| <a name="input_root_volume_size"></a> [root\_volume\_size](#input\_root\_volume\_size) | Size of root volume for the Power BI On-Premises Gateway instances - 30GB minimum for Windows Server | `number` | `80` | no |
+| <a name="input_spot_max_price"></a> [spot\_max\_price](#input\_spot\_max\_price) | max spot price for Power BI On-Premises Gateway instances | `string` | `"0.3"` | no |
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_powerbi_gateway_vpc"></a> [powerbi\_gateway\_vpc](#module\_powerbi\_gateway\_vpc) | terraform-aws-modules/vpc/aws | 5.5.1 |
+## Outputs
+
+No outputs.
+<!-- vale on -->
+<!-- markdownlint-enable -->
+<!-- END_TF_DOCS -->

infrastructure/terraform/components/powerbi-gateway/cloudwatch_metric_alarm_patch_task_failed.tf

@@ -0,0 +1,18 @@
+resource "aws_cloudwatch_metric_alarm" "patch_task_failed" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  alarm_name          = "${local.csi}-patch-task-failed"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "FailedCommands"
+  namespace           = "AWS/SSM-RunCommand"
+  period              = 300
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "Alarm when the AWS-RunPatchBaseline maintenance window task reports a failed run"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    DocumentName = "AWS-RunPatchBaseline"
+  }
+}

infrastructure/terraform/components/powerbi-gateway/cloudwatch_metric_alarm_powerbi_gateway_standalone_status.tf

@@ -0,0 +1,25 @@
+resource "aws_cloudwatch_metric_alarm" "powerbi_gateway_standalone_status_check_failed" {
+  for_each = var.enable_powerbi_gateway ? {
+    for idx, instance in aws_instance.powerbi_gateway_standalone :
+    idx => {
+      id   = instance.id
+      name = format("%s-standalone-%02d-status-check-failed", local.csi, idx + 1)
+    }
+  } : {}
+
+  alarm_name          = each.value.name
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 2
+  datapoints_to_alarm = 2
+  metric_name         = "StatusCheckFailed"
+  namespace           = "AWS/EC2"
+  period              = 300
+  statistic           = "Maximum"
+  threshold           = 1
+  alarm_description   = "Instance or system status check failed for a standalone Power BI gateway host"
+  treat_missing_data  = "breaching"
+
+  dimensions = {
+    InstanceId = each.value.id
+  }
+}

infrastructure/terraform/components/powerbi-gateway/data_cloudinit_config_powerbi_gateway.tf

@@ -0,0 +1,11 @@
+data "cloudinit_config" "powerbi_gateway" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  gzip          = false
+  base64_encode = true
+
+  part {
+    content_type = "text/cloud-config"
+    content      = local.powerbi_gateway_script
+  }
+}

infrastructure/terraform/components/powerbi-gateway/ec2_instances_powerbi_gateway.tf

@@ -0,0 +1,14 @@
+resource "aws_instance" "powerbi_gateway_standalone" {
+  count = var.enable_powerbi_gateway ? var.powerbi_gateway_instance_count : 0
+
+  associate_public_ip_address = false
+  launch_template {
+    id      = aws_launch_template.powerbi_gateway_standalone[0].id
+    version = "$Latest"
+  }
+
+  tags = {
+    "Name"        = format("%s-standalone-%02d", local.csi, count.index + 1)
+    "Patch Group" = aws_ssm_patch_group.windows_patch_group[0].patch_group
+  }
+}

infrastructure/terraform/components/powerbi-gateway/iam_instance_profile_powerbi_gateway.tf

@@ -0,0 +1,226 @@
+resource "aws_iam_instance_profile" "powerbi_gateway" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  name = local.csi
+  role = aws_iam_role.powerbi_gateway_role[0].name
+}
+
+data "aws_iam_policy_document" "powerbi_gateway_assume_role_policy" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+resource "aws_iam_role" "powerbi_gateway_role" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  name               = local.csi
+  description        = "PowerBI Gateway Instance Role"
+  path               = "/"
+  assume_role_policy = data.aws_iam_policy_document.powerbi_gateway_assume_role_policy[0].json
+}
+
+
+resource "aws_iam_policy" "powerbi_gateway_permissions_policy" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  name        = local.csi
+  description = "PowerBI Gateway Instance Permissions"
+  path        = "/"
+  policy      = data.aws_iam_policy_document.powerbi_gateway_permissions_policy[0].json
+}
+
+resource "aws_iam_role_policy_attachment" "powerbi_gateway_permissions_policy_attachment" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  role       = aws_iam_role.powerbi_gateway_role[0].name
+  policy_arn = aws_iam_policy.powerbi_gateway_permissions_policy[0].arn
+}
+
+resource "aws_iam_role_policy_attachment" "powerbi_gateway_ssm_policy_attachment" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  role       = aws_iam_role.powerbi_gateway_role[0].name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+data "aws_iam_policy_document" "powerbi_gateway_permissions_policy" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  statement {
+    sid    = "AllowLogs"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+
+    resources = [
+      "arn:aws:logs:${local.parameter_bundle.region}:${local.this_account}:log-group:*",
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3DataBucket"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket"
+    ]
+
+    resources = [
+      local.reporting.powerbi_data_bucket_arn,
+      "${local.reporting.powerbi_data_bucket_arn}/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3ResultsBucket"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:PutObject"
+    ]
+
+    resources = [
+      local.reporting.powerbi_results_bucket_arn,
+      "${local.reporting.powerbi_results_bucket_arn}/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowAthenaAccess1"
+    effect = "Allow"
+
+    actions = [
+      "athena:GetQueryResults",
+      "athena:GetQueryResultsStream",
+      "athena:GetQueryExecution",
+      "athena:StartQueryExecution",
+      "athena:GetWorkGroup",
+      "athena:GetNamedQuery"
+    ]
+
+    resources = [
+      local.reporting.powerbi_user_workgroup_arn
+    ]
+  }
+
+  statement {
+    sid    = "AllowAthenaAccess2"
+    effect = "Allow"
+
+    actions = [
+      "athena:GetDatabase",
+      "athena:GetTableMetadata",
+      "athena:GetDataCatalog",
+      "athena:GetTable"
+    ]
+
+    resources = [
+      "arn:aws:athena:${var.region}:${local.this_account}:datacatalog/AWSDataCatalog"
+    ]
+  }
+
+  statement {
+    sid    = "AllowAthenaAccess3"
+    effect = "Allow"
+
+    actions = [
+      "athena:ListDataCatalogs",
+      "athena:ListDatabases",
+      "athena:ListTableMetadata",
+      "athena:ListWorkGroups"
+    ]
+
+    resources = ["*"] # Access to List all above is required. Condition keys not supported for these resources.
+  }
+
+  statement {
+    sid    = "AllowGlueAccess"
+    effect = "Allow"
+
+    actions = [
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:BatchGetTable",
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:GetPartition",
+      "glue:GetPartitions"
+    ]
+
+    resources = concat(
+      local.core_glue_catalog_resources, # Access to all core account catalogs is required as they are all accessible via the default catalog in the environment's account
+      [
+        local.reporting.powerbi_reporting_database_arn,
+        "arn:aws:glue:${var.region}:${var.core_account_id}:catalog",
+        "arn:aws:glue:${var.region}:${local.this_account}:catalog",
+        # Tables
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_completed_summary",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_completed_summary_batch",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_status",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_summary",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_summary_batch",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/client_latest_name",
+        # Views
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_completed_summary_all",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_summary_all",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_summary_all_email_filter",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_smsnudge_staging",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_status_smsnudge",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_plan_read_status_smsnudge",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/request_item_status_smsnudge",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/dates",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/letters_invoice_units_monthly",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/letters_invoice_units_weekly",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/latency_percentiles",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/daily_recipient_count",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/daily_recipient_distribution",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/raw_latency_3m",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/delivered_messages",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_app_recipients_distribution",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_app_recipients_multiple_clients",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_messages_per_recipient",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_recipient_with_more_than_five_messages",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_recipients_distribution",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/monthly_recipients_distribution_by_integrator",
+        "arn:aws:glue:${var.region}:${local.this_account}:table/${local.reporting.powerbi_reporting_database_name}/billing_transactions",
+      ]
+    )
+  }
+
+  statement {
+    sid    = "AllowS3KMSAccess"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:Encrypt",
+      "kms:GenerateDataKey",
+      "kms:GenerateDataKeyWithoutPlaintext",
+      "kms:DescribeKey"
+    ]
+
+    resources = [
+      local.reporting.powerbi_s3_kms_key_arn
+    ]
+  }
+}