feat: [DTOSS-12403] merge audit and core pipelines

.azuredevops/pipelines/cd-infrastructure-sandbox.yaml

@@ -0,0 +1,127 @@
+---
+
+name: $(Build.SourceBranchName)-$(Date:yyyyMMdd)_$(Rev:r)
+trigger: none
+pr: none
+
+pool:
+  #vmImage: ubuntu-latest
+  name: private-pool-dev-uks
+
+resources:
+  repositories:
+    - repository: dtos-devops-templates
+      type: github
+      name: NHSDigital/dtos-devops-templates
+      ref: 8d9323116e7e0442e3f89750df39af27a2e1a4a7
+      endpoint: NHSDigital
+
+parameters:
+  - name: terraformActionsAudit
+    # This parameter allows the user to run only the plan for testing purposes
+    displayName: 'Audit: Apply Terraform Plan'
+    type: string
+    values:
+      - 'PlanOnly'
+      - 'Apply'
+    default: 'Apply'
+
+  - name: dockerImageTag
+    # Used to set the Docker image tag via Terraform in the Continuous Deployment settings of function and container apps
+    displayName: 'Core: Docker Image Tag'
+    type: string
+    default: sbrk
+
+  - name: retagImages
+    # Only required until commit hashes are passed in by calling pipeline, except for the
+    # PreProd pipeline which needs to retag images to move them from the Dev ACR to the Prod ACR
+    displayName: 'Core: Re-tag Images'
+    type: boolean
+    default: false
+
+  - name: dockerImageReTagFrom
+    # Only required until commit hashes are passed in by calling pipeline
+    displayName: 'Core: Docker Image Retag from value'
+    type: string
+    default: development
+
+  - name: dockerImageReTagTo
+    # Only required until commit hashes are passed in by calling pipeline
+    displayName: 'Core: Docker Image Retag to value'
+    type: string
+    default: sbrk
+
+  - name: terraformActionsCore
+    # This parameter allows the user to run only the plan for testing purposes
+    displayName: 'Core: Apply Terraform Plan'
+    type: string
+    values:
+      - 'PlanOnly'
+      - 'Apply'
+    default: 'Apply'
+
+  - name: testTypes
+    type: object
+    default:
+      - smoke_e2e
+
+variables:
+  - name: hostPoolName
+    value: private-pool-dev-uks
+  - group: SBRK_backend
+  - group: SBRK_audit_backend_remote_state
+  - group: SBRK_image_pipelines
+  - group: SBRK_automation_testing_pipeline
+  - group: DEV_hub_backend_remote_state
+  - name: TF_VERSION
+    value: 1.14.4
+  - name: TF_PLAN_ARTIFACT
+    value: tf_plan_core_SANDBOX
+  - name: TF_DIRECTORY_CORE
+    value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-core
+  - name: TF_DIRECTORY_AUDIT
+    value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-audit
+  - name: ENVIRONMENT
+    value: sandbox
+
+
+stages:
+  # STAGE 1: AUDIT DEPLOYMENT
+  - template: ../templates/cd-infrastructure-audit-common.yaml
+    parameters:
+      terraformActions: ${{ parameters.terraformActionsAudit }}
+      environment: $(ENVIRONMENT)
+
+  # STAGE 2: CORE DEPLOYMENT
+  - template: ../templates/cd-infrastructure-core-common.yaml
+    parameters:
+      hostPoolName: $(hostPoolName)
+      serviceConnection: $(SERVICE_CONNECTION)
+      targetSubscriptionId: $(TF_VAR_TARGET_SUBSCRIPTION_ID_CORE)
+      terraformActions: ${{ parameters.terraformActionsCore }}
+      tfVarsFile: environments/$(ENVIRONMENT).tfvars
+      environment: $(ENVIRONMENT)
+      resourceGroupNameSql: $(RESOURCE_GROUP_NAME_SQL)
+      databaseManagementJobName: $(DATABASE_MANAGEMENT_JOB_NAME)
+      dockerImageTag: ${{ parameters.dockerImageTag }}
+      retagImages: ${{ parameters.retagImages }}
+      retagImagesFrom: ${{ parameters.dockerImageReTagFrom }}
+      retagImagesTo: ${{ parameters.dockerImageReTagTo }}
+      slackWebHook: $(SLACK_WEBHOOK_URL_WORKFLOWS)
+
+  - ${{ each testType in parameters.testTypes }}:
+      - stage: ${{ testType }}_stage
+        displayName: "Core: Run ${{ testType }} Tests"
+        condition: always()
+        jobs:
+          - job: ${{ testType }}_job
+            pool:
+              name: $(hostPoolName)
+            steps:
+              - template: .azuredevops/templates/steps/run-post-deployment-pw-tests.yaml@dtos-devops-templates
+                parameters:
+                  serviceConnection: $(SERVICE_CONNECTION)
+                  testProjectDirectory: "tests/playwright-tests"
+                  testfileDirectory: "src/tests/e2e/testFiles"
+                  testProjectName: "tests"
+                  testType: ${{ testType }}

.azuredevops/pipelines/cd-infrastructure.yaml

@@ -0,0 +1,136 @@
+---
+name: $(Build.SourceBranchName)-$(Date:yyyyMMdd)_$(Rev:r)
+trigger: none
+pr: none
+
+pool:
+  name: private-pool-dev-uks
+
+resources:
+  repositories:
+    - repository: dtos-devops-templates
+      type: github
+      name: NHSDigital/dtos-devops-templates
+      ref: 8d9323116e7e0442e3f89750df39af27a2e1a4a7
+      endpoint: NHSDigital
+
+# 1. RUNTIME PARAMETERS
+parameters:
+  - name: targetEnvironment
+    displayName: 'Target Environment'
+    type: string
+    default: 'development'
+    values:
+      - sandbox
+      - development
+      - nft
+      - integration
+      - preprod
+      - production
+
+  - name: terraformActionsAudit
+    displayName: 'Audit: Apply Terraform Plan'
+    type: string
+    values:
+      - 'PlanOnly'
+      - 'Apply'
+    default: 'Apply'
+
+  - name: dockerImageTag
+    displayName: 'Core: Docker Image Tag'
+    type: string
+    default: development
+
+  - name: retagImages
+    displayName: 'Core: Re-tag Images'
+    type: boolean
+    default: false
+
+  - name: dockerImageReTagFrom
+    displayName: 'Core: Docker Image Retag from value'
+    type: string
+    default: development
+
+  - name: dockerImageReTagTo
+    displayName: 'Core: Docker Image Retag to value'
+    type: string
+    default: development
+
+  - name: terraformActionsCore
+    displayName: 'Core: Apply Terraform Plan'
+    type: string
+    values:
+      - 'PlanOnly'
+      - 'Apply'
+    default: 'Apply'
+
+  - name: testTypes
+    type: object
+    default:
+      - smoke_e2e
+
+# 2. DYNAMIC VARIABLES
+variables:
+  - name: hostPoolName
+    value: private-pool-dev-uks
+  - name: TF_VERSION
+    value: 1.14.4
+  - name: TF_PLAN_ARTIFACT
+    value: tf_plan_core_${{ upper(parameters.targetEnvironment) }}
+  - name: TF_DIRECTORY_CORE
+    value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-core
+  - name: TF_DIRECTORY_AUDIT
+    value: $(System.DefaultWorkingDirectory)/$(System.TeamProject)/infrastructure/tf-audit
+
+  # Variable Template
+  - template: ../variables/infrastructure-vars.yaml
+    parameters:
+      environmentName: ${{ parameters.targetEnvironment }}
+
+  # New Secret groups
+  - group: ${{ parameters.targetEnvironment }}-secrets
+  - group: ${{ parameters.targetEnvironment }}-automation-testing
+
+
+# 3. DEPLOYMENT STAGES
+stages:
+  # --- STAGE 1: AUDIT ---
+  - template: ../templates/cd-infrastructure-audit-common.yaml
+    parameters:
+      terraformActions: ${{ parameters.terraformActionsAudit }}
+      environment: $(ENVIRONMENT)
+
+  # --- STAGE 2: CORE ---
+  - template: ../templates/cd-infrastructure-core-common.yaml
+    parameters:
+      hostPoolName: $(hostPoolName)
+      serviceConnection: $(SERVICE_CONNECTION)
+      targetSubscriptionId: $(TF_VAR_TARGET_SUBSCRIPTION_ID_CORE)
+      terraformActions: ${{ parameters.terraformActionsCore }}
+      tfVarsFile: environments/$(ENVIRONMENT).tfvars
+      environment: $(ENVIRONMENT)
+      resourceGroupNameSql: $(RESOURCE_GROUP_NAME_SQL)
+      databaseManagementJobName: $(DATABASE_MANAGEMENT_JOB_NAME)
+      dockerImageTag: ${{ parameters.dockerImageTag }}
+      retagImages: ${{ parameters.retagImages }}
+      retagImagesFrom: ${{ parameters.dockerImageReTagFrom }}
+      retagImagesTo: ${{ parameters.dockerImageReTagTo }}
+      slackWebHook: $(SLACK_WEBHOOK_URL_WORKFLOWS)
+
+  # --- STAGE 3: AUTOMATED TESTING ---
+  - ${{ each testType in parameters.testTypes }}:
+      - stage: ${{ testType }}_stage
+        displayName: "Core: Run ${{ testType }} Tests"
+        condition: always()
+        jobs:
+          - job: ${{ testType }}_job
+            pool:
+              name: $(hostPoolName)
+            steps:
+              - template: .azuredevops/templates/steps/run-post-deployment-pw-tests.yaml@dtos-devops-templates
+                parameters:
+                  serviceConnection: $(SERVICE_CONNECTION)
+                  testProjectDirectory: "tests/playwright-tests"
+                  testfileDirectory: "src/tests/e2e/testFiles"
+                  testProjectName: "tests"
+                  testType: ${{ testType }}

.azuredevops/templates/cd-infrastructure-audit-common.yaml

@@ -0,0 +1,50 @@
+parameters:
+  - name: pipelineAction
+    displayName: 'Pipeline Action'
+    type: string
+    values:
+      - 'PlanOnly'
+      - 'Apply'
+    default: 'Apply'
+  - name: terraformActions
+    type: string
+    default: 'Apply'
+  - name: environment
+    type: string
+
+stages:
+- stage: terraform_plan
+  displayName: "Audit: Terraform Plan"
+  condition: and(in('${{ parameters.pipelineAction }}', 'Apply', 'PlanOnly'), eq(variables['Build.Reason'], 'Manual'))
+  variables:
+    tfVarsFile: environments/$(ENVIRONMENT).tfvars
+    BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME: $(BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME_AUDIT)
+    TF_VAR_TARGET_SUBSCRIPTION_ID: $(TF_VAR_TARGET_SUBSCRIPTION_ID_AUDIT)
+    TF_DIRECTORY: $(TF_DIRECTORY_AUDIT)
+  jobs:
+    - job: init_and_plan
+      displayName: Init, plan, store artifact
+      steps:
+        - checkout: self
+        - checkout: dtos-devops-templates
+        - template: .azuredevops/templates/steps/tf_plan.yaml@dtos-devops-templates
+
+- stage: terraform_apply
+  displayName: "Audit: Terraform Apply"
+  dependsOn: [terraform_plan]
+  condition: and(eq('${{ parameters.pipelineAction }}', 'Apply'), eq(dependencies.terraform_plan.outputs['init_and_plan.TerraformPlan.changesPresent'], 'true'), eq(variables['Build.Reason'], 'Manual'))
+  variables:
+    BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME: $(BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME_AUDIT)
+    TF_VAR_TARGET_SUBSCRIPTION_ID: $(TF_VAR_TARGET_SUBSCRIPTION_ID_AUDIT)
+    TF_DIRECTORY: $(TF_DIRECTORY_AUDIT)
+  jobs:
+    - deployment: terraform_apply
+      displayName: Init, get plan artifact, apply
+      environment: $(ENVIRONMENT)
+      strategy:
+        runOnce:
+          deploy:
+            steps:
+              - checkout: self
+              - checkout: dtos-devops-templates
+              - template: .azuredevops/templates/steps/tf_apply.yaml@dtos-devops-templates

.azuredevops/templates/cd-infrastructure-core-common.yaml

@@ -31,7 +31,7 @@ stages:
 - stage: retag_images_stage
   # Only required until commit hashes are passed in by calling pipeline, except for the
   # PreProd pipeline which needs to retag images to move them from the Dev ACR to the Prod ACR
-  displayName: ACR re-tag
+  displayName: "Core: ACR re-tag"
   pool:
     name: ${{ parameters.hostPoolName }}
   jobs:
@@ -68,12 +68,15 @@ stages:
 
 - stage: terraform_deploy_stage
   ${{ if eq(parameters.terraformActions, 'Apply') }}:
-    displayName: Terraform Deploy
+    displayName: "Core: Terraform Deploy"
   ${{ else }}:
-    displayName: Terraform Plan Only
+    displayName: "Core: Terraform Plan Only"
   condition: in(dependencies.retag_images_stage.result, 'Succeeded', 'Skipped')
   variables:
     tfVarsFile: ${{ parameters.tfVarsFile }}
+    BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME: $(BACKEND_AZURE_STORAGE_ACCOUNT_CONTAINER_NAME_CORE)
+    TF_VAR_TARGET_SUBSCRIPTION_ID: $(TF_VAR_TARGET_SUBSCRIPTION_ID_CORE)
+    TF_DIRECTORY: $(TF_DIRECTORY_CORE)
   pool:
     name: ${{ parameters.hostPoolName }}
   jobs:
@@ -126,9 +129,11 @@ stages:
           displayName: Set Stage Status
 
 - stage: db_changes_stage
-  displayName: Database changes
+  displayName: "Core: Database changes"
   dependsOn: terraform_deploy_stage
   condition: in(dependencies.terraform_deploy_stage.result, 'Succeeded', 'Skipped')
+  variables:
+    TF_VAR_TARGET_SUBSCRIPTION_ID: $(TF_VAR_TARGET_SUBSCRIPTION_ID_CORE)
   pool:
     name: ${{ parameters.hostPoolName }}
   jobs:
@@ -165,7 +170,7 @@ stages:
           displayName: Set Stage Status
 
 - stage: restart_functions_stage
-  displayName: Restart Function Apps
+  displayName: "Core: Restart Function Apps"
   dependsOn:
   - terraform_deploy_stage
   - db_changes_stage
@@ -174,6 +179,8 @@ stages:
       in(dependencies.terraform_deploy_stage.result, 'Succeeded', 'Skipped'),
       in(dependencies.db_changes_stage.result, 'Succeeded', 'Skipped')
     )
+  variables:
+    TF_VAR_TARGET_SUBSCRIPTION_ID: $(TF_VAR_TARGET_SUBSCRIPTION_ID_CORE)
   pool:
     name: ${{ parameters.hostPoolName }}
   jobs:
@@ -205,7 +212,7 @@ stages:
           displayName: Set Stage Status
 
 - stage: notify_stage
-  displayName: Send Slack Notification
+  displayName: "Core: Send Slack Notification"
   dependsOn:
     - retag_images_stage
     - terraform_deploy_stage