From 04c045161d2c067252eee6cfd9da037354cd3882 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:27:28 +0100 Subject: [PATCH 01/50] test --- .env | 4 +- .github/workflows/build.yml | 94 +++++ .github/workflows/publish.yml | 50 +++ .github/workflows/test.yml | 31 ++ Jenkinsfile | 693 +++++++++++++++++----------------- mhs/outbound/pycurl | 1 + 6 files changed, 530 insertions(+), 343 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml create mode 160000 mhs/outbound/pycurl diff --git a/.env b/.env index 42caf8cf4..ccab03ec6 100644 --- a/.env +++ b/.env @@ -10,4 +10,6 @@ MHS_SECRET_CA_CERTS="" INBOUND_BUILD_TAG=latest OUTBOUND_BUILD_TAG=latest ROUTE_BUILD_TAG=latest -WEB_SERVICE_BUILD_TAG=latest \ No newline at end of file +WEB_SERVICE_BUILD_TAG=latest + +ENVIRONMENT_ID=build diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..7e73ccb84 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,94 @@ +name: "Build" + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + push: + branches: + - main + - TEST-GITHUB-ACTIONS + +jobs: + build_common: + name: "Common & MHS Common Tests" + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: "Build & test Common directory" + working-directory: ./common + run: | + python3 -m pip install -r requirements.txt + python3 -m pytest + - name: "Build & test MHS Common directory" + working-directory: ./mhs/common + run: | + python3 -m pip install -r requirements.txt + python3 -m pytest + + inbound_tests: + name: "Inbound Tests" + uses: ./.github/workflows/test.yml + with: + name: Inbound + path: ./mhs/inbound + secrets: inherit + + outbound_tests: + name: "Outbound Tests" + uses: ./.github/workflows/test.yml + with: + name: Outbound + path: ./mhs/outbound + secrets: inherit + + route_tests: + name: "Route Tests" + uses: ./.github/workflows/test.yml + with: + name: Route + path: ./mhs/spineroutelookup + secrets: inherit + + generate-build-id: + name: "Generate Build Id" + needs: [ build_common, inbound_tests, outbound_tests, route_tests ] + runs-on: ubuntu-latest + outputs: + build-id: ${{ steps.generate.outputs.buildId }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - id: generate + run: | + BUILD_ID=$(python3 pipeline/scripts/tag.py ${{ github.ref }} ${{ github.run_number }} ${{ github.sha }}) + echo "Generated the build tag: $BUILD_ID" + echo "buildId=$BUILD_ID" >> "$GITHUB_OUTPUT" + + publish-docker-images: + name: "Publish Docker Images" + needs: [ generate-build-id ] + strategy: + matrix: + config: + - directory: mhs/inbound + repository: mhs/inbound + dockerfile: docker/inbound/Dockerfile + - directory: mhs/outbound + repository: mhs/outbound + dockerfile: docker/outbound/Dockerfile + - directory: mhs/spineroutelookup + repository: mhs/route + dockerfile: docker/spineroutelookup/Dockerfile + - directory: integration-tests/fake_spine + repository: fake-spine + dockerfile: integration-tests/fake_spine/Dockerfile + uses: ./.github/workflows/publish.yml + with: + directory: ${{ matrix.config.directory }} + repository: ${{ matrix.config.repository }} + dockerfile: ${{ matrix.config.dockerfile }} + build-id: ${{ needs.generate-build-id.outputs.build-id }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..bcbe93d33 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,50 @@ +name: "Publish" + +on: + workflow_call: + inputs: + directory: + required: true + type: string + repository: + required: true + type: string + dockerfile: + required: true + type: string + build-id: + required: true + type: string + secrets: + AWS_ACCOUNT_ID: + required: true + AWS_ROLE_TO_ASSUME: + required: true + AWS_REGION: + required: true + +jobs: + publish_image: + name: "Publish ${{ inputs.repository }} Image" + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Build and Push Docker Image + run: | + ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" + DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" + + docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} . + + docker push $DOCKER_IMAGE \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..d04edcc92 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: "Test" + +on: + workflow_call: + inputs: + name: + required: true + type: string + path: + required: true + type: string + +jobs: + unit_tests: + name: ${{ inputs.name }} Unit Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install Dependencies + working-directory: ${{ inputs.path }} + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + - name: Execute Unit Tests + working-directory: ${{ inputs.path }} + run: python3 -m pytest \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index d18f5c307..e9e504406 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,20 +3,20 @@ pipeline { label 'jenkins-workers' } - environment { - BUILD_TAG = sh label: 'Generating build tag', returnStdout: true, script: 'python3 pipeline/scripts/tag.py ${GIT_BRANCH} ${BUILD_NUMBER} ${GIT_COMMIT}' - BUILD_TAG_LOWER = sh label: 'Lowercase build tag', returnStdout: true, script: "echo -n ${BUILD_TAG} | tr '[:upper:]' '[:lower:]'" - ENVIRONMENT_ID = "build" - MHS_INBOUND_QUEUE_NAME = "${ENVIRONMENT_ID}-inbound" - LOCAL_INBOUND_IMAGE_NAME = "local/mhs-inbound:${BUILD_TAG}" - LOCAL_OUTBOUND_IMAGE_NAME = "local/mhs-outbound:${BUILD_TAG}" - LOCAL_ROUTE_IMAGE_NAME = "local/mhs-route:${BUILD_TAG}" - LOCAL_FAKE_SPINE_IMAGE_NAME = "local/fake-spine:${BUILD_TAG}" - INBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/inbound:${BUILD_TAG}" - OUTBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/outbound:${BUILD_TAG}" - ROUTE_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/route:${BUILD_TAG}" - FAKE_SPINE_IMAGE_NAME = "${DOCKER_REGISTRY}/fake-spine:${BUILD_TAG}" - } +// environment { +// BUILD_TAG = sh label: 'Generating build tag', returnStdout: true, script: 'python3 pipeline/scripts/tag.py ${GIT_BRANCH} ${BUILD_NUMBER} ${GIT_COMMIT}' +// BUILD_TAG_LOWER = sh label: 'Lowercase build tag', returnStdout: true, script: "echo -n ${BUILD_TAG} | tr '[:upper:]' '[:lower:]'" +// ENVIRONMENT_ID = "build" +// MHS_INBOUND_QUEUE_NAME = "${ENVIRONMENT_ID}-inbound" +// LOCAL_INBOUND_IMAGE_NAME = "local/mhs-inbound:${BUILD_TAG}" +// LOCAL_OUTBOUND_IMAGE_NAME = "local/mhs-outbound:${BUILD_TAG}" +// LOCAL_ROUTE_IMAGE_NAME = "local/mhs-route:${BUILD_TAG}" +// LOCAL_FAKE_SPINE_IMAGE_NAME = "local/fake-spine:${BUILD_TAG}" +// INBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/inbound:${BUILD_TAG}" +// OUTBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/outbound:${BUILD_TAG}" +// ROUTE_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/route:${BUILD_TAG}" +// FAKE_SPINE_IMAGE_NAME = "${DOCKER_REGISTRY}/fake-spine:${BUILD_TAG}" +// } stages { stage('Build & test Common directory') { @@ -127,336 +127,345 @@ pipeline { } } - stage('Run Tests') { - stages { - stage('Run Component Tests (SpineRouteLookup)') { - stages { - stage('Deploy component locally (SpineRouteLookup)') { - steps { - sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' - sh label: 'Start containers', script: ''' - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml down -v - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p custom_network down -v - . ./component-test-source.sh - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d''' - } - } - stage('Component Tests (SpineRouteLookup)') { - steps { - sh label: 'Run component tests', script: '''docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile .''' - sh label: 'Run component tests', script:''' - - docker run --network "${BUILD_TAG_LOWER}_default" \ - --env "MHS_ADDRESS=http://outbound" \ - --env "AWS_ACCESS_KEY_ID=test" \ - --env "AWS_SECRET_ACCESS_KEY=test" \ - --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ - --env "FAKE_SPINE_ADDRESS=http://fakespine" \ - --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ - --env "MHS_INBOUND_QUEUE_NAME=inbound" \ - --env "SCR_ADDRESS=http://scradaptor" \ - --name "${BUILD_TAG_LOWER}_component_test" \ - local/mhs-componenttest:$BUILD_TAG - ''' - } - } - } - post { - always { - sh label: 'Docker status', script: 'docker ps --all' - sh label: 'Dump container logs to files', script: ''' - mkdir -p logs - docker logs ${BUILD_TAG_LOWER}_route_1 > logs/route_1.log - docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log - docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_1.log - docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_1.log - docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_1.log - docker logs ${BUILD_TAG_LOWER}_redis_1 > logs/redis_1.log - docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_1.log - docker logs ${BUILD_TAG_LOWER}_mongodb_1 > logs/mongodb_1.log - ''' - archiveArtifacts artifacts: 'logs/*.log', fingerprint: true - sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' - } - } - } - stage('Run Component Tests (SDS API)') { - stages { - stage('Deploy component locally (SDS API)') { - steps { - sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' - sh label: 'Start containers', script: ''' - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v - . ./component-test-source.sh - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d - ''' - } - } - stage('Component Tests (SDS API)') { - steps { - sh label: 'Run component tests', script: ''' - docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . - docker run --rm --network "${BUILD_TAG_LOWER}_default" \ - --env "MHS_ADDRESS=http://outbound" \ - --env "AWS_ACCESS_KEY_ID=test" \ - --env "AWS_SECRET_ACCESS_KEY=test" \ - --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ - --env "FAKE_SPINE_ADDRESS=http://fakespine" \ - --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ - --env "MHS_INBOUND_QUEUE_NAME=inbound" \ - --env "SCR_ADDRESS=http://scradaptor" \ - local/mhs-componenttest:$BUILD_TAG - ''' - } - } - } - post { - always { - sh label: 'Docker status', script: 'docker ps --all' - sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' - sh label: 'Dump container logs to files', script: ''' - mkdir -p logs - docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log - docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log - docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log - docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log - docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log - docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log - ''' - archiveArtifacts artifacts: 'logs/*.log', fingerprint: true - sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' - } - } - } - - stage('Run Integration Tests (SpineRouteLookup)') { - options { - lock('exemplar-test-environment') - } - stages { - stage('Deploy MHS (SpineRouteLookup)') { - steps { - dir('pipeline/terraform/mhs-environment') { - script { - terraformBinPath = tfEnv() - } - sh label: 'Initialising Terraform', script: """ - ${terraformBinPath} init \ - -backend-config="bucket=${TF_STATE_BUCKET}" \ - -backend-config="region=${TF_STATE_BUCKET_REGION}" \ - -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ - -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ - -input=false -no-color - """ - sh label: 'Applying Terraform configuration', script: """ - ${terraformBinPath} apply -no-color -auto-approve \ - -var environment_id=${ENVIRONMENT_ID} \ - -var build_id=${BUILD_TAG} \ - -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ - -var opentest_vpc_id=${OPENTEST_VPC_ID} \ - -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ - -var mhs_outbound_service_minimum_instance_count=3 \ - -var mhs_outbound_service_maximum_instance_count=9 \ - -var mhs_inbound_service_minimum_instance_count=3 \ - -var mhs_inbound_service_maximum_instance_count=9 \ - -var mhs_route_service_minimum_instance_count=3 \ - -var mhs_route_service_maximum_instance_count=9 \ - -var task_role_arn=${TASK_ROLE} \ - -var execution_role_arn=${TASK_EXECUTION_ROLE} \ - -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ - -var ecr_address=${DOCKER_REGISTRY} \ - -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ - -var mhs_log_level=DEBUG \ - -var mhs_outbound_spineroutelookup_verify_certificate="False" \ - -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ - -var mhs_state_table_read_capacity=5 \ - -var mhs_state_table_write_capacity=5 \ - -var mhs_sync_async_table_read_capacity=5 \ - -var mhs_sync_async_table_write_capacity=5 \ - -var mhs_spine_org_code=${SPINE_ORG_CODE} \ - -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ - -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ - -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ - -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ - -var party_key_arn=${PARTY_KEY_ARN} \ - -var client_cert_arn=${CLIENT_CERT_ARN} \ - -var client_key_arn=${CLIENT_KEY_ARN} \ - -var ca_certs_arn=${CA_CERTS_ARN} \ - -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ - -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ - -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ - -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ - -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ - -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ - -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ - -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ - -var elasticache_node_type="cache.t2.micro" \ - -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ - -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ - -var mhs_sds_api_url="" \ - -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ - """ - script { - env.MHS_ADDRESS = sh ( - label: 'Obtaining outbound LB DNS name', - returnStdout: true, - script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" - ).trim() - env.MHS_OUTBOUND_TARGET_GROUP = sh ( - label: 'Obtaining outbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output outbound_lb_target_group_arn" - ).trim() - env.MHS_INBOUND_TARGET_GROUP = sh ( - label: 'Obtaining inbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output inbound_lb_target_group_arn" - ).trim() - env.MHS_ROUTE_TARGET_GROUP = sh ( - label: 'Obtaining route LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output route_lb_target_group_arn" - ).trim() - env.MHS_STATE_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS state', - returnStdout: true, - script: "${terraformBinPath} output mhs_state_table_name" - ).trim() - env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS sync/async state', - returnStdout: true, - script: "${terraformBinPath} output mhs_sync_async_table_name" - ).trim() - } - } - } - } - } - } - stage('Run Integration Tests (SDS API)') { - options { - lock('exemplar-test-environment') - } - stages { - stage('Deploy MHS (SDS API)') { - steps { - dir('pipeline/terraform/mhs-environment') { - script { - terraformBinPath = tfEnv() - } - sh label: 'Initialising Terraform', script: """ - ${terraformBinPath} init \ - -backend-config="bucket=${TF_STATE_BUCKET}" \ - -backend-config="region=${TF_STATE_BUCKET_REGION}" \ - -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ - -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ - -input=false -no-color - """ - sh label: 'Applying Terraform configuration', script: """ - ${terraformBinPath} apply -no-color -auto-approve \ - -var environment_id=${ENVIRONMENT_ID} \ - -var build_id=${BUILD_TAG} \ - -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ - -var opentest_vpc_id=${OPENTEST_VPC_ID} \ - -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ - -var mhs_outbound_service_minimum_instance_count=3 \ - -var mhs_outbound_service_maximum_instance_count=9 \ - -var mhs_inbound_service_minimum_instance_count=3 \ - -var mhs_inbound_service_maximum_instance_count=9 \ - -var mhs_route_service_minimum_instance_count=3 \ - -var mhs_route_service_maximum_instance_count=9 \ - -var task_role_arn=${TASK_ROLE} \ - -var execution_role_arn=${TASK_EXECUTION_ROLE} \ - -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ - -var ecr_address=${DOCKER_REGISTRY} \ - -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ - -var mhs_log_level=DEBUG \ - -var mhs_outbound_spineroutelookup_verify_certificate="False" \ - -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ - -var mhs_state_table_read_capacity=5 \ - -var mhs_state_table_write_capacity=5 \ - -var mhs_sync_async_table_read_capacity=5 \ - -var mhs_sync_async_table_write_capacity=5 \ - -var mhs_spine_org_code=${SPINE_ORG_CODE} \ - -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ - -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ - -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ - -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ - -var party_key_arn=${PARTY_KEY_ARN} \ - -var client_cert_arn=${CLIENT_CERT_ARN} \ - -var client_key_arn=${CLIENT_KEY_ARN} \ - -var ca_certs_arn=${CA_CERTS_ARN} \ - -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ - -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ - -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ - -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ - -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ - -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ - -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ - -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ - -var elasticache_node_type="cache.t2.micro" \ - -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ - -var mhs_outbound_routing_lookup_method="SDS_API" \ - -var mhs_sds_api_url=${MHS_SDS_API_URL} \ - -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ - """ - script { - env.MHS_ADDRESS = sh ( - label: 'Obtaining outbound LB DNS name', - returnStdout: true, - script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" - ).trim() - env.MHS_OUTBOUND_TARGET_GROUP = sh ( - label: 'Obtaining outbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output outbound_lb_target_group_arn" - ).trim() - env.MHS_INBOUND_TARGET_GROUP = sh ( - label: 'Obtaining inbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output inbound_lb_target_group_arn" - ).trim() - env.MHS_ROUTE_TARGET_GROUP = sh ( - label: 'Obtaining route LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output route_lb_target_group_arn" - ).trim() - env.MHS_STATE_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS state', - returnStdout: true, - script: "${terraformBinPath} output mhs_state_table_name" - ).trim() - env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS sync/async state', - returnStdout: true, - script: "${terraformBinPath} output mhs_sync_async_table_name" - ).trim() - } - } - } - } - } - } - } - } - } - post { - always { - cobertura coberturaReportFile: '**/coverage.xml' - junit '**/test-reports/*.xml' - sh 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' - sh 'docker volume prune --force' - // Prune Docker images for current CI build. - // Note that the * in the glob patterns doesn't match / - sh 'docker image rm -f $(docker images "*/*:*${BUILD_TAG}" -q) $(docker images "*/*/*:*${BUILD_TAG}" -q) || true' - } - } -} +// +// +// stage('Run Tests') { +// stages { +// stage('Run Component Tests (SpineRouteLookup)') { +// stages { +// stage('Deploy component locally (SpineRouteLookup)') { +// steps { +// sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' +// sh label: 'Start containers', script: ''' +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml down -v +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p custom_network down -v +// . ./component-test-source.sh +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d''' +// } +// } +// stage('Component Tests (SpineRouteLookup)') { +// steps { +// sh label: 'Run component tests', script: '''docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile .''' +// sh label: 'Run component tests', script:''' +// +// docker run --network "${BUILD_TAG_LOWER}_default" \ +// --env "MHS_ADDRESS=http://outbound" \ +// --env "AWS_ACCESS_KEY_ID=test" \ +// --env "AWS_SECRET_ACCESS_KEY=test" \ +// --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ +// --env "FAKE_SPINE_ADDRESS=http://fakespine" \ +// --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ +// --env "MHS_INBOUND_QUEUE_NAME=inbound" \ +// --env "SCR_ADDRESS=http://scradaptor" \ +// --name "${BUILD_TAG_LOWER}_component_test" \ +// local/mhs-componenttest:$BUILD_TAG +// ''' +// } +// } +// } +// post { +// always { +// sh label: 'Docker status', script: 'docker ps --all' +// sh label: 'Dump container logs to files', script: ''' +// mkdir -p logs +// docker logs ${BUILD_TAG_LOWER}_route_1 > logs/route_1.log +// docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log +// docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_1.log +// docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_1.log +// docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_1.log +// docker logs ${BUILD_TAG_LOWER}_redis_1 > logs/redis_1.log +// docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_1.log +// docker logs ${BUILD_TAG_LOWER}_mongodb_1 > logs/mongodb_1.log +// ''' +// archiveArtifacts artifacts: 'logs/*.log', fingerprint: true +// sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' +// } +// } +// } +// +// stage('Run Component Tests (SDS API)') { +// stages { +// stage('Deploy component locally (SDS API)') { +// steps { +// sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' +// sh label: 'Start containers', script: ''' +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v +// . ./component-test-source.sh +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d +// ''' +// } +// } +// stage('Component Tests (SDS API)') { +// steps { +// sh label: 'Run component tests', script: ''' +// docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . +// docker run --rm --network "${BUILD_TAG_LOWER}_default" \ +// --env "MHS_ADDRESS=http://outbound" \ +// --env "AWS_ACCESS_KEY_ID=test" \ +// --env "AWS_SECRET_ACCESS_KEY=test" \ +// --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ +// --env "FAKE_SPINE_ADDRESS=http://fakespine" \ +// --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ +// --env "MHS_INBOUND_QUEUE_NAME=inbound" \ +// --env "SCR_ADDRESS=http://scradaptor" \ +// local/mhs-componenttest:$BUILD_TAG +// ''' +// } +// } +// } +// post { +// always { +// sh label: 'Docker status', script: 'docker ps --all' +// sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' +// sh label: 'Dump container logs to files', script: ''' +// mkdir -p logs +// docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log +// docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log +// docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log +// docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log +// docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log +// docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log +// +// ''' +// archiveArtifacts artifacts: 'logs/*.log', fingerprint: true +// sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' +// } +// } +// } +// +// +// +// +// +// +// stage('Run Integration Tests (SpineRouteLookup)') { +// options { +// lock('exemplar-test-environment') +// } +// stages { +// stage('Deploy MHS (SpineRouteLookup)') { +// steps { +// dir('pipeline/terraform/mhs-environment') { +// script { +// terraformBinPath = tfEnv() +// } +// sh label: 'Initialising Terraform', script: """ +// ${terraformBinPath} init \ +// -backend-config="bucket=${TF_STATE_BUCKET}" \ +// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ +// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ +// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ +// -input=false -no-color +// """ +// sh label: 'Applying Terraform configuration', script: """ +// ${terraformBinPath} apply -no-color -auto-approve \ +// -var environment_id=${ENVIRONMENT_ID} \ +// -var build_id=${BUILD_TAG} \ +// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ +// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ +// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ +// -var mhs_outbound_service_minimum_instance_count=3 \ +// -var mhs_outbound_service_maximum_instance_count=9 \ +// -var mhs_inbound_service_minimum_instance_count=3 \ +// -var mhs_inbound_service_maximum_instance_count=9 \ +// -var mhs_route_service_minimum_instance_count=3 \ +// -var mhs_route_service_maximum_instance_count=9 \ +// -var task_role_arn=${TASK_ROLE} \ +// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ +// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ +// -var ecr_address=${DOCKER_REGISTRY} \ +// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ +// -var mhs_log_level=DEBUG \ +// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ +// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ +// -var mhs_state_table_read_capacity=5 \ +// -var mhs_state_table_write_capacity=5 \ +// -var mhs_sync_async_table_read_capacity=5 \ +// -var mhs_sync_async_table_write_capacity=5 \ +// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ +// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ +// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ +// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ +// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ +// -var party_key_arn=${PARTY_KEY_ARN} \ +// -var client_cert_arn=${CLIENT_CERT_ARN} \ +// -var client_key_arn=${CLIENT_KEY_ARN} \ +// -var ca_certs_arn=${CA_CERTS_ARN} \ +// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ +// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ +// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ +// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ +// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ +// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ +// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ +// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ +// -var elasticache_node_type="cache.t2.micro" \ +// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ +// -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ +// -var mhs_sds_api_url="" \ +// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ +// """ +// script { +// env.MHS_ADDRESS = sh ( +// label: 'Obtaining outbound LB DNS name', +// returnStdout: true, +// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" +// ).trim() +// env.MHS_OUTBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining outbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output outbound_lb_target_group_arn" +// ).trim() +// env.MHS_INBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining inbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output inbound_lb_target_group_arn" +// ).trim() +// env.MHS_ROUTE_TARGET_GROUP = sh ( +// label: 'Obtaining route LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output route_lb_target_group_arn" +// ).trim() +// env.MHS_STATE_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_state_table_name" +// ).trim() +// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS sync/async state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_sync_async_table_name" +// ).trim() +// } +// } +// } +// } +// } +// } +// stage('Run Integration Tests (SDS API)') { +// options { +// lock('exemplar-test-environment') +// } +// stages { +// stage('Deploy MHS (SDS API)') { +// steps { +// dir('pipeline/terraform/mhs-environment') { +// script { +// terraformBinPath = tfEnv() +// } +// sh label: 'Initialising Terraform', script: """ +// ${terraformBinPath} init \ +// -backend-config="bucket=${TF_STATE_BUCKET}" \ +// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ +// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ +// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ +// -input=false -no-color +// """ +// sh label: 'Applying Terraform configuration', script: """ +// ${terraformBinPath} apply -no-color -auto-approve \ +// -var environment_id=${ENVIRONMENT_ID} \ +// -var build_id=${BUILD_TAG} \ +// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ +// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ +// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ +// -var mhs_outbound_service_minimum_instance_count=3 \ +// -var mhs_outbound_service_maximum_instance_count=9 \ +// -var mhs_inbound_service_minimum_instance_count=3 \ +// -var mhs_inbound_service_maximum_instance_count=9 \ +// -var mhs_route_service_minimum_instance_count=3 \ +// -var mhs_route_service_maximum_instance_count=9 \ +// -var task_role_arn=${TASK_ROLE} \ +// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ +// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ +// -var ecr_address=${DOCKER_REGISTRY} \ +// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ +// -var mhs_log_level=DEBUG \ +// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ +// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ +// -var mhs_state_table_read_capacity=5 \ +// -var mhs_state_table_write_capacity=5 \ +// -var mhs_sync_async_table_read_capacity=5 \ +// -var mhs_sync_async_table_write_capacity=5 \ +// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ +// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ +// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ +// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ +// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ +// -var party_key_arn=${PARTY_KEY_ARN} \ +// -var client_cert_arn=${CLIENT_CERT_ARN} \ +// -var client_key_arn=${CLIENT_KEY_ARN} \ +// -var ca_certs_arn=${CA_CERTS_ARN} \ +// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ +// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ +// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ +// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ +// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ +// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ +// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ +// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ +// -var elasticache_node_type="cache.t2.micro" \ +// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ +// -var mhs_outbound_routing_lookup_method="SDS_API" \ +// -var mhs_sds_api_url=${MHS_SDS_API_URL} \ +// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ +// """ +// script { +// env.MHS_ADDRESS = sh ( +// label: 'Obtaining outbound LB DNS name', +// returnStdout: true, +// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" +// ).trim() +// env.MHS_OUTBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining outbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output outbound_lb_target_group_arn" +// ).trim() +// env.MHS_INBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining inbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output inbound_lb_target_group_arn" +// ).trim() +// env.MHS_ROUTE_TARGET_GROUP = sh ( +// label: 'Obtaining route LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output route_lb_target_group_arn" +// ).trim() +// env.MHS_STATE_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_state_table_name" +// ).trim() +// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS sync/async state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_sync_async_table_name" +// ).trim() +// } +// } +// } +// } +// } +// } +// } +// } +// } +// post { +// always { +// cobertura coberturaReportFile: '**/coverage.xml' +// junit '**/test-reports/*.xml' +// sh 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' +// sh 'docker volume prune --force' +// // Prune Docker images for current CI build. +// // Note that the * in the glob patterns doesn't match / +// sh 'docker image rm -f $(docker images "*/*:*${BUILD_TAG}" -q) $(docker images "*/*/*:*${BUILD_TAG}" -q) || true' +// } +// } +// } String tfEnv(String tfEnvRepo="https://github.com/tfutils/tfenv.git", String tfEnvPath="~/.tfenv") { sh(label: "Get tfenv" , script: "git clone ${tfEnvRepo} ${tfEnvPath}", returnStatus: true) diff --git a/mhs/outbound/pycurl b/mhs/outbound/pycurl new file mode 160000 index 000000000..016c3835e --- /dev/null +++ b/mhs/outbound/pycurl @@ -0,0 +1 @@ +Subproject commit 016c3835e4d5a41af89395a6e84261dd4c693e17 From b51c3732ce51b5ccea268832a251c0211e2ed9c3 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:29:36 +0100 Subject: [PATCH 02/50] pip --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e73ccb84..8ed698db5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,13 +20,13 @@ jobs: - name: "Build & test Common directory" working-directory: ./common run: | - python3 -m pip install -r requirements.txt - python3 -m pytest + pipenv install --dev + pipenv run pytest - name: "Build & test MHS Common directory" working-directory: ./mhs/common run: | - python3 -m pip install -r requirements.txt - python3 -m pytest + pipenv install --dev + pipenv run pytest inbound_tests: name: "Inbound Tests" From 59d2064ab840a21c523ac650f10f89aadda47601 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:30:45 +0100 Subject: [PATCH 03/50] pip --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d04edcc92..1940da522 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: "Test" +name: "Reusable Python Test Workflow" on: workflow_call: @@ -21,11 +21,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.x' + - name: Install pipenv + run: python3 -m pip install pipenv - name: Install Dependencies working-directory: ${{ inputs.path }} - run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt + run: pipenv install --dev - name: Execute Unit Tests working-directory: ${{ inputs.path }} - run: python3 -m pytest \ No newline at end of file + run: pipenv run pytest \ No newline at end of file From 5e8480b086e2b8cf0e8c860f7f8c363b6aa300eb Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:33:07 +0100 Subject: [PATCH 04/50] pip --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1940da522..2798fca5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.11' - name: Install pipenv run: python3 -m pip install pipenv - name: Install Dependencies From ce180303a7333ad589625457a880ff3eabd5058a Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:37:28 +0100 Subject: [PATCH 05/50] pip --- .github/workflows/build.yml | 6 ++++++ .github/workflows/test.yml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8ed698db5..99326012f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,12 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install pipenv + run: python3 -m pip install pipenv - name: "Build & test Common directory" working-directory: ./common run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2798fca5e..0198c80cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: "Reusable Python Test Workflow" +name: "Test" on: workflow_call: @@ -28,4 +28,4 @@ jobs: run: pipenv install --dev - name: Execute Unit Tests working-directory: ${{ inputs.path }} - run: pipenv run pytest \ No newline at end of file + run: pipenv run unittests-cov \ No newline at end of file From 9493c99c675f1de2d4d6a443794b5e13354dd424 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:41:04 +0100 Subject: [PATCH 06/50] pip --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99326012f..4036db1d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,12 +27,12 @@ jobs: working-directory: ./common run: | pipenv install --dev - pipenv run pytest + pipenv run unittests-cov - name: "Build & test MHS Common directory" working-directory: ./mhs/common run: | pipenv install --dev - pipenv run pytest + pipenv run unittests-cov inbound_tests: name: "Inbound Tests" From c6b68ae0451070b86abd76409fcf4ede4ef0ec4c Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:45:40 +0100 Subject: [PATCH 07/50] pip --- .github/workflows/build.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4036db1d9..e508bfaa9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,16 +23,18 @@ jobs: python-version: '3.11' - name: Install pipenv run: python3 -m pip install pipenv - - name: "Build & test Common directory" + - name: "Install Common directory dependencies" working-directory: ./common - run: | - pipenv install --dev - pipenv run unittests-cov - - name: "Build & test MHS Common directory" + run: pipenv install --dev + - name: "Run Common directory tests" + working-directory: ./common + run: pipenv run unittests-cov + - name: "Install MHS Common directory dependencies" working-directory: ./mhs/common - run: | - pipenv install --dev - pipenv run unittests-cov + run: pipenv install --dev + - name: "Run MHS Common directory tests" + working-directory: ./mhs/common + run: pipenv run unittests-cov inbound_tests: name: "Inbound Tests" @@ -97,4 +99,7 @@ jobs: repository: ${{ matrix.config.repository }} dockerfile: ${{ matrix.config.dockerfile }} build-id: ${{ needs.generate-build-id.outputs.build-id }} - secrets: inherit \ No newline at end of file + secrets: + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} + AWS_REGION: ${{ secrets.AWS_REGION }} \ No newline at end of file From 7327b9c7c28738c8e6bdcff6063cc8f1dc32ba61 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 10:57:41 +0100 Subject: [PATCH 08/50] post build --- .github/workflows/build.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e508bfaa9..c86f473b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,16 +25,24 @@ jobs: run: python3 -m pip install pipenv - name: "Install Common directory dependencies" working-directory: ./common - run: pipenv install --dev + run: pipenv install --dev --ignore-pipfile - name: "Run Common directory tests" working-directory: ./common run: pipenv run unittests-cov - name: "Install MHS Common directory dependencies" working-directory: ./mhs/common - run: pipenv install --dev + run: pipenv install --dev --ignore-pipfile - name: "Run MHS Common directory tests" working-directory: ./mhs/common run: pipenv run unittests-cov + - name: Upload Test Reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: Common-Test-Reports + path: | + common/test-reports/** + mhs/common/test-reports/** inbound_tests: name: "Inbound Tests" @@ -102,4 +110,16 @@ jobs: secrets: AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} - AWS_REGION: ${{ secrets.AWS_REGION }} \ No newline at end of file + AWS_REGION: ${{ secrets.AWS_REGION }} + + post-build-cleanup: + name: "Post-build Cleanup" + needs: [ build_common, inbound_tests, outbound_tests, route_tests, generate-build-id, publish-docker-images ] + if: always() + runs-on: ubuntu-latest + steps: + - name: "Prune Docker Images" + run: | + docker image prune --force + docker container prune --force + docker volume prune --force \ No newline at end of file From 0334db3e95b1b39324b9fe3eb2c95b210903073e Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:02:16 +0100 Subject: [PATCH 09/50] checkout repo --- .github/workflows/publish.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bcbe93d33..68be829cb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,18 +33,16 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 - + with: + submodules: 'false' - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} - - name: Build and Push Docker Image run: | ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" - - docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} . - + docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} ${{ inputs.directory }} docker push $DOCKER_IMAGE \ No newline at end of file From 5f356f5bb3502d5eae57a3085f4eb8415addc8a5 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:08:17 +0100 Subject: [PATCH 10/50] checkout repo --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 68be829cb..71decf1a6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,4 +45,4 @@ jobs: ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} ${{ inputs.directory }} - docker push $DOCKER_IMAGE \ No newline at end of file + docker push $DOCKER_IMAGE From 4a628aa157ad1dc602218c9f63524dcb33144e79 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:13:55 +0100 Subject: [PATCH 11/50] kicking it off again --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 71decf1a6..68be829cb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,4 +45,4 @@ jobs: ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} ${{ inputs.directory }} - docker push $DOCKER_IMAGE + docker push $DOCKER_IMAGE \ No newline at end of file From 06847095b5d7d9f39eced2553e70e59abc8e5f0c Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:19:03 +0100 Subject: [PATCH 12/50] kicking it off again --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 68be829cb..71decf1a6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,4 +45,4 @@ jobs: ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} ${{ inputs.directory }} - docker push $DOCKER_IMAGE \ No newline at end of file + docker push $DOCKER_IMAGE From e121c073e148457dc534752566a206f7ad03f1f6 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:22:53 +0100 Subject: [PATCH 13/50] kicking it off again --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 71decf1a6..d609e9197 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -44,5 +44,5 @@ jobs: run: | ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" DOCKER_IMAGE="$ECR_REGISTRY/${{ inputs.repository }}:${{ inputs.build-id }}" - docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} ${{ inputs.directory }} - docker push $DOCKER_IMAGE + docker build -t $DOCKER_IMAGE -f ${{ inputs.dockerfile }} . + docker push $DOCKER_IMAGE \ No newline at end of file From 36285ae516b19d4c32bc521d07c8d5ac728463a1 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:29:19 +0100 Subject: [PATCH 14/50] logging into ecr --- .github/workflows/publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d609e9197..1e43bfe8b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,6 +40,10 @@ jobs: with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} + - name: Login to ECR + run: | + DOCKER_REGISTRY="https://${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" + aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin $DOCKER_REGISTRY - name: Build and Push Docker Image run: | ECR_REGISTRY="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com" From 1e569471937bfb07f3de3b0b36ea097541b9e60e Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:35:22 +0100 Subject: [PATCH 15/50] adding other tests + hoping --- .github/workflows/build.yml | 89 +++++++++++++++++-- Jenkinsfile | 168 ++---------------------------------- 2 files changed, 87 insertions(+), 170 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c86f473b2..f5273a86a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,13 +25,13 @@ jobs: run: python3 -m pip install pipenv - name: "Install Common directory dependencies" working-directory: ./common - run: pipenv install --dev --ignore-pipfile + run: pipenv install --dev - name: "Run Common directory tests" working-directory: ./common run: pipenv run unittests-cov - name: "Install MHS Common directory dependencies" working-directory: ./mhs/common - run: pipenv install --dev --ignore-pipfile + run: pipenv install --dev - name: "Run MHS Common directory tests" working-directory: ./mhs/common run: pipenv run unittests-cov @@ -70,7 +70,7 @@ jobs: generate-build-id: name: "Generate Build Id" - needs: [ build_common, inbound_tests, outbound_tests, route_tests ] + needs: [build_common, inbound_tests, outbound_tests, route_tests] runs-on: ubuntu-latest outputs: build-id: ${{ steps.generate.outputs.buildId }} @@ -85,7 +85,7 @@ jobs: publish-docker-images: name: "Publish Docker Images" - needs: [ generate-build-id ] + needs: [generate-build-id] strategy: matrix: config: @@ -112,14 +112,89 @@ jobs: AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} AWS_REGION: ${{ secrets.AWS_REGION }} + component_tests: + name: "Component Tests" + needs: [publish-docker-images] + runs-on: ubuntu-latest + strategy: + matrix: + component: ['SpineRouteLookup', 'SDS API'] + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Setup Docker Environment + run: | + ./integration-tests/setup_component_test_env.sh + if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d + elif [[ "${{ matrix.component }}" == "SDS API" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d + fi + - name: Run Component Tests + run: | + docker build -t local/mhs-componenttest:${{ needs.generate-build-id.outputs.build-id }} -f ./component-test.Dockerfile . + docker run --rm --network "${BUILD_TAG_LOWER}_default" \ + --env "MHS_ADDRESS=http://outbound" \ + --env "FAKE_SPINE_ADDRESS=http://fakespine" \ + local/mhs-componenttest:${{ needs.generate-build-id.outputs.build-id }} + - name: Dump Logs and Cleanup + if: always() + run: | + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v + mkdir -p logs + docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log + - name: Archive Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.component }} Component Test Logs + path: logs/ + + integration_tests: + name: "Integration Tests" + needs: [component_tests] + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.0.0 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + - name: Terraform Init + run: terraform init -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" -backend-config="region=${{ secrets.TF_STATE_BUCKET_REGION }}" -backend-config="key=${{ env.ENVIRONMENT_ID }}-mhs.tfstate" + working-directory: ./pipeline/terraform/mhs-environment + - name: Terraform Apply + run: | + terraform apply -auto-approve \ + -var environment_id=build \ + -var build_id=${{ needs.generate-build-id.outputs.build-id }} + working-directory: ./pipeline/terraform/mhs-environment + - name: Run Integration Tests + run: | + LB_DNS=$(terraform output -raw outbound_lb_domain_name) + echo "MHS_ADDRESS=https://$LB_DNS" >> $GITHUB_ENV + # Execute your integration test script here + working-directory: ./pipeline/terraform/mhs-environment + - name: Clean up Terraform + if: always() + run: terraform destroy -auto-approve + working-directory: ./pipeline/terraform/mhs-environment + post-build-cleanup: name: "Post-build Cleanup" - needs: [ build_common, inbound_tests, outbound_tests, route_tests, generate-build-id, publish-docker-images ] + needs: [build_common, inbound_tests, outbound_tests, route_tests, generate-build-id, publish-docker-images, component_tests, integration_tests] if: always() runs-on: ubuntu-latest steps: - name: "Prune Docker Images" run: | - docker image prune --force - docker container prune --force + docker system prune --force docker volume prune --force \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index e9e504406..011232756 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,134 +1,6 @@ -pipeline { - agent{ - label 'jenkins-workers' - } - -// environment { -// BUILD_TAG = sh label: 'Generating build tag', returnStdout: true, script: 'python3 pipeline/scripts/tag.py ${GIT_BRANCH} ${BUILD_NUMBER} ${GIT_COMMIT}' -// BUILD_TAG_LOWER = sh label: 'Lowercase build tag', returnStdout: true, script: "echo -n ${BUILD_TAG} | tr '[:upper:]' '[:lower:]'" -// ENVIRONMENT_ID = "build" -// MHS_INBOUND_QUEUE_NAME = "${ENVIRONMENT_ID}-inbound" -// LOCAL_INBOUND_IMAGE_NAME = "local/mhs-inbound:${BUILD_TAG}" -// LOCAL_OUTBOUND_IMAGE_NAME = "local/mhs-outbound:${BUILD_TAG}" -// LOCAL_ROUTE_IMAGE_NAME = "local/mhs-route:${BUILD_TAG}" -// LOCAL_FAKE_SPINE_IMAGE_NAME = "local/fake-spine:${BUILD_TAG}" -// INBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/inbound:${BUILD_TAG}" -// OUTBOUND_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/outbound:${BUILD_TAG}" -// ROUTE_IMAGE_NAME = "${DOCKER_REGISTRY}/mhs/route:${BUILD_TAG}" -// FAKE_SPINE_IMAGE_NAME = "${DOCKER_REGISTRY}/fake-spine:${BUILD_TAG}" -// } - - stages { - stage('Build & test Common directory') { - steps { - dir('common') { - buildModules('Installing common dependencies') - executeUnitTestsWithCoverage() - } - } - } - stage('Build & test MHS Common directory') { - steps { - dir('mhs/common') { - buildModules('Installing mhs common dependencies') - executeUnitTestsWithCoverage() - } - } - } - stage('Build MHS') { - parallel { - stage('Inbound') { - stages { - stage('Build') { - steps { - dir('mhs/inbound') { - buildModules('Installing inbound dependencies') - } - } - } - stage('Unit test') { - steps { - dir('mhs/inbound') { - executeUnitTestsWithCoverage() - } - } - } - stage('Build and Push image') { - when { - expression { currentBuild.resultIsBetterOrEqualTo('SUCCESS') } - } - steps { - buildAndPushImage('${LOCAL_INBOUND_IMAGE_NAME}', '${INBOUND_IMAGE_NAME}', 'docker/inbound/Dockerfile') - } - } - } - } - stage('Outbound') { - stages { - stage('Build') { - steps { - dir('mhs/outbound') { - buildModules('Installing outbound dependencies') - } - } - } - stage('Unit test') { - steps { - dir('mhs/outbound') { - executeUnitTestsWithCoverage() - } - } - } - stage('Build and Push image') { - when { - expression { currentBuild.resultIsBetterOrEqualTo('SUCCESS') } - } - steps { - buildAndPushImage('${LOCAL_OUTBOUND_IMAGE_NAME}', '${OUTBOUND_IMAGE_NAME}', 'docker/outbound/Dockerfile') - } - } - } - } - stage('Route') { - stages { - stage('Build') { - steps { - dir('mhs/spineroutelookup') { - buildModules('Installing route lookup dependencies') - } - } - } - stage('Unit test') { - steps { - dir('mhs/spineroutelookup') { - executeUnitTestsWithCoverage() - } - } - } - stage('Build and Push image') { - when { - expression { currentBuild.resultIsBetterOrEqualTo('SUCCESS') } - } - steps { - buildAndPushImage('${LOCAL_ROUTE_IMAGE_NAME}', '${ROUTE_IMAGE_NAME}', 'docker/spineroutelookup/Dockerfile') - } - } - } - } - stage('Fake Spine') { - stages { - stage('Build and Push image') { - steps { - buildAndPushImage('${LOCAL_FAKE_SPINE_IMAGE_NAME}', '${FAKE_SPINE_IMAGE_NAME}', 'integration-tests/fake_spine/Dockerfile') - } - } - } - } - } - } - - - +// +// +// // // // stage('Run Tests') { @@ -466,35 +338,5 @@ pipeline { // } // } // } - -String tfEnv(String tfEnvRepo="https://github.com/tfutils/tfenv.git", String tfEnvPath="~/.tfenv") { - sh(label: "Get tfenv" , script: "git clone ${tfEnvRepo} ${tfEnvPath}", returnStatus: true) - sh(label: "Install TF", script: "${tfEnvPath}/bin/tfenv install" , returnStatus: true) - return "${tfEnvPath}/bin/terraform" -} - -void executeUnitTestsWithCoverage() { - sh label: 'Running unit tests', script: 'pipenv run unittests-cov' - sh label: 'Displaying code coverage report', script: 'pipenv run coverage-report' - sh label: 'Exporting code coverage report', script: 'pipenv run coverage-report-xml' -// SonarQube disabled as atm it's not set up on AWS -// sh label: 'Running SonarQube analysis', script: "sonar-scanner -Dsonar.host.url=${SONAR_HOST} -Dsonar.login=${SONAR_TOKEN}" -} - -void buildModules(String action) { - sh label: action, script: 'pipenv install --dev --deploy --ignore-pipfile' -} - -int ecrLogin(String aws_region) { - String dockerLogin = "aws ecr get-login-password --region ${aws_region} | docker login -u AWS --password-stdin \"https://\$(aws sts get-caller-identity --query 'Account' --output text).dkr.ecr.${aws_region}.amazonaws.com\"" - return sh(label: "Logging in with Docker", script: dockerLogin, returnStatus: true) -} - -void buildAndPushImage(String localImageName, String imageName, String dockerFile, String context = '.') { - sh label: 'Running docker build', script: 'docker build -t ' + localImageName + ' -f ' + dockerFile + ' ' + context - if (ecrLogin(TF_STATE_BUCKET_REGION) != 0 ) { error("Docker login to ECR failed") } - sh label: 'Tag ecr image', script: 'docker tag ' + localImageName + ' ' + imageName - String dockerPushCommand = "docker push " + imageName - if (sh (label: "Pushing image", script: dockerPushCommand, returnStatus: true) !=0) { error("Docker push image failed") } - sh label: 'Deleting local ECR image', script: 'docker rmi ' + imageName -} +// +// From a45286a6a911a5186201e8e2e0a5d33ba5f1305b Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:45:24 +0100 Subject: [PATCH 16/50] Trying to access docker --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5273a86a..606915d86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,14 +114,18 @@ jobs: component_tests: name: "Component Tests" - needs: [publish-docker-images] + needs: [ publish-docker-images ] runs-on: ubuntu-latest strategy: matrix: - component: ['SpineRouteLookup', 'SDS API'] + component: [ 'SpineRouteLookup', 'SDS API' ] steps: - name: Checkout Repository uses: actions/checkout@v4 + - name: Install docker-compose + run: | + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose - name: Setup Docker Environment run: | ./integration-tests/setup_component_test_env.sh From 0257dcae92718c9e8be87ebc62b38a46d95424bb Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:53:52 +0100 Subject: [PATCH 17/50] Component Tests need generate-build-id to run --- .github/workflows/build.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 606915d86..f0d1db3eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,8 +114,11 @@ jobs: component_tests: name: "Component Tests" - needs: [ publish-docker-images ] + needs: [ publish-docker-images, generate-build-id ] runs-on: ubuntu-latest + env: + BUILD_TAG: ${{ needs.generate-build-id.outputs.build-id }} + BUILD_TAG_LOWER: ${{ toLower(needs.generate-build-id.outputs.build-id) }} strategy: matrix: component: [ 'SpineRouteLookup', 'SDS API' ] @@ -131,24 +134,24 @@ jobs: ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d fi - name: Run Component Tests run: | - docker build -t local/mhs-componenttest:${{ needs.generate-build-id.outputs.build-id }} -f ./component-test.Dockerfile . - docker run --rm --network "${BUILD_TAG_LOWER}_default" \ + docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . + docker run --rm --network "${{ env.BUILD_TAG_LOWER }}_default" \ --env "MHS_ADDRESS=http://outbound" \ --env "FAKE_SPINE_ADDRESS=http://fakespine" \ - local/mhs-componenttest:${{ needs.generate-build-id.outputs.build-id }} + local/mhs-componenttest:${{ env.BUILD_TAG }} - name: Dump Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v mkdir -p logs - docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_outbound_1 > logs/outbound_1.log - name: Archive Logs if: always() uses: actions/upload-artifact@v4 From e456eed7c9e083fd769adab9d92108ebf0465947 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:55:28 +0100 Subject: [PATCH 18/50] Component Tests need generate-build-id to run --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0d1db3eb..651deae13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-latest env: BUILD_TAG: ${{ needs.generate-build-id.outputs.build-id }} - BUILD_TAG_LOWER: ${{ toLower(needs.generate-build-id.outputs.build-id) }} + BUILD_TAG_LOWER: ${{ needs.generate-build-id.outputs.build-id | toLower }} strategy: matrix: component: [ 'SpineRouteLookup', 'SDS API' ] From 3f423883dfbcf3205606c9e5afea46b2e71e42ac Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 11:56:36 +0100 Subject: [PATCH 19/50] Component Tests need generate-build-id to run --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 651deae13..37d224830 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,6 @@ jobs: runs-on: ubuntu-latest env: BUILD_TAG: ${{ needs.generate-build-id.outputs.build-id }} - BUILD_TAG_LOWER: ${{ needs.generate-build-id.outputs.build-id | toLower }} strategy: matrix: component: [ 'SpineRouteLookup', 'SDS API' ] @@ -129,6 +128,10 @@ jobs: run: | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose + - name: Set Lowercase Build Tag + run: | + BUILD_TAG_LOWER=$(echo -n ${{ env.BUILD_TAG }} | tr '[:upper:]' '[:lower:]') + echo "BUILD_TAG_LOWER=$BUILD_TAG_LOWER" >> $GITHUB_ENV - name: Setup Docker Environment run: | ./integration-tests/setup_component_test_env.sh From a8025079f4a67a40b67156ad6201966ba64d5d1d Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 13:39:54 +0100 Subject: [PATCH 20/50] Debug component and SDS API tests --- .github/workflows/build.yml | 70 ++-- Jenkinsfile | 684 ++++++++++++++++++------------------ 2 files changed, 377 insertions(+), 377 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37d224830..f31db56ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -162,41 +162,41 @@ jobs: name: ${{ matrix.component }} Component Test Logs path: logs/ - integration_tests: - name: "Integration Tests" - needs: [component_tests] - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.0.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Terraform Init - run: terraform init -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" -backend-config="region=${{ secrets.TF_STATE_BUCKET_REGION }}" -backend-config="key=${{ env.ENVIRONMENT_ID }}-mhs.tfstate" - working-directory: ./pipeline/terraform/mhs-environment - - name: Terraform Apply - run: | - terraform apply -auto-approve \ - -var environment_id=build \ - -var build_id=${{ needs.generate-build-id.outputs.build-id }} - working-directory: ./pipeline/terraform/mhs-environment - - name: Run Integration Tests - run: | - LB_DNS=$(terraform output -raw outbound_lb_domain_name) - echo "MHS_ADDRESS=https://$LB_DNS" >> $GITHUB_ENV - # Execute your integration test script here - working-directory: ./pipeline/terraform/mhs-environment - - name: Clean up Terraform - if: always() - run: terraform destroy -auto-approve - working-directory: ./pipeline/terraform/mhs-environment +# integration_tests: +# name: "Integration Tests" +# needs: [component_tests] +# runs-on: ubuntu-latest +# steps: +# - name: Checkout Repository +# uses: actions/checkout@v4 +# - name: Setup Terraform +# uses: hashicorp/setup-terraform@v3 +# with: +# terraform_version: 1.0.0 +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_TO_ASSUME }} +# aws-region: ${{ secrets.AWS_REGION }} +# - name: Terraform Init +# run: terraform init -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" -backend-config="region=${{ secrets.TF_STATE_BUCKET_REGION }}" -backend-config="key=${{ env.ENVIRONMENT_ID }}-mhs.tfstate" +# working-directory: ./pipeline/terraform/mhs-environment +# - name: Terraform Apply +# run: | +# terraform apply -auto-approve \ +# -var environment_id=build \ +# -var build_id=${{ needs.generate-build-id.outputs.build-id }} +# working-directory: ./pipeline/terraform/mhs-environment +# - name: Run Integration Tests +# run: | +# LB_DNS=$(terraform output -raw outbound_lb_domain_name) +# echo "MHS_ADDRESS=https://$LB_DNS" >> $GITHUB_ENV +# # Execute your integration test script here +# working-directory: ./pipeline/terraform/mhs-environment +# - name: Clean up Terraform +# if: always() +# run: terraform destroy -auto-approve +# working-directory: ./pipeline/terraform/mhs-environment post-build-cleanup: name: "Post-build Cleanup" diff --git a/Jenkinsfile b/Jenkinsfile index 011232756..f5154e79a 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,342 +1,342 @@ -// -// -// -// -// -// stage('Run Tests') { -// stages { -// stage('Run Component Tests (SpineRouteLookup)') { -// stages { -// stage('Deploy component locally (SpineRouteLookup)') { -// steps { -// sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' -// sh label: 'Start containers', script: ''' -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml down -v -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p custom_network down -v -// . ./component-test-source.sh -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d''' -// } -// } -// stage('Component Tests (SpineRouteLookup)') { -// steps { -// sh label: 'Run component tests', script: '''docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile .''' -// sh label: 'Run component tests', script:''' -// -// docker run --network "${BUILD_TAG_LOWER}_default" \ -// --env "MHS_ADDRESS=http://outbound" \ -// --env "AWS_ACCESS_KEY_ID=test" \ -// --env "AWS_SECRET_ACCESS_KEY=test" \ -// --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ -// --env "FAKE_SPINE_ADDRESS=http://fakespine" \ -// --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ -// --env "MHS_INBOUND_QUEUE_NAME=inbound" \ -// --env "SCR_ADDRESS=http://scradaptor" \ -// --name "${BUILD_TAG_LOWER}_component_test" \ -// local/mhs-componenttest:$BUILD_TAG -// ''' -// } -// } -// } -// post { -// always { -// sh label: 'Docker status', script: 'docker ps --all' -// sh label: 'Dump container logs to files', script: ''' -// mkdir -p logs -// docker logs ${BUILD_TAG_LOWER}_route_1 > logs/route_1.log -// docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log -// docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_1.log -// docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_1.log -// docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_1.log -// docker logs ${BUILD_TAG_LOWER}_redis_1 > logs/redis_1.log -// docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_1.log -// docker logs ${BUILD_TAG_LOWER}_mongodb_1 > logs/mongodb_1.log -// ''' -// archiveArtifacts artifacts: 'logs/*.log', fingerprint: true -// sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' -// } -// } -// } -// -// stage('Run Component Tests (SDS API)') { -// stages { -// stage('Deploy component locally (SDS API)') { -// steps { -// sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' -// sh label: 'Start containers', script: ''' -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v -// . ./component-test-source.sh -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build -// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d -// ''' -// } -// } -// stage('Component Tests (SDS API)') { -// steps { -// sh label: 'Run component tests', script: ''' -// docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . -// docker run --rm --network "${BUILD_TAG_LOWER}_default" \ -// --env "MHS_ADDRESS=http://outbound" \ -// --env "AWS_ACCESS_KEY_ID=test" \ -// --env "AWS_SECRET_ACCESS_KEY=test" \ -// --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ -// --env "FAKE_SPINE_ADDRESS=http://fakespine" \ -// --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ -// --env "MHS_INBOUND_QUEUE_NAME=inbound" \ -// --env "SCR_ADDRESS=http://scradaptor" \ -// local/mhs-componenttest:$BUILD_TAG -// ''' -// } -// } -// } -// post { -// always { -// sh label: 'Docker status', script: 'docker ps --all' -// sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' -// sh label: 'Dump container logs to files', script: ''' -// mkdir -p logs -// docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log -// docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log -// docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log -// docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log -// docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log -// docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log -// -// ''' -// archiveArtifacts artifacts: 'logs/*.log', fingerprint: true -// sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' -// } -// } -// } -// -// -// -// -// -// -// stage('Run Integration Tests (SpineRouteLookup)') { -// options { -// lock('exemplar-test-environment') -// } -// stages { -// stage('Deploy MHS (SpineRouteLookup)') { -// steps { -// dir('pipeline/terraform/mhs-environment') { -// script { -// terraformBinPath = tfEnv() -// } -// sh label: 'Initialising Terraform', script: """ -// ${terraformBinPath} init \ -// -backend-config="bucket=${TF_STATE_BUCKET}" \ -// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ -// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ -// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ -// -input=false -no-color -// """ -// sh label: 'Applying Terraform configuration', script: """ -// ${terraformBinPath} apply -no-color -auto-approve \ -// -var environment_id=${ENVIRONMENT_ID} \ -// -var build_id=${BUILD_TAG} \ -// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ -// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ -// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ -// -var mhs_outbound_service_minimum_instance_count=3 \ -// -var mhs_outbound_service_maximum_instance_count=9 \ -// -var mhs_inbound_service_minimum_instance_count=3 \ -// -var mhs_inbound_service_maximum_instance_count=9 \ -// -var mhs_route_service_minimum_instance_count=3 \ -// -var mhs_route_service_maximum_instance_count=9 \ -// -var task_role_arn=${TASK_ROLE} \ -// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ -// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ -// -var ecr_address=${DOCKER_REGISTRY} \ -// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ -// -var mhs_log_level=DEBUG \ -// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ -// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ -// -var mhs_state_table_read_capacity=5 \ -// -var mhs_state_table_write_capacity=5 \ -// -var mhs_sync_async_table_read_capacity=5 \ -// -var mhs_sync_async_table_write_capacity=5 \ -// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ -// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ -// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ -// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ -// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ -// -var party_key_arn=${PARTY_KEY_ARN} \ -// -var client_cert_arn=${CLIENT_CERT_ARN} \ -// -var client_key_arn=${CLIENT_KEY_ARN} \ -// -var ca_certs_arn=${CA_CERTS_ARN} \ -// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ -// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ -// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ -// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ -// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ -// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ -// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ -// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ -// -var elasticache_node_type="cache.t2.micro" \ -// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ -// -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ -// -var mhs_sds_api_url="" \ -// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ -// """ -// script { -// env.MHS_ADDRESS = sh ( -// label: 'Obtaining outbound LB DNS name', -// returnStdout: true, -// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" -// ).trim() -// env.MHS_OUTBOUND_TARGET_GROUP = sh ( -// label: 'Obtaining outbound LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output outbound_lb_target_group_arn" -// ).trim() -// env.MHS_INBOUND_TARGET_GROUP = sh ( -// label: 'Obtaining inbound LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output inbound_lb_target_group_arn" -// ).trim() -// env.MHS_ROUTE_TARGET_GROUP = sh ( -// label: 'Obtaining route LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output route_lb_target_group_arn" -// ).trim() -// env.MHS_STATE_TABLE_NAME = sh ( -// label: 'Obtaining the table name used for the MHS state', -// returnStdout: true, -// script: "${terraformBinPath} output mhs_state_table_name" -// ).trim() -// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( -// label: 'Obtaining the table name used for the MHS sync/async state', -// returnStdout: true, -// script: "${terraformBinPath} output mhs_sync_async_table_name" -// ).trim() -// } -// } -// } -// } -// } -// } -// stage('Run Integration Tests (SDS API)') { -// options { -// lock('exemplar-test-environment') -// } -// stages { -// stage('Deploy MHS (SDS API)') { -// steps { -// dir('pipeline/terraform/mhs-environment') { -// script { -// terraformBinPath = tfEnv() -// } -// sh label: 'Initialising Terraform', script: """ -// ${terraformBinPath} init \ -// -backend-config="bucket=${TF_STATE_BUCKET}" \ -// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ -// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ -// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ -// -input=false -no-color -// """ -// sh label: 'Applying Terraform configuration', script: """ -// ${terraformBinPath} apply -no-color -auto-approve \ -// -var environment_id=${ENVIRONMENT_ID} \ -// -var build_id=${BUILD_TAG} \ -// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ -// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ -// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ -// -var mhs_outbound_service_minimum_instance_count=3 \ -// -var mhs_outbound_service_maximum_instance_count=9 \ -// -var mhs_inbound_service_minimum_instance_count=3 \ -// -var mhs_inbound_service_maximum_instance_count=9 \ -// -var mhs_route_service_minimum_instance_count=3 \ -// -var mhs_route_service_maximum_instance_count=9 \ -// -var task_role_arn=${TASK_ROLE} \ -// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ -// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ -// -var ecr_address=${DOCKER_REGISTRY} \ -// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ -// -var mhs_log_level=DEBUG \ -// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ -// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ -// -var mhs_state_table_read_capacity=5 \ -// -var mhs_state_table_write_capacity=5 \ -// -var mhs_sync_async_table_read_capacity=5 \ -// -var mhs_sync_async_table_write_capacity=5 \ -// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ -// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ -// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ -// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ -// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ -// -var party_key_arn=${PARTY_KEY_ARN} \ -// -var client_cert_arn=${CLIENT_CERT_ARN} \ -// -var client_key_arn=${CLIENT_KEY_ARN} \ -// -var ca_certs_arn=${CA_CERTS_ARN} \ -// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ -// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ -// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ -// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ -// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ -// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ -// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ -// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ -// -var elasticache_node_type="cache.t2.micro" \ -// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ -// -var mhs_outbound_routing_lookup_method="SDS_API" \ -// -var mhs_sds_api_url=${MHS_SDS_API_URL} \ -// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ -// """ -// script { -// env.MHS_ADDRESS = sh ( -// label: 'Obtaining outbound LB DNS name', -// returnStdout: true, -// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" -// ).trim() -// env.MHS_OUTBOUND_TARGET_GROUP = sh ( -// label: 'Obtaining outbound LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output outbound_lb_target_group_arn" -// ).trim() -// env.MHS_INBOUND_TARGET_GROUP = sh ( -// label: 'Obtaining inbound LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output inbound_lb_target_group_arn" -// ).trim() -// env.MHS_ROUTE_TARGET_GROUP = sh ( -// label: 'Obtaining route LB target group ARN', -// returnStdout: true, -// script: "${terraformBinPath} output route_lb_target_group_arn" -// ).trim() -// env.MHS_STATE_TABLE_NAME = sh ( -// label: 'Obtaining the table name used for the MHS state', -// returnStdout: true, -// script: "${terraformBinPath} output mhs_state_table_name" -// ).trim() -// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( -// label: 'Obtaining the table name used for the MHS sync/async state', -// returnStdout: true, -// script: "${terraformBinPath} output mhs_sync_async_table_name" -// ).trim() -// } -// } -// } -// } -// } -// } -// } -// } -// } -// post { -// always { -// cobertura coberturaReportFile: '**/coverage.xml' -// junit '**/test-reports/*.xml' -// sh 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' -// sh 'docker volume prune --force' -// // Prune Docker images for current CI build. -// // Note that the * in the glob patterns doesn't match / -// sh 'docker image rm -f $(docker images "*/*:*${BUILD_TAG}" -q) $(docker images "*/*/*:*${BUILD_TAG}" -q) || true' -// } -// } -// } -// -// + + + + + + stage('Run Tests') { + stages { + stage('Run Component Tests (SpineRouteLookup)') { + stages { + stage('Deploy component locally (SpineRouteLookup)') { + steps { + sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' + sh label: 'Start containers', script: ''' + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml down -v + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p custom_network down -v + . ./component-test-source.sh + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} up -d''' + } + } + stage('Component Tests (SpineRouteLookup)') { + steps { + sh label: 'Run component tests', script: '''docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile .''' + sh label: 'Run component tests', script:''' + + docker run --network "${BUILD_TAG_LOWER}_default" \ + --env "MHS_ADDRESS=http://outbound" \ + --env "AWS_ACCESS_KEY_ID=test" \ + --env "AWS_SECRET_ACCESS_KEY=test" \ + --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ + --env "FAKE_SPINE_ADDRESS=http://fakespine" \ + --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ + --env "MHS_INBOUND_QUEUE_NAME=inbound" \ + --env "SCR_ADDRESS=http://scradaptor" \ + --name "${BUILD_TAG_LOWER}_component_test" \ + local/mhs-componenttest:$BUILD_TAG + ''' + } + } + } + post { + always { + sh label: 'Docker status', script: 'docker ps --all' + sh label: 'Dump container logs to files', script: ''' + mkdir -p logs + docker logs ${BUILD_TAG_LOWER}_route_1 > logs/route_1.log + docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_1.log + docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_1.log + docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_1.log + docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_1.log + docker logs ${BUILD_TAG_LOWER}_redis_1 > logs/redis_1.log + docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_1.log + docker logs ${BUILD_TAG_LOWER}_mongodb_1 > logs/mongodb_1.log + ''' + archiveArtifacts artifacts: 'logs/*.log', fingerprint: true + sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' + } + } + } + + stage('Run Component Tests (SDS API)') { + stages { + stage('Deploy component locally (SDS API)') { + steps { + sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' + sh label: 'Start containers', script: ''' + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v + . ./component-test-source.sh + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d + ''' + } + } + stage('Component Tests (SDS API)') { + steps { + sh label: 'Run component tests', script: ''' + docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . + docker run --rm --network "${BUILD_TAG_LOWER}_default" \ + --env "MHS_ADDRESS=http://outbound" \ + --env "AWS_ACCESS_KEY_ID=test" \ + --env "AWS_SECRET_ACCESS_KEY=test" \ + --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ + --env "FAKE_SPINE_ADDRESS=http://fakespine" \ + --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ + --env "MHS_INBOUND_QUEUE_NAME=inbound" \ + --env "SCR_ADDRESS=http://scradaptor" \ + local/mhs-componenttest:$BUILD_TAG + ''' + } + } + } + post { + always { + sh label: 'Docker status', script: 'docker ps --all' + sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' + sh label: 'Dump container logs to files', script: ''' + mkdir -p logs + docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log + docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log + docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log + docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log + docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log + docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log + + ''' + archiveArtifacts artifacts: 'logs/*.log', fingerprint: true + sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' + } + } + } + + + + + + + stage('Run Integration Tests (SpineRouteLookup)') { + options { + lock('exemplar-test-environment') + } + stages { + stage('Deploy MHS (SpineRouteLookup)') { + steps { + dir('pipeline/terraform/mhs-environment') { + script { + terraformBinPath = tfEnv() + } + sh label: 'Initialising Terraform', script: """ + ${terraformBinPath} init \ + -backend-config="bucket=${TF_STATE_BUCKET}" \ + -backend-config="region=${TF_STATE_BUCKET_REGION}" \ + -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ + -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ + -input=false -no-color + """ + sh label: 'Applying Terraform configuration', script: """ + ${terraformBinPath} apply -no-color -auto-approve \ + -var environment_id=${ENVIRONMENT_ID} \ + -var build_id=${BUILD_TAG} \ + -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ + -var opentest_vpc_id=${OPENTEST_VPC_ID} \ + -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ + -var mhs_outbound_service_minimum_instance_count=3 \ + -var mhs_outbound_service_maximum_instance_count=9 \ + -var mhs_inbound_service_minimum_instance_count=3 \ + -var mhs_inbound_service_maximum_instance_count=9 \ + -var mhs_route_service_minimum_instance_count=3 \ + -var mhs_route_service_maximum_instance_count=9 \ + -var task_role_arn=${TASK_ROLE} \ + -var execution_role_arn=${TASK_EXECUTION_ROLE} \ + -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ + -var ecr_address=${DOCKER_REGISTRY} \ + -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ + -var mhs_log_level=DEBUG \ + -var mhs_outbound_spineroutelookup_verify_certificate="False" \ + -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ + -var mhs_state_table_read_capacity=5 \ + -var mhs_state_table_write_capacity=5 \ + -var mhs_sync_async_table_read_capacity=5 \ + -var mhs_sync_async_table_write_capacity=5 \ + -var mhs_spine_org_code=${SPINE_ORG_CODE} \ + -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ + -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ + -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ + -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ + -var party_key_arn=${PARTY_KEY_ARN} \ + -var client_cert_arn=${CLIENT_CERT_ARN} \ + -var client_key_arn=${CLIENT_KEY_ARN} \ + -var ca_certs_arn=${CA_CERTS_ARN} \ + -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ + -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ + -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ + -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ + -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ + -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ + -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ + -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ + -var elasticache_node_type="cache.t2.micro" \ + -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ + -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ + -var mhs_sds_api_url="" \ + -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ + """ + script { + env.MHS_ADDRESS = sh ( + label: 'Obtaining outbound LB DNS name', + returnStdout: true, + script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" + ).trim() + env.MHS_OUTBOUND_TARGET_GROUP = sh ( + label: 'Obtaining outbound LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output outbound_lb_target_group_arn" + ).trim() + env.MHS_INBOUND_TARGET_GROUP = sh ( + label: 'Obtaining inbound LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output inbound_lb_target_group_arn" + ).trim() + env.MHS_ROUTE_TARGET_GROUP = sh ( + label: 'Obtaining route LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output route_lb_target_group_arn" + ).trim() + env.MHS_STATE_TABLE_NAME = sh ( + label: 'Obtaining the table name used for the MHS state', + returnStdout: true, + script: "${terraformBinPath} output mhs_state_table_name" + ).trim() + env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( + label: 'Obtaining the table name used for the MHS sync/async state', + returnStdout: true, + script: "${terraformBinPath} output mhs_sync_async_table_name" + ).trim() + } + } + } + } + } + } + stage('Run Integration Tests (SDS API)') { + options { + lock('exemplar-test-environment') + } + stages { + stage('Deploy MHS (SDS API)') { + steps { + dir('pipeline/terraform/mhs-environment') { + script { + terraformBinPath = tfEnv() + } + sh label: 'Initialising Terraform', script: """ + ${terraformBinPath} init \ + -backend-config="bucket=${TF_STATE_BUCKET}" \ + -backend-config="region=${TF_STATE_BUCKET_REGION}" \ + -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ + -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ + -input=false -no-color + """ + sh label: 'Applying Terraform configuration', script: """ + ${terraformBinPath} apply -no-color -auto-approve \ + -var environment_id=${ENVIRONMENT_ID} \ + -var build_id=${BUILD_TAG} \ + -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ + -var opentest_vpc_id=${OPENTEST_VPC_ID} \ + -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ + -var mhs_outbound_service_minimum_instance_count=3 \ + -var mhs_outbound_service_maximum_instance_count=9 \ + -var mhs_inbound_service_minimum_instance_count=3 \ + -var mhs_inbound_service_maximum_instance_count=9 \ + -var mhs_route_service_minimum_instance_count=3 \ + -var mhs_route_service_maximum_instance_count=9 \ + -var task_role_arn=${TASK_ROLE} \ + -var execution_role_arn=${TASK_EXECUTION_ROLE} \ + -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ + -var ecr_address=${DOCKER_REGISTRY} \ + -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ + -var mhs_log_level=DEBUG \ + -var mhs_outbound_spineroutelookup_verify_certificate="False" \ + -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ + -var mhs_state_table_read_capacity=5 \ + -var mhs_state_table_write_capacity=5 \ + -var mhs_sync_async_table_read_capacity=5 \ + -var mhs_sync_async_table_write_capacity=5 \ + -var mhs_spine_org_code=${SPINE_ORG_CODE} \ + -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ + -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ + -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ + -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ + -var party_key_arn=${PARTY_KEY_ARN} \ + -var client_cert_arn=${CLIENT_CERT_ARN} \ + -var client_key_arn=${CLIENT_KEY_ARN} \ + -var ca_certs_arn=${CA_CERTS_ARN} \ + -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ + -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ + -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ + -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ + -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ + -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ + -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ + -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ + -var elasticache_node_type="cache.t2.micro" \ + -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ + -var mhs_outbound_routing_lookup_method="SDS_API" \ + -var mhs_sds_api_url=${MHS_SDS_API_URL} \ + -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ + """ + script { + env.MHS_ADDRESS = sh ( + label: 'Obtaining outbound LB DNS name', + returnStdout: true, + script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" + ).trim() + env.MHS_OUTBOUND_TARGET_GROUP = sh ( + label: 'Obtaining outbound LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output outbound_lb_target_group_arn" + ).trim() + env.MHS_INBOUND_TARGET_GROUP = sh ( + label: 'Obtaining inbound LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output inbound_lb_target_group_arn" + ).trim() + env.MHS_ROUTE_TARGET_GROUP = sh ( + label: 'Obtaining route LB target group ARN', + returnStdout: true, + script: "${terraformBinPath} output route_lb_target_group_arn" + ).trim() + env.MHS_STATE_TABLE_NAME = sh ( + label: 'Obtaining the table name used for the MHS state', + returnStdout: true, + script: "${terraformBinPath} output mhs_state_table_name" + ).trim() + env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( + label: 'Obtaining the table name used for the MHS sync/async state', + returnStdout: true, + script: "${terraformBinPath} output mhs_sync_async_table_name" + ).trim() + } + } + } + } + } + } + } + } + } + post { + always { + cobertura coberturaReportFile: '**/coverage.xml' + junit '**/test-reports/*.xml' + sh 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${BUILD_TAG_LOWER} down -v' + sh 'docker volume prune --force' + // Prune Docker images for current CI build. + // Note that the * in the glob patterns doesn't match / + sh 'docker image rm -f $(docker images "*/*:*${BUILD_TAG}" -q) $(docker images "*/*/*:*${BUILD_TAG}" -q) || true' + } + } +} + + From 4610c4d47afd02ad1d6992861ada39127fb3a990 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 13:40:31 +0100 Subject: [PATCH 21/50] Debug component and SDS API tests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f31db56ca..9871a19fa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -200,7 +200,7 @@ jobs: post-build-cleanup: name: "Post-build Cleanup" - needs: [build_common, inbound_tests, outbound_tests, route_tests, generate-build-id, publish-docker-images, component_tests, integration_tests] + needs: [build_common, inbound_tests, outbound_tests, route_tests, generate-build-id, publish-docker-images, component_tests ] if: always() runs-on: ubuntu-latest steps: From f7b40616584e709fd709f2be463d7cb0943018b1 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 13:54:32 +0100 Subject: [PATCH 22/50] Debug component and SDS API tests --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9871a19fa..280675e59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,10 +137,10 @@ jobs: ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up elif [[ "${{ matrix.component }}" == "SDS API" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up fi - name: Run Component Tests run: | From dbd85e6b3221d46b5a781a76e66c0c7133f63e02 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:01:32 +0100 Subject: [PATCH 23/50] Debug component and SDS API tests --- .github/workflows/build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 280675e59..95e7f2b36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,11 +136,9 @@ jobs: run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --build -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --build -d fi - name: Run Component Tests run: | From 5a0f450d93252e0da95cc411f52754ffca113a45 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:10:08 +0100 Subject: [PATCH 24/50] Trying to build before docker-compose runs tests. --- .github/workflows/build.yml | 20 ++++++++++++-------- docker-compose.yml | 9 +++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95e7f2b36..9120f1be6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,14 +132,18 @@ jobs: run: | BUILD_TAG_LOWER=$(echo -n ${{ env.BUILD_TAG }} | tr '[:upper:]' '[:lower:]') echo "BUILD_TAG_LOWER=$BUILD_TAG_LOWER" >> $GITHUB_ENV - - name: Setup Docker Environment - run: | - ./integration-tests/setup_component_test_env.sh - if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --build -d - elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --build -d - fi + - name: Build Docker Images + run: | + chmod +x ./build.sh + ./build.sh + - name: Setup Docker Environment + run: | + ./integration-tests/setup_component_test_env.sh + if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + elif [[ "${{ matrix.component }}" == "SDS API" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + fi - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . diff --git a/docker-compose.yml b/docker-compose.yml index 8ec8238e0..0fe150a95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,9 @@ version: '3' services: inbound: + build: + context: . + dockerfile: ./docker/outbound/Dockerfile image: local/mhs-inbound:${BUILD_TAG} ports: - "443" @@ -29,6 +32,9 @@ services: - SERVICE_PORTS=443,80 - SUPPORTED_FILE_TYPES outbound: + build: + context: . + dockerfile: ./docker/outbound/Dockerfile image: local/mhs-outbound:${BUILD_TAG} ports: - "80" @@ -57,6 +63,9 @@ services: - MHS_SDS_API_URL - MHS_SDS_API_KEY route: + build: + context: . + dockerfile: ./docker/outbound/Dockerfile image: local/mhs-route:${BUILD_TAG} ports: - "8080:80" From dcc645fc68b3bd2c42c9f0aa9f215d092c8bcda0 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:15:17 +0100 Subject: [PATCH 25/50] Trying to build before docker-compose runs tests. --- .github/workflows/build.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9120f1be6..192d4c127 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,20 +130,19 @@ jobs: sudo chmod +x /usr/local/bin/docker-compose - name: Set Lowercase Build Tag run: | - BUILD_TAG_LOWER=$(echo -n ${{ env.BUILD_TAG }} | tr '[:upper:]' '[:lower:]') - echo "BUILD_TAG_LOWER=$BUILD_TAG_LOWER" >> $GITHUB_ENV - - name: Build Docker Images - run: | - chmod +x ./build.sh - ./build.sh - - name: Setup Docker Environment - run: | - ./integration-tests/setup_component_test_env.sh - if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d - elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d - fi + echo "BUILD_TAG_LOWER=$(echo -n ${{ env.BUILD_TAG }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Build Docker Images + run: | + chmod +x ./build.sh + ./build.sh + - name: Setup Docker Environment + run: | + ./integration-tests/setup_component_test_env.sh + if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + elif [[ "${{ matrix.component }}" == "SDS API" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + fi - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . From bb0e0ce265a273109c6b07c1d44ac996cc40977e Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:25:42 +0100 Subject: [PATCH 26/50] No credentials error fix --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 192d4c127..b8ea252fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,6 +149,8 @@ jobs: docker run --rm --network "${{ env.BUILD_TAG_LOWER }}_default" \ --env "MHS_ADDRESS=http://outbound" \ --env "FAKE_SPINE_ADDRESS=http://fakespine" \ + --env "AWS_ACCESS_KEY_ID=test" \ + --env "AWS_SECRET_ACCESS_KEY=test" \ local/mhs-componenttest:${{ env.BUILD_TAG }} - name: Dump Logs and Cleanup if: always() From fe9963a3ca3c5606801e5afe09f9baeb95bcb2c9 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:33:24 +0100 Subject: [PATCH 27/50] No credentials error fix --- .github/workflows/build.yml | 4 ++-- docker-compose.yml | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8ea252fe..1c1c410a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,9 +139,9 @@ jobs: run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests run: | diff --git a/docker-compose.yml b/docker-compose.yml index 0fe150a95..eeb727267 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: inbound: @@ -80,6 +80,12 @@ services: image: nhsdev/nia-dynamodb-local:1.0.3 ports: - "8000:8000" + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8000 || exit 1" ] + interval: 5s + timeout: 5s + retries: 5 + start_period: 20s mongodb: image: mongo:5.0.16 ports: From 39decbb9cd0382f78afe186a8f41184742287239 Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:44:35 +0100 Subject: [PATCH 28/50] Update docker --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c1c410a2..1f55be5ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,7 +126,7 @@ jobs: uses: actions/checkout@v4 - name: Install docker-compose run: | - sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo curl -L "https://github.com/docker/compose/releases/download/v2.24.5/docker-compose-linux-x86_64" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose - name: Set Lowercase Build Tag run: | From 3f527a51bc904772b7ca8fc99ebbcacb468aaf0e Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 14:57:37 +0100 Subject: [PATCH 29/50] Unhealthy dynamodb fix --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f55be5ff..5ea6a8a14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,6 +83,7 @@ jobs: echo "Generated the build tag: $BUILD_ID" echo "buildId=$BUILD_ID" >> "$GITHUB_OUTPUT" + publish-docker-images: name: "Publish Docker Images" needs: [generate-build-id] @@ -143,6 +144,8 @@ jobs: elif [[ "${{ matrix.component }}" == "SDS API" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi + - name: Get Unhealthy Container Logs + run: docker-compose -p ${{ env.BUILD_TAG_LOWER }} logs dynamodb - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . From e1a0e0c1340a112658e186fef949b86f0aac168a Mon Sep 17 00:00:00 2001 From: Chiara Date: Mon, 18 Aug 2025 15:27:19 +0100 Subject: [PATCH 30/50] Unhealthy outbound container, fix SPINE_ROUTE_LOOKUP_URL --- .github/workflows/build.yml | 2 -- docker-compose.yml | 14 ++++++++------ mhs-config.yaml | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ea6a8a14..d2fdd753d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,8 +144,6 @@ jobs: elif [[ "${{ matrix.component }}" == "SDS API" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - - name: Get Unhealthy Container Logs - run: docker-compose -p ${{ env.BUILD_TAG_LOWER }} logs dynamodb - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . diff --git a/docker-compose.yml b/docker-compose.yml index eeb727267..a1d5a6918 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: context: . dockerfile: ./docker/outbound/Dockerfile image: local/mhs-outbound:${BUILD_TAG} + volumes: + - ./mhs-config.yaml:/usr/src/app/mhs-config.yaml ports: - "80" environment: @@ -80,12 +82,12 @@ services: image: nhsdev/nia-dynamodb-local:1.0.3 ports: - "8000:8000" - healthcheck: - test: [ "CMD-SHELL", "curl -f http://localhost:8000 || exit 1" ] - interval: 5s - timeout: 5s - retries: 5 - start_period: 20s +# healthcheck: +# test: [ "CMD-SHELL", "curl -f http://localhost:8000 || exit 1" ] +# interval: 5s +# timeout: 5s +# retries: 5 +# start_period: 20s mongodb: image: mongo:5.0.16 ports: diff --git a/mhs-config.yaml b/mhs-config.yaml index 6dc7cd17e..24b5a3e1f 100644 --- a/mhs-config.yaml +++ b/mhs-config.yaml @@ -26,6 +26,7 @@ AWS_SECRET_ACCESS_KEY: test MHS_RESYNC_INTERVAL: "1" MAX_RESYNC_RETRIES: "20" MHS_SPINE_ROUTE_LOOKUP_URL: http://localhost:8088 +SPINE_ROUTE_LOOKUP_URL: http://localhost:8088 MHS_SPINE_ORG_CODE: "YES" MHS_SPINE_REQUEST_MAX_SIZE: "4999600" From 1bc82e55c2df0af9d0cd4f59791c2f8f935b78f1 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 10:29:47 +0100 Subject: [PATCH 31/50] revert some changes --- docker-compose.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1d5a6918..6cc42e7d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,6 @@ version: '3.8' services: inbound: - build: - context: . - dockerfile: ./docker/outbound/Dockerfile image: local/mhs-inbound:${BUILD_TAG} ports: - "443" @@ -32,12 +29,7 @@ services: - SERVICE_PORTS=443,80 - SUPPORTED_FILE_TYPES outbound: - build: - context: . - dockerfile: ./docker/outbound/Dockerfile image: local/mhs-outbound:${BUILD_TAG} - volumes: - - ./mhs-config.yaml:/usr/src/app/mhs-config.yaml ports: - "80" environment: @@ -65,12 +57,9 @@ services: - MHS_SDS_API_URL - MHS_SDS_API_KEY route: - build: - context: . - dockerfile: ./docker/outbound/Dockerfile image: local/mhs-route:${BUILD_TAG} ports: - - "8080:80" + - "8080:80" environment: - MHS_LOG_LEVEL=NOTSET - MHS_SDS_URL=ldap://192.168.128.11 @@ -82,12 +71,6 @@ services: image: nhsdev/nia-dynamodb-local:1.0.3 ports: - "8000:8000" -# healthcheck: -# test: [ "CMD-SHELL", "curl -f http://localhost:8000 || exit 1" ] -# interval: 5s -# timeout: 5s -# retries: 5 -# start_period: 20s mongodb: image: mongo:5.0.16 ports: From a2d9ee578967711d029333d3d2c265f76948e527 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:05:39 +0100 Subject: [PATCH 32/50] More similar to Jenkinsfile --- .github/workflows/build.yml | 34 ++- Jenkinsfile | 536 ++++++++++++++++++------------------ 2 files changed, 298 insertions(+), 272 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2fdd753d..fab47a0a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,40 +125,58 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install docker-compose + + - name: Install Docker Compose v2 run: | - sudo curl -L "https://github.com/docker/compose/releases/download/v2.24.5/docker-compose-linux-x86_64" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose + sudo apt-get update + sudo apt-get install -y docker-compose-plugin + - name: Set Lowercase Build Tag run: | echo "BUILD_TAG_LOWER=$(echo -n ${{ env.BUILD_TAG }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Build Docker Images run: | chmod +x ./build.sh ./build.sh + - name: Setup Docker Environment run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi + - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . docker run --rm --network "${{ env.BUILD_TAG_LOWER }}_default" \ + --env-file ./component-test-source.sh \ --env "MHS_ADDRESS=http://outbound" \ - --env "FAKE_SPINE_ADDRESS=http://fakespine" \ --env "AWS_ACCESS_KEY_ID=test" \ --env "AWS_SECRET_ACCESS_KEY=test" \ + --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ + --env "FAKE_SPINE_ADDRESS=http://fakespine" \ + --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ + --env "MHS_INBOUND_QUEUE_NAME=inbound" \ + --env "SCR_ADDRESS=http://scradaptor" \ local/mhs-componenttest:${{ env.BUILD_TAG }} + - name: Dump Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v mkdir -p logs docker logs ${{ env.BUILD_TAG_LOWER }}_outbound_1 > logs/outbound_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_inbound_1 > logs/inbound_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_fakespine_1 > logs/fakespine_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_rabbitmq_1 > logs/rabbitmq_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_redis_1 > logs/redis_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_dynamodb_1 > logs/dynamodb_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}_mongodb_1 > logs/mongodb_1.log + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v + - name: Archive Logs if: always() uses: actions/upload-artifact@v4 @@ -166,6 +184,8 @@ jobs: name: ${{ matrix.component }} Component Test Logs path: logs/ + + # integration_tests: # name: "Integration Tests" # needs: [component_tests] diff --git a/Jenkinsfile b/Jenkinsfile index f5154e79a..be9e4c66f 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,274 +58,280 @@ } } - stage('Run Component Tests (SDS API)') { - stages { - stage('Deploy component locally (SDS API)') { - steps { - sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' - sh label: 'Start containers', script: ''' - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v - . ./component-test-source.sh - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d - ''' - } - } - stage('Component Tests (SDS API)') { - steps { - sh label: 'Run component tests', script: ''' - docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . - docker run --rm --network "${BUILD_TAG_LOWER}_default" \ - --env "MHS_ADDRESS=http://outbound" \ - --env "AWS_ACCESS_KEY_ID=test" \ - --env "AWS_SECRET_ACCESS_KEY=test" \ - --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ - --env "FAKE_SPINE_ADDRESS=http://fakespine" \ - --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ - --env "MHS_INBOUND_QUEUE_NAME=inbound" \ - --env "SCR_ADDRESS=http://scradaptor" \ - local/mhs-componenttest:$BUILD_TAG - ''' - } - } - } - post { - always { - sh label: 'Docker status', script: 'docker ps --all' - sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' - sh label: 'Dump container logs to files', script: ''' - mkdir -p logs - docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log - docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log - docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log - docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log - docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log - docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log - - ''' - archiveArtifacts artifacts: 'logs/*.log', fingerprint: true - sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' - } - } - } - - - - stage('Run Integration Tests (SpineRouteLookup)') { - options { - lock('exemplar-test-environment') - } - stages { - stage('Deploy MHS (SpineRouteLookup)') { - steps { - dir('pipeline/terraform/mhs-environment') { - script { - terraformBinPath = tfEnv() - } - sh label: 'Initialising Terraform', script: """ - ${terraformBinPath} init \ - -backend-config="bucket=${TF_STATE_BUCKET}" \ - -backend-config="region=${TF_STATE_BUCKET_REGION}" \ - -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ - -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ - -input=false -no-color - """ - sh label: 'Applying Terraform configuration', script: """ - ${terraformBinPath} apply -no-color -auto-approve \ - -var environment_id=${ENVIRONMENT_ID} \ - -var build_id=${BUILD_TAG} \ - -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ - -var opentest_vpc_id=${OPENTEST_VPC_ID} \ - -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ - -var mhs_outbound_service_minimum_instance_count=3 \ - -var mhs_outbound_service_maximum_instance_count=9 \ - -var mhs_inbound_service_minimum_instance_count=3 \ - -var mhs_inbound_service_maximum_instance_count=9 \ - -var mhs_route_service_minimum_instance_count=3 \ - -var mhs_route_service_maximum_instance_count=9 \ - -var task_role_arn=${TASK_ROLE} \ - -var execution_role_arn=${TASK_EXECUTION_ROLE} \ - -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ - -var ecr_address=${DOCKER_REGISTRY} \ - -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ - -var mhs_log_level=DEBUG \ - -var mhs_outbound_spineroutelookup_verify_certificate="False" \ - -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ - -var mhs_state_table_read_capacity=5 \ - -var mhs_state_table_write_capacity=5 \ - -var mhs_sync_async_table_read_capacity=5 \ - -var mhs_sync_async_table_write_capacity=5 \ - -var mhs_spine_org_code=${SPINE_ORG_CODE} \ - -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ - -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ - -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ - -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ - -var party_key_arn=${PARTY_KEY_ARN} \ - -var client_cert_arn=${CLIENT_CERT_ARN} \ - -var client_key_arn=${CLIENT_KEY_ARN} \ - -var ca_certs_arn=${CA_CERTS_ARN} \ - -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ - -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ - -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ - -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ - -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ - -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ - -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ - -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ - -var elasticache_node_type="cache.t2.micro" \ - -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ - -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ - -var mhs_sds_api_url="" \ - -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ - """ - script { - env.MHS_ADDRESS = sh ( - label: 'Obtaining outbound LB DNS name', - returnStdout: true, - script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" - ).trim() - env.MHS_OUTBOUND_TARGET_GROUP = sh ( - label: 'Obtaining outbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output outbound_lb_target_group_arn" - ).trim() - env.MHS_INBOUND_TARGET_GROUP = sh ( - label: 'Obtaining inbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output inbound_lb_target_group_arn" - ).trim() - env.MHS_ROUTE_TARGET_GROUP = sh ( - label: 'Obtaining route LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output route_lb_target_group_arn" - ).trim() - env.MHS_STATE_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS state', - returnStdout: true, - script: "${terraformBinPath} output mhs_state_table_name" - ).trim() - env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS sync/async state', - returnStdout: true, - script: "${terraformBinPath} output mhs_sync_async_table_name" - ).trim() - } - } - } - } - } - } - stage('Run Integration Tests (SDS API)') { - options { - lock('exemplar-test-environment') - } - stages { - stage('Deploy MHS (SDS API)') { - steps { - dir('pipeline/terraform/mhs-environment') { - script { - terraformBinPath = tfEnv() - } - sh label: 'Initialising Terraform', script: """ - ${terraformBinPath} init \ - -backend-config="bucket=${TF_STATE_BUCKET}" \ - -backend-config="region=${TF_STATE_BUCKET_REGION}" \ - -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ - -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ - -input=false -no-color - """ - sh label: 'Applying Terraform configuration', script: """ - ${terraformBinPath} apply -no-color -auto-approve \ - -var environment_id=${ENVIRONMENT_ID} \ - -var build_id=${BUILD_TAG} \ - -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ - -var opentest_vpc_id=${OPENTEST_VPC_ID} \ - -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ - -var mhs_outbound_service_minimum_instance_count=3 \ - -var mhs_outbound_service_maximum_instance_count=9 \ - -var mhs_inbound_service_minimum_instance_count=3 \ - -var mhs_inbound_service_maximum_instance_count=9 \ - -var mhs_route_service_minimum_instance_count=3 \ - -var mhs_route_service_maximum_instance_count=9 \ - -var task_role_arn=${TASK_ROLE} \ - -var execution_role_arn=${TASK_EXECUTION_ROLE} \ - -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ - -var ecr_address=${DOCKER_REGISTRY} \ - -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ - -var mhs_log_level=DEBUG \ - -var mhs_outbound_spineroutelookup_verify_certificate="False" \ - -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ - -var mhs_state_table_read_capacity=5 \ - -var mhs_state_table_write_capacity=5 \ - -var mhs_sync_async_table_read_capacity=5 \ - -var mhs_sync_async_table_write_capacity=5 \ - -var mhs_spine_org_code=${SPINE_ORG_CODE} \ - -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ - -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ - -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ - -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ - -var party_key_arn=${PARTY_KEY_ARN} \ - -var client_cert_arn=${CLIENT_CERT_ARN} \ - -var client_key_arn=${CLIENT_KEY_ARN} \ - -var ca_certs_arn=${CA_CERTS_ARN} \ - -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ - -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ - -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ - -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ - -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ - -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ - -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ - -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ - -var elasticache_node_type="cache.t2.micro" \ - -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ - -var mhs_outbound_routing_lookup_method="SDS_API" \ - -var mhs_sds_api_url=${MHS_SDS_API_URL} \ - -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ - """ - script { - env.MHS_ADDRESS = sh ( - label: 'Obtaining outbound LB DNS name', - returnStdout: true, - script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" - ).trim() - env.MHS_OUTBOUND_TARGET_GROUP = sh ( - label: 'Obtaining outbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output outbound_lb_target_group_arn" - ).trim() - env.MHS_INBOUND_TARGET_GROUP = sh ( - label: 'Obtaining inbound LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output inbound_lb_target_group_arn" - ).trim() - env.MHS_ROUTE_TARGET_GROUP = sh ( - label: 'Obtaining route LB target group ARN', - returnStdout: true, - script: "${terraformBinPath} output route_lb_target_group_arn" - ).trim() - env.MHS_STATE_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS state', - returnStdout: true, - script: "${terraformBinPath} output mhs_state_table_name" - ).trim() - env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( - label: 'Obtaining the table name used for the MHS sync/async state', - returnStdout: true, - script: "${terraformBinPath} output mhs_sync_async_table_name" - ).trim() - } - } - } - } - } - } - } - } - } +// +// +// +// stage('Run Component Tests (SDS API)') { +// stages { +// stage('Deploy component locally (SDS API)') { +// steps { +// sh label: 'Setup component test environment', script: './integration-tests/setup_component_test_env.sh' +// sh label: 'Start containers', script: ''' +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml down -v +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p custom_network down -v +// . ./component-test-source.sh +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml build +// docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} up -d +// ''' +// } +// } +// stage('Component Tests (SDS API)') { +// steps { +// sh label: 'Run component tests', script: ''' +// docker build -t local/mhs-componenttest:$BUILD_TAG -f ./component-test.Dockerfile . +// docker run --rm --network "${BUILD_TAG_LOWER}_default" \ +// --env "MHS_ADDRESS=http://outbound" \ +// --env "AWS_ACCESS_KEY_ID=test" \ +// --env "AWS_SECRET_ACCESS_KEY=test" \ +// --env "MHS_DB_ENDPOINT_URL=http://dynamodb:8000" \ +// --env "FAKE_SPINE_ADDRESS=http://fakespine" \ +// --env "MHS_INBOUND_QUEUE_BROKERS=amqp://rabbitmq:5672" \ +// --env "MHS_INBOUND_QUEUE_NAME=inbound" \ +// --env "SCR_ADDRESS=http://scradaptor" \ +// local/mhs-componenttest:$BUILD_TAG +// ''' +// } +// } +// } +// post { +// always { +// sh label: 'Docker status', script: 'docker ps --all' +// sh label: 'Docker inspect network', script: 'docker network inspect ${BUILD_TAG_LOWER}_default' +// sh label: 'Dump container logs to files', script: ''' +// mkdir -p logs +// docker logs ${BUILD_TAG_LOWER}_outbound_1 > logs/outbound_2.log +// docker logs ${BUILD_TAG_LOWER}_inbound_1 > logs/inbound_2.log +// docker logs ${BUILD_TAG_LOWER}_fakespine_1 > logs/fakespine_2.log +// docker logs ${BUILD_TAG_LOWER}_rabbitmq_1 > logs/rabbitmq_2.log +// docker logs ${BUILD_TAG_LOWER}_dynamodb_1 > logs/dynamodb_2.log +// docker logs ${BUILD_TAG_LOWER}_sds-api-mock_1 > logs/sdsapimock_2.log +// +// ''' +// archiveArtifacts artifacts: 'logs/*.log', fingerprint: true +// sh label: 'Docker compose down', script: 'docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${BUILD_TAG_LOWER} down -v' +// } +// } +// } +// +// +// +// +// +// +// stage('Run Integration Tests (SpineRouteLookup)') { +// options { +// lock('exemplar-test-environment') +// } +// stages { +// stage('Deploy MHS (SpineRouteLookup)') { +// steps { +// dir('pipeline/terraform/mhs-environment') { +// script { +// terraformBinPath = tfEnv() +// } +// sh label: 'Initialising Terraform', script: """ +// ${terraformBinPath} init \ +// -backend-config="bucket=${TF_STATE_BUCKET}" \ +// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ +// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ +// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ +// -input=false -no-color +// """ +// sh label: 'Applying Terraform configuration', script: """ +// ${terraformBinPath} apply -no-color -auto-approve \ +// -var environment_id=${ENVIRONMENT_ID} \ +// -var build_id=${BUILD_TAG} \ +// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ +// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ +// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ +// -var mhs_outbound_service_minimum_instance_count=3 \ +// -var mhs_outbound_service_maximum_instance_count=9 \ +// -var mhs_inbound_service_minimum_instance_count=3 \ +// -var mhs_inbound_service_maximum_instance_count=9 \ +// -var mhs_route_service_minimum_instance_count=3 \ +// -var mhs_route_service_maximum_instance_count=9 \ +// -var task_role_arn=${TASK_ROLE} \ +// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ +// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ +// -var ecr_address=${DOCKER_REGISTRY} \ +// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ +// -var mhs_log_level=DEBUG \ +// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ +// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ +// -var mhs_state_table_read_capacity=5 \ +// -var mhs_state_table_write_capacity=5 \ +// -var mhs_sync_async_table_read_capacity=5 \ +// -var mhs_sync_async_table_write_capacity=5 \ +// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ +// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ +// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ +// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ +// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ +// -var party_key_arn=${PARTY_KEY_ARN} \ +// -var client_cert_arn=${CLIENT_CERT_ARN} \ +// -var client_key_arn=${CLIENT_KEY_ARN} \ +// -var ca_certs_arn=${CA_CERTS_ARN} \ +// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ +// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ +// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ +// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ +// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ +// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ +// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ +// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ +// -var elasticache_node_type="cache.t2.micro" \ +// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ +// -var mhs_outbound_routing_lookup_method="SPINE_ROUTE_LOOKUP" \ +// -var mhs_sds_api_url="" \ +// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ +// """ +// script { +// env.MHS_ADDRESS = sh ( +// label: 'Obtaining outbound LB DNS name', +// returnStdout: true, +// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" +// ).trim() +// env.MHS_OUTBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining outbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output outbound_lb_target_group_arn" +// ).trim() +// env.MHS_INBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining inbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output inbound_lb_target_group_arn" +// ).trim() +// env.MHS_ROUTE_TARGET_GROUP = sh ( +// label: 'Obtaining route LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output route_lb_target_group_arn" +// ).trim() +// env.MHS_STATE_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_state_table_name" +// ).trim() +// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS sync/async state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_sync_async_table_name" +// ).trim() +// } +// } +// } +// } +// } +// } +// stage('Run Integration Tests (SDS API)') { +// options { +// lock('exemplar-test-environment') +// } +// stages { +// stage('Deploy MHS (SDS API)') { +// steps { +// dir('pipeline/terraform/mhs-environment') { +// script { +// terraformBinPath = tfEnv() +// } +// sh label: 'Initialising Terraform', script: """ +// ${terraformBinPath} init \ +// -backend-config="bucket=${TF_STATE_BUCKET}" \ +// -backend-config="region=${TF_STATE_BUCKET_REGION}" \ +// -backend-config="key=${ENVIRONMENT_ID}-mhs.tfstate" \ +// -backend-config="dynamodb_table=${ENVIRONMENT_ID}-${TF_MHS_LOCK_TABLE_NAME}" \ +// -input=false -no-color +// """ +// sh label: 'Applying Terraform configuration', script: """ +// ${terraformBinPath} apply -no-color -auto-approve \ +// -var environment_id=${ENVIRONMENT_ID} \ +// -var build_id=${BUILD_TAG} \ +// -var supplier_vpc_id=${SUPPLIER_VPC_ID} \ +// -var opentest_vpc_id=${OPENTEST_VPC_ID} \ +// -var internal_root_domain=${INTERNAL_ROOT_DOMAIN} \ +// -var mhs_outbound_service_minimum_instance_count=3 \ +// -var mhs_outbound_service_maximum_instance_count=9 \ +// -var mhs_inbound_service_minimum_instance_count=3 \ +// -var mhs_inbound_service_maximum_instance_count=9 \ +// -var mhs_route_service_minimum_instance_count=3 \ +// -var mhs_route_service_maximum_instance_count=9 \ +// -var task_role_arn=${TASK_ROLE} \ +// -var execution_role_arn=${TASK_EXECUTION_ROLE} \ +// -var task_scaling_role_arn=${TASK_SCALING_ROLE} \ +// -var ecr_address=${DOCKER_REGISTRY} \ +// -var mhs_outbound_validate_certificate=${MHS_OUTBOUND_VALIDATE_CERTIFICATE} \ +// -var mhs_log_level=DEBUG \ +// -var mhs_outbound_spineroutelookup_verify_certificate="False" \ +// -var mhs_outbound_http_proxy=${MHS_OUTBOUND_HTTP_PROXY} \ +// -var mhs_state_table_read_capacity=5 \ +// -var mhs_state_table_write_capacity=5 \ +// -var mhs_sync_async_table_read_capacity=5 \ +// -var mhs_sync_async_table_write_capacity=5 \ +// -var mhs_spine_org_code=${SPINE_ORG_CODE} \ +// -var inbound_queue_brokers="${MHS_INBOUND_QUEUE_BROKERS}" \ +// -var inbound_queue_name="${MHS_INBOUND_QUEUE_NAME}" \ +// -var inbound_queue_username_arn=${INBOUND_QUEUE_USERNAME_ARN} \ +// -var inbound_queue_password_arn=${INBOUND_QUEUE_PASSWORD_ARN} \ +// -var party_key_arn=${PARTY_KEY_ARN} \ +// -var client_cert_arn=${CLIENT_CERT_ARN} \ +// -var client_key_arn=${CLIENT_KEY_ARN} \ +// -var ca_certs_arn=${CA_CERTS_ARN} \ +// -var route_ca_certs_arn=${ROUTE_CA_CERTS_ARN} \ +// -var outbound_alb_certificate_arn=${OUTBOUND_ALB_CERT_ARN} \ +// -var route_alb_certificate_arn=${ROUTE_ALB_CERT_ARN} \ +// -var mhs_resynchroniser_max_retries=${MHS_RESYNC_RETRIES} \ +// -var mhs_resynchroniser_interval=${MHS_RESYNC_INTERVAL} \ +// -var spineroutelookup_service_sds_url=${SPINEROUTELOOKUP_SERVICE_LDAP_URL} \ +// -var spineroutelookup_service_search_base=${SPINEROUTELOOKUP_SERVICE_SEARCH_BASE} \ +// -var spineroutelookup_service_disable_sds_tls=${SPINEROUTELOOKUP_SERVICE_DISABLE_TLS} \ +// -var elasticache_node_type="cache.t2.micro" \ +// -var mhs_forward_reliable_endpoint_url=${MHS_FORWARD_RELIABLE_ENDPOINT_URL} \ +// -var mhs_outbound_routing_lookup_method="SDS_API" \ +// -var mhs_sds_api_url=${MHS_SDS_API_URL} \ +// -var mhs_sds_api_key_arn=${MHS_SDS_API_KEY_ARN} \ +// """ +// script { +// env.MHS_ADDRESS = sh ( +// label: 'Obtaining outbound LB DNS name', +// returnStdout: true, +// script: "echo \"https://\$(${terraformBinPath} output outbound_lb_domain_name)\"" +// ).trim() +// env.MHS_OUTBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining outbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output outbound_lb_target_group_arn" +// ).trim() +// env.MHS_INBOUND_TARGET_GROUP = sh ( +// label: 'Obtaining inbound LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output inbound_lb_target_group_arn" +// ).trim() +// env.MHS_ROUTE_TARGET_GROUP = sh ( +// label: 'Obtaining route LB target group ARN', +// returnStdout: true, +// script: "${terraformBinPath} output route_lb_target_group_arn" +// ).trim() +// env.MHS_STATE_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_state_table_name" +// ).trim() +// env.MHS_SYNC_ASYNC_TABLE_NAME = sh ( +// label: 'Obtaining the table name used for the MHS sync/async state', +// returnStdout: true, +// script: "${terraformBinPath} output mhs_sync_async_table_name" +// ).trim() +// } +// } +// } +// } +// } +// } +// } +// } +// } post { always { cobertura coberturaReportFile: '**/coverage.xml' From 9206759c77f6e85f994b740e840c4806fa8eb6f6 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:12:51 +0100 Subject: [PATCH 33/50] fixing docker compose version --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fab47a0a2..e14eb5c7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,10 +126,13 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install Docker Compose v2 + - name: Install Docker Compose + uses: docker/setup-docker-compose@v2 + + - name: Install other dependencies run: | sudo apt-get update - sudo apt-get install -y docker-compose-plugin + sudo apt-get install -y docker-compose-plugin # or other packages you need - name: Set Lowercase Build Tag run: | From 0e5e7deec4c8d7da8a5915a52943c8e6f329d1df Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:18:49 +0100 Subject: [PATCH 34/50] fixing docker compose version --- .github/workflows/build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e14eb5c7c..d90d83638 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,9 +130,7 @@ jobs: uses: docker/setup-docker-compose@v2 - name: Install other dependencies - run: | - sudo apt-get update - sudo apt-get install -y docker-compose-plugin # or other packages you need + run: sudo apt-get update - name: Set Lowercase Build Tag run: | @@ -147,9 +145,9 @@ jobs: run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests @@ -187,8 +185,6 @@ jobs: name: ${{ matrix.component }} Component Test Logs path: logs/ - - # integration_tests: # name: "Integration Tests" # needs: [component_tests] From 6acdbd1ff8e2aedf5f6dc5b1f23f2fd942d88c53 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:26:37 +0100 Subject: [PATCH 35/50] fixing docker compose version --- .github/workflows/build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d90d83638..8dcf9dfe6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,11 +126,10 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install Docker Compose - uses: docker/setup-docker-compose@v2 - - - name: Install other dependencies - run: sudo apt-get update + - name: Install Docker Compose v2 + run: | + sudo apt-get update + sudo apt-get install -y docker-compose-plugin - name: Set Lowercase Build Tag run: | @@ -145,16 +144,15 @@ jobs: run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose --env-file ./component-test-source.sh -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . docker run --rm --network "${{ env.BUILD_TAG_LOWER }}_default" \ - --env-file ./component-test-source.sh \ --env "MHS_ADDRESS=http://outbound" \ --env "AWS_ACCESS_KEY_ID=test" \ --env "AWS_SECRET_ACCESS_KEY=test" \ From b44df5fc8292d62d871e273f1b9172e1b8d2c3d2 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:33:17 +0100 Subject: [PATCH 36/50] fixing docker compose version --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8dcf9dfe6..ce1023305 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -145,8 +145,8 @@ jobs: ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d - elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d +# elif [[ "${{ matrix.component }}" == "SDS API" ]]; then +# docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests From 7015144b3574a76314fcd2dd1ca25f713c4ace5a Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:36:52 +0100 Subject: [PATCH 37/50] fixing docker compose version --- .github/workflows/build.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce1023305..352e2398a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,10 +126,11 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install Docker Compose v2 - run: | - sudo apt-get update - sudo apt-get install -y docker-compose-plugin + - name: Install Docker Compose + uses: docker/setup-compose-action@v2 + + - name: Install other dependencies + run: sudo apt-get update - name: Set Lowercase Build Tag run: | @@ -145,8 +146,8 @@ jobs: ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d -# elif [[ "${{ matrix.component }}" == "SDS API" ]]; then -# docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + elif [[ "${{ matrix.component }}" == "SDS API" ]]; then + docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests From 897219395beda714075945463ada75773fb14f04 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:41:53 +0100 Subject: [PATCH 38/50] fixing docker compose version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 352e2398a..b7d64280c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -127,7 +127,7 @@ jobs: uses: actions/checkout@v4 - name: Install Docker Compose - uses: docker/setup-compose-action@v2 + uses: docker/setup-docker-compose@v2 - name: Install other dependencies run: sudo apt-get update From bd31d56020f8bf009d4e56762023b63f99a58fd8 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 11:52:07 +0100 Subject: [PATCH 39/50] fixing docker compose version --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7d64280c..fdfea282d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,8 +126,8 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Install Docker Compose - uses: docker/setup-docker-compose@v2 + - name: Check Docker Compose Version + run: docker compose version - name: Install other dependencies run: sudo apt-get update From 9b44af1b563ab03d5298cfe808b429328b92e794 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 13:52:41 +0100 Subject: [PATCH 40/50] fixing docker compose from docker-compose --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fdfea282d..dfa268f18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -145,9 +145,9 @@ jobs: run: | ./integration-tests/setup_component_test_env.sh if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d + docker compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi - name: Run Component Tests @@ -175,7 +175,7 @@ jobs: docker logs ${{ env.BUILD_TAG_LOWER }}_redis_1 > logs/redis_1.log docker logs ${{ env.BUILD_TAG_LOWER }}_dynamodb_1 > logs/dynamodb_1.log docker logs ${{ env.BUILD_TAG_LOWER }}_mongodb_1 > logs/mongodb_1.log - docker-compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v + docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v - name: Archive Logs if: always() From f768eda160339014c286bfb10a71ce63426107f1 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 14:08:36 +0100 Subject: [PATCH 41/50] Saving env vars for other actions. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dfa268f18..56d3ba527 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,6 +144,8 @@ jobs: - name: Setup Docker Environment run: | ./integration-tests/setup_component_test_env.sh + cat component-test-source.sh >> $GITHUB_ENV + if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then @@ -176,6 +178,7 @@ jobs: docker logs ${{ env.BUILD_TAG_LOWER }}_dynamodb_1 > logs/dynamodb_1.log docker logs ${{ env.BUILD_TAG_LOWER }}_mongodb_1 > logs/mongodb_1.log docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v + rm -f .integration-tests/component-test.source.sh - name: Archive Logs if: always() From aa110f44a578b56814eeabfc26a5da39c142216f Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 14:19:58 +0100 Subject: [PATCH 42/50] sourcing vars --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56d3ba527..7385a25aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,8 +144,10 @@ jobs: - name: Setup Docker Environment run: | ./integration-tests/setup_component_test_env.sh - cat component-test-source.sh >> $GITHUB_ENV - + if [ -f component-test-source.sh ]; then + source component-test-source.sh + fi + if [[ "${{ matrix.component }}" == "SpineRouteLookup" ]]; then docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d elif [[ "${{ matrix.component }}" == "SDS API" ]]; then From 6ad1256ff8bc91f0247dc32c04a5afb384eb640a Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 19 Aug 2025 14:30:34 +0100 Subject: [PATCH 43/50] running only spineroutelookup for now --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7385a25aa..6f479f3b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,7 @@ jobs: BUILD_TAG: ${{ needs.generate-build-id.outputs.build-id }} strategy: matrix: - component: [ 'SpineRouteLookup', 'SDS API' ] + component: [ 'SpineRouteLookup' ] steps: - name: Checkout Repository uses: actions/checkout@v4 From 77fd95d4041e371b324c811c8e0ec232daa13dba Mon Sep 17 00:00:00 2001 From: Chiara Date: Wed, 20 Aug 2025 15:05:51 +0100 Subject: [PATCH 44/50] docker ps --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f479f3b4..7b08841b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -180,6 +180,8 @@ jobs: docker logs ${{ env.BUILD_TAG_LOWER }}_dynamodb_1 > logs/dynamodb_1.log docker logs ${{ env.BUILD_TAG_LOWER }}_mongodb_1 > logs/mongodb_1.log docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v + docker ps + rm -f .integration-tests/component-test.source.sh - name: Archive Logs From 56142d03b052ccb669febea66f33f05793397565 Mon Sep 17 00:00:00 2001 From: Chiara Date: Wed, 20 Aug 2025 15:14:56 +0100 Subject: [PATCH 45/50] changing containers names --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b08841b1..1e9918fc4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -153,6 +153,7 @@ jobs: elif [[ "${{ matrix.component }}" == "SDS API" ]]; then docker compose -f docker-compose.yml -f docker-compose.component.override.yml -f docker-compose.component-sds.override.yml -p ${{ env.BUILD_TAG_LOWER }} up --wait -d fi + docker ps - name: Run Component Tests run: | @@ -172,15 +173,14 @@ jobs: if: always() run: | mkdir -p logs - docker logs ${{ env.BUILD_TAG_LOWER }}_outbound_1 > logs/outbound_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_inbound_1 > logs/inbound_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_fakespine_1 > logs/fakespine_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_rabbitmq_1 > logs/rabbitmq_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_redis_1 > logs/redis_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_dynamodb_1 > logs/dynamodb_1.log - docker logs ${{ env.BUILD_TAG_LOWER }}_mongodb_1 > logs/mongodb_1.log - docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v - docker ps + docker logs ${{ env.BUILD_TAG_LOWER }}-outbound-1 > logs/outbound_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-inbound-1 > logs/inbound_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-fakespine-1 > logs/fakespine_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-rabbitmq-1 > logs/rabbitmq_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-redis-1 > logs/redis_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-dynamodb-1 > logs/dynamodb_1.log + docker logs ${{ env.BUILD_TAG_LOWER }}-mongodb-1 > logs/mongodb_1.log + docker compose -f docker-compose.yml -f docker-compose.component.override.yml -p ${{ env.BUILD_TAG_LOWER }} down -v rm -f .integration-tests/component-test.source.sh From dbac14db2983f029605451827ddc710ff7a863a7 Mon Sep 17 00:00:00 2001 From: Chiara Date: Fri, 22 Aug 2025 13:09:38 +0100 Subject: [PATCH 46/50] Changing Docker stuff to make it work. --- docker-compose.component.override.yml | 2 -- docker-compose.release.image.override.yml | 2 -- docker-compose.yml | 15 ++++++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docker-compose.component.override.yml b/docker-compose.component.override.yml index a3c7e21bd..a66607126 100644 --- a/docker-compose.component.override.yml +++ b/docker-compose.component.override.yml @@ -1,5 +1,3 @@ -version: '3' - services: route: build: diff --git a/docker-compose.release.image.override.yml b/docker-compose.release.image.override.yml index 4d58c5ade..1fda33440 100644 --- a/docker-compose.release.image.override.yml +++ b/docker-compose.release.image.override.yml @@ -1,5 +1,3 @@ -version: '3' - services: inbound: image: nhsdev/nia-mhs-inbound:${BUILD_TAG} diff --git a/docker-compose.yml b/docker-compose.yml index 6cc42e7d7..bc97728a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,8 @@ -version: '3.8' - services: inbound: + build: + context: . + dockerfile: ./docker/inbound/Dockerfile image: local/mhs-inbound:${BUILD_TAG} ports: - "443" @@ -20,8 +21,6 @@ services: - MHS_SYNC_ASYNC_STATE_TABLE_NAME=sync_async_state - MHS_DB_ENDPOINT_URL=http://dynamodb:8000 - MHS_INBOUND_QUEUE_MESSAGE_TTL_IN_SECONDS=0 - # boto3 requires some AWS creds to be provided, even - # when connecting to local DynamoDB - AWS_ACCESS_KEY_ID=test - AWS_SECRET_ACCESS_KEY=test - MHS_INBOUND_USE_SSL=True @@ -29,6 +28,9 @@ services: - SERVICE_PORTS=443,80 - SUPPORTED_FILE_TYPES outbound: + build: + context: . + dockerfile: ./docker/outbound/Dockerfile image: local/mhs-outbound:${BUILD_TAG} ports: - "80" @@ -47,16 +49,15 @@ services: - MAX_RESYNC_RETRIES=20 - MHS_SPINE_ROUTE_LOOKUP_URL=http://route - MHS_SPINE_ORG_CODE=YES - - MHS_SPINE_REQUEST_MAX_SIZE=4999600 # 5 000 000 - 400 - # Note that this endpoint URL is Opentest-specific + - MHS_SPINE_REQUEST_MAX_SIZE=4999600 - MHS_FORWARD_RELIABLE_ENDPOINT_URL=https://192.168.128.11/reliablemessaging/forwardreliable - # This is for disabling hostname validation so OpenTest ip address will work - MHS_OUTBOUND_VALIDATE_CERTIFICATE=${MHS_OUTBOUND_VALIDATE_CERTIFICATE:-False} - SERVICE_PORTS=80 - MHS_OUTBOUND_ROUTING_LOOKUP_METHOD - MHS_SDS_API_URL - MHS_SDS_API_KEY route: + build: ./docker/spineroutelookup image: local/mhs-route:${BUILD_TAG} ports: - "8080:80" From e5b192f340ca5745d1f4c7e1eaaabdad7871262f Mon Sep 17 00:00:00 2001 From: ole4ryb <12736698+ole4ryb@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:39:53 +0100 Subject: [PATCH 47/50] docker network checks --- .github/workflows/build.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e9918fc4..dddf02765 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -155,6 +155,18 @@ jobs: fi docker ps + - name: Docker Setup debug + run: | + docker network ls + + - name: Ensure Network Exists + run: | + BUILD_TAG_LOWER_NETWORK=${{ env.BUILD_TAG_LOWER }}_default + if ! docker network ls --format '{{.Name}}' | grep -q "^$BUILD_TAG_LOWER_NETWORK$"; then + echo "Network $BUILD_TAG_LOWER_NETWORK not found, creating it." + docker network create $BUILD_TAG_LOWER_NETWORK + fi + - name: Run Component Tests run: | docker build -t local/mhs-componenttest:${{ env.BUILD_TAG }} -f ./component-test.Dockerfile . @@ -236,4 +248,4 @@ jobs: - name: "Prune Docker Images" run: | docker system prune --force - docker volume prune --force \ No newline at end of file + docker volume prune --force From 4cb2ce3572853160c670267d180275f9cd3f8c0b Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 26 Aug 2025 13:58:22 +0100 Subject: [PATCH 48/50] Disabling test files to only have 1 (quicker feedback). --- ...> x_component_asynchronous_express_message_pattern_tests.py} | 0 ... x_component_asynchronous_reliable_message_pattern_tests.py} | 0 ...daptor_tests.py => x_component_persistence_adaptor_tests.py} | 0 ...ests.py => x_component_summary_care_record_adaptor_tests.py} | 0 ...ts.py => x_component_synchronous_messaging_pattern_tests.py} | 0 .../component_tests/{test_tests.py => x_test_tests.py} | 0 integration-tests/setup_component_test_env.sh | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename integration-tests/integration_tests/integration_tests/component_tests/{component_asynchronous_express_message_pattern_tests.py => x_component_asynchronous_express_message_pattern_tests.py} (100%) rename integration-tests/integration_tests/integration_tests/component_tests/{component_asynchronous_reliable_message_pattern_tests.py => x_component_asynchronous_reliable_message_pattern_tests.py} (100%) rename integration-tests/integration_tests/integration_tests/component_tests/{component_persistence_adaptor_tests.py => x_component_persistence_adaptor_tests.py} (100%) rename integration-tests/integration_tests/integration_tests/component_tests/{component_summary_care_record_adaptor_tests.py => x_component_summary_care_record_adaptor_tests.py} (100%) rename integration-tests/integration_tests/integration_tests/component_tests/{component_synchronous_messaging_pattern_tests.py => x_component_synchronous_messaging_pattern_tests.py} (100%) rename integration-tests/integration_tests/integration_tests/component_tests/{test_tests.py => x_test_tests.py} (100%) diff --git a/integration-tests/integration_tests/integration_tests/component_tests/component_asynchronous_express_message_pattern_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_component_asynchronous_express_message_pattern_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/component_asynchronous_express_message_pattern_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_component_asynchronous_express_message_pattern_tests.py diff --git a/integration-tests/integration_tests/integration_tests/component_tests/component_asynchronous_reliable_message_pattern_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_component_asynchronous_reliable_message_pattern_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/component_asynchronous_reliable_message_pattern_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_component_asynchronous_reliable_message_pattern_tests.py diff --git a/integration-tests/integration_tests/integration_tests/component_tests/component_persistence_adaptor_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_component_persistence_adaptor_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/component_persistence_adaptor_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_component_persistence_adaptor_tests.py diff --git a/integration-tests/integration_tests/integration_tests/component_tests/component_summary_care_record_adaptor_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_component_summary_care_record_adaptor_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/component_summary_care_record_adaptor_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_component_summary_care_record_adaptor_tests.py diff --git a/integration-tests/integration_tests/integration_tests/component_tests/component_synchronous_messaging_pattern_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_component_synchronous_messaging_pattern_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/component_synchronous_messaging_pattern_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_component_synchronous_messaging_pattern_tests.py diff --git a/integration-tests/integration_tests/integration_tests/component_tests/test_tests.py b/integration-tests/integration_tests/integration_tests/component_tests/x_test_tests.py similarity index 100% rename from integration-tests/integration_tests/integration_tests/component_tests/test_tests.py rename to integration-tests/integration_tests/integration_tests/component_tests/x_test_tests.py diff --git a/integration-tests/setup_component_test_env.sh b/integration-tests/setup_component_test_env.sh index baa4960b7..c6b39f315 100755 --- a/integration-tests/setup_component_test_env.sh +++ b/integration-tests/setup_component_test_env.sh @@ -23,4 +23,4 @@ echo -e "export INBOUND_CA_STORE=\"$(cat ./generated-certs/fake-spine/cert.pem)\ echo -e "export MHS_SECRET_PARTY_KEY=\"test-party-key\"" >> component-test-source.sh echo -e "export MHS_OUTBOUND_VALIDATE_CERTIFICATE=\"False\"" >> component-test-source.sh -rm -rf ./generated-certs \ No newline at end of file +#rm -rf ./generated-certs \ No newline at end of file From 05dba8449d910823a32c1acceebd58a730d091a4 Mon Sep 17 00:00:00 2001 From: Chiara Date: Tue, 26 Aug 2025 13:58:53 +0100 Subject: [PATCH 49/50] Disabling test files to only have 1 (quicker feedback). --- mhs/outbound/pycurl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mhs/outbound/pycurl b/mhs/outbound/pycurl index 016c3835e..b0afc76fc 160000 --- a/mhs/outbound/pycurl +++ b/mhs/outbound/pycurl @@ -1 +1 @@ -Subproject commit 016c3835e4d5a41af89395a6e84261dd4c693e17 +Subproject commit b0afc76fc251ebb0e784328ce0f6c915a16fdf42 From d720e7bfb0bf2d267f36ab9763991f977ee5ad5d Mon Sep 17 00:00:00 2001 From: ole4ryb <12736698+ole4ryb@users.noreply.github.com> Date: Tue, 26 Aug 2025 16:13:04 +0100 Subject: [PATCH 50/50] adding url log --- mhs/common/mhs_common/routing/spine_route_lookup_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mhs/common/mhs_common/routing/spine_route_lookup_client.py b/mhs/common/mhs_common/routing/spine_route_lookup_client.py index 8d2162328..4fe058549 100644 --- a/mhs/common/mhs_common/routing/spine_route_lookup_client.py +++ b/mhs/common/mhs_common/routing/spine_route_lookup_client.py @@ -64,6 +64,8 @@ async def get_end_point(self, interaction_id: str, ods_code: str = None) -> Dict logger.info("Proxy setup: {proxy_host} & {proxy_port}.", fparams={"proxy_host": self._proxy_host, "proxy_port": self._proxy_port}) + logger.info("URL: {url}.", fparams={"url": url}) + http_response = await common_https.CommonHttps.make_request(url=url, method="GET", headers=build_tracking_headers(), body=None,