diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0d6b8c0551..0357b30637 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -121,11 +121,6 @@ functions: export OCSP_CONNECTIVITY="${OCSP_CONNECTIVITY}" export OCSP_VERIFIER="${OCSP_VERIFIER}" - export ATLAS_REPLICA_SET_URI="${atlas_replica_set_uri}" - export ATLAS_SHARDED_URI="${atlas_sharded_uri}" - export ATLAS_FREE_TIER_URI="${atlas_free_tier_uri}" - export ATLAS_TLS11_URI="${atlas_tls11_uri}" - export ATLAS_TLS12_URI="${atlas_tls12_uri}" export RVM_RUBY="${RVM_RUBY}" EOT @@ -173,13 +168,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi export CSOT_SPEC_TESTS=1 unset TOPOLOGY export TOPOLOGY=${MLAUNCH_TOPOLOGY} @@ -201,33 +189,14 @@ functions: .evergreen/run-tests.sh "export FLE credentials": - - command: shell.exec + - command: subprocess.exec type: test params: - silent: true + binary: bash working_dir: "src" - script: | - cat < .env.private - MONGO_RUBY_DRIVER_AWS_KEY="${fle_aws_key}" - MONGO_RUBY_DRIVER_AWS_SECRET="${fle_aws_secret}" - MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" - MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" - - MONGO_RUBY_DRIVER_AZURE_TENANT_ID="${fle_azure_tenant_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_ID="${fle_azure_client_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET="${fle_azure_client_secret}" - MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" - - MONGO_RUBY_DRIVER_GCP_EMAIL="${fle_gcp_email}" - MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY="${fle_gcp_private_key}" - MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" - MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" - MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" - MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" - MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" - EOT + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, DRIVERS_TOOLS] + args: + - "${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh" "export Kerberos credentials": - command: shell.exec @@ -368,12 +337,17 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" + if [ -n "${FLE}" ]; then + export MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" + export MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" + export MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" + export MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" + export MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" + export MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" + export MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" + export MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" + export MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" + export MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" fi unset TOPOLOGY export TOPOLOGY=${MLAUNCH_TOPOLOGY} @@ -401,6 +375,17 @@ functions: ${PREPARE_SHELL} .evergreen/run-tests-kerberos-unit.sh + "export Atlas credentials": + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: "src" + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, DRIVERS_TOOLS] + args: + - "${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh" + - "drivers/atlas_connect" + "run Atlas tests": - command: shell.exec type: test @@ -410,15 +395,7 @@ functions: script: | ${PREPARE_SHELL} AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" \ - ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} \ - ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} \ - ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} \ - ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} \ - ATLAS_X509_CERT_BASE64="${atlas_x509_cert_base64}" \ - ATLAS_X509_URI="${atlas_x509}" \ - ATLAS_X509_DEV_CERT_BASE64="${atlas_x509_dev_cert_base64}" \ - ATLAS_X509_DEV_URI="${atlas_x509_dev}" \ - .evergreen/run-tests-atlas.sh + .evergreen/run-tests-atlas.sh pre: - func: assume-test-secrets-ec2-role @@ -620,6 +597,7 @@ tasks: - func: "run tests with orchestration and drivers tools" - name: "test-atlas" commands: + - func: "export Atlas credentials" - func: "run Atlas tests" - name: "test-mlaunch" commands: diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 53262db080..6176951194 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -118,11 +118,6 @@ functions: export OCSP_CONNECTIVITY="${OCSP_CONNECTIVITY}" export OCSP_VERIFIER="${OCSP_VERIFIER}" - export ATLAS_REPLICA_SET_URI="${atlas_replica_set_uri}" - export ATLAS_SHARDED_URI="${atlas_sharded_uri}" - export ATLAS_FREE_TIER_URI="${atlas_free_tier_uri}" - export ATLAS_TLS11_URI="${atlas_tls11_uri}" - export ATLAS_TLS12_URI="${atlas_tls12_uri}" export RVM_RUBY="${RVM_RUBY}" EOT @@ -170,13 +165,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi export CSOT_SPEC_TESTS=1 unset TOPOLOGY export TOPOLOGY=${MLAUNCH_TOPOLOGY} @@ -198,33 +186,14 @@ functions: .evergreen/run-tests.sh "export FLE credentials": - - command: shell.exec + - command: subprocess.exec type: test params: - silent: true + binary: bash working_dir: "src" - script: | - cat < .env.private - MONGO_RUBY_DRIVER_AWS_KEY="${fle_aws_key}" - MONGO_RUBY_DRIVER_AWS_SECRET="${fle_aws_secret}" - MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" - MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" - - MONGO_RUBY_DRIVER_AZURE_TENANT_ID="${fle_azure_tenant_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_ID="${fle_azure_client_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET="${fle_azure_client_secret}" - MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" - - MONGO_RUBY_DRIVER_GCP_EMAIL="${fle_gcp_email}" - MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY="${fle_gcp_private_key}" - MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" - MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" - MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" - MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" - MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" - EOT + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, DRIVERS_TOOLS] + args: + - "${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh" "export Kerberos credentials": - command: shell.exec @@ -365,12 +334,17 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" + if [ -n "${FLE}" ]; then + export MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" + export MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" + export MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" + export MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" + export MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" + export MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" + export MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" + export MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" + export MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" + export MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" fi unset TOPOLOGY export TOPOLOGY=${MLAUNCH_TOPOLOGY} @@ -398,6 +372,17 @@ functions: ${PREPARE_SHELL} .evergreen/run-tests-kerberos-unit.sh + "export Atlas credentials": + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: "src" + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, DRIVERS_TOOLS] + args: + - "${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh" + - "drivers/atlas_connect" + "run Atlas tests": - command: shell.exec type: test @@ -407,15 +392,7 @@ functions: script: | ${PREPARE_SHELL} AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" \ - ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} \ - ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} \ - ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} \ - ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} \ - ATLAS_X509_CERT_BASE64="${atlas_x509_cert_base64}" \ - ATLAS_X509_URI="${atlas_x509}" \ - ATLAS_X509_DEV_CERT_BASE64="${atlas_x509_dev_cert_base64}" \ - ATLAS_X509_DEV_URI="${atlas_x509_dev}" \ - .evergreen/run-tests-atlas.sh + .evergreen/run-tests-atlas.sh pre: - func: assume-test-secrets-ec2-role @@ -617,6 +594,7 @@ tasks: - func: "run tests with orchestration and drivers tools" - name: "test-atlas" commands: + - func: "export Atlas credentials" - func: "run Atlas tests" - name: "test-mlaunch" commands: diff --git a/.evergreen/run-tests-atlas.sh b/.evergreen/run-tests-atlas.sh index 2b2298c48a..038adcb56c 100755 --- a/.evergreen/run-tests-atlas.sh +++ b/.evergreen/run-tests-atlas.sh @@ -28,5 +28,18 @@ echo "Running specs" export ATLAS_TESTING=1 +if test -f secrets-export.sh; then + # shellcheck disable=SC1091 + . ./secrets-export.sh + # Map from vault variable names (shared with Python/Node) to Ruby driver expected names. + export ATLAS_REPLICA_SET_URI="${ATLAS_REPL}" + export ATLAS_SHARDED_URI="${ATLAS_SHRD}" + export ATLAS_FREE_TIER_URI="${ATLAS_FREE}" + export ATLAS_TLS11_URI="${ATLAS_TLS11}" + export ATLAS_TLS12_URI="${ATLAS_TLS12}" + export ATLAS_X509_URI="${ATLAS_X509}" + export ATLAS_X509_DEV_URI="${ATLAS_X509_DEV}" +fi + bundle exec rspec spec/atlas \ --format Rfc::Riff --format RspecJunitFormatter --out tmp/rspec.xml diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index f1bd48a125..3b13f4f1bd 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -236,8 +236,21 @@ if test -n "$FLE"; then python3 -u .evergreen/csfle/fake_azure.py & python3 -u .evergreen/csfle/kms_failpoint_server.py --port 9003 & - # Obtain temporary AWS credentials - PYTHON=python3 . .evergreen/csfle/set-temp-creds.sh + # Source FLE credentials generated by csfle/setup-secrets.sh. + if test -f secrets-export.sh; then + # shellcheck disable=SC1091 + . ./secrets-export.sh + # setup-secrets.sh sets AWS_SESSION_TOKEN="" for long-lived keys. Unset it + # so the driver does not include an empty security token in KMS requests. + [ -z "${AWS_SESSION_TOKEN:-}" ] && unset AWS_SESSION_TOKEN + export MONGO_RUBY_DRIVER_AWS_KEY="${FLE_AWS_KEY}" + export MONGO_RUBY_DRIVER_AWS_SECRET="${FLE_AWS_SECRET}" + export MONGO_RUBY_DRIVER_AZURE_TENANT_ID="${FLE_AZURE_TENANTID}" + export MONGO_RUBY_DRIVER_AZURE_CLIENT_ID="${FLE_AZURE_CLIENTID}" + export MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET="${FLE_AZURE_CLIENTSECRET}" + export MONGO_RUBY_DRIVER_GCP_EMAIL="${FLE_GCP_EMAIL}" + export MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY="${FLE_GCP_PRIVATEKEY}" + fi if [[ "$FLE" == "helper" || "$FLE" == "mongocryptd" ]]; then echo "Using helper gem"