diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0357b30637..903569080d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -93,7 +93,6 @@ functions: export AUTH=${AUTH} export SSL=${SSL} export TOPOLOGY=${TOPOLOGY} - export MLAUNCH_TOPOLOGY=${MLAUNCH_TOPOLOGY} export COMPRESSOR=${COMPRESSOR} export RVM_RUBY="${RVM_RUBY}" export MONGODB_VERSION=${MONGODB_VERSION} @@ -169,8 +168,6 @@ functions: script: | ${PREPARE_SHELL} export CSOT_SPEC_TESTS=1 - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rspec spec/spec_tests/client_side_operations_timeout_spec.rb --format Rfc::Riff --format RspecJunitFormatter --out tmp/rspec.xml" \ .evergreen/run-tests.sh @@ -183,8 +180,6 @@ functions: script: | ${PREPARE_SHELL} export OTEL_SPEC_TESTS=1 - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rspec spec/spec_tests/open_telemetry_spec.rb --format Rfc::Riff --format RspecJunitFormatter --out tmp/rspec.xml" \ .evergreen/run-tests.sh @@ -313,8 +308,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rake driver_bench" PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf.json" .evergreen/run-tests.sh - command: perf.send params: @@ -349,8 +342,6 @@ functions: export MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" export MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" fi - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} .evergreen/run-tests.sh "run AWS auth tests": @@ -361,8 +352,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} .evergreen/run-tests-aws-auth.sh "run Kerberos unit tests": @@ -599,7 +588,7 @@ tasks: commands: - func: "export Atlas credentials" - func: "run Atlas tests" - - name: "test-mlaunch" + - name: "test" commands: - func: "run tests" - name: "driver-bench" @@ -776,27 +765,23 @@ axes: display_name: Standalone variables: TOPOLOGY: server - MLAUNCH_TOPOLOGY: standalone - id: "replica-set" display_name: Replica Set variables: TOPOLOGY: replica_set - MLAUNCH_TOPOLOGY: replica-set - id: "replica-set-single-node" display_name: Replica Set (Single Node) variables: - TOPOLOGY: replica-set-single-node - MLAUNCH_TOPOLOGY: replica-set-single-node + TOPOLOGY: replica_set + ORCHESTRATION_FILE: single-node.json - id: "sharded-cluster" display_name: Sharded variables: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster - id: "load-balanced" display_name: Load Balanced variables: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster LOAD_BALANCED: 'true' - id: "single-mongos" @@ -1205,7 +1190,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} single-lb ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "mongo-api-version" matrix_spec: @@ -1217,7 +1202,7 @@ buildvariants: display_name: "${mongodb-version} api-version-required ${topology} ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "single-mongos" matrix_spec: @@ -1229,7 +1214,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} single-mongos ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: CSOT matrix_spec: @@ -1263,7 +1248,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} ${retry-reads} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "no-retry-writes" matrix_spec: @@ -1275,7 +1260,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} ${retry-writes} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "lint" matrix_spec: @@ -1286,7 +1271,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${lint} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "fork" matrix_spec: @@ -1297,7 +1282,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} fork ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "solo" matrix_spec: @@ -1308,7 +1293,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} solo ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "stress older" matrix_spec: @@ -1319,7 +1304,7 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "stress" matrix_spec: @@ -1330,7 +1315,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" # - matrix_name: "x509-tests" # matrix_spec: @@ -1341,7 +1326,7 @@ buildvariants: # os: ubuntu2204 # display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" # tasks: -# - name: "test-mlaunch" +# - name: "test" - matrix_name: zlib-"ruby-4.0" matrix_spec: @@ -1396,7 +1381,7 @@ buildvariants: display_name: "AS ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: bson-"ruby-4.0" matrix_spec: @@ -1408,7 +1393,7 @@ buildvariants: display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: zlib-"ruby-2.7" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] @@ -1462,7 +1447,7 @@ buildvariants: display_name: "AS ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: bson-"ruby-2.7" matrix_spec: @@ -1474,7 +1459,7 @@ buildvariants: display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "fle" matrix_spec: @@ -1533,7 +1518,6 @@ buildvariants: run_on: ubuntu2204-small expansions: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster RVM_RUBY: "ruby-4.0" MONGODB_VERSION: 'latest' tasks: @@ -1566,7 +1550,7 @@ buildvariants: display_name: "OCSP verifier: ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-must-staple matrix_spec: @@ -1580,7 +1564,7 @@ buildvariants: auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - must staple: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-unknown matrix_spec: @@ -1593,7 +1577,7 @@ buildvariants: auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - unknown: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: @@ -1609,7 +1593,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1624,7 +1608,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1639,7 +1623,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1654,7 +1638,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1669,7 +1653,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1684,7 +1668,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1699,7 +1683,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1714,7 +1698,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1729,7 +1713,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-connectivity-jruby matrix_spec: @@ -1750,7 +1734,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test # https://jira.mongodb.org/browse/RUBY-3540 #- matrix_name: testgcpkms-variant diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index f621c72f4f..b7bad1c93a 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -45,27 +45,23 @@ axes: display_name: Standalone variables: TOPOLOGY: server - MLAUNCH_TOPOLOGY: standalone - id: "replica-set" display_name: Replica Set variables: TOPOLOGY: replica_set - MLAUNCH_TOPOLOGY: replica-set - id: "replica-set-single-node" display_name: Replica Set (Single Node) variables: - TOPOLOGY: replica-set-single-node - MLAUNCH_TOPOLOGY: replica-set-single-node + TOPOLOGY: replica_set + ORCHESTRATION_FILE: single-node.json - id: "sharded-cluster" display_name: Sharded variables: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster - id: "load-balanced" display_name: Load Balanced variables: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster LOAD_BALANCED: 'true' - id: "single-mongos" diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 6176951194..106f81fe68 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -90,7 +90,6 @@ functions: export AUTH=${AUTH} export SSL=${SSL} export TOPOLOGY=${TOPOLOGY} - export MLAUNCH_TOPOLOGY=${MLAUNCH_TOPOLOGY} export COMPRESSOR=${COMPRESSOR} export RVM_RUBY="${RVM_RUBY}" export MONGODB_VERSION=${MONGODB_VERSION} @@ -166,8 +165,6 @@ functions: script: | ${PREPARE_SHELL} export CSOT_SPEC_TESTS=1 - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rspec spec/spec_tests/client_side_operations_timeout_spec.rb --format Rfc::Riff --format RspecJunitFormatter --out tmp/rspec.xml" \ .evergreen/run-tests.sh @@ -180,8 +177,6 @@ functions: script: | ${PREPARE_SHELL} export OTEL_SPEC_TESTS=1 - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rspec spec/spec_tests/open_telemetry_spec.rb --format Rfc::Riff --format RspecJunitFormatter --out tmp/rspec.xml" \ .evergreen/run-tests.sh @@ -310,8 +305,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} TEST_CMD="bundle exec rake driver_bench" PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf.json" .evergreen/run-tests.sh - command: perf.send params: @@ -346,8 +339,6 @@ functions: export MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" export MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" fi - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} .evergreen/run-tests.sh "run AWS auth tests": @@ -358,8 +349,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - unset TOPOLOGY - export TOPOLOGY=${MLAUNCH_TOPOLOGY} .evergreen/run-tests-aws-auth.sh "run Kerberos unit tests": @@ -596,7 +585,7 @@ tasks: commands: - func: "export Atlas credentials" - func: "run Atlas tests" - - name: "test-mlaunch" + - name: "test" commands: - func: "run tests" - name: "driver-bench" diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 0b9d91610c..037d884ff4 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -123,7 +123,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} single-lb ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "mongo-api-version" matrix_spec: @@ -135,7 +135,7 @@ buildvariants: display_name: "${mongodb-version} api-version-required ${topology} ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "single-mongos" matrix_spec: @@ -147,7 +147,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} single-mongos ${auth-and-ssl} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: CSOT matrix_spec: @@ -181,7 +181,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} ${retry-reads} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "no-retry-writes" matrix_spec: @@ -193,7 +193,7 @@ buildvariants: display_name: "${mongodb-version} ${topology} ${retry-writes} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "lint" matrix_spec: @@ -204,7 +204,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${lint} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "fork" matrix_spec: @@ -215,7 +215,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} fork ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "solo" matrix_spec: @@ -226,7 +226,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} solo ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "stress older" matrix_spec: @@ -237,7 +237,7 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: "stress" matrix_spec: @@ -248,7 +248,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test" # - matrix_name: "x509-tests" # matrix_spec: @@ -259,7 +259,7 @@ buildvariants: # os: ubuntu2204 # display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" # tasks: -# - name: "test-mlaunch" +# - name: "test" <% [ [latest_ruby, stable_and_rapid, 'ubuntu2204'], @@ -319,7 +319,7 @@ buildvariants: display_name: "AS ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" - matrix_name: <%= "bson-#{rubies}" %> matrix_spec: @@ -331,7 +331,7 @@ buildvariants: display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tags: ["pr"] tasks: - - name: "test-mlaunch" + - name: "test" <% end %> - matrix_name: "fle" @@ -391,7 +391,6 @@ buildvariants: run_on: ubuntu2204-small expansions: TOPOLOGY: sharded_cluster - MLAUNCH_TOPOLOGY: sharded-cluster RVM_RUBY: <%= latest_ruby %> MONGODB_VERSION: 'latest' tasks: @@ -424,7 +423,7 @@ buildvariants: display_name: "OCSP verifier: ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-must-staple matrix_spec: @@ -438,7 +437,7 @@ buildvariants: auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - must staple: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test - matrix_name: ocsp-unknown matrix_spec: @@ -451,7 +450,7 @@ buildvariants: auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - unknown: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test <% [ %w(valid none pass), @@ -481,7 +480,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test <% end %> - matrix_name: ocsp-connectivity-jruby @@ -503,7 +502,7 @@ buildvariants: display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${mongodb-version} ${ruby}" tags: ["pr"] tasks: - - name: test-mlaunch + - name: test # https://jira.mongodb.org/browse/RUBY-3540 #- matrix_name: testgcpkms-variant diff --git a/.evergreen/lib/server_setup.rb b/.evergreen/lib/server_setup.rb index fbadef3c5d..79937358c2 100644 --- a/.evergreen/lib/server_setup.rb +++ b/.evergreen/lib/server_setup.rb @@ -99,8 +99,13 @@ def client end def bootstrap_client - @bootstrap_client ||= Mongo::Client.new(ENV['MONGODB_URI'] || %w(localhost), - user: 'bootstrap', password: 'bootstrap', auth_mech: :scram, auth_mech_properties: nil, - ) + @bootstrap_client ||= if ENV['MONGODB_URI'] + # In orchestration mode MONGODB_URI already has admin credentials; use them. + Mongo::Client.new(ENV['MONGODB_URI'], auth_mech: :scram, auth_mech_properties: nil) + else + Mongo::Client.new(%w(localhost), + user: 'bootstrap', password: 'bootstrap', auth_mech: :scram, auth_mech_properties: nil, + ) + end end end diff --git a/.evergreen/orchestration-configs/replica_sets/single-node.json b/.evergreen/orchestration-configs/replica_sets/single-node.json new file mode 100644 index 0000000000..6ca22b95f0 --- /dev/null +++ b/.evergreen/orchestration-configs/replica_sets/single-node.json @@ -0,0 +1,13 @@ +{ + "id": "repl0", + "members": [ + { + "procParams": { + "ipv6": true, + "bind_ip": "127.0.0.1,::1", + "oplogSize": 500, + "port": 27017 + } + } + ] +} diff --git a/.evergreen/orchestration-configs/sharded_clusters/single-mongos.json b/.evergreen/orchestration-configs/sharded_clusters/single-mongos.json new file mode 100644 index 0000000000..645848c1ac --- /dev/null +++ b/.evergreen/orchestration-configs/sharded_clusters/single-mongos.json @@ -0,0 +1,43 @@ +{ + "id": "shard_cluster_1", + "shards": [ + { + "id": "sh01", + "shardParams": { + "members": [ + { + "procParams": { + "ipv6": true, + "bind_ip": "127.0.0.1,::1", + "shardsvr": true, + "port": 27217 + } + }, + { + "procParams": { + "ipv6": true, + "bind_ip": "127.0.0.1,::1", + "shardsvr": true, + "port": 27218 + } + }, + { + "procParams": { + "ipv6": true, + "bind_ip": "127.0.0.1,::1", + "shardsvr": true, + "port": 27219 + } + } + ] + } + } + ], + "routers": [ + { + "ipv6": true, + "bind_ip": "127.0.0.1,::1", + "port": 27017 + } + ] +} diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh new file mode 100755 index 0000000000..342207d1e4 --- /dev/null +++ b/.evergreen/run-orchestration.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Translates Ruby driver environment variables to drivers-tools format, +# then starts MongoDB via drivers-evergreen-tools run-mongodb.sh. + +set -e + +if test -z "${DRIVERS_TOOLS:-}"; then + echo "DRIVERS_TOOLS must be set" >&2 + exit 1 +fi + +if test -z "${MONGO_ORCHESTRATION_HOME:-}"; then + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" +fi + +# Translate topology names to orchestration format. +case "${TOPOLOGY:-server}" in + replica-set-single-node) + export TOPOLOGY=replica_set + export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-single-node.json}" + ;; + standalone) + export TOPOLOGY=server + ;; +esac + +# Single mongos: use a 1-router sharded cluster config. +if test "${SINGLE_MONGOS:-}" = 'true' && test "${TOPOLOGY:-}" = sharded_cluster; then + export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-single-mongos.json}" +fi + +# Load balancer support. +if test "${LOAD_BALANCED:-}" = 'true'; then + export LOAD_BALANCER=1 +fi + +# x509 auth: start server with auth+ssl; the x509 user is created later in run-tests.sh. +if test "${AUTH:-}" = x509; then + export AUTH=auth + export SSL=yes +fi + +# AWS auth: use the auth-aws orchestration file which enables MONGODB-AWS mechanism. +if echo "${AUTH:-}" | grep -q ^aws; then + export AUTH_AWS=1 + export AUTH=auth +fi + +# OCSP: select orchestration file based on algorithm and mustStaple flag. +# Without mustStaple the server does not staple, so use the disableStapling variant. +if test -n "${OCSP_ALGORITHM:-}"; then + if test "${OCSP_MUST_STAPLE:-}" = 1; then + _ocsp_file="${OCSP_ALGORITHM}-basic-tls-ocsp-mustStaple.json" + else + _ocsp_file="${OCSP_ALGORITHM}-basic-tls-ocsp-disableStapling.json" + fi + export ORCHESTRATION_FILE="${ORCHESTRATION_FILE:-$_ocsp_file}" +fi + +# If prepare_server already downloaded MongoDB, reuse those binaries. +if test -n "${BINDIR:-}"; then + export EXISTING_BINARIES_DIR="$BINDIR" +fi + +# Copy Ruby-driver-specific orchestration configs that are not (yet) in drivers-evergreen-tools. +_configs_src="$(dirname "$0")/orchestration-configs" +_configs_dst="$MONGO_ORCHESTRATION_HOME/configs" +mkdir -p "$_configs_dst/replica_sets" "$_configs_dst/sharded_clusters" +cp "$_configs_src"/replica_sets/single-node.json "$_configs_dst/replica_sets/" +cp "$_configs_src"/sharded_clusters/single-mongos.json "$_configs_dst/sharded_clusters/" + +"$DRIVERS_TOOLS"/.evergreen/run-mongodb.sh start + +# Export MONGODB_URI written by the orchestration tool. +. ./mo-expansion.sh +export MONGODB_URI diff --git a/.evergreen/run-tests-aws-auth.sh b/.evergreen/run-tests-aws-auth.sh index 55827d5984..931acf41c0 100755 --- a/.evergreen/run-tests-aws-auth.sh +++ b/.evergreen/run-tests-aws-auth.sh @@ -13,8 +13,6 @@ elif test -f .env.private; then . ./.env.private fi -export TOPOLOGY=${MLAUNCH_TOPOLOGY} - # The AWS auth-related Evergreen variables are set the same way for most/all # drivers. Therefore we don't want to change the variable names in order to # transparently benefit from possible updates to these credentials in diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 3b13f4f1bd..a020e984fc 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,17 +1,5 @@ #!/bin/bash -# Note that mlaunch is executed with (and therefore installed with) Python 2. -# The reason for this is that in the past, some of the distros we tested on -# had an ancient version of Python 3 that was unusable (e.g. it couldn't -# install anything from PyPI due to outdated TLS/SSL implementation). -# It is likely that all of the current distros we use have a recent enough -# and working Python 3 implementation, such that we could use Python 3 for -# everything. -# -# Note that some distros (e.g. ubuntu2004) do not contain a `python' binary -# at all, thus python2 or python3 must be explicitly specified depending on -# the desired version. - set -e set -o pipefail @@ -59,73 +47,54 @@ export JAVACMD=$JAVA_HOME/bin/java prepare_server -if test "$DOCKER_PRELOAD" != 1; then - install_mlaunch_venv - pip3 install waitress -fi - # Make sure cmake is installed (in case we need to install the libmongocrypt # helper) if [ -n "$FLE" ]; then install_cmake fi -if test "$TOPOLOGY" = load-balanced; then +if test "${LOAD_BALANCED:-}" = 'true'; then install_haproxy fi -# Launching mongod under $MONGO_ORCHESTRATION_HOME -# makes its log available through log collecting machinery - -export dbdir="$MONGO_ORCHESTRATION_HOME"/db -mkdir -p "$dbdir" +# Compute OCSP mock server arguments before starting MongoDB. +if test -n "${OCSP_ALGORITHM:-}"; then + _ocsp_ca="spec/support/ocsp/$OCSP_ALGORITHM/ca.crt" + OCSP_ARGS="--ca_file $_ocsp_ca" + if test "${OCSP_DELEGATE:-}" = 1; then + OCSP_ARGS="$OCSP_ARGS \ +--ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \ +--ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key" + else + OCSP_ARGS="$OCSP_ARGS \ +--ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \ +--ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key" + fi + if test -n "${OCSP_STATUS:-}"; then + OCSP_ARGS="$OCSP_ARGS --fault $OCSP_STATUS" + fi + export OCSP_ARGS +fi -if test -z "$TOPOLOGY"; then - export TOPOLOGY=standalone +if test -n "${OCSP_ALGORITHM:-}" || test -n "${OCSP_VERIFIER:-}"; then + python3 -m pip install asn1crypto oscrypto flask waitress fi -calculate_server_args launch_ocsp_mock -launch_server "$dbdir" +export TOPOLOGY="${TOPOLOGY:-server}" -uri_options="$URI_OPTIONS" +.evergreen/run-orchestration.sh +. ./mo-expansion.sh +export MONGODB_URI bundle_install -if test "$TOPOLOGY" = sharded-cluster; then - if test -n "$SINGLE_MONGOS"; then - # Some tests may run into https://jira.mongodb.org/browse/SERVER-16836 - # when executing against a multi-sharded mongos. - # At the same time, due to pinning in sharded transactions, - # it is beneficial to test a single shard to ensure that server - # monitoring and selection are working correctly and recover the driver's - # ability to operate in reasonable time after errors and fail points trigger - # on a single shard - echo Restricting to a single mongos - hosts=localhost:27017 - else - hosts=localhost:27017,localhost:27018 - fi -elif test "$TOPOLOGY" = replica-set; then - # To set FCV we use mongo shell, it needs to be placed in replica set topology - # or it can try to send the commands to secondaries. - hosts=localhost:27017,localhost:27018 - uri_options="$uri_options&replicaSet=test-rs" -elif test "$TOPOLOGY" = replica-set-single-node; then - hosts=localhost:27017 - uri_options="$uri_options&replicaSet=test-rs" -else - hosts=localhost:27017 -fi - -if test "$AUTH" = auth; then - hosts="bob:pwd123@$hosts" -elif test "$AUTH" = x509; then +if test "$AUTH" = x509; then create_user_cmd="`cat <<'EOT' db.getSiblingDB("$external").runCommand( { - createUser: "C=US,ST=New York,L=New York City,O=MongoDB,OU=x509,CN=localhost", + createUser: "CN=client,OU=Drivers,O=MDB,L=New York City,ST=New York,C=US", roles: [ { role: "root", db: "admin" }, ], @@ -136,16 +105,17 @@ EOT `" "$BINDIR"/mongosh --tls \ - --tlsCAFile spec/support/certificates/ca.crt \ - --tlsCertificateKeyFile spec/support/certificates/client-x509.pem \ - -u bootstrap -p bootstrap \ + --tlsCAFile .evergreen/x509gen/ca.pem \ + -u bob -p pwd123 \ + --authenticationDatabase admin \ --eval "$create_user_cmd" elif test "$AUTH" = aws-regular; then clear_instance_profile ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth - hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts" + _mongo_host=$(echo "$MONGODB_URI" | sed 's|mongodb://[^@]*@||' | sed 's|/.*||') + export MONGODB_URI="mongodb://$(uri_escape "$MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID"):$(uri_escape "$MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY")@${_mongo_host}/?authMechanism=MONGODB-AWS&authSource=\$external" elif test "$AUTH" = aws-assume-role; then clear_instance_profile @@ -165,10 +135,8 @@ elif test "$AUTH" = aws-assume-role; then aws sts get-caller-identity - hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts" - - uri_options="$uri_options&"\ -"authMechanismProperties=AWS_SESSION_TOKEN:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN`" + _mongo_host=$(echo "$MONGODB_URI" | sed 's|mongodb://[^@]*@||' | sed 's|/.*||') + export MONGODB_URI="mongodb://$(uri_escape "$MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID"):$(uri_escape "$MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY")@${_mongo_host}/?authMechanism=MONGODB-AWS&authSource=\$external&authMechanismProperties=AWS_SESSION_TOKEN:$(uri_escape "$MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN")" elif test "$AUTH" = aws-ec2; then ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth @@ -178,6 +146,9 @@ elif test "$AUTH" = aws-ec2; then # The EC2 credential retrieval tests clears the instance profile as part # of one of the tests. ruby -Ispec -Ilib -I.evergreen/lib -rec2_setup -e Ec2Setup.new.assign_instance_profile + + _mongo_host=$(echo "$MONGODB_URI" | sed 's|mongodb://[^@]*@||' | sed 's|/.*||') + export MONGODB_URI="mongodb://${_mongo_host}/?authMechanism=MONGODB-AWS&authSource=\$external" elif test "$AUTH" = aws-ecs; then if test -z "$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; then # drivers-evergreen-tools performs this operation in its ECS E2E tester. @@ -185,10 +156,16 @@ elif test "$AUTH" = aws-ecs; then fi ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth + + _mongo_host=$(echo "$MONGODB_URI" | sed 's|mongodb://[^@]*@||' | sed 's|/.*||') + export MONGODB_URI="mongodb://${_mongo_host}/?authMechanism=MONGODB-AWS&authSource=\$external" elif test "$AUTH" = aws-web-identity; then clear_instance_profile ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth + + _mongo_host=$(echo "$MONGODB_URI" | sed 's|mongodb://[^@]*@||' | sed 's|/.*||') + export MONGODB_URI="mongodb://${_mongo_host}/?authMechanism=MONGODB-AWS&authSource=\$external" elif test "$AUTH" = kerberos; then export MONGO_RUBY_DRIVER_KERBEROS=1 fi @@ -218,14 +195,6 @@ if test -n "$FLE"; then # Start the KMS servers first so that they are launching while we are # fetching libmongocrypt. if test "$DOCKER_PRELOAD" != 1; then - # We already have a virtualenv activated for mlaunch, - # install kms dependencies into it. - #. .evergreen/csfle/activate_venv.sh - - # Adjusted package versions: - # cryptography 3.4 requires rust, see - # https://github.com/pyca/cryptography/issues/5771. - #pip install boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0 pip3 install boto3~=1.19 'cryptography<3.4' pykmip~=0.10.0 'sqlalchemy<2.0.0' fi python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 & @@ -308,23 +277,21 @@ if test -n "$FLE"; then fi if test -n "$OCSP_CONNECTIVITY"; then - # TODO Maybe OCSP_CONNECTIVITY=* should set SSL=ssl instead. - uri_options="$uri_options&tls=true" + add_uri_option tls=true + add_uri_option "tlsCAFile=$DRIVERS_TOOLS/.evergreen/ocsp/$OCSP_ALGORITHM/ca.pem" fi if test -n "$EXTRA_URI_OPTIONS"; then - uri_options="$uri_options&$EXTRA_URI_OPTIONS" + add_uri_option "$EXTRA_URI_OPTIONS" fi -export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options" - if echo "$AUTH" |grep -q ^aws-assume-role; then $BINDIR/mongosh "$MONGODB_URI" --eval 'db.runCommand({serverStatus: 1})' | wc fi set_fcv -if test "$TOPOLOGY" = replica-set || test "$TOPOLOGY" = replica-set-single-node; then +if test "$TOPOLOGY" = replica_set; then ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.setup_tags fi @@ -338,13 +305,13 @@ if ! test "$OCSP_VERIFIER" = 1 && ! test -n "$OCSP_CONNECTIVITY"; then bundle exec rake spec:prepare fi -if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then +if test "$TOPOLOGY" = sharded_cluster && test $MONGODB_VERSION = 3.6; then # On 3.6 server the sessions collection is not immediately available, # wait for it to spring into existence bundle exec rake spec:wait_for_sessions fi -export MONGODB_URI="mongodb://$hosts/?appName=test-suite$uri_options" +add_uri_option "appName=test-suite" # Compression is handled via an environment variable, convert to URI option if test "$COMPRESSOR" = zlib && ! echo $MONGODB_URI |grep -q compressors=; then @@ -407,7 +374,7 @@ if test -n "$OCSP_MOCK_PID"; then kill "$OCSP_MOCK_PID" fi -python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir" || true +"$DRIVERS_TOOLS"/.evergreen/run-mongodb.sh stop || true if test -n "$FLE" && test "$DOCKER_PRELOAD" != 1; then # Terminate all kmip servers... and whatever else happens to be running diff --git a/spec/support/spec_config.rb b/spec/support/spec_config.rb index 3e1015d73a..524bd94b74 100644 --- a/spec/support/spec_config.rb +++ b/spec/support/spec_config.rb @@ -330,7 +330,11 @@ def local_client_pem_path end def client_x509_pem_path - "#{ssl_certs_dir}/client-x509.pem" + if drivers_tools? + evergreen_client_pem_path + else + "#{ssl_certs_dir}/client-x509.pem" + end end def second_level_cert_path