From b72204f15a13a579c1ee5971cad40116b599c853 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 25 Jan 2024 13:51:03 +0100 Subject: [PATCH] RUBY-3384 Update serverless tests --- .evergreen/config.yml | 150 ++++++++++++++++++----------- .evergreen/config/axes.yml.erb | 9 -- .evergreen/config/common.yml.erb | 125 +++++++++++++++--------- .evergreen/config/standard.yml.erb | 16 ++- .mod/drivers-evergreen-tools | 2 +- 5 files changed, 189 insertions(+), 113 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 934829aae4..e4c8f1e25c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -124,25 +124,8 @@ functions: export ATLAS_SERVERLESS_LB_URI="${atlas_serverless_lb_uri}" export RVM_RUBY="${RVM_RUBY}" - if [[ -n "${USE_PROXY_SERVERLESS}" ]]; - then - export SERVERLESS_DRIVERS_GROUP="${PROXY_SERVERLESS_DRIVERS_GROUP}" - else - export SERVERLESS_DRIVERS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - - export SERVERLESS_API_PUBLIC_KEY="${SERVERLESS_API_PUBLIC_KEY}" - export SERVERLESS_API_PRIVATE_KEY="${SERVERLESS_API_PRIVATE_KEY}" export SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" export SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" - - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi EOT # See what we've done @@ -154,17 +137,6 @@ functions: params: file: src/expansion.yml - "delete serverless instance": - - command: shell.exec - params: - working_dir: "src" - shell: bash - script: | - ${PREPARE_SHELL} - if [[ -n "${SERVERLESS_INSTANCE_NAME}" ]]; then - SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} .evergreen/serverless/delete-instance.sh - fi - "export AWS auth credentials": - command: shell.exec type: test @@ -385,6 +357,13 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi .evergreen/run-tests.sh "run tests via docker": @@ -395,6 +374,13 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi .evergreen/run-tests-docker.sh "run AWS auth tests": @@ -438,21 +424,6 @@ functions: AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} .evergreen/run-tests-atlas.sh "run serverless tests": - - command: shell.exec - params: - shell: bash - working_dir: "src" - script: | - ${PREPARE_SHELL} - # This hack with PROJECT env is necessary while serverless - # rejects creating instances with too long names. - PROJECT="mongo-ruby-driver" LOADBALANCED="ON" .evergreen/serverless/create-instance.sh - cat serverless-expansion.yml - - - command: expansions.update - params: - file: src/serverless-expansion.yml - - command: shell.exec type: test params: @@ -460,6 +431,14 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi + CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}" SERVERLESS=1 SSL=ssl RVM_RUBY="${RVM_RUBY}" SINGLE_MONGOS="${SINGLE_MONGOS}" SERVERLESS_URI="${SERVERLESS_URI}" FLE="${FLE}" SERVERLESS_MONGODB_VERSION="${SERVERLESS_MONGODB_VERSION}" .evergreen/run-tests-serverless.sh pre: @@ -473,9 +452,67 @@ post: - func: "upload mo artifacts" #- func: "upload test results" - func: "upload test results to s3" - - func: "delete serverless instance" task_groups: + - name: serverless_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: "fetch source" + - func: "create expansions" + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_task: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + - func: "upload test results" + tasks: + - "test-serverless" + + - name: serverless_next_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: "fetch source" + - func: "create expansions" + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh serverless_next + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_task: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh serverless_next + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + - func: "upload test results" + tasks: + - "test-serverless" + - name: testatlas_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -1204,15 +1241,6 @@ axes: - id: no display_name: No - - id: serverless-project - display_name: serverless project - values: - - id: original - display_name: Original - - id: proxy - display_name: Proxy - variables: - USE_PROXY_SERVERLESS: 1 buildvariants: - matrix_name: "auth/ssl" @@ -1788,10 +1816,20 @@ buildvariants: ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5"] fle: path os: rhel8 - serverless-project: [ original, proxy ] - display_name: "Atlas serverless ${ruby} ${serverless-project}" + display_name: "Atlas serverless ${ruby}" + tasks: + - name: serverless_task_group + + - matrix_name: "serverless-next" + matrix_spec: + # https://jira.mongodb.org/browse/RUBY-3217 + # ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5", "jruby-9.3", "jruby-9.2"] + ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5"] + fle: path + os: rhel8 + display_name: "Atlas serverless-next ${ruby}" tasks: - - name: test-serverless + - name: serverless_next_task_group - matrix_name: "aws-lambda" matrix_spec: diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index 074cba7a06..136b835ff8 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -406,12 +406,3 @@ axes: - id: no display_name: No - - id: serverless-project - display_name: serverless project - values: - - id: original - display_name: Original - - id: proxy - display_name: Proxy - variables: - USE_PROXY_SERVERLESS: 1 diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 54eee122a1..915510136f 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -121,25 +121,8 @@ functions: export ATLAS_SERVERLESS_LB_URI="${atlas_serverless_lb_uri}" export RVM_RUBY="${RVM_RUBY}" - if [[ -n "${USE_PROXY_SERVERLESS}" ]]; - then - export SERVERLESS_DRIVERS_GROUP="${PROXY_SERVERLESS_DRIVERS_GROUP}" - else - export SERVERLESS_DRIVERS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - - export SERVERLESS_API_PUBLIC_KEY="${SERVERLESS_API_PUBLIC_KEY}" - export SERVERLESS_API_PRIVATE_KEY="${SERVERLESS_API_PRIVATE_KEY}" export SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" export SERVERLESS_ATLAS_PASSWORD="${SERVERLESS_ATLAS_PASSWORD}" - - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi EOT # See what we've done @@ -151,17 +134,6 @@ functions: params: file: src/expansion.yml - "delete serverless instance": - - command: shell.exec - params: - working_dir: "src" - shell: bash - script: | - ${PREPARE_SHELL} - if [[ -n "${SERVERLESS_INSTANCE_NAME}" ]]; then - SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} .evergreen/serverless/delete-instance.sh - fi - "export AWS auth credentials": - command: shell.exec type: test @@ -382,6 +354,13 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi .evergreen/run-tests.sh "run tests via docker": @@ -392,6 +371,13 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi .evergreen/run-tests-docker.sh "run AWS auth tests": @@ -435,21 +421,6 @@ functions: AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} .evergreen/run-tests-atlas.sh "run serverless tests": - - command: shell.exec - params: - shell: bash - working_dir: "src" - script: | - ${PREPARE_SHELL} - # This hack with PROJECT env is necessary while serverless - # rejects creating instances with too long names. - PROJECT="mongo-ruby-driver" LOADBALANCED="ON" .evergreen/serverless/create-instance.sh - cat serverless-expansion.yml - - - command: expansions.update - params: - file: src/serverless-expansion.yml - - command: shell.exec type: test params: @@ -457,6 +428,14 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} + # Needed for generating temporary aws credentials. + if [ -n "${FLE}" ]; + then + export AWS_ACCESS_KEY_ID="${fle_aws_key}" + export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" + export AWS_DEFAULT_REGION="${fle_aws_region}" + fi + CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}" SERVERLESS=1 SSL=ssl RVM_RUBY="${RVM_RUBY}" SINGLE_MONGOS="${SINGLE_MONGOS}" SERVERLESS_URI="${SERVERLESS_URI}" FLE="${FLE}" SERVERLESS_MONGODB_VERSION="${SERVERLESS_MONGODB_VERSION}" .evergreen/run-tests-serverless.sh pre: @@ -470,9 +449,67 @@ post: - func: "upload mo artifacts" #- func: "upload test results" - func: "upload test results to s3" - - func: "delete serverless instance" task_groups: + - name: serverless_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: "fetch source" + - func: "create expansions" + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_task: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + - func: "upload test results" + tasks: + - "test-serverless" + + - name: serverless_next_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: "fetch source" + - func: "create expansions" + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh serverless_next + bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_task: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh serverless_next + bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + - func: "upload test results" + tasks: + - "test-serverless" + - name: testatlas_task_group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 1964946d4c..dc4ee4a3be 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -517,10 +517,20 @@ buildvariants: ruby: <%= supported_mri_rubies %> fle: path os: rhel8 - serverless-project: [ original, proxy ] - display_name: "Atlas serverless ${ruby} ${serverless-project}" + display_name: "Atlas serverless ${ruby}" tasks: - - name: test-serverless + - name: serverless_task_group + + - matrix_name: "serverless-next" + matrix_spec: + # https://jira.mongodb.org/browse/RUBY-3217 + # ruby: <%= supported_rubies %> + ruby: <%= supported_mri_rubies %> + fle: path + os: rhel8 + display_name: "Atlas serverless-next ${ruby}" + tasks: + - name: serverless_next_task_group - matrix_name: "aws-lambda" matrix_spec: diff --git a/.mod/drivers-evergreen-tools b/.mod/drivers-evergreen-tools index c3335b46e7..8cb042bf7f 160000 --- a/.mod/drivers-evergreen-tools +++ b/.mod/drivers-evergreen-tools @@ -1 +1 @@ -Subproject commit c3335b46e7190beeab6b65eb9a76bd0e09f69903 +Subproject commit 8cb042bf7fda753ec19eda19f02c441d4220a8dc