From e5bb44934c1135c36ffce9d23d05f520c41d03c7 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 1 Jul 2024 14:42:23 +0300 Subject: [PATCH] Update x-pack/metricbeat pipelline to match Jenkins on 7.17 (#39744) Updated x-pack/metricbeat pipeline to match Jenkins steps and common styling --- .buildkite/deploy/docker/docker-compose.yml | 23 ----- .buildkite/hooks/pre-command | 10 --- .buildkite/scripts/initCloudEnv.sh | 85 +++++++++++++++++++ .buildkite/scripts/setup_cloud_env.sh | 50 ----------- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 56 +++--------- .../x-pack/pipeline.xpack.metricbeat.yml | 85 +++++++------------ 6 files changed, 127 insertions(+), 182 deletions(-) delete mode 100644 .buildkite/deploy/docker/docker-compose.yml create mode 100755 .buildkite/scripts/initCloudEnv.sh delete mode 100644 .buildkite/scripts/setup_cloud_env.sh diff --git a/.buildkite/deploy/docker/docker-compose.yml b/.buildkite/deploy/docker/docker-compose.yml deleted file mode 100644 index c27417158a3..00000000000 --- a/.buildkite/deploy/docker/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: '2.3' -services: - # This is a proxy used to block beats until all services are healthy. - # See: https://github.com/docker/compose/issues/4369 - proxy_dep: - image: busybox - depends_on: - localstack: { condition: service_healthy } - - localstack: - container_name: "${localstack_integration_test_container}" - image: localstack/localstack:3.1.0 # Latest stable release - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - environment: - - DEBUG=1 - - DOCKER_HOST=unix:///var/run/docker.sock - - LOCALSTACK_HOST=localhost - - S3_HOSTNAME=localhost - - PROVIDER_OVERRIDE_S3=asf - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 587d10688e9..8453ed5b373 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,7 +6,6 @@ source .buildkite/env-scripts/util.sh # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables -AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" @@ -72,15 +71,6 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do fi done -if [[ "$BUILDKITE_PIPELINE_SLUG" == *"xpack-metricbeat"* || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == *"extended-cloud-test"* ]]; then - BEATS_AWS_SECRET_KEY=$(retry_with_count 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_SECRET_KEY - BEATS_AWS_ACCESS_KEY=$(retry_with_count 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) - export BEATS_AWS_ACCESS_KEY - fi -fi - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh new file mode 100755 index 00000000000..47e607dab93 --- /dev/null +++ b/.buildkite/scripts/initCloudEnv.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_DIR=$(pwd) +AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" + +exportAwsSecrets() { + local awsSecretKey + local awsAccessKey + + awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + awsAccessKey=$(retry -t 5 -- vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + + echo "~~~ Exporting AWS secrets" + export AWS_ACCESS_KEY_ID=$awsAccessKey + export AWS_SECRET_ACCESS_KEY=$awsSecretKey + + # AWS_REGION is not set here, since AWS region is taken from *.tf file: + # - x-pack/metricbeat/module/aws/terraform.tf + # - x-pack/filebeat/input/awscloudwatch/_meta/terraform/variables.tf +} + +terraformApply() { + TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') + TF_VAR_CREATED_DATE=$(date +%s) + export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" + export TF_VAR_ENVIRONMENT="ci" + export TF_VAR_REPO="${REPO}" + export TF_VAR_BRANCH + export TF_VAR_CREATED_DATE + + echo "Terraform Init on $MODULE_DIR" + terraform -chdir="$MODULE_DIR" init + + echo "Terraform Apply on $MODULE_DIR" + terraform -chdir="$MODULE_DIR" apply -auto-approve +} + +terraformDestroy() { + echo "--- Terraform Cleanup" + cd $REPO_DIR + find "$MODULE_DIR" -name terraform.tfstate -print0 | while IFS= read -r -d '' tfstate; do + cd "$(dirname "$tfstate")" + echo "Uploading terraform.tfstate to Buildkite artifacts" + buildkite-agent artifact upload "**/terraform.tfstate" + if ! terraform destroy -auto-approve; then + return 1 + fi + cd - + done + return 0 +} + +trap 'terraformDestroy' EXIT +exportAwsSecrets + +max_retries=2 +timeout=5 +retries=0 + +while true; do + echo "~~~ Setting up Terraform" + out=$(terraformApply 2>&1) + exit_code=$? + + echo "$out" + + if [ $exit_code -eq 0 ]; then + break + else + retries=$((retries + 1)) + + if [ $retries -gt $max_retries ]; then + terraformDestroy + echo "+++ Terraform init & apply failed: $out" + exit 1 + fi + + terraformDestroy + + sleep_time=$((timeout * retries)) + echo "~~~~ Retry #$retries failed. Retrying after ${sleep_time}s..." + sleep $sleep_time + fi +done diff --git a/.buildkite/scripts/setup_cloud_env.sh b/.buildkite/scripts/setup_cloud_env.sh deleted file mode 100644 index a13a8446dc7..00000000000 --- a/.buildkite/scripts/setup_cloud_env.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO_DIR=$(pwd) - -teardown() { - # reset the directory to the root of the project - cd $REPO_DIR - # Teardown resources after using them - echo "~~~ Terraform Cleanup" - tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK - - echo "~~~ Docker Compose Cleanup" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v -} - -tf_cleanup() { - DIRECTORY=${1:-.} - - for tfstate in $(find $DIRECTORY -name terraform.tfstate); do - cd $(dirname $tfstate) - terraform init - if ! terraform destroy -auto-approve; then - echo "+++ Failed to Terraform destroy the resources" - fi - cd - - done -} - -trap 'teardown' EXIT - -# Prepare the cloud resources using Terraform -echo "~~~ Loading creds" -set +o xtrace -export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY -export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY -export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -set -o xtrace - -echo "~~~ Run docker-compose services for emulated cloud env" -docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d -echo "~~~ Initialize TF cloud resources" -cd "$MODULE_DIR" -export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') -export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" -export TF_VAR_CREATED_DATE=$(date +%s) -export TF_VAR_ENVIRONMENT="ci" -export TF_VAR_REPO="${REPO}" -terraform init && terraform apply -auto-approve -cd - diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 83d3d554279..1aac5ca1f6a 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -43,7 +43,7 @@ steps: notify: - github_commit_status: context: "x-pack-filebeat: check/update" - + - wait: ~ # with PRs, we want to run mandatory tests only if check/update step succeed # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests @@ -129,7 +129,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Python Integration Tests" - - label: "Filebeat: Module compat tests - previous minor" + - label: ":ubuntu: x-pack/filebeat Module compat tests - previous minor" # Run module integration tests under previous minor of Elastic stack. env: STACK_ENVIRONMENT: "prev-minor" @@ -154,7 +154,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Module compat tests / previous minor" + context: "x-pack/filebeat: Module compat tests / previous minor" - label: ":windows: x-pack/filebeat Win-2019 Unit Tests" command: | @@ -248,53 +248,23 @@ steps: - github_commit_status: context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/filebeat AWS Tests" key: "x-pack-filebeat-extended-cloud-test" + skip: "skipping due to elastic/ingest-dev#3467" + # https://github.com/elastic/ingest-dev/issues/3467 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat - echo "~~~ Running tests" - source .buildkite/scripts/setup_cloud_env.sh - cd x-pack/filebeat - mage build test - env: - ASDF_TERRAFORM_VERSION: 1.0.2 - AWS_REGION: "eu-central-1" - MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" - REPO: beats - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Cloud (MODULE) Tests" - - - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests" - key: "x-pack-filebeat-extended-cloud-test-aws" - skip: "Skipping due to elastic/beats#36425" - # https://github.com/elastic/beats/issues/36425 - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/filebeat + source .buildkite/scripts/initCloudEnv.sh echo "~~~ Running tests" - source .buildkite/scripts/setup_cloud_env.sh cd x-pack/filebeat mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 - AWS_REGION: "eu-central-1" MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" - REPO: beats + MODULE: "aws" + # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 + TEST_TAGS: "aws" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -304,7 +274,7 @@ steps: - "x-pack/filebeat/build/*.json" notify: - github_commit_status: - context: "x-pack/filebeat: Cloud AWS (MODULE) Tests" + context: "x-pack/filebeat: AWS Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -317,7 +287,7 @@ steps: - group: "x-pack/filebeat Packaging" key: "x-pack-filebeat-packaging" steps: - - label: ":linux: x-pack/filebeat Packaging Linux" + - label: ":ubuntu: x-pack/filebeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/filebeat @@ -338,7 +308,7 @@ steps: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - - label: ":linux: x-pack/filebeat Packaging arm64" + - label: ":ubuntu: x-pack/filebeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/filebeat diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 5b55f668616..668d6a0ee13 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -55,8 +55,9 @@ steps: - group: "x-pack/metricbeat Mandatory Tests" key: "x-pack-metricbeat-mandatory-tests" + steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/metricbeat Linux x86_64 Unit Tests" key: "mandatory-linux-unit-test" command: | cd x-pack/metricbeat @@ -73,12 +74,16 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Ubuntu Unit Tests" + context: "x-pack/metricbeat: Linux x86_64 Unit Tests" - - label: ":go: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat Go (MODULE) Integration Tests" key: "mandatory-int-test" command: | set -euo pipefail + # docker-compose problem debugging + which docker-compose + docker-compose --version + pwd # defines the MODULE env var based on what's changed in a PR source .buildkite/scripts/changesets.sh defineModuleFromTheChangeSet x-pack/metricbeat @@ -86,7 +91,7 @@ steps: cd x-pack/metricbeat && mage -v goIntegTest retry: automatic: - - limit: 3 + - limit: 1 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -98,7 +103,7 @@ steps: - github_commit_status: context: "x-pack/metricbeat: Go (MODULE) Integration Tests" - - label: ":python: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat Python (MODULE) Integration Tests" key: "mandatory-python-int-test" command: | set -euo pipefail @@ -121,7 +126,7 @@ steps: - github_commit_status: context: "x-pack/metricbeat: Python (MODULE) Integration Tests" - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: x-pack/metricbeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -140,14 +145,14 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Windows 2019 Unit Tests" + context: "x-pack/metricbeat: Win-2019 Unit Tests" - group: "x-pack/metricbeat Extended Windows Tests" key: "x-pack-metricbeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Windows 2016 Unit Tests" + - label: ":windows: x-pack/metricbeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -166,9 +171,9 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Windows 2016 Unit Tests" + context: "x-pack/metricbeat: Win-2016 Unit Tests" - - label: ":windows: Windows 10 Unit Tests" + - label: ":windows: x-pack/metricbeat Win 10 Unit Tests" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -187,13 +192,14 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Windows 10 Unit Tests" + context: "x-pack/metricbeat: Win 10 Unit Tests" - group: "x-pack/metricbeat Extended Tests" key: "x-pack-metricbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ + steps: - - label: ":mac: MacOS x86_64 Unit Tests" + - label: ":mac: x-pack/metricbeat macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail @@ -210,56 +216,23 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: MacOS x86_64 Unit Tests" + context: "x-pack/metricbeat: macOS x86_64 Unit Tests" - - label: ":linux: Cloud (MODULE) Tests" + - label: ":ubuntu: x-pack/metricbeat AWS Tests" key: "x-pack-metricbeat-extended-cloud-test" if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests" - source .buildkite/scripts/setup_cloud_env.sh - cd x-pack/metricbeat - mage build test - env: - ASDF_TERRAFORM_VERSION: 1.0.2 - AWS_REGION: "eu-central-1" - MODULE_DIR: x-pack/metricbeat/module/aws - REPO: beats - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "x-pack/metricbeat/build/*.xml" - - "x-pack/metricbeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/metricbeat: Cloud (MODULE) Tests" - - - label: ":linux: Cloud AWS (MODULE) Tests" - key: "x-pack-metricbeat-extended-cloud-test-aws" - skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170" - # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170 - if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - + source .buildkite/scripts/initCloudEnv.sh echo "~~~ Running tests" - source .buildkite/scripts/setup_cloud_env.sh cd x-pack/metricbeat mage build test env: ASDF_TERRAFORM_VERSION: 1.0.2 - AWS_REGION: "eu-central-1" - MODULE_DIR: x-pack/metricbeat/module/aws - REPO: beats + MODULE_DIR: "x-pack/metricbeat/module/aws" + MODULE: "aws" + # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 + TEST_TAGS: "aws" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -269,7 +242,7 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Cloud AWS (MODULE) Tests" + context: "x-pack/metricbeat: AWS Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -282,7 +255,7 @@ steps: - group: "x-pack/metricbeat Packaging" key: "x-pack-metricbeat-packaging" steps: - - label: ":linux: Packaging Linux" + - label: ":ubuntu: x-pack/metricbeat Packaging Linux" key: "packaging-linux" command: | cd x-pack/metricbeat @@ -303,7 +276,7 @@ steps: - github_commit_status: context: "x-pack/metricbeat: Packaging Linux" - - label: ":linux: Packaging ARM" + - label: ":ubuntu: x-pack/metricbeat Packaging arm64" key: "packaging-arm" command: | cd x-pack/metricbeat @@ -321,4 +294,4 @@ steps: PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/metricbeat: Packaging Linux ARM" + context: "x-pack/metricbeat: Packaging Linux arm64"