From 8bda9b26e2068ae0fff68a4082531f8ba3fd717a Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:27:39 +0200 Subject: [PATCH] migrate x-pack/heartbeat pipeline (#38490) * migrate x-pack/heartbeat pipeline --- .buildkite/hooks/pre-command | 4 +- .buildkite/scripts/common.sh | 61 +++++++- .../generate_xpack_heartbeat_pipeline.sh | 145 ++++++++++++++++++ .buildkite/scripts/install_tools.sh | 10 ++ .buildkite/scripts/setenv.sh | 14 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 2 +- .../x-pack/pipeline.xpack.heartbeat.yml | 52 ++++++- 7 files changed, 279 insertions(+), 9 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_heartbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a68d98227d48..2c69f322ad81 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -62,12 +62,12 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then - source .buildkite/scripts/setenv.sh +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" exit 0 fi + source .buildkite/scripts/setenv.sh fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 060be87613c0..201fa8a42c78 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -22,6 +22,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" +[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat @@ -39,6 +40,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" +[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" [ -z "${run_xpack_osquerybeat_macos_tests+x}" ] && run_xpack_osquerybeat_macos_tests="$(buildkite-agent meta-data get run_xpack_osquerybeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat @@ -69,6 +71,10 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) +xpack_heartbeat_changeset=( + "^x-pack/heartbeat/.*" + ) + xpack_filebeat_changeset=( "^x-pack/filebeat/.*" ) @@ -140,11 +146,14 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-auditbeat") BEAT_CHANGESET_REFERENCE=${xpack_auditbeat_changeset[@]} ;; + "beats-xpack-dockerlogbeat") + BEAT_CHANGESET_REFERENCE=${xpack_dockerlogbeat_changeset[@]} + ;; "beats-xpack-filebeat") BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]} ;; - "beats-xpack-dockerlogbeat") - BEAT_CHANGESET_REFERENCE=${xpack_dockerlogbeat_changeset[@]} + "beats-xpack-heartbeat") + BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]} ;; "beats-xpack-libbeat") BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} @@ -419,7 +428,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -520,6 +529,52 @@ startCloudTestEnv() { popd > /dev/null } +withNodeJSEnv() { + # HOME="${WORKSPACE}" + local version=$1 + # local nvmPath="${HOME}/.nvm/versions/node/${version}/bin" + echo "Installing nvm" + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + echo "Installing the NodeJs version $version" + nvm install "$version" + # export PATH="${nvmPath}:${PATH}" + nvm use "$version" + node --version +} + +installNodeJsDependencies() { + # Install dependencies to run browsers + if [ "${platform_type}" == "Linux" ]; then + sudo apt-get install -y \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libxkbcommon0 \ + libatspi2.0-0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxrandr2 \ + libgbm1 \ + libpango-1.0-0 \ + libcairo2 \ + libasound2 + if [ $? -ne 0 ]; then + echo "Error: Failed to install dependencies." + exit 1 + else + echo "Dependencies installed successfully." + fi + elif [ "${platform_type}" == "Darwin" ]; then + echo "TBD" + else + echo "Unsupported platform type." + exit 1 + fi +} + teardown() { # Teardown resources after using them echo "---Terraform Cleanup" diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh new file mode 100755 index 000000000000..b51ca0ab3b9e --- /dev/null +++ b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-heartbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +# ## TODO: there are windows test failures already reported +# ## https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 +# ## waiting for being fixed. + +# - label: ":windows: Windows Unit Tests - {{matrix.image}}" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# key: "mandatory-win-unit-tests" +# agents: +# provider: "gcp" +# image: "{{matrix.image}}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# matrix: +# setup: +# image: +# - "${IMAGE_WIN_2016}" +# - "${IMAGE_WIN_2022}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + +# - group: "Extended Windows Tests" +# key: "extended-win-tests" +# steps: + +# - label: ":windows: Windows Unit Tests - {{matrix.image}}" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# key: "extended-win-unit-tests" +# agents: +# provider: "gcp" +# image: "{{matrix.image}}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# matrix: +# setup: +# image: +# - "${IMAGE_WIN_10}" +# - "${IMAGE_WIN_11}" +# - "${IMAGE_WIN_2019}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 80e70ae96c53..3d25cf8e5c98 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -41,6 +41,16 @@ with_mage with_python with_dependencies config_git + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then + # Install NodeJS + withNodeJSEnv "${NODEJS_VERSION}" + installNodeJsDependencies + + echo "Install @elastic/synthetics" + npm i -g @elastic/synthetics +fi + mage dumpVariables #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index b11bc4fe1683..8da3796aa1ec 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -15,6 +15,7 @@ PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 dar PACKAGING_ARM_PLATFORMS="linux/arm64" ASDF_TERRAFORM_VERSION="1.0.2" AWS_REGION="eu-central-1" +NODEJS_VERSION="18.17.1" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -30,6 +31,7 @@ export TMP_FOLDER export DOCKER_REGISTRY export ASDF_TERRAFORM_VERSION export AWS_REGION +export NODEJS_VERSION exportVars() { local platform_type="$(uname)" @@ -64,7 +66,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then + source .buildkite/scripts/common.sh + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then + export ELASTIC_SYNTHETICS_CAPABLE=true + else + # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. + defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" + fi +fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. source .buildkite/scripts/common.sh defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index b0dd795c811e..a324353b65f9 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -66,4 +66,4 @@ steps: command: ".buildkite/scripts/generate_xpack_filebeat_pipeline.sh" notify: - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" \ No newline at end of file + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 65175d3b029a..2804e98996a1 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -1,6 +1,54 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-heartbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/heartbeat" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Heartbeat - run_xpack_heartbeat" + key: "run_xpack_heartbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Heartbeat - run_xpack_heartbeat_macos_tests" + key: "run_xpack_heartbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack heartbeat pipeline" + key: "xpack-heartbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_heartbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"