Merge branch 'main' into add_otel_kube_stack

elastic · Nov 29, 2024 · b6b592b · b6b592b
2 parents 6f6bb66 + a4bf7d0
commit b6b592b
Show file tree

Hide file tree

Showing 235 changed files with 51,065 additions and 21,344 deletions.
diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml
@@ -4,8 +4,6 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   ASDF_MAGE_VERSION: 1.14.0
-  ASDF_GOLANG_VERSION: 1.22.6
-  ASDF_TERRAFORM_VERSION: 1.9.2
 
 steps:
   - label: "Integration tests: packaging"
@@ -19,6 +17,8 @@ steps:
 
   - label: Start ESS stack for integration tests
     key: integration-ess
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
     depends_on:
       - package-it
     command: |
@@ -32,124 +32,181 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
 
-  - group: "Stateful IT (Sudo): Ubuntu"
-    key: integration-tests
+  - group: "Stateful: Windows"
+    key: integration-tests-win
     depends_on:
-      - package-it
       - integration-ess
+      - package-it
     steps:
-      - label: "Default"
-        key: stateful-ubuntu-default-sudo
-        command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "default" "^(TestAPMConfig|TestDiagnosticsOptionalValues|TestIsolatedUnitsDiagnosticsOptionalValues|TestDiagnosticsCommand|TestIsolatedUnitsDiagnosticsCommand|TestEventLogFile|TestFakeComponent|TestFakeIsolatedUnitsComponent|TestOtelFileProcessing|TestOtelLogsIngestion|TestOtelAPMIngestion|TestPackageVersion)$$"
-        artifact_paths:
-          - build/**
-        agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
-
-      - label: "Upgrade"
-        key: stateful-ubuntu-upgrade-sudo
+      - label: "Win2022:sudo:{{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "upgrade" "^(TestUpgradeBrokenPackageVersion|TestStandaloneUpgradeWithGPGFallback|TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing|TestStandaloneUpgradeRollback|TestStandaloneUpgradeRollbackOnRestarts|TestStandaloneUpgradeFailsWhenUpgradeIsInProgress|TestStandaloneUpgradeRetryDownload|TestStandaloneUpgradeSameCommit|TestStandaloneUpgrade|TestStandaloneUpgradeUninstallKillWatcher)$$"
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
+          - fleet
+          - fleet-privileged
+          - upgrade
 
-      - label: "Fleet"
-        key: stateful-ubuntu-fleet-sudo
+      - label: "Win2022:non-sudo:{{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet" "^(TestLongRunningAgentForLeaks|TestDelayEnroll|TestDelayEnrollUnprivileged|TestInstallAndCLIUninstallWithEndpointSecurity|TestInstallAndUnenrollWithEndpointSecurity|TestInstallWithEndpointSecurityAndRemoveEndpointIntegration|TestEndpointSecurityNonDefaultBasePath|TestEndpointSecurityUnprivileged|TestEndpointSecurityCannotSwitchToUnprivileged|TestEndpointLogsAreCollectedInDiagnostics|TestForceInstallOverProtectedPolicy|TestSetLogLevelFleetManaged|TestLogIngestionFleetManaged|TestMetricsMonitoringCorrectBinaries|TestEndpointAgentServiceMonitoring|TestMonitoringPreserveTextConfig|TestMonitoringLivenessReloadable|TestComponentBuildHashInDiagnostics|TestProxyURL|TestFleetManagedUpgradeUnprivileged)$$"
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
 
-      - label: "FQDN"
-        key: stateful-ubuntu-fqdn-sudo
+  - group: "Stateful:Ubuntu"
+    key: integration-tests-ubuntu
+    depends_on:
+      - integration-ess
+      - package-it
+    steps:
+      - label: "x86_64:non-sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fqdn" "^(TestFQDN)$$"
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
 
-      - label: "Deb"
-        key: stateful-ubuntu-deb-sudo
+      - label: "x86_64:sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "deb" "^(TestDebLogIngestFleetManaged|TestDebFleetUpgrade)$$"
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
 
-      - label: "Fleet Airgapped"
-        key: stateful-ubuntu-fleet-airgapped-sudo
+      - label: "arm:sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped" "^(TestFleetAirGappedUpgradeUnprivileged)$$"
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
         agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.2xlarge"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
+          - container
+          - fleet-upgrade-to-pr-build
+          - upgrade
+          - fleet
+          - fqdn
+          - deb
+          - fleet-airgapped
+          - fleet-privileged
+          - fleet-airgapped-privileged
 
-      - label: "Fleet Privileged"
-        key: stateful-ubuntu-fleet-privileged-sudo
+      - label: "arm:non-sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-privileged" "^(TestInstallFleetServerBootstrap|TestFleetManagedUpgradePrivileged)$$"
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
         agents:
-          provider: "gcp"
-          imageProject: elastic-images-qa
-          machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.xlarge"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+        matrix:
+          - default
 
-      - label: "Fleet Airgapped Privileged"
-        key: stateful-ubuntu-fleet-airgapped-privileged-sudo
+  - group: "Stateful(Sudo):RHEL8"
+    key: integration-tests-rhel8
+    depends_on:
+      - package-it
+      - integration-ess
+    steps:
+      - label: "x86_64:sudo:rpm"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          ls -lah build/distributions/
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh "fleet-airgapped-privileged" "^(TestFleetAirGappedUpgradePrivileged)$$"
+          .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**
         agents:
           provider: "gcp"
-          imageProject: elastic-images-qa
           machineType: "n1-standard-8"
-          image: "family/platform-ingest-elastic-agent-ubuntu-2204"
+          image: "family/platform-ingest-elastic-agent-rhel-8"
+        plugins:
+          - test-collector#v1.10.1:
+              files: "build/TEST-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
 
   - label: ESS stack cleanup
-    depends_on: integration-tests
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
     allow_dependency_failure: true
     command: |
       buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -63,7 +63,6 @@ if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   echo "--- Prepare BK test analytics token :vault:"
   BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_analytics_token)
   export BUILDKITE_ANALYTICS_TOKEN
-
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then

diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
@@ -1,32 +1,2 @@
 # Shorten BUILDKITE_MESSAGE if needed to avoid filling the Windows env var buffer
 $env:BUILDKITE_MESSAGE = $env:BUILDKITE_MESSAGE.Substring(0, [System.Math]::Min(2048, $env:BUILDKITE_MESSAGE.Length))
-
-# Install gcc TODO: Move to the VM image
-choco install mingw
-Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
-refreshenv
-
-# Install gvm and go
-# TODO: Move GVM download to the base VM image
-$env:GvmVersion = "0.5.2"
-[Net.ServicePointManager]::SecurityProtocol = "tls12"
-$env:GoVersion = Get-Content -Path .go-version
-Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
-gvm --format=powershell $env:GoVersion | Invoke-Expression
-go version
-
-$GOPATH = $(go env GOPATH)
-$env:Path = "$GOPATH\bin;" + $env:Path
-$env:GOTMPDIR = "$GOPATH\tmp"
-New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
-[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
-[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
-go env
-
-# Install tools
-go install github.com/magefile/mage
-go install github.com/elastic/go-licenser
-go install golang.org/x/tools/cmd/goimports
-go install github.com/jstemmer/go-junit-report
-go install gotest.tools/gotestsum