From 1fad81b4e7ee4789dd372cba5f4704e07a8dce05 Mon Sep 17 00:00:00 2001 From: paolino Date: Thu, 9 Jan 2025 17:11:52 +0000 Subject: [PATCH] Avoid tag clashes when testing the release pipeline --- .buildkite/pipeline.yml | 36 +++++++------- .buildkite/release.yml | 49 +++++++++++++++---- scripts/buildkite/main/linux-package.sh | 13 ++++- scripts/buildkite/main/macos-intel-package.sh | 8 +++ .../buildkite/main/macos-silicon-package.sh | 9 ++++ .../buildkite/release/push-to-dockerhub.sh | 42 ++++++++++++++++ .../buildkite/release/release-candidate.sh | 26 +++++++--- 7 files changed, 149 insertions(+), 34 deletions(-) create mode 100755 scripts/buildkite/release/push-to-dockerhub.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ccd5621505a..dde1a8c17da 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -541,7 +541,7 @@ steps: build.branch !~ /^gh-readonly-queue\/master/ && build.branch != "master" && build.branch != "rc-latest" - && build.env("RELEASE_CANDIDATE") == null + && (build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "TRUE") key: trigger-build-windows-artifacts - label: Build Package (windows) @@ -587,7 +587,7 @@ steps: - block: Windows E2E Tests depends_on: [] - if: build.env("RELEASE_CANDIDATE") == null + if: build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "TRUE" key: trigger-windows-e2e-tests - label: ⚙️ Windows E2E Tests @@ -630,21 +630,23 @@ steps: agents: system: x86_64-linux - - block: Docker Build - depends_on: [] - if: build.env("TEST_RC") == "TRUE" - key: docker-build-block - - - label: Push Docker Image - depends_on: - - docker-build - key: build-docker - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" - agents: - system: x86_64-linux + + + # - block: Docker Build + # depends_on: [] + # if: build.env("TEST_RC") == "TRUE" + # key: docker-build-block + + # - label: Push Docker Image + # depends_on: + # - docker-build + # key: build-docker + # command: + # - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" + # - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" + # - "./docker-build-push" + # agents: + # system: x86_64-linux - group: Docker Checks depends_on: diff --git a/.buildkite/release.yml b/.buildkite/release.yml index 20f1f8ffd25..5cdd4b46847 100644 --- a/.buildkite/release.yml +++ b/.buildkite/release.yml @@ -38,6 +38,7 @@ steps: key: nightly depends_on: main-pipeline-build steps: + - label: Push swagger nightly commands: - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-bump.sh @@ -76,6 +77,29 @@ steps: env: RELEASE: false + - label: Push image to dockerhub + depends_on: define-tag + commands: + - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-dockerhub.sh + agents: + system: x86_64-linux + env: + RELEASE: false + + + # - block: Push test image to dockerhub + # key: push-dockerhub + # depends_on: create-release + + # - label: Push test image to dockerhub + # depends_on: push-dockerhub + # commands: + # - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-dockerhub.sh + # agents: + # system: x86_64-linux + # env: + # RELEASE: false + - group: Release depends_on: nightly if: build.branch == "master" @@ -123,17 +147,22 @@ steps: env: RELEASE: true - - label: Push Docker Image - depends_on: - - create-release - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" - agents: - system: x86_64-linux - - label: Update Documentation Links depends_on: create-release commands: - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/update-documentation-links.sh + + - block: Push to dockerhub + key: push-dockerhub + depends_on: create-release + + - label: Push to dockerhub + depends_on: push-dockerhub + artifact_paths: + - ./artifacts/*.tgz + commands: + - nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-dockerhub.sh + agents: + system: x86_64-linux + env: + RELEASE: true \ No newline at end of file diff --git a/scripts/buildkite/main/linux-package.sh b/scripts/buildkite/main/linux-package.sh index ea8fc328411..40893b6f53c 100755 --- a/scripts/buildkite/main/linux-package.sh +++ b/scripts/buildkite/main/linux-package.sh @@ -9,4 +9,15 @@ git fetch --all git checkout "$RELEASE_CANDIDATE_COMMIT" rm -rf ./result/* -nix build -o result/linux .#ci.artifacts.linux64.release \ No newline at end of file +nix build -o result/linux .#ci.artifacts.linux64.release + +TEST_RC_OR_FALSE=${TEST_RC:-FALSE} +OLD_GIT_TAG=$( git tag -l "v2*-*-*" | sort | tail -n1) +GIT_TAG=$(buildkite-agent meta-data get "release-version" --default="${OLD_GIT_TAG}") + +# intercept test artifacts and rename them +if [ "$TEST_RC_OR_FALSE" == "TRUE" ]; then + right="cardano-wallet-$GIT_TAG-test-linux64.tar.gz" + wrong="cardano-wallet-$GIT_TAG.1-linux64.tar.gz" + mv "./result/$wrong" "./result/$right" +fi \ No newline at end of file diff --git a/scripts/buildkite/main/macos-intel-package.sh b/scripts/buildkite/main/macos-intel-package.sh index 556df4225c0..2a894d32ddf 100755 --- a/scripts/buildkite/main/macos-intel-package.sh +++ b/scripts/buildkite/main/macos-intel-package.sh @@ -10,3 +10,11 @@ git checkout "$RELEASE_CANDIDATE_COMMIT" rm -rf ./result/* nix build -o result/macos-intel .#packages.x86_64-darwin.ci.artifacts.macos-intel.release + +TEST_RC_OR_FALSE=${TEST_RC:-FALSE} + +if [ "$TEST_RC_OR_FALSE" == "TRUE" ]; then + right="cardano-wallet-$NEW_GIT_TAG-test-macos-intel.tar.gz" + wrong="cardano-wallet-$NEW_GIT_TAG.1-macos-intel.tar.gz" + mv "./result/$wrong" "./result/$right" +fi \ No newline at end of file diff --git a/scripts/buildkite/main/macos-silicon-package.sh b/scripts/buildkite/main/macos-silicon-package.sh index 5e592155e8c..6b7ebe14f7f 100755 --- a/scripts/buildkite/main/macos-silicon-package.sh +++ b/scripts/buildkite/main/macos-silicon-package.sh @@ -10,3 +10,12 @@ git checkout "$RELEASE_CANDIDATE_COMMIT" rm -rf ./result/* nix build -o result/macos-silicon .#packages.aarch64-darwin.ci.artifacts.macos-silicon.release + +# interpret '.1' as '-test' +TEST_RC_OR_FALSE=${TEST_RC:-FALSE} + +if [ "$TEST_RC_OR_FALSE" == "TRUE" ]; then + right="cardano-wallet-$NEW_GIT_TAG-test-macos-silicon.tar.gz" + wrong="cardano-wallet-$NEW_GIT_TAG.1-macos-silicon.tar.gz" + mv "./result/$wrong" "./result/$right" +fi \ No newline at end of file diff --git a/scripts/buildkite/release/push-to-dockerhub.sh b/scripts/buildkite/release/push-to-dockerhub.sh new file mode 100755 index 00000000000..f128d1cb849 --- /dev/null +++ b/scripts/buildkite/release/push-to-dockerhub.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euox pipefail + +TRIGGERED_BY=$(buildkite-agent meta-data get base-build) +NEW_GIT_TAG=$(buildkite-agent meta-data get release-version) + +if [ "$RELEASE" == "false" ]; then + if [ "$TEST_RC" == "TRUE" ]; then + TAG="test" + else + TAG="nightly" + fi +else + TAG=$NEW_GIT_TAG +fi + +main_build=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -X GET "https://api.buildkite.com/v2/builds" \ + | jq ".[] | select(.meta_data.\"triggered-by\" == \"$TRIGGERED_BY\")" \ + | jq .number) + +mkdir -p artifacts + +artifact() { + local artifact_name=$1 + # shellcheck disable=SC2155 + local artifact_value=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \ + -X GET "https://api.buildkite.com/v2/organizations/cardano-foundation/pipelines/cardano-wallet/builds/$main_build/artifacts?per_page=100" \ + | jq -r " [.[] | select(.filename == \"$artifact_name\")][0] \ + | .download_url") + curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -L \ + -o "artifacts/$artifact_name" \ + "$artifact_value" + local image_name="cardanofoundation/cardano-wallet:$TAG" + docker login -u cfhal -p "$DOCKER_HUB_TOKEN" + docker load -i "artifacts/$artifact_name" + # docker push "$image_name" + echo "$image_name" +} + +artifact "cardano-wallet-$NEW_GIT_TAG-docker-image.tgz" \ No newline at end of file diff --git a/scripts/buildkite/release/release-candidate.sh b/scripts/buildkite/release/release-candidate.sh index c52d5bf26ba..e3aa0726eb3 100755 --- a/scripts/buildkite/release/release-candidate.sh +++ b/scripts/buildkite/release/release-candidate.sh @@ -3,17 +3,29 @@ set -euox pipefail +if [ "$BUILDKITE_BRANCH" == "master" ]; then + TEST_RC="FALSE" +else + TEST_RC="TRUE" +fi + # date from git tag # example v2023-04-04 -> 2023-04-04 tag_date() { echo "${1##v}" } + +tag_today() { + sed -e 's/-0/-/g' -e 's/-/./g' <<< "$1" +} + # cabal version from git tag # example v2023-04-04 -> 2023.4.4 tag_cabal_ver() { - tag_date "$1" | sed -e s/-0/-/g -e s/-/./g + tag_today "$(tag_date "$1")" } + git tag -l | xargs git tag -d git fetch --tags @@ -23,9 +35,13 @@ git checkout "$BASE_COMMIT" today=$(date +%Y-%m-%d) -NEW_GIT_TAG=v$today - -NEW_CABAL_VERSION=$(tag_cabal_ver "$NEW_GIT_TAG") +if [ $TEST_RC == "TRUE" ]; then + NEW_GIT_TAG="v$today" + NEW_CABAL_VERSION=$(tag_today "$today").1 +else + NEW_GIT_TAG="v$today" + NEW_CABAL_VERSION=$(tag_today "$today") +fi OLD_GIT_TAG=$( git tag -l "v2*-*-*" | sort | tail -n1) @@ -47,10 +63,8 @@ CARDANO_NODE_TAG=$(cardano-node version | head -n1 | awk '{print $2}') if [ "$BUILDKITE_BRANCH" == "master" ]; then RELEASE_CANDIDATE_BRANCH="release-candidate/$NEW_GIT_TAG" - TEST_RC="FALSE" else RELEASE_CANDIDATE_BRANCH="test-rc/$BUILDKITE_BRANCH" - TEST_RC="TRUE" fi git config --global user.email "gha@cardanofoundation.org"