From 63526d8a37ea527b366ae50996614ce5b1e51b71 Mon Sep 17 00:00:00 2001 From: paolino Date: Sat, 6 Jul 2024 16:21:51 +0000 Subject: [PATCH 1/7] Add docker steps to main pipeline --- .buildkite/pipeline.yml | 78 ++++++++++++++---- scripts/buildkite/release/docker-build.sh | 16 ++++ .../buildkite/release/docker-smoke-test.sh | 81 +++++++++++++++++++ 3 files changed, 158 insertions(+), 17 deletions(-) create mode 100755 scripts/buildkite/release/docker-build.sh create mode 100755 scripts/buildkite/release/docker-smoke-test.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ed2373da3ee..d18dd779b5f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -431,20 +431,64 @@ steps: agents: system: ${linux} - - wait: ~ - - - block: Push Docker Image - if: build.env("RELEASE_CANDIDATE") == null - key: docker-push-block - - - label: Push Docker Image - depends_on: - - linux-nix - - docker-push-block - key: build-docker - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" - agents: - system: x86_64-linux + - group: Docker Artifacts + key: docker-artifacts + steps: + - block: 'Docker build' + if: | + build.branch !~ /^gh-readonly-queue\/master/ + && build.branch != "master" + && build.env("RELEASE_CANDIDATE") == null + key: docker-artifacts-block + + - label: 'Build Docker Image' + key: docker-build + depends_on: + - docker-artifacts-block + - linux-nix + commands: + ./scripts/buildkite/release/docker-build.sh + agents: + system: x86_64-linux + + - group: "Docker Checks" + key: "docker-e2e" + steps: + + - block: 'Docker build and test' + key: docker-tests-block + if: | + build.branch !~ /^gh-readonly-queue\/master/ + && build.branch != "master" + && build.env("RELEASE_CANDIDATE") == null + + - label: 'Smoke test docker-compose' + depends_on: + - docker-tests-block + - docker-build + timeout_in_minutes: 2 + commands: + - ./scripts/buildkite/release/docker-smoke-test.sh + artifact_paths: + - "./logs/*" + env: + NODE_STATE_DIR: "${STATE_DIR?}/node/preprod" + agents: + system: x86_64-linux + + - block: Push Docker Image + if: build.env("RELEASE_CANDIDATE") == null + key: docker-push-block + + - label: Push Docker Image + depends_on: + - linux-nix + - docker-build + - docker-push-block + key: build-docker + command: + - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" + - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" + - "./docker-build-push" + agents: + system: x86_64-linux diff --git a/scripts/buildkite/release/docker-build.sh b/scripts/buildkite/release/docker-build.sh new file mode 100755 index 00000000000..c4469abba3e --- /dev/null +++ b/scripts/buildkite/release/docker-build.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env -S nix shell --command bash +# shellcheck shell=bash + +set -euox pipefail + +git fetch --all + +RELEASE_CANDIDATE_COMMIT=$(buildkite-agent meta-data get "release-candidate-commit") + +git checkout "$RELEASE_CANDIDATE_COMMIT" + +mkdir -p result + +nix build .#dockerImage -o result/docker-image + +docker load < result/docker-image diff --git a/scripts/buildkite/release/docker-smoke-test.sh b/scripts/buildkite/release/docker-smoke-test.sh new file mode 100755 index 00000000000..c5ea7e229e2 --- /dev/null +++ b/scripts/buildkite/release/docker-smoke-test.sh @@ -0,0 +1,81 @@ +#! /usr/bin/env -S nix shell 'nixpkgs#docker-compose' 'nixpkgs#rsync' 'nixpkgs#jq' --command bash +# shellcheck shell=bash + +set -euox pipefail + + +NETWORK=preprod +export NETWORK + +TESTS_NODE_DB="$(pwd)/state/node_db" + +mkdir -p "$TESTS_NODE_DB" +export TESTS_NODE_DB + +rsync -a --delete "$NODE_STATE_DIR/db/" "$TESTS_NODE_DB" + +WALLET_TAG=$(buildkite-agent meta-data get "release-cabal-version") +export WALLET_TAG + +NODE_TAG="8.9.4" +export NODE_TAG + +NODE_DB="$TESTS_NODE_DB" +export NODE_DB + +WALLET_DB="$(pwd)/state/wallet_db" +mkdir -p "$WALLET_DB" +export WALLET_DB + +WALLET_PORT=$(shuf -i 2000-65000 -n 1) +export WALLET_PORT + +USER_ID=$(id -u) +export USER_ID + +tmpfile=$(mktemp -d /tmp/node-preprod.XXXXXX) + +# set the node socket path +NODE_SOCKET_DIR="$tmpfile" +export NODE_SOCKET_DIR + +NODE_SOCKET_NAME="node.socket" +export NODE_SOCKET_NAME + +COMPOSE_PROJECT_NAME="docker-smoke-test-$WALLET_PORT" +export COMPOSE_PROJECT_NAME + +docker compose down || true + +docker-compose up -d + +n=0 +while : +do + result=$(curl --connect-timeout 1 localhost:"$WALLET_PORT"/v2/network/information || echo "wait") + echo "$result" + if [ "$result" != "wait" ]; + then + echo "$result" | jq + break + else + sleep 4 + n=$((n+1)) + fi + if [ "$n" -ge 30 ] + then break + fi +done + + +mkdir -p logs +docker-compose logs > logs/docker-compose.log +docker-compose down + +rm -rf "$tmpfile" + +if [ "$result" == "wait" ]; + then + echo "Failed to start the wallet server" + exit 1 +fi From 662bb7d2f32bd57f28f7ec439701cb8cafda4b85 Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 10:41:48 +0000 Subject: [PATCH 2/7] Add 10 minutes of timeout to latency benchmark --- .buildkite/pipeline.yml | 48 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d18dd779b5f..f4f9d0af599 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -220,7 +220,7 @@ steps: export BENCHMARK_CSV_FILE="`pwd`/bench-results.csv" ./scripts/buildkite/main/bench-latency.sh depends_on: trigger-benchmarks - timeout_in_minutes: 20 + timeout_in_minutes: 30 agents: system: x86_64-linux queue: adrestia-bench @@ -434,14 +434,14 @@ steps: - group: Docker Artifacts key: docker-artifacts steps: - - block: 'Docker build' + - block: Build Docker Artifacts if: | build.branch !~ /^gh-readonly-queue\/master/ && build.branch != "master" && build.env("RELEASE_CANDIDATE") == null key: docker-artifacts-block - - label: 'Build Docker Image' + - label: Build Docker Image key: docker-build depends_on: - docker-artifacts-block @@ -451,18 +451,35 @@ steps: agents: system: x86_64-linux - - group: "Docker Checks" - key: "docker-e2e" + - block: Push Docker Image + if: build.env("RELEASE_CANDIDATE") == null + key: docker-push-block + + - label: Push Docker Image + depends_on: + - linux-nix + - docker-build + - docker-push-block + key: build-docker + command: + - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" + - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" + - "./docker-build-push" + agents: + system: x86_64-linux + + - group: Docker Checks + key: docker-e2e steps: - - block: 'Docker build and test' + - block: Run Docker Checks key: docker-tests-block if: | build.branch !~ /^gh-readonly-queue\/master/ && build.branch != "master" && build.env("RELEASE_CANDIDATE") == null - - label: 'Smoke test docker-compose' + - label: Smoke Test docker-compose depends_on: - docker-tests-block - docker-build @@ -475,20 +492,3 @@ steps: NODE_STATE_DIR: "${STATE_DIR?}/node/preprod" agents: system: x86_64-linux - - - block: Push Docker Image - if: build.env("RELEASE_CANDIDATE") == null - key: docker-push-block - - - label: Push Docker Image - depends_on: - - linux-nix - - docker-build - - docker-push-block - key: build-docker - command: - - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" - - "nix build .#pushDockerImage --override-input hostNixpkgs \"path:$(nix eval --impure -I $NIX_PATH --expr '(import {}).path')\" --override-input customConfig path:./config -o docker-build-push" - - "./docker-build-push" - agents: - system: x86_64-linux From dfe979ac9a597900706ed3bb9dd827ec2cc770cf Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 11:16:06 +0000 Subject: [PATCH 3/7] Add a default commit in build docker image step --- scripts/buildkite/release/docker-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/buildkite/release/docker-build.sh b/scripts/buildkite/release/docker-build.sh index c4469abba3e..dc0980a6ad4 100755 --- a/scripts/buildkite/release/docker-build.sh +++ b/scripts/buildkite/release/docker-build.sh @@ -5,7 +5,7 @@ set -euox pipefail git fetch --all -RELEASE_CANDIDATE_COMMIT=$(buildkite-agent meta-data get "release-candidate-commit") +RELEASE_CANDIDATE_COMMIT=$(buildkite-agent meta-data get "release-candidate-commit" --default="${BUILDKITE_COMMIT}") git checkout "$RELEASE_CANDIDATE_COMMIT" From e0a28738cf06f3e53547620b423dfccdb0dfb343 Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 12:30:14 +0000 Subject: [PATCH 4/7] Always push rc-latest docker images --- .buildkite/pipeline.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f4f9d0af599..fdc7fabafcf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -439,6 +439,7 @@ steps: build.branch !~ /^gh-readonly-queue\/master/ && build.branch != "master" && build.env("RELEASE_CANDIDATE") == null + && build.tag != "rc-latest" key: docker-artifacts-block - label: Build Docker Image @@ -451,15 +452,11 @@ steps: agents: system: x86_64-linux - - block: Push Docker Image - if: build.env("RELEASE_CANDIDATE") == null - key: docker-push-block - - label: Push Docker Image + if: build.tag == "rc-latest" depends_on: - linux-nix - docker-build - - docker-push-block key: build-docker command: - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" From 707661670c00d97f5dfec2ab7a557cead328a621 Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 12:53:22 +0000 Subject: [PATCH 5/7] Push commit tag as docker image tag --- .buildkite/docker-build-push.nix | 8 ++++---- .buildkite/pipeline.yml | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.buildkite/docker-build-push.nix b/.buildkite/docker-build-push.nix index 8a3b761c597..ccefc5a1294 100644 --- a/.buildkite/docker-build-push.nix +++ b/.buildkite/docker-build-push.nix @@ -71,16 +71,16 @@ in # Apply tagging scheme orig_tag="${image.imageName}:${image.imageTag}" - git_branch="''${BUILDKITE_BRANCH:-}" git_tag="''${BUILDKITE_TAG:-}" + git_branch="''${BUILDKITE_BRANCH:-}" tags=() if [[ "$git_tag" =~ ^v20 ]]; then tags+=( "${image.imageTag}" ) tags+=( "latest" ) - elif [[ "$git_tag" = "rc-latest" ]]; then - tags+=( "$git_tag") + elif [[ "$git_branch" =~ ^release-candidate ]]; then + tags+=( "release-candidate" ) else - echo 'Not pushing docker image because this is neither a rc-latest nor a v20* tag build.' + tags+=( "test") fi echo diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fdc7fabafcf..4a058d862da 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -439,7 +439,6 @@ steps: build.branch !~ /^gh-readonly-queue\/master/ && build.branch != "master" && build.env("RELEASE_CANDIDATE") == null - && build.tag != "rc-latest" key: docker-artifacts-block - label: Build Docker Image @@ -452,11 +451,15 @@ steps: agents: system: x86_64-linux + - block: Push Docker Image + if: build.env("RELEASE_CANDIDATE") == null + key: push-docker-block + - label: Push Docker Image - if: build.tag == "rc-latest" depends_on: - linux-nix - docker-build + - push-docker-block key: build-docker command: - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" From 62da2e10efce011edae4138ee727b7a0deb3d2f6 Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 13:48:14 +0000 Subject: [PATCH 6/7] Make docker push depend on docker smoke test --- .buildkite/pipeline.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4a058d862da..d7582a49a9f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -451,15 +451,12 @@ steps: agents: system: x86_64-linux - - block: Push Docker Image - if: build.env("RELEASE_CANDIDATE") == null - key: push-docker-block - - label: Push Docker Image + if: build.env("RELEASE_CANDIDATE") != null depends_on: - linux-nix - docker-build - - push-docker-block + - docker-smoke-test key: build-docker command: - "mkdir -p config && echo '{ outputs = _: { dockerHubRepoName = \"cardanofoundation/cardano-wallet\"; }; }' > config/flake.nix" @@ -471,17 +468,10 @@ steps: - group: Docker Checks key: docker-e2e steps: - - - block: Run Docker Checks - key: docker-tests-block - if: | - build.branch !~ /^gh-readonly-queue\/master/ - && build.branch != "master" - && build.env("RELEASE_CANDIDATE") == null - - label: Smoke Test docker-compose + if: build.env("RELEASE_CANDIDATE") != null + key: docker-smoke-test depends_on: - - docker-tests-block - docker-build timeout_in_minutes: 2 commands: From d57ca1756a31a204eac4c09378350d0aab3aa05f Mon Sep 17 00:00:00 2001 From: paolino Date: Sun, 7 Jul 2024 15:15:43 +0000 Subject: [PATCH 7/7] Increase timeout of docker smoke test , in case the node is replaying --- .buildkite/pipeline.yml | 2 +- scripts/buildkite/release/docker-smoke-test.sh | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d7582a49a9f..c0d68831424 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -473,7 +473,7 @@ steps: key: docker-smoke-test depends_on: - docker-build - timeout_in_minutes: 2 + timeout_in_minutes: 120 commands: - ./scripts/buildkite/release/docker-smoke-test.sh artifact_paths: diff --git a/scripts/buildkite/release/docker-smoke-test.sh b/scripts/buildkite/release/docker-smoke-test.sh index c5ea7e229e2..7f812004831 100755 --- a/scripts/buildkite/release/docker-smoke-test.sh +++ b/scripts/buildkite/release/docker-smoke-test.sh @@ -59,12 +59,9 @@ do echo "$result" | jq break else - sleep 4 + sleep 30 n=$((n+1)) fi - if [ "$n" -ge 30 ] - then break - fi done