[ADP-3224] Fix workflows cache issues and add slack reporting for them (

#4283) - [x] Add blockchain snapshot download on cache-hit fail - [x] Remove support for other networks than preprod - [x] Add slack reporting to release-relevant workflows The result is that tests are running correctly but we still have failures
cardano-foundation · Nov 30, 2023 · a3e908a · a3e908a
2 parents 3eb7067 + 8584171
commit a3e908a
Show file tree

Hide file tree

Showing 14 changed files with 413 additions and 142 deletions.
diff --git a/.github/workflows/docker_linux.yml b/.github/workflows/docker_linux.yml
@@ -3,11 +3,7 @@ on:
   schedule:
   - cron:  "0 23 * * *"
   workflow_dispatch:
-    inputs:
-      network:
-        description: 'NETWORK'
-        required: true
-        default: 'mainnet'
+
 
 jobs:
   build:
@@ -27,4 +23,30 @@ jobs:
           docker-compose up -d
           ./scripts/connect_wallet.rb
         env:
-          NETWORK: ${{ github.event.inputs.network || 'mainnet' }}
+          NETWORK: preprod
+  report:
+    needs: [build]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification on failure
+      if: needs.build.result == 'failure'
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':poop:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#FF0000'
+
+    - name: Slack Notification on success
+      if: needs.build.result == 'success'
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':rocket:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#00FF00'
diff --git a/.github/workflows/docker_macos.yml b/.github/workflows/docker_macos.yml
@@ -3,11 +3,7 @@ on:
   schedule:
   - cron:  "0 23 * * *"
   workflow_dispatch:
-    inputs:
-      network:
-        description: 'NETWORK'
-        required: true
-        default: 'mainnet'
+
 
 jobs:
   build:
@@ -17,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3.2.0
-      - uses: docker-practice/actions-setup-docker@1.0.11
+      - uses: docker-practice/actions-setup-docker@master
       - uses: ruby/setup-ruby@v1.127.0
         with:
           ruby-version: 2.7.1
@@ -28,4 +24,30 @@ jobs:
           docker-compose up -d
           ./scripts/connect_wallet.rb
         env:
-          NETWORK: ${{ github.event.inputs.network || 'mainnet' }}
+          NETWORK: preprod
+  report:
+    needs: [build]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification on failure
+      if: needs.build.result == 'failure'
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':poop:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#FF0000'
+
+    - name: Slack Notification on success
+      if: needs.build.result == 'success'
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':rocket:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#00FF00'
diff --git a/.github/workflows/e2e-docker.yml b/.github/workflows/e2e-docker.yml
@@ -13,10 +13,6 @@ on:
         description: 'Wallet tag (docker)'
         required: true
         default: 'dev-master'
-      network:
-        description: 'Network'
-        required: true
-        default: 'preprod'
       tags:
         description: 'Test tags (all, light, offchain...)'
         default: 'all'
@@ -30,6 +26,14 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    env:
+      TESTS_E2E_FIXTURES: ${{ secrets.TESTS_E2E_FIXTURES }}
+      BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_TOKEN_READ_BUILDS_ARTIFACTS }}
+      WALLET: ${{ github.event.inputs.walletTag || 'dev-master' }}
+      TESTS_E2E_TOKEN_METADATA: https://metadata.cardano-testnet.iohkdev.io/
+      TAGS: ${{ github.event.inputs.tags || 'all' }}
+      E2E_DOCKER_RUN: 1
+
     steps:
     - uses: actions/checkout@v3.2.0
 
@@ -55,41 +59,47 @@ jobs:
       run: bundle install
 
     - name: ⚙️ Setup (get latest bins and configs and decode fixtures)
-      run: rake setup[$NETWORK]
+      run: rake setup[preprod]
 
-    - name: 🕒 Get Date/Time
-      id: date-time
-      shell: bash
-      run: |
-        echo "value=$(rake datetime)" >> $GITHUB_OUTPUT
 
     - name: 💾 Cache node db
-      id: cache
+      id: cache-node
       uses: actions/cache@v3
       with:
-        path: test/e2e/state/node_db/${{ env.NETWORK }}
-        key: node-db-docker-${{ env.NETWORK }}-${{ steps.date-time.outputs.value }}
-        restore-keys: |
-          node-db-docker-${{ env.NETWORK }}-
-          node-db-Linux-${{ env.NETWORK }}-
+        path: test/e2e/state/node_db/preprod
+        key: node-db-docker-linux-preprod
+
+    - name: Fetch preprod snapshot
+      if: steps.cache-node.outputs.cache-hit != 'true'
+      run: |
+        cd state
+        mkdir node_db
+        cd node_db
+        mkdir preprod
+        cd preprod
+        curl -s https://downloads.csnapshots.io/snapshots/testnet/testnet-db-snapshot.json \
+            | jq -r .[].file_name > snapshot.json
+        curl -o - \
+            https://downloads.csnapshots.io/snapshots/testnet/$(cat snapshot.json) \
+            | lz4 -c -d - | tar -x -C .
+        mv db/* .
 
     - name: 💾 Cache wallet db
       id: cache-wallet
       uses: actions/cache@v3
       with:
-        path: test/e2e/state/wallet_db/${{ env.NETWORK }}
-        key: wallet-db3-${{ runner.os }}-${{ env.NETWORK }}-${{ steps.date-time.outputs.value }}
-        restore-keys: wallet-db3-docker-${{ env.NETWORK }}-
+        path: test/e2e/state/wallet_db/preprod
+        key: wallet-db-docker-linux-preprod
 
     - name: 🚀 Start node and wallet
       run: |
         echo "Wallet: $WALLET"
         echo "Node: ${{steps.cardano-node-tag.outputs.NODE_TAG}}"
 
         NODE=${{steps.cardano-node-tag.outputs.NODE_TAG}} \
-        NODE_CONFIG_PATH=`pwd`/state/configs/$NETWORK \
-        DATA=`pwd`/state/node_db/$NETWORK \
-        WALLET_DATA=`pwd`/state/wallet_db/$NETWORK \
+        NODE_CONFIG_PATH=`pwd`/state/configs/preprod \
+        DATA=`pwd`/state/node_db/preprod \
+        WALLET_DATA=`pwd`/state/wallet_db/preprod \
         docker-compose -f docker-compose-test.yml up --detach
 
     - name: 🔍 Display versions
@@ -111,17 +121,44 @@ jobs:
       uses: actions/upload-artifact@v3
       if: always()
       with:
-        name: ${{ runner.os }}-docker-logs
+        name: linux-docker-logs
         path: test/e2e/state/logs
 
     - name: Stop docker-compose
-      run: NODE_CONFIG_PATH=`pwd`/state/configs/$NETWORK docker-compose -f docker-compose-test.yml down
-    env:
-      TESTS_E2E_FIXTURES: ${{ secrets.TESTS_E2E_FIXTURES }}
-      BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_TOKEN_READ_BUILDS_ARTIFACTS }}
-      NETWORK: ${{ github.event.inputs.network || 'preprod' }}
-      WALLET: ${{ github.event.inputs.walletTag || 'dev-master' }}
-      TESTS_E2E_TOKEN_METADATA: https://metadata.cardano-testnet.iohkdev.io/
-      NODE_DB_CACHE: ${{ github.event.inputs.node_db_cache || 'GH' }}
-      TAGS: ${{ github.event.inputs.tags || 'all' }}
-      E2E_DOCKER_RUN: 1
+      run: NODE_CONFIG_PATH=`pwd`/state/configs/preprod docker-compose -f docker-compose-test.yml down
+
+    - name: 💾 GH Save Cache of node db
+      if: always()
+      uses: actions/cache/save@v3
+      with:
+        path: test/e2e/state/node_db/preprod
+        key: node-db-docker-linux-preprod
+
+    - name: 💾 GH Save Cache of wallet db
+      if: always()
+      uses: actions/cache/save@v3
+      with:
+        path: test/e2e/state/wallet_db/preprod
+        key: wallet-db-docker-linux-preprod
+
+    - name: Slack Notification on failure
+      if: failure()
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':poop:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#FF0000'
+
+    - name: Slack Notification on success
+      if: success()
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':rocket:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#00FF00'
diff --git a/.github/workflows/e2e-linux.yml b/.github/workflows/e2e-linux.yml
@@ -5,10 +5,7 @@ on:
   - cron:  "0 20 * * *"
   workflow_dispatch:
     inputs:
-      network:
-        description: 'Network'
-        required: true
-        default: 'preprod'
+
       branch:
         description: 'Run tests against branch'
         default: 'master'
@@ -24,6 +21,12 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    env:
+      TESTS_E2E_FIXTURES: ${{ secrets.TESTS_E2E_FIXTURES }}
+      BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_TOKEN_READ_BUILDS_ARTIFACTS }}
+      BRANCH: ${{ github.event.inputs.branch || '' }}
+      TAGS: ${{ github.event.inputs.tags || 'all' }}
+
     steps:
     - uses: actions/checkout@v3.2.0
 
@@ -36,38 +39,44 @@ jobs:
     - name: Install dependencies
       run: bundle install
 
-    - name: 🕒 Get Date/Time
-      id: date-time
-      shell: bash
-      run: |
-        echo "value=$(rake datetime)" >> $GITHUB_OUTPUT
-
-    - name: 💾 GH Cache node db
-      id: cache
+    - name: 💾 GH Restore Cache of node db
+      id: cache-node
       uses: actions/cache@v3
       with:
-        path: test/e2e/state/node_db/${{ env.NETWORK }}
-        key: node-db-${{ runner.os }}-${{ env.NETWORK }}-${{ steps.date-time.outputs.value }}
-        restore-keys: |
-          node-db-${{ runner.os }}-${{ env.NETWORK }}-
+        path: test/e2e/state/node_db/preprod
+        key: node-db-e2e-linux-preprod
 
     - name: 💾 Cache wallet db
       id: cache-wallet
       uses: actions/cache@v3
       with:
-        path: test/e2e/state/wallet_db/${{ env.NETWORK }}
-        key: wallet-db3-${{ runner.os }}-${{ env.NETWORK }}-${{ steps.date-time.outputs.value }}
-        restore-keys: |
-          wallet-db3-${{ runner.os }}-${{ env.NETWORK }}-
+        path: test/e2e/state/wallet_db/preprod
+        key: wallet-db-e2e-linux-preprod
+
+    - name: Fetch preprod snapshot
+      if: steps.cache-node.outputs.cache-hit != 'true'
+      run: |
+        mkdir state
+        cd state
+        mkdir node_db
+        cd node_db
+        mkdir preprod
+        cd preprod
+        curl -s https://downloads.csnapshots.io/snapshots/testnet/testnet-db-snapshot.json \
+            | jq -r .[].file_name > snapshot.json
+        curl -o - \
+            https://downloads.csnapshots.io/snapshots/testnet/$(cat snapshot.json) \
+            | lz4 -c -d - | tar -x -C .
+        mv db/* .
 
     - name: ⚙️ Setup (get latest bins and configs and decode fixtures)
-      run: rake setup[$NETWORK,$BRANCH]
+      run: rake setup[preprod,$BRANCH]
 
     - name: 🔍 Display versions
       run: rake display_versions
 
     - name: 🚀 Start node and wallet
-      run: rake start_node_and_wallet[$NETWORK]
+      run: rake start_node_and_wallet[preprod]
 
     - name: ⏳ Wait until node is synced
       run: rake wait_until_node_synced
@@ -76,22 +85,50 @@ jobs:
       run: rake spec SPEC_OPTS="-t $TAGS"
 
     - name: 🏁 Stop node and wallet
-      run: rake stop_node_and_wallet[$NETWORK]
+      run: rake stop_node_and_wallet[preprod]
+
+    - name: 💾 GH Save Cache of node db
+      if: always()
+      uses: actions/cache/save@v3
+      with:
+        path: test/e2e/state/node_db/preprod
+        key: node-db-e2e-linux-preprod
+
+    - name: 💾 GH Save Cache of wallet db
+      if: always()
+      uses: actions/cache/save@v3
+      with:
+        path: test/e2e/state/wallet_db/preprod
+        key: wallet-db-e2e-linux-preprod
 
     - name: 📎 Upload state
       uses: actions/upload-artifact@v3
       if: always()
       with:
-        name: ${{ runner.os }}-state
+        name: linux-state
         path: |
           test/e2e/state/logs
           test/e2e/state/configs
           test/e2e/state/wallet_db
 
-    env:
-      TESTS_E2E_FIXTURES: ${{ secrets.TESTS_E2E_FIXTURES }}
-      BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_TOKEN_READ_BUILDS_ARTIFACTS }}
-      NETWORK: ${{ github.event.inputs.network || 'preprod' }}
-      BRANCH: ${{ github.event.inputs.branch || '' }}
-      NODE_DB_CACHE: ${{ github.event.inputs.node_db_cache || 'GH' }}
-      TAGS: ${{ github.event.inputs.tags || 'all' }}
+    - name: Slack Notification on failure
+      if: failure()
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':poop:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#FF0000'
+
+    - name: Slack Notification on success
+      if: success()
+      uses: rtCamp/action-slack-notify@v2
+      env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_ICON_EMOJI: ':rocket:'
+          SLACK_USERNAME: 'GitHub Action'
+          SLACK_MESSAGE: |
+              *Job Link:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_COLOR: '#00FF00'