From a358d9e1ba03373cd4858218324db2df5a60a496 Mon Sep 17 00:00:00 2001 From: Guilherme Caponetto <638737+caponetto@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:10:28 -0300 Subject: [PATCH] Build a notebook-based image with the current Elyra code (#78) --- .github/workflows/build.yml | 73 +++++++++++++++++++-- .github/workflows/purge-ghcr.yaml | 31 +++++++++ .github/workflows/scripts/free_gha_space.sh | 23 +++++++ 3 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/purge-ghcr.yaml create mode 100755 .github/workflows/scripts/free_gha_space.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11910defc..707764ba7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: Elyra Tests +name: Elyra CI on: push: # all branches pull_request: # all branches @@ -26,6 +26,8 @@ env: NODE_VERSION: 20.11.0 YARN_VERSION: 3.5.0 PYTHON_VERSION: 3.11 + QUAY_WORKBENCH_IMAGES_REPOSITORY: 'opendatahub/workbench-images' + QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV: 'jupyter-trustyai-ubi9-python-3.11-20241,cuda-jupyter-tensorflow-ubi9-python-3.11-20241' jobs: prepare-yarn-cache: @@ -230,8 +232,10 @@ jobs: - name: Validate runtime images run: make REMOVE_RUNTIME_IMAGE=1 validate-runtime-images - upload-artifacts: - name: Upload Artifacts + publish-artifacts: + permissions: + packages: write + name: Publish Artifacts needs: prepare-yarn-cache runs-on: ubuntu-latest steps: @@ -254,10 +258,71 @@ jobs: corepack prepare yarn@${{ env.YARN_VERSION }} --activate yarn set version ${{ env.YARN_VERSION }} yarn --version + - name: Check whether to build container images + id: should-build-images + run: | + echo "result=$([[ '${{ github.event_name }}' != 'pull_request' || '${{ github.event.pull_request.head.repo.full_name }}' == '${{ github.repository }}' ]] && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT + - name: Free up additional disk space + if: steps.should-build-images.outputs.result == 'true' + run: | + ./.github/workflows/scripts/free_gha_space.sh - name: Build run: | make install-prod - - name: Upload artifacts + - name: Login to GitHub Container Registry + if: steps.should-build-images.outputs.result == 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build notebook images + if: steps.should-build-images.outputs.result == 'true' + id: build-notebook-images + run: | + TAG_SUFFIX=$([ -n "${{ github.event.pull_request.number }}" ] && echo "-pr-${{ github.event.pull_request.number }}" || echo "-sha-$(echo "${{ github.sha }}" | cut -c1-8)") + echo "Tag suffix: $TAG_SUFFIX" + + WHL_FILE_PATH=$(ls dist/*.whl | head -n 1) + WHL_FILE_NAME=$(basename $WHL_FILE_PATH) + echo "Wheel file: $WHL_FILE_PATH" + + URLS="" + for tag_prefix in $(echo "${{ env.QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV }}" | tr ',' '\n'); do + LATEST_TAG=$(curl -s "https://quay.io/api/v1/repository/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}/tag/?onlyActiveTags=true" | \ + jq -r ".tags | map(select(.name | startswith(\"${tag_prefix}\"))) | .[].name" | \ + sort -r | head -n 1) + echo "Latest tag: $LATEST_TAG" + + CONTAINER_IMAGE=ghcr.io/${{ github.repository }}/workbench-images:${LATEST_TAG}${TAG_SUFFIX} + echo "Build container image: $CONTAINER_IMAGE" + + docker build -t ${CONTAINER_IMAGE} -f - . <> "$GITHUB_OUTPUT" + done + - name: Print container image URLs + if: steps.should-build-images.outputs.result == 'true' + run: | + echo ${{ steps.build-notebook-images.outputs.urls }} | tr ',' '\n' + - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: elyra_build_artifacts diff --git a/.github/workflows/purge-ghcr.yaml b/.github/workflows/purge-ghcr.yaml new file mode 100644 index 000000000..288bc93b3 --- /dev/null +++ b/.github/workflows/purge-ghcr.yaml @@ -0,0 +1,31 @@ +name: 'Purge old ghcr.io test images periodically' + +on: + workflow_dispatch: + inputs: + dry_run: + type: boolean + default: true + description: 'Do a dry run?' + schedule: + - cron: '0 5 * * *' # at 05:00 UTC every day + +permissions: + packages: write + +jobs: + clean: + runs-on: ubuntu-latest + name: Delete old test images + steps: + - uses: snok/container-retention-policy@4f22ef80902ad409ed55a99dc5133cc1250a0d03 # v3.0.0 + with: + account: ${{ (github.repository_owner == github.actor) && 'user' || github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: | + ${{ github.event.repository.name }}/workbench-images + image-tags: '*' + cut-off: '3w' + dry-run: ${{ inputs.dry_run || false }} + env: + RUST_BACKTRACE: 1 diff --git a/.github/workflows/scripts/free_gha_space.sh b/.github/workflows/scripts/free_gha_space.sh new file mode 100755 index 000000000..a6aeb5fed --- /dev/null +++ b/.github/workflows/scripts/free_gha_space.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +df -h + +sudo apt-get update +sudo apt-get remove -y '^dotnet-.*' +sudo apt-get remove -y '^llvm-.*' +sudo apt-get remove -y 'php.*' +sudo apt-get remove -y '^mongodb-.*' +sudo apt-get autoremove -y +sudo apt-get clean +sudo rm -rf /usr/local/.ghcup & +sudo rm -rf /usr/local/lib/android & +sudo rm -rf /usr/local/share/boost & +sudo rm -rf /usr/share/dotnet & +sudo rm -rf /opt/ghc & +sudo rm -rf /opt/hostedtoolcache/CodeQL & + +sudo docker image prune --all --force & + +wait + +df -h