[pre-commit.ci] pre-commit autoupdate #1712
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [master] | |
| tags: [v*.*.*] | |
| pull_request: | |
| branches: [master] | |
| pull_request_target: | |
| branches: [master] | |
| concurrency: CI | |
| jobs: | |
| build: | |
| if: | | |
| (github.event_name != 'pull_request_target' && github.actor != 'dependabot[bot]') || | |
| (github.event_name == 'pull_request_target' && github.actor == 'dependabot[bot]') | |
| strategy: | |
| matrix: | |
| base-image: | |
| - nvidia/cuda:12.6.2-cudnn-devel-ubuntu24.04 | |
| - nvidia/cuda:12.6.2-cudnn-runtime-ubuntu24.04 | |
| dockerfile: | |
| - Dockerfile | |
| - Dockerfile.minimal | |
| env: | |
| IMAGE_NAME: ghcr.io/neuro-inc/base | |
| BASE_IMAGE: ${{ matrix.base-image }} | |
| name: Build, test and publish image | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| steps: | |
| - name: Checkout commit | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Install dependencies | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -U apolo-cli | |
| - name: Configure environment | |
| run: | | |
| source venv/bin/activate | |
| apolo config login-with-token ${{ secrets.CLIENT_TEST_E2E_USER_NAME }} https://api.dev.apolo.us/api/v1 | |
| apolo config show | |
| - name: Login ghcr.io | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Get image tags | |
| id: get-image-tags | |
| run: | | |
| if [[ ${{ matrix.base-image }} =~ runtime ]]; then | |
| export BASE_IMAGE_TYPE="runtime"; | |
| elif [[ ${{ matrix.base-image }} =~ devel ]]; then | |
| export BASE_IMAGE_TYPE="devel"; | |
| fi | |
| if [[ ${{ matrix.dockerfile }} =~ minimal ]]; then | |
| export BASE_IMAGE_TYPE="$BASE_IMAGE_TYPE-minimal"; | |
| fi | |
| echo "::set-output name=BASE_IMAGE_TYPE::$BASE_IMAGE_TYPE" | |
| echo "::set-output name=platform_image_tag::ghcr.io/neuro-inc/base:built-$BASE_IMAGE_TYPE" | |
| - name: Cleanup tool cache | |
| run: | | |
| du -h -d 1 /opt/hostedtoolcache | |
| rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL | |
| - name: Build image | |
| env: | |
| BASE_IMAGE_TYPE: ${{ steps.get-image-tags.outputs.BASE_IMAGE_TYPE }} | |
| DOCKERFILE: ${{ matrix.dockerfile }} | |
| run: | | |
| make image_build | |
| - name: List pip packages minimal | |
| if: ${{ matrix.dockerfile == 'Dockerfile.minimal' }} | |
| id: list-pip-packages-minimal | |
| run: | | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} pip list | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} pip freeze > pip_dependencies.txt | |
| env: | |
| IMAGE_TAG: ${{ steps.get-image-tags.outputs.platform_image_tag }} | |
| - name: List pip packages | |
| if: ${{ matrix.dockerfile == 'Dockerfile' }} | |
| id: list-pip-packages | |
| run: | | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} pip list | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} pip freeze > pip_dependencies.txt | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} bash -c "source /opt/conda/bin/activate tf && pip freeze" > pip_dependencies_tf.txt | |
| docker run --tty --rm ${{ steps.get-image-tags.outputs.platform_image_tag }} bash -c "source /opt/conda/bin/activate torch && pip freeze" > pip_dependencies_torch.txt | |
| env: | |
| IMAGE_TAG: ${{ steps.get-image-tags.outputs.platform_image_tag }} | |
| - name: Test image | |
| env: | |
| BASE_IMAGE_TYPE: ${{ steps.get-image-tags.outputs.BASE_IMAGE_TYPE }} | |
| TEST_PRESET: gpu-l4-x1 | |
| run: | | |
| source venv/bin/activate | |
| make e2e_neuro_push | |
| if [[ ${{ matrix.dockerfile }} =~ minimal ]]; then | |
| export TEST_CMD="bash /var/storage/dependencies.minimal.sh"; | |
| fi | |
| make test_dependencies | |
| - name: Push release | |
| env: | |
| BASE_IMAGE_TYPE: ${{ steps.get-image-tags.outputs.BASE_IMAGE_TYPE }} | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| export TARGET_IMAGE_TAGS="latest-$BASE_IMAGE_TYPE,${GITHUB_REF#refs/tags/}-$BASE_IMAGE_TYPE"; | |
| if [[ $BASE_IMAGE_TYPE == runtime ]]; then | |
| export TARGET_IMAGE_TAGS="latest,$TARGET_IMAGE_TAGS"; | |
| fi | |
| echo "$TARGET_IMAGE_TAGS" > released_tags.txt | |
| make image_deploy | |
| if [[ $BASE_IMAGE_TYPE == runtime ]]; then | |
| # publish legacy images | |
| TARGET_IMAGE_NAME="neuromation/base" | |
| make image_deploy | |
| fi | |
| - name: Append Docker images and dependencies to release notes (with truncation) | |
| if: startsWith(github.ref, 'refs/tags/') && matrix.dockerfile == 'Dockerfile' | |
| id: update-release | |
| continue-on-error: true | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const MAX_BODY_LENGTH = 125000; | |
| const fs = require('fs'); | |
| const pipDependencies = fs.readFileSync('pip_dependencies.txt', 'utf8'); | |
| const pipDependenciesTorch = fs.readFileSync('pip_dependencies_torch.txt', 'utf8'); | |
| const pipDependenciesTF = fs.readFileSync('pip_dependencies_tf.txt', 'utf8'); | |
| const baseImage = "ghcr.io/neuro-inc/base"; | |
| const releasedTagsContent = fs.readFileSync('released_tags.txt', 'utf8'); | |
| console.log('Raw released_tags.txt content:', releasedTagsContent); | |
| const dockerImages = releasedTagsContent | |
| .split(',') | |
| .map(tag => `- \`${baseImage}:${tag.trim()}\``) | |
| .join('\n'); | |
| console.log('Generated Docker images list:\n', dockerImages); | |
| const newReleaseContent = ` | |
| ## Docker Images Released | |
| \`\`\` | |
| ${dockerImages} | |
| \`\`\` | |
| ## Pip Dependencies | |
| \`\`\` | |
| ${pipDependencies} | |
| \`\`\` | |
| ## Pip Dependencies torch env | |
| \`\`\` | |
| ${pipDependenciesTorch} | |
| \`\`\` | |
| ## Pip Dependencies tf env | |
| \`\`\` | |
| ${pipDependenciesTF} | |
| \`\`\` | |
| `; | |
| const tagRef = process.env.GITHUB_REF; | |
| if (!tagRef || !tagRef.startsWith('refs/tags/')) { | |
| throw new Error('GITHUB_REF does not contain a tag reference. Ensure this action runs on a tag event.'); | |
| } | |
| const tagName = tagRef.replace('refs/tags/', ''); | |
| console.log('Tag name:', tagName); | |
| try { | |
| // Fetch the existing release | |
| const existingRelease = await github.rest.repos.getReleaseByTag({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| tag: tagName | |
| }); | |
| console.log('Existing release found:', existingRelease.data.id); | |
| // Get the current release body | |
| let currentBody = existingRelease.data.body || ''; | |
| console.log('Current release body length:', currentBody.length); | |
| // Append the new content to the existing body | |
| let updatedBody = `${currentBody}\n${newReleaseContent}`; | |
| // Truncate if the updated body exceeds the limit | |
| if (updatedBody.length > MAX_BODY_LENGTH) { | |
| console.log('Release body exceeds the maximum allowed length. Truncating...'); | |
| currentBody = currentBody.slice(-(MAX_BODY_LENGTH - newReleaseContent.length)); // Keep the most recent content | |
| updatedBody = `${currentBody}\n${newReleaseContent}`; | |
| } | |
| console.log('Updated release body length:', updatedBody.length); | |
| // Update the release with the appended and (if necessary) truncated content | |
| await github.rest.repos.updateRelease({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| release_id: existingRelease.data.id, | |
| body: updatedBody | |
| }); | |
| console.log('Release updated successfully with appended content.'); | |
| } catch (error) { | |
| if (error.status === 404) { | |
| console.log('Release not found, creating a new release...'); | |
| // Create a new release if none exists | |
| await github.rest.repos.createRelease({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| tag_name: tagName, | |
| name: `Release ${tagName}`, | |
| body: newReleaseContent, | |
| draft: false, | |
| prerelease: false | |
| }); | |
| console.log('New release created successfully.'); | |
| } else { | |
| console.error('Error checking or creating release:', error); | |
| throw error; | |
| } | |
| } | |
| - name: Append Docker images and dependencies to release notes minimal | |
| if: startsWith(github.ref, 'refs/tags/') && matrix.dockerfile == 'Dockerfile.minimal' | |
| id: update-release-minimal | |
| continue-on-error: true | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const MAX_BODY_LENGTH = 125000; | |
| const fs = require('fs'); | |
| const pipDependencies = fs.readFileSync('pip_dependencies.txt', 'utf8'); | |
| const baseImage = "ghcr.io/neuro-inc/base"; | |
| const releasedTagsContent = fs.readFileSync('released_tags.txt', 'utf8'); | |
| console.log('Raw released_tags.txt content:', releasedTagsContent); | |
| const dockerImages = releasedTagsContent | |
| .split(',') | |
| .map(tag => `- \`${baseImage}:${tag.trim()}\``) | |
| .join('\n'); | |
| console.log('Generated Docker images list:\n', dockerImages); | |
| const tagRef = process.env.GITHUB_REF; | |
| if (!tagRef || !tagRef.startsWith('refs/tags/')) { | |
| throw new Error('GITHUB_REF does not contain a tag reference. Ensure this action runs on a tag event.'); | |
| } | |
| const tagName = tagRef.replace('refs/tags/', '').trim(); | |
| console.log('Tag name:', tagName); | |
| const newReleaseContent = ` | |
| ## Docker Images Released | |
| \`\`\` | |
| ${dockerImages} | |
| \`\`\` | |
| ## Pip Dependencies | |
| \`\`\` | |
| ${pipDependencies} | |
| \`\`\` | |
| `; | |
| try { | |
| const existingRelease = await github.rest.repos.getReleaseByTag({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| tag: tagName | |
| }); | |
| console.log('Existing release found:', existingRelease.data.id); | |
| let currentBody = existingRelease.data.body || ''; | |
| console.log('Current release body length:', currentBody.length); | |
| let updatedBody = `${currentBody}\n${newReleaseContent}`; | |
| if (updatedBody.length > MAX_BODY_LENGTH) { | |
| console.log('Release body exceeds the maximum allowed length. Truncating...'); | |
| currentBody = currentBody.slice(-(MAX_BODY_LENGTH - newReleaseContent.length)); | |
| updatedBody = `${currentBody}\n${newReleaseContent}`; | |
| } | |
| console.log('Updated release body length:', updatedBody.length); | |
| await github.rest.repos.updateRelease({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| release_id: existingRelease.data.id, | |
| body: updatedBody | |
| }); | |
| console.log('Release updated successfully with appended content.'); | |
| } catch (error) { | |
| if (error.status === 404) { | |
| console.log('Release not found, creating a new release...'); | |
| await github.rest.repos.createRelease({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| tag_name: tagName, | |
| name: `Release ${tagName}`, | |
| body: newReleaseContent, | |
| draft: false, | |
| prerelease: false | |
| }); | |
| console.log('New release created successfully.'); | |
| } else { | |
| console.error('Error checking or creating release:', error); | |
| throw error; | |
| } | |
| } |