kaito-e2e.yaml

name: e2e-test

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  push:
    branches: [main]
    paths-ignore: ['docs/**', '**.md', '**.mdx', '**.png', '**.jpg']
  pull_request:
    branches: [main]
    paths-ignore: ['docs/**', '**.md', '**.mdx', '**.png', '**.jpg']
  repository_dispatch:
    types: [ release-tag ]
    branches: [ release-** ]

env:
  GO_VERSION: "1.20"

permissions:
  id-token: write # This is required for requesting the JWT
  contents: read # This is required for actions/checkout

jobs:
  e2e-tests:
    runs-on: ubuntu-latest
    environment: e2e-test
    steps:
      - name: Shorten SHA
        if: ${{ !github.event.client_payload.isRelease }}
        id: vars
        run: echo "pr_sha_short=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})" >> $GITHUB_OUTPUT

      - name: Set e2e Resource and Cluster Name
        run: |
          rand=${{ steps.vars.outputs.pr_sha_short }}

          if [ "$rand" = "" ]; then
             rand=$RANDOM
          fi

          echo "VERSION=${rand}" >> $GITHUB_ENV
          echo "CLUSTER_NAME=kaito${rand}" >> $GITHUB_ENV
          echo "RUN_LLAMA_13B=false" >> $GITHUB_ENV
          echo "REGISTRY=kaito${rand}.azurecr.io" >> $GITHUB_ENV

      - name: Set Registry
        if: ${{ github.event.client_payload.isRelease }}
        run: |
              echo "REGISTRY=${{ github.event.client_payload.registry }}" >> $GITHUB_ENV
              echo "VERSION=$(echo ${{ github.event.client_payload.tag }} | tr -d v)" >> $GITHUB_ENV

      - name: Set up Go ${{ env.GO_VERSION }}
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION  }}

      - name: Checkout
        if: ${{ !github.event.client_payload.isRelease }}
        uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 0
      
      - name: Checkout
        uses: actions/checkout@v4
        if: ${{ github.event.client_payload.isRelease }}
        with:
          fetch-depth: 0
          submodules: true
          ref: ${{ env.REPO_TAG }}
          
      - uses: azure/login@v1.6.1
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - uses: azure/setup-helm@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
        id: install

      - name: Create Resource Group
        shell: bash
        run: |
          make create-rg
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}

      - name: Create ACR
        shell: bash
        run: |
          make create-acr
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}

      - name: Create Azure Identity
        uses: azure/CLI@v1.0.9
        with:
          inlineScript: |
            az identity create --name gpuIdentity --resource-group  ${{ env.CLUSTER_NAME }}

      - name: build KAITO image
        if: ${{ !github.event.client_payload.isRelease }}
        shell: bash
        run: |
          make docker-build-kaito
        env:
          REGISTRY: ${{ env.REGISTRY }}
          VERSION: ${{ env.VERSION }}

      - name: create cluster
        shell: bash
        run: |
          make create-aks-cluster
        env:
          AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}

      - name: Install gpu-provisioner helm chart
        shell: bash
        run: |
          make gpu-provisioner-helm
          kubectl wait --for=condition=available deploy "kaito-gpu-provisioner" -n gpu-provisioner --timeout=300s
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}

      - uses: azure/login@v1.6.1
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

      - name: Create Role Assignment
        uses: azure/CLI@v1.0.9
        with:
          inlineScript: |
            IDENTITY_PRINCIPAL_ID="$(az identity show --name gpuIdentity --resource-group ${{ env.CLUSTER_NAME }} --query 'principalId' -otsv)"
            az role assignment create --assignee ${IDENTITY_PRINCIPAL_ID} --scope "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.CLUSTER_NAME }}"  --role "Contributor"

      - name: Create Azure Federated Identity
        uses: azure/CLI@v1.0.9
        with:
          inlineScript: |
            AKS_OIDC_ISSUER="$(az aks show -n "${{ env.CLUSTER_NAME }}" -g "${{ env.CLUSTER_NAME }}" --query 'oidcIssuerProfile.issuerUrl' -otsv)"
            az identity federated-credential create --name gpu-fed-credential --identity-name gpuIdentity --resource-group "${{ env.CLUSTER_NAME }}" \
            --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"gpu-provisioner:gpu-provisioner" --audience api://AzureADTokenExchange

      - name: Install KAITO Workspace helm chart
        shell: bash
        run: |
          make az-patch-install-helm
          kubectl wait --for=condition=available deploy "kaito-workspace" -n kaito-workspace --timeout=300s
        env:
          AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          REGISTRY: ${{ env.REGISTRY }}
          VERSION: ${{ env.VERSION }}
            
      - name: Add Secret Credentials
        run: |
          kubectl create secret docker-registry ${{secrets.AMRT_SECRET_NAME}} \
          --docker-server=${{secrets.ACR_AMRT_USERNAME}}.azurecr.io \
          --docker-username=${{secrets.ACR_AMRT_USERNAME}} \
          --docker-password=${{secrets.ACR_AMRT_PASSWORD}}
            
      - name: Log kaito-workspace
        run: |
          kubectl get pods -n kaito-workspace -o name | grep "^pod/kaito-workspace" | sed 's/^pod\///' | xargs -I {} kubectl logs -n kaito-workspace {}

      - name: Run e2e test
        run: |
          make kaito-workspace-e2e-test
        env:
          AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
          RUN_LLAMA_13B: ${{ env.RUN_LLAMA_13B }}
          AI_MODELS_REGISTRY: ${{secrets.ACR_AMRT_USERNAME}}.azurecr.io
          AI_MODELS_REGISTRY_SECRET: ${{secrets.AMRT_SECRET_NAME}}

      - name: Cleanup e2e resources
        if: ${{ always() }}
        uses: azure/CLI@v1.0.9
        with:
          inlineScript: |
            set +e
            az group delete --name "${{ env.CLUSTER_NAME }}" --yes --no-wait || true