fix: numerous fixes, including syntax fixes, fixed filepaths and requ…

…irements
kaito-project · Dec 8, 2023 · 542d8ea · 542d8ea
1 parent e2fecf0
commit 542d8ea
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 99 deletions.
diff --git a/.github/matrix-configs.json b/.github/matrix-configs.json
@@ -6,7 +6,7 @@
             "dockerfile": "docker/presets/tfs/Dockerfile",
             "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-7b"
         },
-        "shouldBuildFalcon": "true"
+        "shouldBuildTfs": "true"
     },
     {
         "model": {
@@ -15,25 +15,27 @@
             "dockerfile": "docker/presets/tfs/Dockerfile",
             "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-7b-instruct"
         },
-        "shouldBuildFalcon": "true"
+        "shouldBuildTfs": "true"
     },
+
     {
         "model": {
             "runs_on": "self-hosted",
-            "name": "falcon-7b-instruct",
+            "name": "falcon-7b-instruct-onnx",
             "dockerfile": "docker/presets/tfs-onnx/Dockerfile",
-            "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-7b-instruct"
+            "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-7b-instruct-onnx"
         },
-        "shouldBuildFalcon": "true"
+        "shouldBuildTfs": "true"
     },
+
     {
         "model": {
             "runs_on": "self-hosted",
             "name": "falcon-40b",
             "dockerfile": "docker/presets/tfs/Dockerfile",
             "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-40b"
         },
-        "shouldBuildFalcon": "true"
+        "shouldBuildTfs": "true"
     },
 
     {
@@ -43,15 +45,35 @@
             "dockerfile": "docker/presets/tfs/Dockerfile",
             "build_args": "--build-arg MODEL_NAME=tiiuae/falcon-40b-instruct"
         },
-        "shouldBuildFalcon": "true"
+        "shouldBuildTfs": "true"
+    },
+
+    {
+        "model": {
+            "runs_on": "self-hosted",
+            "name": "mistralai/Mistral-7B-v0.1",
+            "dockerfile": "docker/presets/tfs/Dockerfile",
+            "build_args": "--build-arg MODEL_NAME=mistralai/Mistral-7B-v0.1"
+        },
+        "shouldBuildTfs": "true"
+    },
+
+    {
+        "model": {
+            "runs_on": "self-hosted",
+            "name": "mistralai/Mistral-7B-Instruct-v0.1",
+            "dockerfile": "docker/presets/tfs/Dockerfile",
+            "build_args": "--build-arg MODEL_NAME=mistralai/Mistral-7B-Instruct-v0.1"
+        },
+        "shouldBuildTfs": "true"
     },
 
     {
         "model": {
             "runs_on": "self-hosted",
             "name": "llama-2-7b",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-7b --build-arg SRC_DIR=/home/presets/llama-2"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-7b --build-arg SRC_DIR=/home/presets/llama-2"
         },
         "shouldBuildLlama2": "true"
     },
@@ -61,7 +83,7 @@
             "runs_on": "self-hosted",
             "name": "llama-2-13b",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-13b --build-arg SRC_DIR=/home/presets/llama-2"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-13b --build-arg SRC_DIR=/home/presets/llama-2"
         },
         "shouldBuildLlama2": "true"
     },
@@ -71,7 +93,7 @@
             "runs_on": "self-hosted",
             "name": "llama-2-70b",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-70b --build-arg SRC_DIR=/home/presets/llama-2"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-70b --build-arg SRC_DIR=/home/presets/llama-2"
         },
         "shouldBuildLlama2": "true"
     },
@@ -81,7 +103,7 @@
             "runs_on": "self-hosted",
             "name": "llama-2-7b-chat",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-7b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-7b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
         },
         "shouldBuildLlama2Chat": "true"
     },
@@ -91,7 +113,7 @@
             "runs_on": "self-hosted",
             "name": "llama-2-13b-chat",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-13b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-13b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
         },
         "shouldBuildLlama2Chat": "true"
     },
@@ -101,7 +123,7 @@
             "runs_on": "self-hosted",
             "name": "llama-2-70b-chat",
             "dockerfile": "docker/presets/llama-2/Dockerfile",
-            "build_args": "--build-arg LLAMA_WEIGHTS=/llama/llama-2-70b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
+            "build_args": "--build-arg LLAMA_WEIGHTS=/home/llama/llama-2-70b-chat --build-arg SRC_DIR=/home/presets/llama-2-chat"
         },
         "shouldBuildLlama2Chat": "true"
     }

diff --git a/.github/workflows/preset-image-build.yml b/.github/workflows/preset-image-build.yml
@@ -32,7 +32,7 @@ jobs:
     runs-on: [self-hosted, 'username:runner-0']
     outputs:
       image_tag: ${{ steps.set_tag.outputs.image_tag }}
-      FALCON_MODIFIED: ${{ steps.check_modified_paths.outputs.FALCON_MODIFIED }}
+      TFS_MODIFIED: ${{ steps.check_modified_paths.outputs.TFS_MODIFIED }}
       LLAMA2_MODIFIED: ${{ steps.check_modified_paths.outputs.LLAMA2_MODIFIED }}
       LLAMA2_CHAT_MODIFIED: ${{ steps.check_modified_paths.outputs.LLAMA2_CHAT_MODIFIED }}
     steps:
@@ -67,15 +67,15 @@ jobs:
       - name: Check Modified Paths
         id: check_modified_paths
         run: |
-          FALCON_MODIFIED=false
+          TFS_MODIFIED=false
           LLAMA2_MODIFIED=false
           LLAMA2_CHAT_MODIFIED=false
           IFS=';' read -ra ADDR <<< "$FILES_MODIFIED"
           for file in "${ADDR[@]}"; do
             echo "Checking file: $file"
-            if [[ "$file" == presets/tfs/* ]] && [[ "$FALCON_MODIFIED" == false ]]; then
-              echo "File matches falcon path: $file"
-              FALCON_MODIFIED=true
+            if [[ "$file" == presets/tfs/* ]] && [[ "$TFS_MODIFIED" == false ]]; then
+              echo "File matches tfs path: $file"
+              TFS_MODIFIED=true
             elif [[ "$file" == presets/llama-2/* ]] && [[ "$LLAMA2_MODIFIED" == false ]]; then
               echo "File matches llama-2 path: $file"
               LLAMA2_MODIFIED=true
@@ -86,13 +86,13 @@ jobs:
               echo "File does not match any paths: $file"
             fi
           done
-          echo "FALCON_MODIFIED=$FALCON_MODIFIED" >> $GITHUB_OUTPUT
+          echo "TFS_MODIFIED=$TFS_MODIFIED" >> $GITHUB_OUTPUT
           echo "LLAMA2_MODIFIED=$LLAMA2_MODIFIED" >> $GITHUB_OUTPUT
           echo "LLAMA2_CHAT_MODIFIED=$LLAMA2_CHAT_MODIFIED" >> $GITHUB_OUTPUT
 
       - name: Models to Build
         run: |
-          echo "FALCON_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.FALCON_MODIFIED }}"
+          echo "TFS_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.TFS_MODIFIED }}"
           echo "LLAMA2_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.LLAMA2_MODIFIED }}"
           echo "LLAMA2_CHAT_MODIFIED for this job: ${{ steps.check_modified_paths.outputs.LLAMA2_CHAT_MODIFIED }}"          
 
@@ -151,8 +151,8 @@ jobs:
     - name: "Set matrix"
       id: set_matrix
       run: |
-        matrix=$(jq --arg FALCON_MODIFIED "${{ needs.setup.outputs.FALCON_MODIFIED }}" --arg LLAMA2_MODIFIED "${{ needs.setup.outputs.LLAMA2_MODIFIED }}" --arg LLAMA2_CHAT_MODIFIED "${{ needs.setup.outputs.LLAMA2_CHAT_MODIFIED }}" 'map( 
-            . | select((.shouldBuildFalcon == $FALCON_MODIFIED) or (.shouldBuildLlama2 == $LLAMA2_MODIFIED) or (.shouldBuildLlama2Chat == $LLAMA2_CHAT_MODIFIED))
+        matrix=$(jq --arg TFS_MODIFIED "${{ needs.setup.outputs.TFS_MODIFIED }}" --arg LLAMA2_MODIFIED "${{ needs.setup.outputs.LLAMA2_MODIFIED }}" --arg LLAMA2_CHAT_MODIFIED "${{ needs.setup.outputs.LLAMA2_CHAT_MODIFIED }}" 'map( 
+            . | select((.shouldBuildTfs == $TFS_MODIFIED) or (.shouldBuildLlama2 == $LLAMA2_MODIFIED) or (.shouldBuildLlama2Chat == $LLAMA2_CHAT_MODIFIED))
         )' .github/matrix-configs.json)
         # Check if matrix is empty and set an output variable
         if [[ $matrix == "[]" ]]; then
@@ -202,21 +202,22 @@ jobs:
           echo "ACR_CLIENT_ID=$ACR_CLIENT_ID" >> $GITHUB_OUTPUT
           echo "ACR_CLIENT_SECRET=$ACR_CLIENT_SECRET" >> $GITHUB_OUTPUT
       
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io
+          username: ${{ secrets.ACR_AMRT_USERNAME }}
+          password: ${{ secrets.ACR_AMRT_PASSWORD }}
       # - name: Docker login
-      #   uses: docker/login-action@v3
-      #   with:
-      #     registry: ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io
-      #     username: ${{ secrets.ACR_AMRT_USERNAME }}
-      #     password: ${{ secrets.ACR_AMRT_PASSWORD }}
-        # run: |
-        #   # Echo the current working directory
-        #   echo "Current Working Directory:"
-        #   pwd
-        #   if [[ "${{ steps.acr_name.outputs.ACR_NAME }}" == "aimodelsregistry" ]]; then
-        #     echo ${{ secrets.ACR_AMR_PASSWORD }} | sudo docker login ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMR_USERNAME }} --password-stdin
-        #   else
-        #     echo ${{ secrets.ACR_AMRT_PASSWORD }} | sudo docker login ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMRT_USERNAME }} --password-stdin
-        #   fi
+      #   run: |
+      #     # Echo the current working directory
+      #     echo "Current Working Directory:"
+      #     pwd
+      #     if [[ "${{ steps.acr_name.outputs.ACR_NAME }}" == "aimodelsregistry" ]]; then
+      #       echo ${{ secrets.ACR_AMR_PASSWORD }} | sudo docker login ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMR_USERNAME }} --password-stdin
+      #     else
+      #       echo ${{ secrets.ACR_AMRT_PASSWORD }} | sudo docker login ${{ steps.acr_name.outputs.ACR_NAME }}.azurecr.io --username ${{ secrets.ACR_AMRT_USERNAME }} --password-stdin
+      #     fi
 
       - name: Build model (with retries)
         run: |

diff --git a/docker/presets/tfs-onnx/Dockerfile b/docker/presets/tfs-onnx/Dockerfile
@@ -9,13 +9,13 @@ WORKDIR /workspace/tfs
 # First, copy just the requirements.txt file and install dependencies
 # This is done before copying the code to utilize Docker's layer caching and
 # avoid reinstalling dependencies unless the requirements file changes.
-COPY /home/presets/tfs-onnx/requirements.txt /workspace/tfs/requirements.txt
+COPY docker/presets/tfs-onnx/requirements.txt /workspace/tfs/requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Convert to ONNX Runtime
-RUN python convert_to_onnx.py ${MODEL_NAME}
+# RUN python convert_to_onnx.py ${MODEL_NAME} 
 
 # Copy the entire onnx model to the weights directory
-COPY /tfs/${MODEL_NAME} /workspace/tfs/weights
+COPY /home/tfs/${MODEL_NAME} /workspace/tfs/weights
 # Copy the entire 'presets/tfs-onnx' folder to the working directory
-COPY /home/presets/tfs-onnx /workspace/tfs
+COPY docker/presets/tfs-onnx /workspace/tfs
diff --git a/docker/presets/tfs/Dockerfile b/docker/presets/tfs/Dockerfile
@@ -8,10 +8,10 @@ WORKDIR /workspace/tfs
 # First, copy just the requirements.txt file and install dependencies
 # This is done before copying the code to utilize Docker's layer caching and
 # avoid reinstalling dependencies unless the requirements file changes.
-COPY /home/presets/tfs/requirements.txt /workspace/tfs/requirements.txt
+COPY docker/presets/tfs/requirements.txt /workspace/tfs/requirements.txt
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy the entire model to the weights directory
-COPY /tfs/${MODEL_NAME} /workspace/tfs/weights
+COPY /home/tfs/${MODEL_NAME} /workspace/tfs/weights
 # Copy the entire 'presets/falcon' folder to the working directory
-COPY /home/presets/tfs /workspace/tfs
+COPY docker/presets/tfs /workspace/tfs
diff --git a/pkg/inference/preset-inference-types.go b/pkg/inference/preset-inference-types.go
@@ -36,7 +36,7 @@ const (
 )
 
 const (
-	DefaultModelId = "MODEL_ID"
+	DefaultModelId  = "MODEL_ID"
 	DefaultPipeline = "text-generation"
 )
 
@@ -49,7 +49,7 @@ var (
 	presetFalcon40bImage         = registryName + fmt.Sprintf("/kaito-%s:0.0.1", kaitov1alpha1.PresetFalcon40BModel)
 	presetFalcon40bInstructImage = registryName + fmt.Sprintf("/kaito-%s:0.0.1", kaitov1alpha1.PresetFalcon40BInstructModel)
 
-	presetMistral7bImage = registryName + fmt.Sprintf("/kaito-%s:0.0.1", kaitov1alpha1.PresetMistral7BModel)
+	presetMistral7bImage         = registryName + fmt.Sprintf("/kaito-%s:0.0.1", kaitov1alpha1.PresetMistral7BModel)
 	presetMistral7bInstructImage = registryName + fmt.Sprintf("/kaito-%s:0.0.1", kaitov1alpha1.PresetMistral7BInstructModel)
 
 	baseCommandPresetLlama = "cd /workspace/llama/llama-2 && torchrun"
@@ -97,7 +97,7 @@ var (
 // PresetInferenceParam defines the preset inference.
 type PresetInferenceParam struct {
 	ModelName              string
-	ModelId				   string
+	ModelId                string
 	Image                  string
 	ImagePullSecrets       []corev1.LocalObjectReference
 	AccessMode             string
@@ -231,7 +231,7 @@ var (
 	FalconPresetInferences = map[kaitov1alpha1.ModelName]PresetInferenceParam{
 		kaitov1alpha1.PresetFalcon7BModel: {
 			ModelName:              "Falcon",
-			ModelId: 				"tiiuae/falcon-7b",
+			ModelId:                "tiiuae/falcon-7b",
 			Image:                  presetFalcon7bImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,
@@ -247,7 +247,7 @@ var (
 		},
 		kaitov1alpha1.PresetFalcon7BInstructModel: {
 			ModelName:              "Falcon",
-			ModelId: 				"tiiuae/falcon-7b-instruct",
+			ModelId:                "tiiuae/falcon-7b-instruct",
 			Image:                  presetFalcon7bInstructImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,
@@ -264,7 +264,7 @@ var (
 
 		kaitov1alpha1.PresetFalcon40BModel: {
 			ModelName:              "Falcon",
-			ModelId:				"tiiuae/falcon-40b",
+			ModelId:                "tiiuae/falcon-40b",
 			Image:                  presetFalcon40bImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,
@@ -281,7 +281,7 @@ var (
 
 		kaitov1alpha1.PresetFalcon40BInstructModel: {
 			ModelName:              "Falcon",
-			ModelId: 				"tiiuae/falcon-40b-instruct",
+			ModelId:                "tiiuae/falcon-40b-instruct",
 			Image:                  presetFalcon40bInstructImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,
@@ -295,13 +295,13 @@ var (
 			BaseCommand:            baseCommandPresetTsf,
 			DefaultVolumeMountPath: "/dev/shm",
 		},
-	},
+	}
 
 	// MistralPresetInferences defines the preset inferences for Mistral.
 	MistralPresetInferences = map[kaitov1alpha1.ModelName]PresetInferenceParam{
 		kaitov1alpha1.PresetMistral7BModel: {
 			ModelName:              "Mistral",
-			ModelId: 				"mistralai/Mistral-7B-v0.1",
+			ModelId:                "mistralai/Mistral-7B-v0.1",
 			Image:                  presetMistral7bImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,
@@ -315,9 +315,10 @@ var (
 			BaseCommand:            baseCommandPresetTsf,
 			DefaultVolumeMountPath: "/dev/shm",
 		},
+
 		kaitov1alpha1.PresetMistral7BInstructModel: {
 			ModelName:              "Mistral",
-			ModelId: 				"mistralai/Mistral-7B-Instruct-v0.1",
+			ModelId:                "mistralai/Mistral-7B-Instruct-v0.1",
 			Image:                  presetMistral7bInstructImage,
 			ImagePullSecrets:       defaultImagePullSecrets,
 			AccessMode:             defaultAccessMode,

diff --git a/presets/test/docker.yaml b/presets/test/docker.yaml