diff --git a/.dockerignore b/.dockerignore index 4c49bd78..bc978290 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,14 @@ +.DS_Store +.dev +.dockerignore .env +.github +.idea +.pre-commit.log +.tekton +.vscode +Dockerfile +devbuild +docs +fvt +temp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b05f93f7..21ced579 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,11 @@ repos: rev: v1.51.2 hooks: - id: golangci-lint + entry: golangci-lint run + exclude: ^(generated/) + args: + - --skip-dirs='^(generated/)' + log_file: .pre-commit.log - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.4.1 hooks: @@ -23,3 +28,4 @@ repos: exclude: ^(.github/|.tekton/) args: - --no-bracket-spacing + log_file: .pre-commit.log diff --git a/.tekton/listener.yaml b/.tekton/listener.yaml index 946dcbd6..f5a58a59 100644 --- a/.tekton/listener.yaml +++ b/.tekton/listener.yaml @@ -17,14 +17,9 @@ spec: - name: region description: the ibmcloud registry region default: us-south - - name: org - description: organization - - name: space - description: space - default: dev - name: resource-group description: resource group - default: default + default: Default - name: docker-username description: ibm container registry username default: iamapikey @@ -112,10 +107,6 @@ spec: value: $(params.dockersandbox-namespace) - name: resource-group value: $(params.resource-group) - - name: org - value: $(params.org) - - name: space - value: $(params.space) - name: kubernetes-cluster value: $(params.kubernetes-cluster) - name: serving-ns diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml index 2aaca8b4..80f65f90 100644 --- a/.tekton/pipeline.yaml +++ b/.tekton/pipeline.yaml @@ -32,13 +32,9 @@ spec: description: dockersandbox namespace - name: resource-group description: resource group - default: default - - name: org - description: organization + default: Default - name: region description: region - - name: space - description: space - name: serving-ns description: modelmesh serving namespace default: modelmesh-serving @@ -80,12 +76,8 @@ spec: value: $(params.docker-password) - name: resource-group value: $(params.resource-group) - - name: org - value: $(params.org) - name: region value: $(params.region) - - name: space - value: $(params.space) workspaces: - name: task-pvc workspace: pipeline-pvc @@ -260,12 +252,8 @@ spec: value: $(params.serving-ns) - name: resource-group value: $(params.resource-group) - - name: org - value: $(params.org) - name: region value: $(params.region) - - name: space - value: $(params.space) - name: test-commands value: | kubectl get ns diff --git a/.tekton/task.yaml b/.tekton/task.yaml index 6f36cd83..65e4c71a 100644 --- a/.tekton/task.yaml +++ b/.tekton/task.yaml @@ -20,13 +20,10 @@ spec: description: iam api key - name: resource-group description: resource group - default: default - - name: org - description: organization + default: Default - name: region description: region - - name: space - description: space + default: us-south - name: archive-dir description: archive directory default: "." @@ -35,7 +32,7 @@ spec: mountPath: /artifacts steps: - name: run-setup-script - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: DOCKER_USERNAME value: $(params.docker-username) @@ -47,10 +44,6 @@ spec: value: $(params.repository) - name: REGION value: $(params.region) - - name: ORG - value: $(params.org) - - name: SPACE - value: $(params.space) - name: RESOURCE_GROUP value: $(params.resource-group) - name: GIT_BRANCH @@ -116,13 +109,12 @@ spec: description: docker sandbox namespace - name: publish-tag description: image publish tag - workspaces: - name: task-pvc mountPath: /artifacts steps: - name: process-image - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: DOCKER_USERNAME value: $(params.docker-username) @@ -243,7 +235,7 @@ spec: mountPath: /artifacts steps: - name: deploy-modelmesh-serving - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: DOCKER_USERNAME value: $(params.docker-username) @@ -295,7 +287,7 @@ spec: mountPath: /artifacts steps: - name: run-test - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: IBM_CLOUD_API_KEY value: $(params.apikey) @@ -347,7 +339,7 @@ spec: mountPath: /artifacts steps: - name: run-performance-test - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: SERVING_KUBERNETES_CLUSTER_NAME value: $(params.kubernetes-cluster) @@ -384,7 +376,7 @@ spec: mountPath: /artifacts steps: - name: undeploy-modelmesh-serving - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: SERVING_KUBERNETES_CLUSTER_NAME value: $(params.kubernetes-cluster) @@ -415,13 +407,10 @@ spec: default: kubeflow - name: resource-group description: resource group - default: default - - name: org - description: organization + default: Default - name: region description: region - - name: space - description: space + default: us-south - name: archive-dir description: archive directory default: "." @@ -433,7 +422,7 @@ spec: mountPath: /artifacts steps: - name: run-cleanup-script - image: docker.io/aipipeline/pipeline-base-image:1.2 + image: docker.io/aipipeline/pipeline-base-image:1.5 env: - name: IBM_CLOUD_API_KEY value: $(params.apikey) @@ -445,10 +434,6 @@ spec: value: $(params.serving-ns) - name: REGION value: $(params.region) - - name: ORG - value: $(params.org) - - name: SPACE - value: $(params.space) - name: RESOURCE_GROUP value: $(params.resource-group) - name: PIPELINE_URL @@ -481,7 +466,7 @@ spec: # Set up kubernetes config retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region - retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" + retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" retry 3 3 ibmcloud ks cluster config -c "$SERVING_KUBERNETES_CLUSTER_NAME" $(params.test-commands) diff --git a/Dockerfile b/Dockerfile index b8335857..3ef1be91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.3 + # Copyright 2021 IBM Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE: for syntax, either use "experimental" or "1.3" (or later) to enable multi-arch build with mount option +# see https://hub.docker.com/r/docker/dockerfile (https://github.com/moby/buildkit/releases/tag/dockerfile%2F1.3.0) + ############################################################################### # Stage 1: Run the go build with go compiler native to the build platform # https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ diff --git a/Dockerfile.develop b/Dockerfile.develop index bdd425bf..5f02b696 100644 --- a/Dockerfile.develop +++ b/Dockerfile.develop @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.3 + # Copyright 2021 IBM Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE: for syntax, either use "experimental" or "1.3" (or later) to enable multi-arch build with mount option +# see https://hub.docker.com/r/docker/dockerfile (https://github.com/moby/buildkit/releases/tag/dockerfile%2F1.3.0) + ############################################################################### # Create the develop, test, and build environment ############################################################################### diff --git a/controllers/suite_test.go b/controllers/suite_test.go index e090467b..39075a83 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -11,21 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ package controllers diff --git a/pkg/predictor_source/inferenceservice_registry.go b/pkg/predictor_source/inferenceservice_registry.go index 726b6784..3c8bc21c 100644 --- a/pkg/predictor_source/inferenceservice_registry.go +++ b/pkg/predictor_source/inferenceservice_registry.go @@ -1,18 +1,16 @@ -/* -Copyright 2021 IBM Corporation - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +// Copyright 2021 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package predictor_source diff --git a/scripts/deploy/iks/build-image-dind.sh b/scripts/deploy/iks/build-image-dind.sh index 7248ad74..4b39f2bc 100644 --- a/scripts/deploy/iks/build-image-dind.sh +++ b/scripts/deploy/iks/build-image-dind.sh @@ -13,17 +13,15 @@ set -xe # The following envs could be loaded from `build.properties` that # `run-setup.sh` generates. # - REGION: cloud region (us-south as default) -# - ORG: target organization (dev-advo as default) -# - SPACE: target space (dev as default) # - GIT_BRANCH: git branch # - GIT_COMMIT: git commit hash # - GIT_COMMIT_SHORT: git commit hash short REGION=${REGION:-"us-south"} -ORG=${ORG:-"dev-advo"} -SPACE=${SPACE:-"dev"} RUN_TASK=${RUN_TASK:-"build"} +export DOCKER_BUILDKIT=1 + retry() { local max=$1; shift local interval=$1; shift @@ -39,19 +37,25 @@ retry() { } retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region -retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" +retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" + ###################################################################################### # Build image # ###################################################################################### build_image() { - echo "=======================Build modelmesh controller image=======================" + echo "=======================Build ModelMesh controller image=======================" # Will build develop and then runtime images. + docker version + # docker pull docker/dockerfile:experimental # syntax=docker/dockerfile:experimental + docker pull docker/dockerfile:1.3 # syntax=docker/dockerfile:1.3 + echo "==============================Build dev image ================================" make build.develop docker images docker inspect "kserve/modelmesh-controller-develop:latest" + echo "==========================Build runtime image ================================" make build docker images diff --git a/scripts/deploy/iks/deploy-mm-serving.sh b/scripts/deploy/iks/deploy-mm-serving.sh index 7f7627b9..2e4cec84 100644 --- a/scripts/deploy/iks/deploy-mm-serving.sh +++ b/scripts/deploy/iks/deploy-mm-serving.sh @@ -18,8 +18,6 @@ echo "GIT_BRANCH=${GIT_BRANCH}" echo "GIT_COMMIT=${GIT_COMMIT}" echo "GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}" echo "REGION=${REGION}" -echo "ORG=${ORG}" -echo "SPACE=${SPACE}" echo "RESOURCE_GROUP=${RESOURCE_GROUP}" # These env vars should come from the pipeline run environment properties @@ -46,7 +44,7 @@ retry() { } retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region -retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" +retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" retry 3 3 ibmcloud ks cluster config -c "$SERVING_KUBERNETES_CLUSTER_NAME" kubectl create ns "$SERVING_NS" diff --git a/scripts/deploy/iks/run-perf-test.sh b/scripts/deploy/iks/run-perf-test.sh index c9616489..374676fe 100755 --- a/scripts/deploy/iks/run-perf-test.sh +++ b/scripts/deploy/iks/run-perf-test.sh @@ -23,7 +23,7 @@ retry() { } retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region -retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" +retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" retry 3 3 ibmcloud ks cluster config -c "$SERVING_KUBERNETES_CLUSTER_NAME" kubectl config set-context --current --namespace=${SERVING_NS} diff --git a/scripts/deploy/iks/run-setup.sh b/scripts/deploy/iks/run-setup.sh index 67798f01..90f7dd85 100644 --- a/scripts/deploy/iks/run-setup.sh +++ b/scripts/deploy/iks/run-setup.sh @@ -5,13 +5,9 @@ set -xe # Environment variables needed by this script: # - REGION: cloud region (us-south as default) -# - ORG: target organization (dev-advo as default) -# - SPACE: target space (dev as default) REGION=${REGION:-"us-south"} -ORG=${ORG:-"dev-advo"} -SPACE=${SPACE:-"dev"} -RESOURCE_GROUP=${RESOURCE_GROUP:-"default"} +RESOURCE_GROUP=${RESOURCE_GROUP:-"Default"} GIT_COMMIT_SHORT=$(git log -n1 --format=format:"%h") # Git repo cloned at $WORKING_DIR, copy into $ARCHIVE_DIR and @@ -33,8 +29,6 @@ fi echo "GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}" echo "BUILD_NUMBER=${BUILD_NUMBER}" echo "REGION=${REGION}" - echo "ORG=${ORG}" - echo "SPACE=${SPACE}" echo "RESOURCE_GROUP=${RESOURCE_GROUP}" } >> "${ARCHIVE_DIR}/build.properties" -grep -v -i password "${ARCHIVE_DIR}/build.properties" \ No newline at end of file +grep -v -i password "${ARCHIVE_DIR}/build.properties" diff --git a/scripts/deploy/iks/test-fvt.sh b/scripts/deploy/iks/test-fvt.sh index 27025d82..4b7e84b3 100644 --- a/scripts/deploy/iks/test-fvt.sh +++ b/scripts/deploy/iks/test-fvt.sh @@ -10,8 +10,6 @@ echo "GIT_BRANCH=${GIT_BRANCH}" echo "GIT_COMMIT=${GIT_COMMIT}" echo "GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}" echo "REGION=${REGION}" -echo "ORG=${ORG}" -echo "SPACE=${SPACE}" echo "RESOURCE_GROUP=${RESOURCE_GROUP}" # These env vars should come from the pipeline run environment properties @@ -49,12 +47,12 @@ run_fvt() { export NAMESPACE=${SERVING_NS} export NAMESPACESCOPEMODE=false - ginkgo -v --progress --fail-fast -p fvt/predictor fvt/scaleToZero --timeout 40m > fvt.out + ginkgo -v --procs=2 --compilers=2 --keep-going fvt/predictor fvt/scaleToZero fvt/storage fvt/hpa --timeout=50m --flake-attempts=3 > fvt.out cat fvt.out if [[ $(grep "Test Suite Passed" fvt.out) ]]; then export NAMESPACE="modelmesh-user" - ginkgo -v --progress --fail-fast -p fvt/predictor fvt/scaleToZero --timeout 40m > fvt.out + ginkgo -v --procs=2 --compilers=2 --keep-going fvt/predictor fvt/scaleToZero fvt/storage fvt/hpa --timeout=50m --flake-attempts=3 > fvt.out cat fvt.out if [[ $(grep "Test Suite Passed" fvt.out) ]]; then REV=0 @@ -65,7 +63,7 @@ run_fvt() { } retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region -retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" +retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" retry 3 3 ibmcloud ks cluster config -c "$SERVING_KUBERNETES_CLUSTER_NAME" RESULT=0 diff --git a/scripts/deploy/iks/undeploy-mm-serving.sh b/scripts/deploy/iks/undeploy-mm-serving.sh index 843b7f6e..8156bf4e 100644 --- a/scripts/deploy/iks/undeploy-mm-serving.sh +++ b/scripts/deploy/iks/undeploy-mm-serving.sh @@ -14,8 +14,6 @@ echo "GIT_BRANCH=${GIT_BRANCH}" echo "GIT_COMMIT=${GIT_COMMIT}" echo "GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}" echo "REGION=${REGION}" -echo "ORG=${ORG}" -echo "SPACE=${SPACE}" echo "RESOURCE_GROUP=${RESOURCE_GROUP}" # These env vars should come from the pipeline run environment properties @@ -37,7 +35,7 @@ retry() { } retry 3 3 ibmcloud login --apikey "${IBM_CLOUD_API_KEY}" --no-region -retry 3 3 ibmcloud target -r "$REGION" -o "$ORG" -s "$SPACE" -g "$RESOURCE_GROUP" +retry 3 3 ibmcloud target -r "$REGION" -g "$RESOURCE_GROUP" retry 3 3 ibmcloud ks cluster config -c "$SERVING_KUBERNETES_CLUSTER_NAME" # Update kustomize diff --git a/scripts/fmt.sh b/scripts/fmt.sh index 2d7c77e4..f8d7ac6c 100755 --- a/scripts/fmt.sh +++ b/scripts/fmt.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License.# +if [ -f .pre-commit.log ]; then + rm -f .pre-commit.log +fi + pre-commit run --all-files RETURN_CODE=$? @@ -35,6 +39,10 @@ if [ $RETURN_CODE -eq 127 ]; then echoError 'how to set up your dev environment. This will automatically format' echoError 'your code when you make a new commit.' elif [ "$RETURN_CODE" -ne 0 ]; then + # cat this file for helping on identifying the root cause when some issue happens + if [ -f .pre-commit.log ]; then + cat .pre-commit.log + fi if [ "${CI}" != "true" ]; then echoError 'Pre-commit linter failed, but it may have automatically formatted your files.' echoError 'Check your changed files and/or manually fix the errors above then stage and commit.'