feat: add NIM flag in Operator (#1420)

* feat: add NIM flag in Operator

- API change
- add function in modelcontroller
- only when kserve is managed and its nim is managed, we set nim-state to managed
- use newer branch of serving which has the new params.env

---------

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

README.md

@@ -315,6 +315,8 @@ spec:
       managementState: Managed
     kserve:
       managementState: Managed
+      nim:
+        managementState: Managed
       serving:
         ingressGateway:
           certificate:

apis/components/v1alpha1/kserve_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	operatorv1 "github.com/openshift/api/operator/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/opendatahub-io/opendatahub-operator/v2/apis/common"
@@ -52,6 +53,15 @@ type KserveCommonSpec struct {
 	// This field is optional. If no default deployment mode is specified, Kserve will use Serverless mode.
 	// +kubebuilder:validation:Enum=Serverless;RawDeployment
 	DefaultDeploymentMode DefaultDeploymentMode `json:"defaultDeploymentMode,omitempty"`
+	// Configures and enables NVIDIA NIM integration
+	NIM NimSpec `json:"nim,omitempty"`
+}
+
+// nimSpec enables NVIDIA NIM integration
+type NimSpec struct {
+	// +kubebuilder:validation:Enum=Managed;Removed
+	// +kubebuilder:default=Managed
+	ManagementState operatorv1.ManagementState `json:"managementState,omitempty"`
 }
 
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

apis/components/v1alpha1/modelcontroller_types.go

@@ -58,6 +58,7 @@ type ModelControllerSpec struct {
 // a mini version of the DSCKserve only keep devflags and management spec
 type ModelControllerKerveSpec struct {
 	ManagementState     operatorv1.ManagementState `json:"managementState,omitempty"`
+	NIM                 NimSpec                    `json:"nim,omitempty"`
 	common.DevFlagsSpec `json:",inline"`
 }
 

bundle/manifests/components.platform.opendatahub.io_kserves.yaml

@@ -84,6 +84,17 @@ spec:
                       type: object
                     type: array
                 type: object
+              nim:
+                description: Configures and enables NVIDIA NIM integration
+                properties:
+                  managementState:
+                    default: Managed
+                    enum:
+                    - Managed
+                    - Removed
+                    pattern: ^(Managed|Unmanaged|Force|Removed)$
+                    type: string
+                type: object
               serving:
                 description: |-
                   Serving configures the KNative-Serving stack used for model serving. A Service

bundle/manifests/components.platform.opendatahub.io_modelcontrollers.yaml

@@ -87,6 +87,17 @@ spec:
                   managementState:
                     pattern: ^(Managed|Unmanaged|Force|Removed)$
                     type: string
+                  nim:
+                    description: nimSpec enables NVIDIA NIM integration
+                    properties:
+                      managementState:
+                        default: Managed
+                        enum:
+                        - Managed
+                        - Removed
+                        pattern: ^(Managed|Unmanaged|Force|Removed)$
+                        type: string
+                    type: object
                 type: object
               modelMeshServing:
                 description: a mini version of the DSCModelMeshServing only keep devflags

bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml

@@ -245,6 +245,17 @@ spec:
                         - Removed
                         pattern: ^(Managed|Unmanaged|Force|Removed)$
                         type: string
+                      nim:
+                        description: Configures and enables NVIDIA NIM integration
+                        properties:
+                          managementState:
+                            default: Managed
+                            enum:
+                            - Managed
+                            - Removed
+                            pattern: ^(Managed|Unmanaged|Force|Removed)$
+                            type: string
+                        type: object
                       serving:
                         description: |-
                           Serving configures the KNative-Serving stack used for model serving. A Service

bundle/manifests/opendatahub-operator.clusterserviceversion.yaml

@@ -30,6 +30,9 @@ metadata:
               },
               "kserve": {
                 "managementState": "Managed",
+                "nim": {
+                  "managementState": "Managed"
+                },
                 "serving": {
                   "ingressGateway": {
                     "certificate": {

config/crd/bases/components.platform.opendatahub.io_kserves.yaml

@@ -84,6 +84,17 @@ spec:
                       type: object
                     type: array
                 type: object
+              nim:
+                description: Configures and enables NVIDIA NIM integration
+                properties:
+                  managementState:
+                    default: Managed
+                    enum:
+                    - Managed
+                    - Removed
+                    pattern: ^(Managed|Unmanaged|Force|Removed)$
+                    type: string
+                type: object
               serving:
                 description: |-
                   Serving configures the KNative-Serving stack used for model serving. A Service

config/crd/bases/components.platform.opendatahub.io_modelcontrollers.yaml

@@ -87,6 +87,17 @@ spec:
                   managementState:
                     pattern: ^(Managed|Unmanaged|Force|Removed)$
                     type: string
+                  nim:
+                    description: nimSpec enables NVIDIA NIM integration
+                    properties:
+                      managementState:
+                        default: Managed
+                        enum:
+                        - Managed
+                        - Removed
+                        pattern: ^(Managed|Unmanaged|Force|Removed)$
+                        type: string
+                    type: object
                 type: object
               modelMeshServing:
                 description: a mini version of the DSCModelMeshServing only keep devflags

config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml

@@ -245,6 +245,17 @@ spec:
                         - Removed
                         pattern: ^(Managed|Unmanaged|Force|Removed)$
                         type: string
+                      nim:
+                        description: Configures and enables NVIDIA NIM integration
+                        properties:
+                          managementState:
+                            default: Managed
+                            enum:
+                            - Managed
+                            - Removed
+                            pattern: ^(Managed|Unmanaged|Force|Removed)$
+                            type: string
+                        type: object
                       serving:
                         description: |-
                           Serving configures the KNative-Serving stack used for model serving. A Service

config/samples/datasciencecluster_v1_datasciencecluster.yaml

@@ -18,6 +18,9 @@ spec:
       managementState: "Managed"
     kserve: {
       managementState: "Managed",
+      nim: {
+        managementState: "Managed"
+      },
       serving: {
         ingressGateway: {
           certificate: {

controllers/components/kserve/kserve.go

@@ -31,6 +31,7 @@ func init() { //nolint:gochecknoinits
 	cr.Add(&componentHandler{})
 }
 
+// Init for set images.
 func (s *componentHandler) Init(platform cluster.Platform) error {
 	return nil
 }

controllers/components/modelcontroller/modelcontroller.go

@@ -19,6 +19,8 @@ const (
 	ComponentName = componentApi.ModelControllerComponentName
 )
 
+var DefaultPath = odhdeploy.DefaultManifestPath + "/" + ComponentName + "/base"
+
 type componentHandler struct{}
 
 func init() { //nolint:gochecknoinits
@@ -61,24 +63,21 @@ func (s *componentHandler) NewCRObject(dsc *dscv1.DataScienceCluster) client.Obj
 			Annotations: mcAnnotations,
 		},
 		Spec: componentApi.ModelControllerSpec{
-			// ModelMeshServing:  &componentsv1.DSCModelMeshServing {
-			// 	dsc.Spec.Components.ModelMeshServing,
-			// },
 			ModelMeshServing: &componentApi.ModelControllerMMSpec{
 				ManagementState: mState,
 				DevFlagsSpec:    dsc.Spec.Components.ModelMeshServing.DevFlagsSpec,
 			},
 			Kserve: &componentApi.ModelControllerKerveSpec{
 				ManagementState: kState,
 				DevFlagsSpec:    dsc.Spec.Components.Kserve.DevFlagsSpec,
+				NIM:             dsc.Spec.Components.Kserve.NIM,
 			},
 		},
 	})
 }
 
 // Init for set images.
 func (s *componentHandler) Init(platform cluster.Platform) error {
-	DefaultPath := odhdeploy.DefaultManifestPath + "/" + ComponentName + "/base"
 	var imageParamMap = map[string]string{
 		"odh-model-controller": "RELATED_IMAGE_ODH_MODEL_CONTROLLER_IMAGE",
 	}

controllers/components/modelcontroller/modelcontroller_actions.go

@@ -32,7 +32,7 @@ import (
 
 func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
 	// early exist
-	_, ok := rr.Instance.(*componentApi.ModelController)
+	mc, ok := rr.Instance.(*componentApi.ModelController)
 	if !ok {
 		return fmt.Errorf("resource instance %v is not a componentApi.ModelController)", rr.Instance)
 	}
@@ -41,6 +41,18 @@ func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
 		ContextDir: ComponentName,
 		SourcePath: "base",
 	})
+
+	nimState := operatorv1.Removed
+	if mc.Spec.Kserve.ManagementState == operatorv1.Managed {
+		nimState = mc.Spec.Kserve.NIM.ManagementState
+	}
+	extraParamsMap := map[string]string{
+		"nim-state": strings.ToLower(string(nimState)),
+	}
+	if err := odhdeploy.ApplyParams(rr.Manifests[0].String(), nil, extraParamsMap); err != nil {
+		return fmt.Errorf("failed to update images on path %s: %w", rr.Manifests[0].String(), err)
+	}
+
 	return nil
 }
 

docs/api-overview.md

@@ -200,6 +200,7 @@ _Appears in:_
 | `devFlags` _[DevFlags](#devflags)_ | Add developer fields |  |  |
 | `serving` _[ServingSpec](#servingspec)_ | Serving configures the KNative-Serving stack used for model serving. A Service<br />Mesh (Istio) is prerequisite, since it is used as networking layer. |  |  |
 | `defaultDeploymentMode` _[DefaultDeploymentMode](#defaultdeploymentmode)_ | Configures the default deployment mode for Kserve. This can be set to 'Serverless' or 'RawDeployment'.<br />The value specified in this field will be used to set the default deployment mode in the 'inferenceservice-config' configmap for Kserve.<br />This field is optional. If no default deployment mode is specified, Kserve will use Serverless mode. |  | Enum: [Serverless RawDeployment] <br />Pattern: `^(Serverless\|RawDeployment)$` <br /> |
+| `nim` _[NimSpec](#nimspec)_ | Configures and enables NVIDIA NIM integration |  |  |
 
 
 #### DSCKueue
@@ -585,6 +586,7 @@ _Appears in:_
 | `devFlags` _[DevFlags](#devflags)_ | Add developer fields |  |  |
 | `serving` _[ServingSpec](#servingspec)_ | Serving configures the KNative-Serving stack used for model serving. A Service<br />Mesh (Istio) is prerequisite, since it is used as networking layer. |  |  |
 | `defaultDeploymentMode` _[DefaultDeploymentMode](#defaultdeploymentmode)_ | Configures the default deployment mode for Kserve. This can be set to 'Serverless' or 'RawDeployment'.<br />The value specified in this field will be used to set the default deployment mode in the 'inferenceservice-config' configmap for Kserve.<br />This field is optional. If no default deployment mode is specified, Kserve will use Serverless mode. |  | Enum: [Serverless RawDeployment] <br />Pattern: `^(Serverless\|RawDeployment)$` <br /> |
+| `nim` _[NimSpec](#nimspec)_ | Configures and enables NVIDIA NIM integration |  |  |
 
 
 #### KserveList
@@ -623,6 +625,7 @@ _Appears in:_
 | `devFlags` _[DevFlags](#devflags)_ | Add developer fields |  |  |
 | `serving` _[ServingSpec](#servingspec)_ | Serving configures the KNative-Serving stack used for model serving. A Service<br />Mesh (Istio) is prerequisite, since it is used as networking layer. |  |  |
 | `defaultDeploymentMode` _[DefaultDeploymentMode](#defaultdeploymentmode)_ | Configures the default deployment mode for Kserve. This can be set to 'Serverless' or 'RawDeployment'.<br />The value specified in this field will be used to set the default deployment mode in the 'inferenceservice-config' configmap for Kserve.<br />This field is optional. If no default deployment mode is specified, Kserve will use Serverless mode. |  | Enum: [Serverless RawDeployment] <br />Pattern: `^(Serverless\|RawDeployment)$` <br /> |
+| `nim` _[NimSpec](#nimspec)_ | Configures and enables NVIDIA NIM integration |  |  |
 
 
 #### KserveStatus
@@ -772,6 +775,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `managementState` _[ManagementState](#managementstate)_ |  |  |  |
+| `nim` _[NimSpec](#nimspec)_ |  |  |  |
 | `devFlags` _[DevFlags](#devflags)_ | Add developer fields |  |  |
 
 
@@ -1037,6 +1041,25 @@ _Appears in:_
 | `registriesNamespace` _string_ |  |  |  |
 
 
+#### NimSpec
+
+
+
+nimSpec enables NVIDIA NIM integration
+
+
+
+_Appears in:_
+- [DSCKserve](#dsckserve)
+- [KserveCommonSpec](#kservecommonspec)
+- [KserveSpec](#kservespec)
+- [ModelControllerKerveSpec](#modelcontrollerkervespec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `managementState` _[ManagementState](#managementstate)_ |  | Managed | Enum: [Managed Removed] <br /> |
+
+
 #### Ray
 
 

get_all_manifests.sh

@@ -16,8 +16,8 @@ declare -A COMPONENT_MANIFESTS=(
     ["notebooks"]="opendatahub-io:notebooks:main:manifests:notebooks"
     ["trustyai"]="trustyai-explainability:trustyai-service-operator:main:config:trustyai-service-operator"
     ["model-mesh"]="opendatahub-io:modelmesh-serving:release-0.12.0-rc0:config:model-mesh"
-    ["odh-model-controller"]="opendatahub-io:odh-model-controller:release-0.12.0:config:odh-model-controller"
-    ["kserve"]="opendatahub-io:kserve:release-v0.12.1:config:kserve"
+    ["odh-model-controller"]="opendatahub-io:odh-model-controller:incubating:config:odh-model-controller"
+    ["kserve"]="opendatahub-io:kserve:release-v0.14:config:kserve"
     ["modelregistry"]="opendatahub-io:model-registry-operator:main:config:model-registry-operator"
     ["trainingoperator"]="opendatahub-io:training-operator:dev:manifests:trainingoperator"
 )