tuning spec update change

Signed-off-by: Bangqi Zhu <bangqizhu@microsoft.com>

api/v1alpha1/workspace_validation.go

@@ -6,13 +6,13 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
-	"github.com/azure/kaito/pkg/utils/consts"
 	"reflect"
 	"regexp"
-	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/azure/kaito/pkg/utils/consts"
+
 	"github.com/azure/kaito/pkg/utils"
 	"github.com/azure/kaito/pkg/utils/plugin"
 
@@ -183,7 +183,7 @@ func (r *TuningSpec) validateUpdate(old *TuningSpec) (errs *apis.FieldError) {
 	if r.Output == nil {
 		errs = errs.Also(apis.ErrMissingField("Output"))
 	} else {
-		errs = errs.Also(r.Output.validateUpdate(old.Output).ViaField("Output"))
+		errs = errs.Also(r.Output.validateUpdate().ViaField("Output"))
 	}
 	if !reflect.DeepEqual(old.Preset, r.Preset) {
 		errs = errs.Also(apis.ErrGeneric("Preset cannot be changed", "Preset"))
@@ -235,33 +235,7 @@ func (r *DataSource) validateUpdate(old *DataSource, isTuning bool) (errs *apis.
 	if r.Volume != nil {
 		errs = errs.Also(apis.ErrInvalidValue("Volume support is not implemented yet", "Volume"))
 	}
-	oldURLs := make([]string, len(old.URLs))
-	copy(oldURLs, old.URLs)
-	sort.Strings(oldURLs)
-
-	newURLs := make([]string, len(r.URLs))
-	copy(newURLs, r.URLs)
-	sort.Strings(newURLs)
-
-	if !reflect.DeepEqual(oldURLs, newURLs) {
-		errs = errs.Also(apis.ErrInvalidValue("URLs field cannot be changed once set", "URLs"))
-	}
-	// TODO: check if the Volume is changed
-	if old.Image != r.Image {
-		errs = errs.Also(apis.ErrInvalidValue("Image field cannot be changed once set", "Image"))
-	}
-
-	oldSecrets := make([]string, len(old.ImagePullSecrets))
-	copy(oldSecrets, old.ImagePullSecrets)
-	sort.Strings(oldSecrets)
-
-	newSecrets := make([]string, len(r.ImagePullSecrets))
-	copy(newSecrets, r.ImagePullSecrets)
-	sort.Strings(newSecrets)
 
-	if !reflect.DeepEqual(oldSecrets, newSecrets) {
-		errs = errs.Also(apis.ErrInvalidValue("ImagePullSecrets field cannot be changed once set", "ImagePullSecrets"))
-	}
 	return errs
 }
 
@@ -298,18 +272,12 @@ func (r *DataDestination) validateCreate() (errs *apis.FieldError) {
 	return errs
 }
 
-func (r *DataDestination) validateUpdate(old *DataDestination) (errs *apis.FieldError) {
+func (r *DataDestination) validateUpdate() (errs *apis.FieldError) {
 	// TODO: Implement Volumes
 	if r.Volume != nil {
 		errs = errs.Also(apis.ErrInvalidValue("Volume support is not implemented yet", "Volume"))
 	}
-	if old.Image != r.Image {
-		errs = errs.Also(apis.ErrInvalidValue("Image field cannot be changed once set", "Image"))
-	}
 
-	if old.ImagePushSecret != r.ImagePushSecret {
-		errs = errs.Also(apis.ErrInvalidValue("ImagePushSecret field cannot be changed once set", "ImagePushSecret"))
-	}
 	return errs
 }
 

api/v1alpha1/workspace_validation_test.go

@@ -1302,39 +1302,6 @@ func TestDataSourceValidateUpdate(t *testing.T) {
 			wantErr:   true,
 			errFields: []string{"Name"},
 		},
-		{
-			name: "URLs changed",
-			oldSource: &DataSource{
-				URLs: []string{"http://example.com/old"},
-			},
-			newSource: &DataSource{
-				URLs: []string{"http://example.com/new"},
-			},
-			wantErr:   true,
-			errFields: []string{"URLs"},
-		},
-		{
-			name: "Image changed",
-			oldSource: &DataSource{
-				Image: "old-image:latest",
-			},
-			newSource: &DataSource{
-				Image: "new-image:latest",
-			},
-			wantErr:   true,
-			errFields: []string{"Image"},
-		},
-		{
-			name: "ImagePullSecrets changed",
-			oldSource: &DataSource{
-				ImagePullSecrets: []string{"old-secret"},
-			},
-			newSource: &DataSource{
-				ImagePullSecrets: []string{"new-secret"},
-			},
-			wantErr:   true,
-			errFields: []string{"ImagePullSecrets"},
-		},
 	}
 
 	for _, tt := range tests {
@@ -1450,33 +1417,11 @@ func TestDataDestinationValidateUpdate(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "Image changed",
-			oldDest: &DataDestination{
-				Image: "old-image:latest",
-			},
-			newDest: &DataDestination{
-				Image: "new-image:latest",
-			},
-			wantErr:   true,
-			errFields: []string{"Image"},
-		},
-		{
-			name: "ImagePushSecret changed",
-			oldDest: &DataDestination{
-				ImagePushSecret: "old-secret",
-			},
-			newDest: &DataDestination{
-				ImagePushSecret: "new-secret",
-			},
-			wantErr:   true,
-			errFields: []string{"ImagePushSecret"},
-		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			errs := tt.newDest.validateUpdate(tt.oldDest)
+			errs := tt.newDest.validateUpdate()
 			hasErrs := errs != nil
 
 			if hasErrs != tt.wantErr {

pkg/controllers/workspace_controller.go

@@ -703,15 +703,33 @@ func (c *WorkspaceReconciler) applyTuning(ctx context.Context, wObj *kaitov1alph
 
 			tuningParam := model.GetTuningParameters()
 			existingObj := &batchv1.Job{}
+			revisionStr := wObj.Annotations[kaitov1alpha1.WorkspaceRevisionAnnotation]
 			if err = resources.GetResource(ctx, wObj.Name, wObj.Namespace, c.Client, existingObj); err == nil {
 				klog.InfoS("A tuning workload already exists for workspace", "workspace", klog.KObj(wObj))
+
+				if existingObj.Annotations[kaitov1alpha1.WorkspaceRevisionAnnotation] != revisionStr {
+					deletePolicy := metav1.DeletePropagationForeground
+					if err := c.Delete(ctx, existingObj, &client.DeleteOptions{
+						PropagationPolicy: &deletePolicy,
+					}); err != nil {
+						return
+					}
+
+					var workloadObj client.Object
+					workloadObj, err = tuning.CreatePresetTuning(ctx, wObj, revisionStr, tuningParam, c.Client)
+					if err != nil {
+						return
+					}
+					existingObj = workloadObj.(*batchv1.Job)
+				}
+
 				if err = resources.CheckResourceStatus(existingObj, c.Client, tuningParam.ReadinessTimeout); err != nil {
 					return
 				}
 			} else if apierrors.IsNotFound(err) {
 				var workloadObj client.Object
 				// Need to create a new workload
-				workloadObj, err = tuning.CreatePresetTuning(ctx, wObj, tuningParam, c.Client)
+				workloadObj, err = tuning.CreatePresetTuning(ctx, wObj, revisionStr, tuningParam, c.Client)
 				if err != nil {
 					return
 				}

pkg/resources/manifests.go

@@ -186,7 +186,7 @@ func GenerateStatefulSetManifest(ctx context.Context, workspaceObj *kaitov1alpha
 	return ss
 }
 
-func GenerateTuningJobManifest(ctx context.Context, wObj *kaitov1alpha1.Workspace, imageName string,
+func GenerateTuningJobManifest(ctx context.Context, wObj *kaitov1alpha1.Workspace, revisionNum string, imageName string,
 	imagePullSecretRefs []corev1.LocalObjectReference, replicas int, commands []string, containerPorts []corev1.ContainerPort,
 	livenessProbe, readinessProbe *corev1.Probe, resourceRequirements corev1.ResourceRequirements, tolerations []corev1.Toleration,
 	initContainers []corev1.Container, sidecarContainers []corev1.Container, volumes []corev1.Volume, volumeMounts []corev1.VolumeMount,
@@ -225,6 +225,9 @@ func GenerateTuningJobManifest(ctx context.Context, wObj *kaitov1alpha1.Workspac
 			Name:      wObj.Name,
 			Namespace: wObj.Namespace,
 			Labels:    labels,
+			Annotations: map[string]string{
+				kaitov1alpha1.WorkspaceRevisionAnnotation: revisionNum,
+			},
 			OwnerReferences: []v1.OwnerReference{
 				{
 					APIVersion: kaitov1alpha1.GroupVersion.String(),

pkg/tuning/preset-tuning.go

@@ -284,7 +284,7 @@ func setupDefaultSharedVolumes(workspaceObj *kaitov1alpha1.Workspace, cmName str
 	return volumes, volumeMounts
 }
 
-func CreatePresetTuning(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace,
+func CreatePresetTuning(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, revisionNum string,
 	tuningObj *model.PresetParam, kubeClient client.Client) (client.Object, error) {
 	cm, err := EnsureTuningConfigMap(ctx, workspaceObj, kubeClient)
 	if err != nil {
@@ -348,7 +348,7 @@ func CreatePresetTuning(ctx context.Context, workspaceObj *kaitov1alpha1.Workspa
 			Value: "k_proj,q_proj,v_proj,o_proj,gate_proj,down_proj,up_proj",
 		})
 	}
-	jobObj := resources.GenerateTuningJobManifest(ctx, workspaceObj, tuningImage, imagePullSecrets, *workspaceObj.Resource.Count, commands,
+	jobObj := resources.GenerateTuningJobManifest(ctx, workspaceObj, revisionNum, tuningImage, imagePullSecrets, *workspaceObj.Resource.Count, commands,
 		containerPorts, nil, nil, resourceReq, tolerations, initContainers, sidecarContainers, volumes, volumeMounts, envVars)
 
 	err = resources.CreateResource(ctx, jobObj, kubeClient)