update: make VAPB object user configable

- user can update VAPB object
- user cannot update VAP object ( it get reconciled back to default value)
- only VAP has "owner" set

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

controllers/components/kueue/kueue_controller.go

@@ -57,14 +57,12 @@ func (s *componentHandler) NewComponentReconciler(ctx context.Context, mgr ctrl.
 		Owns(&promv1.PrometheusRule{}).
 		Owns(&admissionregistrationv1.MutatingWebhookConfiguration{}).
 		Owns(&admissionregistrationv1.ValidatingWebhookConfiguration{}).
-		// We need to dynamically "own" these 2 resource types due to OCP version differences
-		// TODO: once we do not support 4.17-, could move it to :
-		// Owns(&admissionregistrationv1.ValidatingAdmissionPolicy{}).
-		// Owns(&admissionregistrationv1.ValidatingAdmissionPolicyBinding{}).
-		WatchesGVK(
+		// We need dynamically "watch" VAPB, because we want it to be configable by user and it can be left behind when kueue is removed.
+		OwnsGVK(
 			gvk.ValidatingAdmissionPolicy,
 			reconciler.Dynamic(vapPredicate),
 		).
+		// We need dynamically "own" VAP, because we want it has owner so when kueue is removed it gets cleaned.
 		WatchesGVK(
 			gvk.ValidatingAdmissionPolicyBinding,
 			reconciler.Dynamic(vapPredicate),
@@ -85,6 +83,7 @@ func (s *componentHandler) NewComponentReconciler(ctx context.Context, mgr ctrl.
 			kustomize.WithLabel(labels.ODH.Component(LegacyComponentName), labels.True),
 			kustomize.WithLabel(labels.K8SCommon.PartOf, LegacyComponentName),
 		)).
+		WithAction(customizeResources).
 		WithAction(deploy.NewAction(
 			deploy.WithCache(),
 		)).

controllers/components/kueue/kueue_controller_actions.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 
 	componentApi "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1alpha1"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster/gvk"
 	odhtypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types"
 	odhdeploy "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/annotations"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/resources"
 )
 
 func initialize(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
@@ -45,3 +48,14 @@ func devFlags(ctx context.Context, rr *odhtypes.ReconciliationRequest) error {
 
 	return nil
 }
+
+func customizeResources(_ context.Context, rr *odhtypes.ReconciliationRequest) error {
+	for i := range rr.Resources {
+		if rr.Resources[i].GroupVersionKind() == gvk.ValidatingAdmissionPolicyBinding {
+			// admin can update this resource
+			resources.SetAnnotation(&rr.Resources[i], annotations.ManagedByODHOperator, "false")
+			break // fast exist function
+		}
+	}
+	return nil
+}

tests/e2e/kueue_test.go

@@ -172,14 +172,14 @@ func (tc *KueueTestCtx) testOwnerReferences() error {
 
 func (tc *KueueTestCtx) validateVAPReady() error {
 	// get OCP version from DSC
-	keydsc := types.NamespacedName{Name: "e2e-test-dsci"}
+	keydsc := types.NamespacedName{Name: "e2e-test-dsc"}
 	dsc := &dscv1.DataScienceCluster{}
 	err := tc.testCtx.customClient.Get(tc.testCtx.ctx, keydsc, dsc)
 	if err != nil {
 		return fmt.Errorf("expect one DSC CR to be found but got error: %w", err)
 	}
 
-	// if ocp is 4.17+ then VAP should be created.
+	// if ocp is 4.17+ then VAP and VAPB should be created
 	if dsc.Status.Release.OCPVersion.Minor > 16 {
 		keyvap := types.NamespacedName{Name: "kueue-validating-admission-policy"}
 		vap := &unstructured.Unstructured{}
@@ -189,6 +189,21 @@ func (tc *KueueTestCtx) validateVAPReady() error {
 		if err != nil {
 			return fmt.Errorf("expect validatingadminssionpolicy to be found but got error: %w", err)
 		}
+		if len(vap.GetOwnerReferences()) == 0 {
+			return errors.New("expect VAP to have ownerref set")
+		}
+
+		keyvapb := types.NamespacedName{Name: "kueue-validating-admission-policy-binding"}
+		vapb := &unstructured.Unstructured{}
+		vapb.SetGroupVersionKind(gvk.ValidatingAdmissionPolicyBinding)
+
+		err = tc.testCtx.customClient.Get(tc.testCtx.ctx, keyvapb, vapb)
+		if err != nil {
+			return fmt.Errorf("expect validatingadminssionpolicybinding to be found but got error: %w", err)
+		}
+		if len(vapb.GetOwnerReferences()) != 0 {
+			return errors.New("expect VAPB have no ownerref set")
+		}
 	}
 	return nil
 }