Test

kubernetes-sigs · Jan 2, 2025 · 712b5a1 · 712b5a1
1 parent 4e83f8b
commit 712b5a1
Show file tree

Hide file tree

Showing 21 changed files with 55 additions and 679 deletions.
diff --git a/Makefile-test.mk b/Makefile-test.mk
@@ -70,7 +70,6 @@ IMAGE_TAG ?= $(IMAGE_REPO):$(GIT_TAG)
 JOBSET_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" sigs.k8s.io/jobset)
 KUBEFLOW_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/kubeflow/training-operator)
 KUBEFLOW_MPI_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/kubeflow/mpi-operator)
-KUBERAY_VERSION = $(shell $(GO_CMD) list -m -f "{{.Version}}" github.com/ray-project/kuberay/ray-operator)
 
 ##@ Tests
 
@@ -80,12 +79,11 @@ test: gotestsum ## Run tests.
 
 .PHONY: test-integration
 test-integration: gomod-download envtest ginkgo dep-crds kueuectl ginkgo-top ## Run tests.
-	ls "$(PROJECT_DIR)/dep-crds/ray-operator/crd/bases"
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \
 	KUEUE_BIN=$(PROJECT_DIR)/bin \
 	ENVTEST_K8S_VERSION=$(ENVTEST_K8S_VERSION) \
 	API_LOG_LEVEL=$(INTEGRATION_API_LOG_LEVEL) \
-	$(GINKGO) $(INTEGRATION_FILTERS) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --output-interceptor-mode=none --race --junit-report=junit.xml --json-report=integration.json --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
+	$(GINKGO) $(INTEGRATION_FILTERS) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit.xml --json-report=integration.json --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
 	$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/integration.json > $(ARTIFACTS)/integration-top.yaml
 
 CREATE_KIND_CLUSTER ?= true
@@ -120,8 +118,7 @@ run-test-multikueue-e2e-%: FORCE
 	@echo Running multikueue e2e for k8s ${K8S_VERSION}
 	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) \
 		ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" \
-		JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) \
-		KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) KUBERAY_VERSION=$(KUBERAY_VERSION) \
+		JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) \
 		./hack/multikueue-e2e-test.sh
 	$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml
 

diff --git a/charts/kueue/templates/rbac/role.yaml b/charts/kueue/templates/rbac/role.yaml
@@ -298,16 +298,9 @@ rules:
       - rayclusters/finalizers
       - rayclusters/status
       - rayjobs/finalizers
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ray.io
-    resources:
       - rayjobs/status
     verbs:
       - get
-      - patch
       - update
   - apiGroups:
       - scheduling.k8s.io

diff --git a/config/components/rbac/role.yaml b/config/components/rbac/role.yaml
@@ -297,16 +297,9 @@ rules:
   - rayclusters/finalizers
   - rayclusters/status
   - rayjobs/finalizers
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ray.io
-  resources:
   - rayjobs/status
   verbs:
   - get
-  - patch
   - update
 - apiGroups:
   - scheduling.k8s.io

diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh
@@ -40,14 +40,6 @@ if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
     export KUBEFLOW_MPI_IMAGE=mpioperator/mpi-operator:${KUBEFLOW_MPI_VERSION/#v}
 fi
 
-if [[ -n ${KUBERAY_VERSION:-} ]]; then
-    export KUBERAY_MANIFEST="${ROOT_DIR}/dep-crds/ray-operator/default/"
-    export KUBERAY_IMAGE=bitnami/kuberay-operator:${KUBERAY_VERSION/#v}
-    export KUBERAY_RAY_IMAGE=rayproject/ray:2.9.0
-    export KUBERAY_RAY_IMAGE_ARM=rayproject/ray:2.9.0-aarch64
-    export KUBERAY_CRDS=${ROOT_DIR}/dep-crds/ray-operator/crd/bases
-fi
-
 # sleep image to use for testing.
 export E2E_TEST_SLEEP_IMAGE_OLD=gcr.io/k8s-staging-perf-tests/sleep:v0.0.3@sha256:00ae8e01dd4439edfb7eb9f1960ac28eba16e952956320cce7f2ac08e3446e6b
 E2E_TEST_SLEEP_IMAGE_OLD_WITHOUT_SHA=${E2E_TEST_SLEEP_IMAGE_OLD%%@*}
@@ -97,17 +89,6 @@ function prepare_docker_images {
     if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
         docker pull "${KUBEFLOW_MPI_IMAGE}"
     fi
-    if [[ -n ${KUBERAY_VERSION:-} ]]; then
-        docker pull "${KUBERAY_IMAGE}"
-
-        # Extra e2e images required for Kuberay
-        unamestr=$(uname)
-        if [[ "$unamestr" == 'Linux' ]]; then
-            docker pull "${KUBERAY_RAY_IMAGE}"
-        elif [[ "$unamestr" == 'Darwin' ]]; then
-            docker pull "${KUBERAY_RAY_IMAGE_ARM}"
-        fi 
-    fi
 }
 
 # $1 cluster
@@ -155,22 +136,6 @@ function install_mpi {
     kubectl apply --server-side -f "${KUBEFLOW_MPI_MANIFEST}"
 }
 
-#$1 - cluster name
-function install_kuberay {
-    # Extra e2e images required for Kuberay
-    unamestr=$(uname)
-    if [[ "$unamestr" == 'Linux' ]]; then
-        cluster_kind_load_image "${1}" "${KUBERAY_RAY_IMAGE}"
-    elif [[ "$unamestr" == 'Darwin' ]]; then
-        cluster_kind_load_image "${1}" "${KUBERAY_RAY_IMAGE_ARM}"
-    fi 
-
-    cluster_kind_load_image "${1}" "${KUBERAY_IMAGE}"
-    kubectl config use-context "kind-${1}"
-    # create used instead of apply - https://github.com/ray-project/kuberay/issues/504
-    kubectl create -k "${KUBERAY_MANIFEST}"
-}
-
 INITIAL_IMAGE=$($YQ '.images[] | select(.name == "controller") | [.newName, .newTag] | join(":")' config/components/manager/kustomization.yaml)
 export INITIAL_IMAGE
 

diff --git a/hack/multikueue-e2e-test.sh b/hack/multikueue-e2e-test.sh
@@ -100,17 +100,10 @@ function kind_load {
     install_kubeflow "$WORKER1_KIND_CLUSTER_NAME"
     install_kubeflow "$WORKER2_KIND_CLUSTER_NAME"
 
-    ## MPI
+     ## MPI
     install_mpi "$MANAGER_KIND_CLUSTER_NAME"
     install_mpi "$WORKER1_KIND_CLUSTER_NAME"
     install_mpi "$WORKER2_KIND_CLUSTER_NAME"
-
-    ## KUBERAY
-    kubectl config use-context "kind-${MANAGER_KIND_CLUSTER_NAME}"
-    kubectl apply --server-side -f "${KUBERAY_CRDS}"
-
-    install_kuberay "$WORKER1_KIND_CLUSTER_NAME"
-    install_kuberay "$WORKER2_KIND_CLUSTER_NAME"
 }
 
 function kueue_deploy {

diff --git a/pkg/controller/jobframework/validation.go b/pkg/controller/jobframework/validation.go
@@ -23,7 +23,6 @@ import (
 
 	kfmpi "github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v2beta1"
 	kftraining "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1"
-	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	apivalidation "k8s.io/apimachinery/pkg/api/validation"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -48,9 +47,7 @@ var (
 		kftraining.SchemeGroupVersion.WithKind(kftraining.PaddleJobKind).String(),
 		kftraining.SchemeGroupVersion.WithKind(kftraining.PyTorchJobKind).String(),
 		kftraining.SchemeGroupVersion.WithKind(kftraining.XGBoostJobKind).String(),
-		kfmpi.SchemeGroupVersion.WithKind(kfmpi.Kind).String(),
-		rayv1.SchemeGroupVersion.WithKind("RayJob").String(),
-		rayv1.SchemeGroupVersion.WithKind("RayCluster").String())
+		kfmpi.SchemeGroupVersion.WithKind(kfmpi.Kind).String())
 )
 
 // ValidateJobOnCreate encapsulates all GenericJob validations that must be performed on a Create operation

diff --git a/pkg/controller/jobs/rayjob/rayjob_controller.go b/pkg/controller/jobs/rayjob/rayjob_controller.go
@@ -26,7 +26,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -56,13 +55,12 @@ func init() {
 		JobType:                &rayv1.RayJob{},
 		AddToScheme:            rayv1.AddToScheme,
 		IsManagingObjectsOwner: isRayJob,
-		MultiKueueAdapter:      &multikueueAdapter{},
 	}))
 }
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;watch;update
 // +kubebuilder:rbac:groups=ray.io,resources=rayjobs,verbs=get;list;watch;update;patch
-// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=ray.io,resources=rayjobs/status,verbs=get;update
 // +kubebuilder:rbac:groups=ray.io,resources=rayjobs/finalizers,verbs=get;update
 // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch
@@ -84,16 +82,12 @@ func (j *RayJob) Object() client.Object {
 	return (*rayv1.RayJob)(j)
 }
 
-func fromObject(obj runtime.Object) *RayJob {
-	return (*RayJob)(obj.(*rayv1.RayJob))
-}
-
 func (j *RayJob) IsSuspended() bool {
 	return j.Spec.Suspend
 }
 
 func (j *RayJob) IsActive() bool {
-	return (j.Status.JobDeploymentStatus != rayv1.JobDeploymentStatusSuspended) && (j.Status.JobDeploymentStatus != rayv1.JobDeploymentStatusNew)
+	return j.Status.JobDeploymentStatus != rayv1.JobDeploymentStatusSuspended
 }
 
 func (j *RayJob) Suspend() {

diff --git a/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go b/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go