test: add metrics e2e tests

test/e2e/config/common/manager_e2e_patch.yaml

@@ -3,4 +3,4 @@
   value: IfNotPresent
 - op: add
   path: /spec/template/spec/containers/0/args/-
-  value: --feature-gates=MultiKueueBatchJobWithManagedBy=true,TopologyAwareScheduling=true
+  value: --feature-gates=MultiKueueBatchJobWithManagedBy=true,TopologyAwareScheduling=true,LocalQueueMetrics=true

test/e2e/singlecluster/metrics_test.go

@@ -22,12 +22,15 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	gomegaformat "github.com/onsi/gomega/format"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	config "sigs.k8s.io/kueue/apis/config/v1beta1"
+	utiltesting "sigs.k8s.io/kueue/pkg/util/testing"
+	testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job"
 	testingjobspod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod"
 	"sigs.k8s.io/kueue/test/util"
 )
@@ -63,6 +66,43 @@ var _ = ginkgo.Describe("Metrics", func() {
 			})
 		})
 
+		ginkgo.By("creating resource flavor", func() {
+			resourceFlavor := utiltesting.MakeResourceFlavor("test-flavor").
+				Obj()
+
+			gomega.Expect(k8sClient.Create(ctx, resourceFlavor)).To(gomega.Succeed())
+		})
+
+		ginkgo.By("Creating a cluster queue", func() {
+			clusterQueue := utiltesting.MakeClusterQueue("test-cq").
+				ResourceGroup(
+					*utiltesting.MakeFlavorQuotas("test-flavor").
+						Resource(corev1.ResourceCPU, "1").
+						Resource(corev1.ResourceMemory, "1Gi").
+						Obj(),
+				).
+				Obj()
+
+			gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed())
+		})
+
+		ginkgo.By("Creating a local queue", func() {
+			localQueue := utiltesting.MakeLocalQueue("test-lq", metav1.NamespaceDefault).
+				ClusterQueue("test-cq").
+				Obj()
+
+			gomega.Expect(k8sClient.Create(ctx, localQueue)).To(gomega.Succeed())
+		})
+
+		ginkgo.By("Creating a job", func() {
+			job := testingjob.MakeJob("test-job", metav1.NamespaceDefault).
+				Queue("test-lq").
+				Request(corev1.ResourceCPU, "1").
+				Obj()
+
+			gomega.Expect(k8sClient.Create(ctx, job)).To(gomega.Succeed())
+		})
+
 		ginkgo.By("Creating the curl-metrics pod to access the metrics endpoint")
 		pod := testingjobspod.MakePod("curl-metrics", config.DefaultNamespace).
 			ServiceAccountName(serviceAccountName).
@@ -88,11 +128,39 @@ var _ = ginkgo.Describe("Metrics", func() {
 			}, util.LongTimeout).Should(gomega.Succeed())
 		})
 
+		metrics := []string{
+			"controller_runtime_reconcile_total",
+
+			"admission_attempts_total",
+			"admission_attempt_duration_seconds",
+			"pending_workloads",
+			"reserving_active_workloads",
+			"admitted_active_workloads",
+			"quota_reserved_workloads_total",
+			"quota_reserved_wait_time_seconds",
+			"admitted_workloads_total",
+			"admission_wait_time_seconds",
+			"cluster_queue_status",
+
+			// LocalQueueMetrics
+			"local_queue_pending_workloads",
+			"local_queue_reserving_active_workloads",
+			"local_queue_admitted_active_workloads",
+			"local_queue_quota_reserved_workloads_total",
+			"local_queue_quota_reserved_wait_time_seconds",
+			"local_queue_admitted_workloads_total",
+			"local_queue_admission_wait_time_seconds",
+			"local_queue_status",
+		}
+
 		ginkgo.By("Getting the metrics by checking curl-metrics logs", func() {
 			cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", config.DefaultNamespace)
 			metricsOutput, err := cmd.CombinedOutput()
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-			gomega.Expect(metricsOutput).To(gomega.ContainSubstring("controller_runtime_reconcile_total"))
+			gomegaformat.MaxLength = 0 // Disable truncation
+			for _, metric := range metrics {
+				gomega.Expect(metricsOutput).To(gomega.ContainSubstring(metric))
+			}
 		})
 	})
 })