✨ Replace kube-rbac-proxy with controller-runtime metrics authentication/authorization

cmd/manager/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"net/http"
@@ -41,9 +42,11 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	crcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	crfinalizer "sigs.k8s.io/controller-runtime/pkg/finalizer"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	catalogd "github.com/operator-framework/catalogd/api/v1"
@@ -70,6 +73,7 @@ import (
 var (
 	setupLog               = ctrl.Log.WithName("setup")
 	defaultSystemNamespace = "olmv1-system"
+	certWatcher            *certwatcher.CertWatcher
 )
 
 const authFilePrefix = "operator-controller-global-pull-secrets"
@@ -89,6 +93,8 @@ func podNamespace() string {
 func main() {
 	var (
 		metricsAddr               string
+		certFile                  string
+		keyFile                   string
 		enableLeaderElection      bool
 		probeAddr                 string
 		cachePath                 string
@@ -97,9 +103,11 @@ func main() {
 		caCertDir                 string
 		globalPullSecret          string
 	)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "", "The address for the metrics endpoint. Requires tls-cert and tls-key. (Default: ':8443')")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
+	flag.StringVar(&certFile, "tls-cert", "", "The certificate file used for the metrics server. Required to enable the metrics server. Requires tls-key.")
+	flag.StringVar(&keyFile, "tls-key", "", "The key file used for the metrics server. Required to enable the metrics server. Requires tls-cert")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -119,6 +127,20 @@ func main() {
 		os.Exit(0)
 	}
 
+	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
+		setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together")
+		os.Exit(1)
+	}
+
+	if metricsAddr != "" && certFile == "" && keyFile == "" {
+		setupLog.Error(nil, "metrics-bind-address requires tls-cert and tls-key flags to be set")
+		os.Exit(1)
+	}
+
+	if certFile != "" && keyFile != "" && metricsAddr == "" {
+		metricsAddr = ":8443"
+	}
+
 	ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
 
 	setupLog.Info("starting up the controller", "version info", version.String())
@@ -161,9 +183,49 @@ func main() {
 			},
 		}
 	}
+
+	metricsServerOptions := server.Options{}
+	if len(certFile) > 0 && len(keyFile) > 0 {
+		setupLog.Info("Starting metrics server with TLS enabled", "addr", metricsAddr, "tls-cert", certFile, "tls-key", keyFile)
+
+		metricsServerOptions.BindAddress = metricsAddr
+		metricsServerOptions.SecureServing = true
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		// If the certificate files change, the watcher will reload them.
+		var err error
+		certWatcher, err = certwatcher.New(certFile, keyFile)
+		if err != nil {
+			setupLog.Error(err, "Failed to initialize certificate watcher")
+			os.Exit(1)
+		}
+
+		metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+			config.GetCertificate = certWatcher.GetCertificate
+			// If the enable-http2 flag is false (the default), http/2 should be disabled
+			// due to its vulnerabilities. More specifically, disabling http/2 will
+			// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+			// Rapid Reset CVEs. For more information see:
+			// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+			// - https://github.com/advisories/GHSA-4374-p667-p6c8
+			// Besides, those CVEs are solved already; the solution is still insufficient, and we need to mitigate
+			// the risks. More info https://github.com/golang/go/issues/63417
+			config.NextProtos = []string{"http/1.1"}
+		})
+	} else {
+		// Note that the metrics server is not serving if the BindAddress is set to "0".
+		// Therefore, the metrics server is disabled by default. It is only enabled
+		// if certFile and keyFile are provided. The intention is not allowing the metrics
+		// be served with the default self-signed certificate generated by controller-runtime.
+		metricsServerOptions.BindAddress = "0"
+
+		setupLog.Info("WARNING: Metrics Server is disabled. " +
+			"Metrics will not be served since the TLS certificate and key file are not provided.")
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme.Scheme,
-		Metrics:                server.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "9c4404e7.operatorframework.io",
@@ -220,6 +282,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	if certWatcher != nil {
+		setupLog.Info("Adding certificate watcher to manager")
+		if err := mgr.Add(certWatcher); err != nil {
+			setupLog.Error(err, "unable to add certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	unpacker := &source.ContainersImageRegistry{
 		BaseCachePath: filepath.Join(cachePath, "unpack"),
 		SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) {

config/base/manager/manager.yaml

@@ -52,7 +52,7 @@ spec:
         - /manager
         args:
         - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--metrics-bind-address=:8443"
         - "--leader-elect"
         image: controller:latest
         imagePullPolicy: IfNotPresent
@@ -84,27 +84,6 @@ spec:
             cpu: 10m
             memory: 64Mi
         terminationMessagePolicy: FallbackToLogsOnError
-      - name: kube-rbac-proxy
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - "ALL"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
-        args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --http2-disable
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-        resources:
-          requests:
-            cpu: 5m
-            memory: 64Mi
-        terminationMessagePolicy: FallbackToLogsOnError
       serviceAccountName: operator-controller-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:

config/base/rbac/auth_proxy_service.yaml

@@ -10,6 +10,6 @@ spec:
   - name: https
     port: 8443
     protocol: TCP
-    targetPort: https
+    targetPort: 8443
   selector:
     control-plane: operator-controller-controller-manager

config/base/rbac/kustomization.yaml

@@ -17,10 +17,14 @@ resources:
 - extension_editor_role.yaml
 - extension_viewer_role.yaml
 
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
 - auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+

config/components/coverage/manager_e2e_coverage_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         env:
         - name: GOCOVERDIR

config/components/registries-conf/manager_e2e_registries_conf_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         volumeMounts:
         - name: e2e-registries-conf

config/components/tls/patches/manager_deployment_cert.yaml

@@ -1,9 +1,15 @@
 - op: add
   path: /spec/template/spec/volumes/-
-  value: {"name":"olmv1-certificate", "secret":{"secretName":"olmv1-cert", "optional": false, "items": [{"key": "ca.crt", "path": "olm-ca.crt"}]}}
+  value: {"name":"olmv1-certificate", "secret":{"secretName":"olmv1-cert", "optional": false, "items": [{"key": "ca.crt", "path": "olm-ca.crt"}, {"key": "tls.crt", "path": "tls.cert"}, {"key": "tls.key", "path": "tls.key"}]}}
 - op: add
   path: /spec/template/spec/containers/0/volumeMounts/-
   value: {"name":"olmv1-certificate", "readOnly": true, "mountPath":"/var/certs/"}
 - op: add
   path: /spec/template/spec/containers/0/args/-
   value: "--ca-certs-dir=/var/certs"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-cert=/var/certs/tls.cert"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-key=/var/certs/tls.key"

config/components/tls/resources/manager_cert.yaml

@@ -7,6 +7,8 @@ spec:
   dnsNames:
     - operator-controller.olmv1-system.svc
     - operator-controller.olmv1-system.svc.cluster.local
+    - operator-controller-controller-manager-metrics-service.olmv1-system.svc
+    - operator-controller-controller-manager-metrics-service.olmv1-system.svc.cluster.local
   privateKey:
     algorithm: ECDSA
     size: 256

testdata/build-test-registry.sh

@@ -42,6 +42,8 @@ spec:
   dnsNames:
     - ${name}.${namespace}.svc
     - ${name}.${namespace}.svc.cluster.local
+    - ${name}-controller-manager-metrics-service.${namespace}.svc
+    - ${name}-controller-manager-metrics-service.${namespace}.svc.cluster.local
   privateKey:
     algorithm: ECDSA
     size: 256