From 48dc64aae68e86c4e05c05c1901dcbea7dba0e4f Mon Sep 17 00:00:00 2001 From: Camila M Date: Sun, 17 Nov 2024 10:18:02 +0000 Subject: [PATCH] Replace kube-rbac-proxy with controller-runtime metrics authentication/authorization This commit removes the use of the kube-rbac-proxy image and replaces it with metrics authentication/authorization provided by controller-runtime. The kube-rbac-proxy image is deprecated and will no longer be maintained, which introduces risks to production environments. For more details, see: https://github.com/kubernetes-sigs/kubebuilder/discussions/3907 Key changes: - Updated to configure metrics server options with secure authentication/authorization using controller-runtime filters. - Added support for disabling HTTP/2 by default to mitigate vulnerabilities (e.g., HTTP/2 Stream Cancellation CVE). - Changed the default metrics endpoint to HTTPS (port 8443) and removed the kube-rbac-proxy container from deployment configurations. - Updated RBAC files to include metrics-specific roles and bindings, ensuring secure access to metrics. This aligns with best practices for security and simplifies the metrics setup by leveraging built-in capabilities of controller-runtime. --- cmd/manager/main.go | 43 ++++++++++++++++++- config/base/manager/manager.yaml | 23 +--------- config/base/rbac/auth_proxy_role.yaml | 17 -------- config/base/rbac/auth_proxy_service.yaml | 15 ------- config/base/rbac/kustomization.yaml | 16 ++++--- config/base/rbac/metrics_auth_role.yaml | 17 ++++++++ ...ng.yaml => metrics_auth_role_binding.yaml} | 10 ++--- ...sterrole.yaml => metrics_reader_role.yaml} | 0 .../coverage/manager_e2e_coverage_patch.yaml | 1 - .../manager_e2e_registries_conf_patch.yaml | 1 - 10 files changed, 73 insertions(+), 70 deletions(-) delete mode 100644 config/base/rbac/auth_proxy_role.yaml delete mode 100644 config/base/rbac/auth_proxy_service.yaml create mode 100644 config/base/rbac/metrics_auth_role.yaml rename config/base/rbac/{auth_proxy_role_binding.yaml => metrics_auth_role_binding.yaml} (53%) rename config/base/rbac/{auth_proxy_client_clusterrole.yaml => metrics_reader_role.yaml} (100%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 38d7534f3..117ca0780 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "crypto/tls" "flag" "fmt" "net/http" @@ -44,6 +45,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" crfinalizer "sigs.k8s.io/controller-runtime/pkg/finalizer" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/metrics/server" catalogd "github.com/operator-framework/catalogd/api/v1" @@ -89,6 +91,9 @@ func podNamespace() string { func main() { var ( metricsAddr string + secureMetrics bool + tlsOpts []func(*tls.Config) + enableHTTP2 bool enableLeaderElection bool probeAddr string cachePath string @@ -97,7 +102,11 @@ func main() { caCertDir string globalPullSecret string ) - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -161,9 +170,39 @@ func main() { }, } } + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + metricsServerOptions := server.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme.Scheme, - Metrics: server.Options{BindAddress: metricsAddr}, + Metrics: metricsServerOptions, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "9c4404e7.operatorframework.io", diff --git a/config/base/manager/manager.yaml b/config/base/manager/manager.yaml index e261c5c3e..140ce8ac8 100644 --- a/config/base/manager/manager.yaml +++ b/config/base/manager/manager.yaml @@ -52,7 +52,7 @@ spec: - /manager args: - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" + - "--metrics-bind-address=127.0.0.1:8443" - "--leader-elect" image: controller:latest imagePullPolicy: IfNotPresent @@ -84,27 +84,6 @@ spec: cpu: 10m memory: 64Mi terminationMessagePolicy: FallbackToLogsOnError - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 - args: - - --secure-listen-address=0.0.0.0:8443 - - --http2-disable - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - requests: - cpu: 5m - memory: 64Mi - terminationMessagePolicy: FallbackToLogsOnError serviceAccountName: operator-controller-controller-manager terminationGracePeriodSeconds: 10 volumes: diff --git a/config/base/rbac/auth_proxy_role.yaml b/config/base/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c5..000000000 --- a/config/base/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/base/rbac/auth_proxy_service.yaml b/config/base/rbac/auth_proxy_service.yaml deleted file mode 100644 index ae58ae83b..000000000 --- a/config/base/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: operator-controller-controller-manager - name: controller-manager-metrics-service - namespace: system -spec: - ports: - - name: https - port: 8443 - protocol: TCP - targetPort: https - selector: - control-plane: operator-controller-controller-manager diff --git a/config/base/rbac/kustomization.yaml b/config/base/rbac/kustomization.yaml index 33b8765d5..b9ffcae57 100644 --- a/config/base/rbac/kustomization.yaml +++ b/config/base/rbac/kustomization.yaml @@ -17,10 +17,12 @@ resources: - extension_editor_role.yaml - extension_viewer_role.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/base/rbac/metrics_auth_role.yaml b/config/base/rbac/metrics_auth_role.yaml new file mode 100644 index 000000000..bd419f4e2 --- /dev/null +++ b/config/base/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/base/rbac/auth_proxy_role_binding.yaml b/config/base/rbac/metrics_auth_role_binding.yaml similarity index 53% rename from config/base/rbac/auth_proxy_role_binding.yaml rename to config/base/rbac/metrics_auth_role_binding.yaml index ec7acc0a1..eb0c3f582 100644 --- a/config/base/rbac/auth_proxy_role_binding.yaml +++ b/config/base/rbac/metrics_auth_role_binding.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: proxy-rolebinding + name: metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-auth-role subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system + - kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/base/rbac/auth_proxy_client_clusterrole.yaml b/config/base/rbac/metrics_reader_role.yaml similarity index 100% rename from config/base/rbac/auth_proxy_client_clusterrole.yaml rename to config/base/rbac/metrics_reader_role.yaml diff --git a/config/components/coverage/manager_e2e_coverage_patch.yaml b/config/components/coverage/manager_e2e_coverage_patch.yaml index bda011daf..f2be3a19a 100644 --- a/config/components/coverage/manager_e2e_coverage_patch.yaml +++ b/config/components/coverage/manager_e2e_coverage_patch.yaml @@ -7,7 +7,6 @@ spec: template: spec: containers: - - name: kube-rbac-proxy - name: manager env: - name: GOCOVERDIR diff --git a/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml b/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml index 7530f9b08..42012d697 100644 --- a/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml +++ b/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml @@ -7,7 +7,6 @@ spec: template: spec: containers: - - name: kube-rbac-proxy - name: manager volumeMounts: - name: e2e-registries-conf