From 8ca7c1d2bfab0d8da3b34466b1cb6c906fcd4871 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:37:55 +0000 Subject: [PATCH] Replace kube-rbac-proxy with controller-runtime metrics authentication/authorization This commit removes the use of the kube-rbac-proxy image and replaces it with metrics authentication/authorization provided by controller-runtime. The kube-rbac-proxy image is deprecated and will no longer be maintained, which introduces risks to production environments. For more details, see: kubernetes-sigs/kubebuilder#3907 Key changes: - Updated to configure metrics server options with secure authentication/authorization using controller-runtime filters. - Added support for disabling HTTP/2 by default to mitigate vulnerabilities (e.g., HTTP/2 Stream Cancellation CVE). - Removed the kube-rbac-proxy container from deployment configurations. - Updated RBAC files to include metrics-specific roles and bindings, ensuring secure access to metrics. This aligns with best practices for security and simplifies the metrics setup by leveraging built-in capabilities of controller-runtime. --- cmd/manager/main.go | 31 ++++++++++++++++--- config/base/manager/manager.yaml | 21 ------------- config/base/rbac/auth_proxy_role.yaml | 20 ------------ config/base/rbac/kustomization.yaml | 15 +++++---- config/base/rbac/metrics_auth_role.yaml | 20 ++++++++++++ ...ng.yaml => metrics_auth_role_binding.yaml} | 4 +-- ...sterrole.yaml => metrics_reader_role.yaml} | 0 .../patches/manager_deployment_cacerts.yaml | 4 +-- .../manager_e2e_registries_conf_patch.yaml | 1 - .../tls/patches/manager_deployment_certs.yaml | 6 ++-- internal/storage/localdir_test.go | 12 ------- 11 files changed, 63 insertions(+), 71 deletions(-) delete mode 100644 config/base/rbac/auth_proxy_role.yaml create mode 100644 config/base/rbac/metrics_auth_role.yaml rename config/base/rbac/{auth_proxy_role_binding.yaml => metrics_auth_role_binding.yaml} (83%) rename config/base/rbac/{auth_proxy_client_clusterrole.yaml => metrics_reader_role.yaml} (100%) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 1ba23302..6a818c3c 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -48,6 +48,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" crwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -83,6 +84,7 @@ func init() { func main() { var ( metricsAddr string + tlsOpts []func(*tls.Config) enableLeaderElection bool probeAddr string pprofAddr string @@ -185,12 +187,33 @@ func main() { } } + // http/2 should be disabled due to its vulnerabilities. More specifically, + // disabling http/2 will prevent from being vulnerable to the HTTP/2 Stream + // Cancellation and Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + tlsOpts = append(tlsOpts, disableHTTP2) + + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: true, + TLSOpts: tlsOpts, + + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. + FilterProvider: filters.WithAuthenticationAndAuthorization, + } + // Create manager mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - }, + Scheme: scheme, + Metrics: metricsServerOptions, PprofBindAddress: pprofAddr, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, diff --git a/config/base/manager/manager.yaml b/config/base/manager/manager.yaml index 4428ba9b..a4d7b776 100644 --- a/config/base/manager/manager.yaml +++ b/config/base/manager/manager.yaml @@ -50,27 +50,6 @@ spec: seccompProfile: type: RuntimeDefault containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 - args: - - --secure-listen-address=0.0.0.0:7443 - - --http2-disable - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - ports: - - containerPort: 7443 - protocol: TCP - name: https - resources: - requests: - cpu: 5m - memory: 64Mi - terminationMessagePolicy: FallbackToLogsOnError - command: - ./manager args: diff --git a/config/base/rbac/auth_proxy_role.yaml b/config/base/rbac/auth_proxy_role.yaml deleted file mode 100644 index 3edf78f5..00000000 --- a/config/base/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/part-of: olm - app.kubernetes.io/name: catalogd - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/base/rbac/kustomization.yaml b/config/base/rbac/kustomization.yaml index 31d315b2..5619aa00 100644 --- a/config/base/rbac/kustomization.yaml +++ b/config/base/rbac/kustomization.yaml @@ -9,9 +9,12 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 3 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/config/base/rbac/metrics_auth_role.yaml b/config/base/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000..13ecbbcd --- /dev/null +++ b/config/base/rbac/metrics_auth_role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/part-of: olm + app.kubernetes.io/name: catalogd + name: metrics-auth-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/base/rbac/auth_proxy_role_binding.yaml b/config/base/rbac/metrics_auth_role_binding.yaml similarity index 83% rename from config/base/rbac/auth_proxy_role_binding.yaml rename to config/base/rbac/metrics_auth_role_binding.yaml index 2efcf8dd..dac0ce96 100644 --- a/config/base/rbac/auth_proxy_role_binding.yaml +++ b/config/base/rbac/metrics_auth_role_binding.yaml @@ -4,11 +4,11 @@ metadata: labels: app.kubernetes.io/part-of: olm app.kubernetes.io/name: catalogd - name: proxy-rolebinding + name: metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-auth-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/config/base/rbac/auth_proxy_client_clusterrole.yaml b/config/base/rbac/metrics_reader_role.yaml similarity index 100% rename from config/base/rbac/auth_proxy_client_clusterrole.yaml rename to config/base/rbac/metrics_reader_role.yaml diff --git a/config/components/ca/patches/manager_deployment_cacerts.yaml b/config/components/ca/patches/manager_deployment_cacerts.yaml index 3ccce591..b5b03633 100644 --- a/config/components/ca/patches/manager_deployment_cacerts.yaml +++ b/config/components/ca/patches/manager_deployment_cacerts.yaml @@ -2,8 +2,8 @@ path: /spec/template/spec/volumes/- value: {"name":"olmv1-certificate", "secret":{"secretName":"catalogd-service-cert-git-version", "optional": false, "items": [{"key": "ca.crt", "path": "olm-ca.crt"}]}} - op: add - path: /spec/template/spec/containers/1/volumeMounts/- + path: /spec/template/spec/containers/0/volumeMounts/- value: {"name":"olmv1-certificate", "readOnly": true, "mountPath":"/var/ca-certs/"} - op: add - path: /spec/template/spec/containers/1/args/- + path: /spec/template/spec/containers/0/args/- value: "--ca-certs-dir=/var/ca-certs" diff --git a/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml b/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml index 7530f9b0..42012d69 100644 --- a/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml +++ b/config/components/registries-conf/manager_e2e_registries_conf_patch.yaml @@ -7,7 +7,6 @@ spec: template: spec: containers: - - name: kube-rbac-proxy - name: manager volumeMounts: - name: e2e-registries-conf diff --git a/config/components/tls/patches/manager_deployment_certs.yaml b/config/components/tls/patches/manager_deployment_certs.yaml index 7efcdcbb..3d8b33ac 100644 --- a/config/components/tls/patches/manager_deployment_certs.yaml +++ b/config/components/tls/patches/manager_deployment_certs.yaml @@ -2,11 +2,11 @@ path: /spec/template/spec/volumes/- value: {"name":"catalogserver-certs", "secret":{"secretName":"catalogd-service-cert-git-version"}} - op: add - path: /spec/template/spec/containers/1/volumeMounts/- + path: /spec/template/spec/containers/0/volumeMounts/- value: {"name":"catalogserver-certs", "mountPath":"/var/certs"} - op: add - path: /spec/template/spec/containers/1/args/- + path: /spec/template/spec/containers/0/args/- value: "--tls-cert=/var/certs/tls.crt" - op: add - path: /spec/template/spec/containers/1/args/- + path: /spec/template/spec/containers/0/args/- value: "--tls-key=/var/certs/tls.key" diff --git a/internal/storage/localdir_test.go b/internal/storage/localdir_test.go index d36d655f..c975c8fc 100644 --- a/internal/storage/localdir_test.go +++ b/internal/storage/localdir_test.go @@ -309,10 +309,6 @@ const testCompressableJSON = `{ } ], "relatedImages": [ - { - "name": "", - "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" - }, { "name": "", "image": "quay.io/helmoperators/cockroachdb:v5.0.3" @@ -346,10 +342,6 @@ const testCompressableJSON = `{ } ], "relatedImages": [ - { - "name": "", - "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" - }, { "name": "", "image": "quay.io/helmoperators/cockroachdb:v5.0.4" @@ -383,10 +375,6 @@ const testCompressableJSON = `{ } ], "relatedImages": [ - { - "name": "", - "image": "gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0" - }, { "name": "", "image": "quay.io/cockroachdb/cockroach-helm-operator:6.0.0"