From 35b6926ef1486de39c968dca1df7c66857c3a958 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 6 Oct 2023 11:13:22 +0200 Subject: [PATCH] [wip] create CA bundle --- .../core/openstackcontrolplane_controller.go | 2 + go.mod | 2 +- pkg/openstack/ca.go | 215 +++++++++++++++--- 3 files changed, 185 insertions(+), 34 deletions(-) diff --git a/controllers/core/openstackcontrolplane_controller.go b/controllers/core/openstackcontrolplane_controller.go index fc74de612..bcc4e8517 100644 --- a/controllers/core/openstackcontrolplane_controller.go +++ b/controllers/core/openstackcontrolplane_controller.go @@ -31,6 +31,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + corev1 "k8s.io/api/core/v1" manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -328,6 +329,7 @@ func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, i func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1beta1.OpenStackControlPlane{}). + Owns(&corev1.Secret{}). Owns(&mariadbv1.MariaDB{}). Owns(&mariadbv1.Galera{}). Owns(&memcachedv1.Memcached{}). diff --git a/go.mod b/go.mod index 0119569f6..c5ab69bb3 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/operator-framework/api v0.17.6 github.com/rabbitmq/cluster-operator/v2 v2.5.0 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.4 k8s.io/client-go v0.27.2 @@ -48,7 +49,6 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 // indirect github.com/metal3-io/baremetal-operator/apis v0.3.1 // indirect github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.2.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/tools v0.13.0 // indirect sigs.k8s.io/gateway-api v0.6.0 // indirect diff --git a/pkg/openstack/ca.go b/pkg/openstack/ca.go index 823a23ae6..6c351e63d 100644 --- a/pkg/openstack/ca.go +++ b/pkg/openstack/ca.go @@ -2,6 +2,11 @@ package openstack import ( "context" + "crypto/x509" + "encoding/pem" + "fmt" + "math" + "os" "time" certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -12,6 +17,8 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + "golang.org/x/exp/slices" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" corev1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1" @@ -25,6 +32,10 @@ const ( DefaultPublicCAName = "rootca-" + string(service.EndpointPublic) // DefaultInternalCAName - DefaultInternalCAName = "rootca-" + string(service.EndpointInternal) + // TLSCABundleFile - + TLSCABundleFile = "tls-ca-bundle.pem" + // TLSCABundlePath - + TLSCABundlePath = "/etc/pki/ca-trust/extracted/pem/" + TLSCABundleFile ) // ReconcileCAs - @@ -72,26 +83,49 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, nil } - caCerts := map[string]string{} + bundle := newBundle() + + // load current CA bundle from secret if exist + currentCASecret, _, err := secret.GetSecret(ctx, helper, CombinedCASecret, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if currentCASecret != nil { + if _, ok := currentCASecret.Data[TLSCABundleFile]; ok { + err = bundle.getCertsFromPEM(currentCASecret.Data[TLSCABundleFile]) + if err != nil { + return ctrl.Result{}, err + } + } + } // create RootCA cert and Issuer that uses the generated CA certificate to issue certs - if instance.Spec.TLS.PublicEndpoints.Enabled && instance.Spec.TLS.PublicEndpoints.Issuer == nil { - caCert, ctrlResult, err := createRootCACertAndIssuer( - ctx, - instance, - helper, - issuerReq, - DefaultPublicCAName, - map[string]string{}, - ) - if err != nil { - return ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - return ctrlResult, nil + if instance.Spec.TLS.PublicEndpoints.Enabled { + var caCert []byte + if instance.Spec.TLS.PublicEndpoints.Issuer == nil { + caCert, ctrlResult, err = createRootCACertAndIssuer( + ctx, + instance, + helper, + issuerReq, + DefaultPublicCAName, + map[string]string{}, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } else { + // TODO get secret name from issuer and get ca.crt } - caCerts[DefaultPublicCAName] = string(caCert) + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } + if instance.Spec.TLS.InternalEndpoints.Enabled { caCert, ctrlResult, err := createRootCACertAndIssuer( ctx, @@ -109,7 +143,10 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, nil } - caCerts[DefaultInternalCAName] = string(caCert) + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } instance.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneCAReadyCondition, corev1.OpenStackControlPlaneCAReadyMessage) @@ -129,12 +166,24 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h return ctrlResult, err } - for key, ca := range caSecret.Data { - key := instance.Spec.TLS.CaSecretName + "-" + key - caCerts[key] = string(ca) + for _, caCert := range caSecret.Data { + err = bundle.getCertsFromPEM(caCert) + if err != nil { + return ctrl.Result{}, err + } } } + // get CA bundle from operator + caBundle, err := getOperatorCABundle(TLSCABundlePath) + if err != nil { + return ctrl.Result{}, err + } + err = bundle.getCertsFromPEM(caBundle) + if err != nil { + return ctrl.Result{}, err + } + saSecretTemplate := []util.Template{ { Name: CombinedCASecret, @@ -147,7 +196,7 @@ func ReconcileCAs(ctx context.Context, instance *corev1.OpenStackControlPlane, h CombinedCASecret: "", }, ConfigOptions: nil, - CustomData: caCerts, + CustomData: map[string]string{TLSCABundleFile: string(caBundle)}, }, } @@ -174,8 +223,7 @@ func createRootCACertAndIssuer( selfsignedIssuerReq *certmgrv1.Issuer, caName string, labels map[string]string, -) (string, ctrl.Result, error) { - var caCert string +) ([]byte, ctrl.Result, error) { // create RootCA Certificate used to sign certificates caCertReq := certmanager.Cert( caName, @@ -208,7 +256,7 @@ func createRootCACertAndIssuer( caCertReq.Name, err.Error())) - return caCert, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -216,7 +264,7 @@ func createRootCACertAndIssuer( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } // create Issuer that uses the generated CA certificate to issue certs @@ -239,7 +287,7 @@ func createRootCACertAndIssuer( issuerReq.GetName(), err.Error())) - return caCert, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -247,14 +295,14 @@ func createRootCACertAndIssuer( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } - caCert, ctrlResult, err = getCAFromSecret(ctx, instance, helper, caName) + caCert, ctrlResult, err := getCAFromSecret(ctx, instance, helper, caName) if err != nil { - return caCert, ctrl.Result{}, err + return nil, ctrl.Result{}, err } else if (ctrlResult != ctrl.Result{}) { - return caCert, ctrlResult, nil + return nil, ctrlResult, nil } return caCert, ctrl.Result{}, nil @@ -265,7 +313,7 @@ func getCAFromSecret( instance *corev1.OpenStackControlPlane, helper *helper.Helper, caName string, -) (string, ctrl.Result, error) { +) ([]byte, ctrl.Result, error) { caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, caName, time.Duration(5), "ca.crt") if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( @@ -277,7 +325,7 @@ func getCAFromSecret( caName, err.Error())) - return caSecret, ctrlResult, err + return nil, ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { instance.Status.Conditions.Set(condition.FalseCondition( corev1.OpenStackControlPlaneCAReadyCondition, @@ -285,8 +333,109 @@ func getCAFromSecret( condition.SeverityInfo, corev1.OpenStackControlPlaneCAReadyRunningMessage)) - return caSecret, ctrlResult, nil + return nil, ctrlResult, nil + } + + return []byte(caSecret), ctrl.Result{}, nil +} + +func getOperatorCABundle(caFile string) ([]byte, error) { + contents, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("File reading error %w", err) + } + + return contents, nil +} + +func days(t time.Time) int { + return int(math.Round(time.Since(t).Hours() / 24)) +} + +type caBundle struct { + certs []caCert +} + +type caCert struct { + hash string + cert *x509.Certificate +} + +// newBundle returns a new, empty Bundle +func newBundle() *caBundle { + return &caBundle{ + certs: make([]caCert, 0), + } +} + +func (ca *caBundle) getCertsFromPEM(PEMdata []byte) error { + if PEMdata == nil { + return fmt.Errorf("certificate data can't be nil") + } + + for { + var block *pem.Block + block, PEMdata = pem.Decode(PEMdata) + + if block == nil { + break + } + + if block.Type != "CERTIFICATE" { + // only certificates are allowed in a bundle + return fmt.Errorf("invalid PEM block in bundle: only CERTIFICATE blocks are permitted but found '%s'", block.Type) + } + + if len(block.Headers) != 0 { + return fmt.Errorf("invalid PEM block in bundle; blocks are not permitted to have PEM headers") + } + + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + // the presence of an invalid cert (including things which aren't certs) + // should cause the bundle to be rejected + return fmt.Errorf("invalid PEM block in bundle; invalid PEM certificate: %w", err) + } + + if certificate == nil { + return fmt.Errorf("failed appending a certificate: certificate is nil") + } + + // validate if the CA expired + if -days(certificate.NotAfter) <= 0 { + continue + } + + blockHash, err := util.ObjectHash(block.Bytes) + if err != nil { + return fmt.Errorf("failed calc hash of PEM block : %w", err) + } + + // if cert is not already in bundle list add it + // validate of nextip is already in a reservation and its not us + f := func(c caCert) bool { + return c.hash == blockHash + } + idx := slices.IndexFunc(ca.certs, f) + if idx == -1 { + ca.certs = append(ca.certs, + caCert{ + hash: blockHash, + cert: certificate, + }) + } + } + + return nil +} + +// Create PEM bundle from certificates +func (ca *caBundle) getBundlePEM() [][]byte { + var certsData = make([][]byte, len(ca.certs)) + + for i, cert := range ca.certs { + certsData[i] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.cert.Raw}) } - return caSecret, ctrl.Result{}, nil + return certsData }