Skip to content

Commit

Permalink
Merge pull request openshift#4502 from dgrisonnet/aws-kms-v2
Browse files Browse the repository at this point in the history
OCPBUGS-25937: Support KMS v2 on AWS
  • Loading branch information
openshift-merge-bot[bot] authored Aug 8, 2024
2 parents 84ea9c4 + 50fbc23 commit 1308e2d
Show file tree
Hide file tree
Showing 9 changed files with 286 additions and 65 deletions.
36 changes: 34 additions & 2 deletions control-plane-operator/controllers/hostedcontrolplane/kas/kms.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/kas/kms"
"github.com/openshift/hypershift/support/api"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
)

func applyKMSConfig(podSpec *corev1.PodSpec, secretEncryptionData *hyperv1.SecretEncryptionSpec, images KubeAPIServerImages) error {
Expand All @@ -21,13 +22,44 @@ func applyKMSConfig(podSpec *corev1.PodSpec, secretEncryptionData *hyperv1.Secre
return provider.ApplyKMSConfig(podSpec)
}

func generateKMSEncryptionConfig(kmsSpec *hyperv1.KMSSpec) ([]byte, error) {
// getKMSAPIVersion returns the KMS API version from the given EncryptionConfig
// secret. If the current state is using the IdentityProvider, the function
// returns v2 as the default version to start with.
func getKMSAPIVersion(config *corev1.Secret) (string, error) {
apiVersion := "v2"
encryptionConfigBytes := config.Data[secretEncryptionConfigurationKey]
if len(encryptionConfigBytes) > 0 {
currentConfig := v1.EncryptionConfiguration{}
gvks, _, err := api.Scheme.ObjectKinds(&currentConfig)
if err != nil || len(gvks) == 0 {
return "", fmt.Errorf("cannot determine gvk of resource: %v", err)
}
if _, _, err = api.YamlSerializer.Decode(encryptionConfigBytes, &gvks[0], &currentConfig); err != nil {
return "", fmt.Errorf("cannot decode resource: %v", err)
}

// Only look at write keys to return the APIVersion currently used.
for _, r := range currentConfig.Resources {
if len(r.Providers) > 0 && r.Providers[0].KMS != nil {
return r.Providers[0].KMS.APIVersion, nil
}
}
}
return apiVersion, nil
}

func generateKMSEncryptionConfig(config *corev1.Secret, kmsSpec *hyperv1.KMSSpec) ([]byte, error) {
provider, err := GetKMSProvider(kmsSpec, KubeAPIServerImages{})
if err != nil {
return nil, err
}

encryptionConfig, err := provider.GenerateKMSEncryptionConfig()
apiVersion, err := getKMSAPIVersion(config)
if err != nil {
return nil, err
}

encryptionConfig, err := provider.GenerateKMSEncryptionConfig(apiVersion)
if err != nil {
return nil, err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
backupAWSKMSUnixSocketFileName = "awskmsbackup.sock"
backupAWSKMSHealthPort = 8081
awsKeyNamePrefix = "awskmskey"
kmsAPIVersionV1 = "v1"
)

var (
Expand Down Expand Up @@ -78,7 +79,7 @@ func NewAWSKMSProvider(kmsSpec *hyperv1.AWSKMSSpec, kmsImage, tokenMinterImage s
}, nil
}

func (p *awsKMSProvider) GenerateKMSEncryptionConfig() (*v1.EncryptionConfiguration, error) {
func (p *awsKMSProvider) GenerateKMSEncryptionConfig(apiVersion string) (*v1.EncryptionConfiguration, error) {
var providerConfiguration []v1.ProviderConfiguration
if len(p.activeKey.ARN) == 0 {
return nil, fmt.Errorf("active key metadata is nil")
Expand All @@ -90,10 +91,10 @@ func (p *awsKMSProvider) GenerateKMSEncryptionConfig() (*v1.EncryptionConfigurat
}
providerConfiguration = append(providerConfiguration, v1.ProviderConfiguration{
KMS: &v1.KMSConfiguration{
Name: fmt.Sprintf("%s-%d", awsKeyNamePrefix, hasher.Sum32()),
Endpoint: activeAWSKMSUnixSocket,
CacheSize: ptr.To[int32](100),
Timeout: &metav1.Duration{Duration: 35 * time.Second},
APIVersion: apiVersion,
Name: fmt.Sprintf("%s-%d", awsKeyNamePrefix, hasher.Sum32()),
Endpoint: activeAWSKMSUnixSocket,
Timeout: &metav1.Duration{Duration: 35 * time.Second},
},
})
if p.backupKey != nil && len(p.backupKey.ARN) > 0 {
Expand All @@ -104,13 +105,20 @@ func (p *awsKMSProvider) GenerateKMSEncryptionConfig() (*v1.EncryptionConfigurat
}
providerConfiguration = append(providerConfiguration, v1.ProviderConfiguration{
KMS: &v1.KMSConfiguration{
Name: fmt.Sprintf("%s-%d", awsKeyNamePrefix, hasher.Sum32()),
Endpoint: backupAWSKMSUnixSocket,
CacheSize: ptr.To[int32](100),
Timeout: &metav1.Duration{Duration: 35 * time.Second},
APIVersion: apiVersion,
Name: fmt.Sprintf("%s-%d", awsKeyNamePrefix, hasher.Sum32()),
Endpoint: backupAWSKMSUnixSocket,
Timeout: &metav1.Duration{Duration: 35 * time.Second},
},
})
}

if apiVersion == kmsAPIVersionV1 {
for _, p := range providerConfiguration {
p.KMS.CacheSize = ptr.To[int32](100)
}
}

providerConfiguration = append(providerConfiguration, v1.ProviderConfiguration{
Identity: &v1.IdentityConfiguration{},
})
Expand Down Expand Up @@ -158,6 +166,9 @@ func (p *awsKMSProvider) ApplyKMSConfig(podSpec *corev1.PodSpec) error {
}
container.VolumeMounts = append(container.VolumeMounts,
awsKMSVolumeMounts.ContainerMounts(KasMainContainerName)...)

container.Args = append(container.Args, "--encryption-provider-config-automatic-reload=false")

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewAzureKMSProvider(kmsSpec *hyperv1.AzureKMSSpec, image string) (*azureKMS
}, nil
}

func (p *azureKMSProvider) GenerateKMSEncryptionConfig() (*v1.EncryptionConfiguration, error) {
func (p *azureKMSProvider) GenerateKMSEncryptionConfig(_ string) (*v1.EncryptionConfiguration, error) {
var providerConfiguration []v1.ProviderConfiguration

activeKeyHash, err := util.HashStruct(p.kmsSpec.ActiveKey)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func NewIBMCloudKMSProvider(ibmCloud *hyperv1.IBMCloudKMSSpec, kmsImage string)
}, nil
}

func (p *ibmCloudKMSProvider) GenerateKMSEncryptionConfig() (*v1.EncryptionConfiguration, error) {
func (p *ibmCloudKMSProvider) GenerateKMSEncryptionConfig(_ string) (*v1.EncryptionConfiguration, error) {

providerConfiguration := []v1.ProviderConfiguration{
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
)

type IKMSProvider interface {
GenerateKMSEncryptionConfig() (*v1.EncryptionConfiguration, error)
GenerateKMSEncryptionConfig(apiVersion string) (*v1.EncryptionConfiguration, error)

ApplyKMSConfig(podSpec *corev1.PodSpec) error
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func ReconcileKMSEncryptionConfig(config *corev1.Secret,
config.Data = map[string][]byte{}
}

encryptionConfigurationBytes, err := generateKMSEncryptionConfig(encryptionSpec)
encryptionConfigurationBytes, err := generateKMSEncryptionConfig(config, encryptionSpec)
if err != nil {
return err
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package kas

import (
"bytes"
"testing"
"time"

hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
"github.com/openshift/hypershift/support/api"
"github.com/openshift/hypershift/support/config"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
"k8s.io/utils/ptr"

"github.com/google/go-cmp/cmp"
)

const (
kmsAPIVersionV1 = "v1"
kmsAPIVersionV2 = "v2"
)

func TestReconcileKMSEncryptionConfigAWS(t *testing.T) {
encryptionSpec := hyperv1.KMSSpec{Provider: hyperv1.AWS, AWS: &hyperv1.AWSKMSSpec{
ActiveKey: hyperv1.AWSKMSKeyEntry{ARN: "test"},
}}

testCases := []struct {
name string
config *v1.EncryptionConfiguration
expectedConfig *v1.EncryptionConfiguration
}{
{
name: "No encryption config",
expectedConfig: generateExpectedEncryptionConfig(kmsAPIVersionV2),
},
{
name: "Encryption config with Identity provider",
config: &v1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"},
Resources: []v1.ResourceConfiguration{
{
Resources: config.KMSEncryptedObjects(),
Providers: []v1.ProviderConfiguration{
{
Identity: &v1.IdentityConfiguration{},
},
},
},
},
},
expectedConfig: generateExpectedEncryptionConfig(kmsAPIVersionV2),
},
{
name: "KMS v1 encryption config",
config: &v1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"},
Resources: []v1.ResourceConfiguration{
{
Resources: config.KMSEncryptedObjects(),
Providers: []v1.ProviderConfiguration{
{
KMS: &v1.KMSConfiguration{
APIVersion: "v1",
},
},
},
},
},
},
expectedConfig: generateExpectedEncryptionConfig(kmsAPIVersionV1),
},
{
name: "KMS v2 encryption config",
config: &v1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"},
Resources: []v1.ResourceConfiguration{
{
Resources: config.KMSEncryptedObjects(),
Providers: []v1.ProviderConfiguration{
{
KMS: &v1.KMSConfiguration{
APIVersion: "v2",
},
},
},
},
},
},
expectedConfig: generateExpectedEncryptionConfig(kmsAPIVersionV2),
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
encryptionConfigFile := manifests.KASSecretEncryptionConfigFile("test-namespace")
encryptionConfigFile.Data = map[string][]byte{}
if tc.config != nil {
buff := bytes.NewBuffer([]byte{})
err := api.YamlSerializer.Encode(tc.config, buff)
if err != nil {
t.Errorf("failed to encode encryption config: %v", err)
}
encryptionConfigFile.Data[secretEncryptionConfigurationKey] = buff.Bytes()
}

err := ReconcileKMSEncryptionConfig(encryptionConfigFile, config.OwnerRef{}, &encryptionSpec)
if err != nil {
t.Errorf("failed to reconcile KMS encryption config: %v", err)
}

encryptionConfigBytes := encryptionConfigFile.Data[secretEncryptionConfigurationKey]
if len(encryptionConfigBytes) == 0 {
t.Error("reconciled empty encryption config")
}
config := v1.EncryptionConfiguration{}
gvks, _, err := api.Scheme.ObjectKinds(&config)
if err != nil || len(gvks) == 0 {
t.Errorf("cannot determine gvk of resource: %v", err)
}
if _, _, err = api.YamlSerializer.Decode(encryptionConfigBytes, &gvks[0], &config); err != nil {
t.Errorf("cannot decode resource: %v", err)
}

if diff := cmp.Diff(config, *tc.expectedConfig); diff != "" {
t.Errorf("reconciled encrytion config differs from expected: %s", diff)
}
})
}
}

func generateExpectedEncryptionConfig(apiVersion string) *v1.EncryptionConfiguration {
config := &v1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"},
Resources: []v1.ResourceConfiguration{
{
Resources: config.KMSEncryptedObjects(),
Providers: []v1.ProviderConfiguration{
{
KMS: &v1.KMSConfiguration{
APIVersion: apiVersion,
Name: "awskmskey-3157003241",
Endpoint: "unix:///var/run/awskmsactive.sock",
Timeout: &metav1.Duration{Duration: 35 * time.Second},
},
},
{
Identity: &v1.IdentityConfiguration{},
},
},
},
},
}

if apiVersion == kmsAPIVersionV1 {
config.Resources[0].Providers[0].KMS.CacheSize = ptr.To[int32](100)
}

return config
}
2 changes: 1 addition & 1 deletion test/e2e/create_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func TestCreateClusterCustomConfig(t *testing.T) {
g.Expect(hostedCluster.Spec.SecretEncryption.KMS.AWS.Auth.AWSKMSRoleARN).ToNot(BeEmpty())

guestClient := e2eutil.WaitForGuestClient(t, testContext, mgtClient, hostedCluster)
e2eutil.EnsureSecretEncryptedUsingKMS(t, ctx, hostedCluster, guestClient)
e2eutil.EnsureSecretEncryptedUsingKMSV2(t, ctx, hostedCluster, guestClient)
// test oauth with identity provider
e2eutil.EnsureOAuthWithIdentityProvider(t, ctx, mgtClient, hostedCluster)
}).Execute(&clusterOpts, globalOpts.Platform, globalOpts.ArtifactDir, globalOpts.ServiceAccountSigningKey)
Expand Down
Loading

0 comments on commit 1308e2d

Please sign in to comment.