From f37f42bb244db4ee67fd89a0155bc6e3c781a737 Mon Sep 17 00:00:00 2001 From: Rui Lopes Date: Tue, 30 Apr 2024 07:56:55 +0100 Subject: [PATCH] get the documentdb connection string from secrets manager --- README.md | 9 +++++++++ documentdb.tf | 24 ++++++++++++++++-------- example/go.mod | 14 ++++++++++++++ example/go.sum | 28 ++++++++++++++++++++++++++++ example/main.go | 35 +++++++++++++++++++++++++++++++++-- lambda.tf | 38 ++++++++++++++++++++++++++++---------- vpc.tf | 44 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 172 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4a13cb3..1ba9b0b 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,15 @@ This will: return its modified value. * Create the `counters` database. * Create the `hits` database collection. + * Get the database credentials from a Secret. + * The Secret is stored in [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). * Upload it to the [Amazon ECR](https://aws.amazon.com/ecr/). * Create an [Amazon API Gateway](https://aws.amazon.com/api-gateway/). * Configure it to use the Go AWS Lambda Function. * Create a VPC and all the required plumbing required for the Go AWS Lambda Function to use an Amazon DocumentDB Database instance. * Make the Document DB Database instance available in a [VPC database subnet](https://docs.aws.amazon.com/documentdb/latest/developerguide/document-db-subnet-groups.html). + * Make the Secrets Manager service endpoint available as a [VPC Endpoint](https://docs.aws.amazon.com/whitepapers/latest/aws-privatelink/what-are-vpc-endpoints.html). # Usage (on a Ubuntu Desktop) @@ -122,3 +125,9 @@ List this repository dependencies (and which have newer versions): ```bash GITHUB_COM_TOKEN='YOUR_GITHUB_PERSONAL_TOKEN' ./renovate.sh ``` + +# Notes + +* There is no way to use an AWS IAM Role to authenticate as a DocumentDB User. + * This means we cannot use the Lambda Function IAM Role as a password-less + authentication mechanism. So, we must manage the DocumentDB User password. diff --git a/documentdb.tf b/documentdb.tf index 2b57f32..3f75930 100644 --- a/documentdb.tf +++ b/documentdb.tf @@ -20,6 +20,19 @@ resource "random_password" "example_docdb_master_password" { override_special = "!#$%&*()-_=+[]{}<>:?" # NB cannot contain /"@ } +# see https://repost.aws/knowledge-center/delete-secrets-manager-secret +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret +resource "aws_secretsmanager_secret" "example_docdb_master_secret" { + name_prefix = "${var.name_prefix}-docdb-master-" + recovery_window_in_days = 0 +} + +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version +resource "aws_secretsmanager_secret_version" "example_docdb_master_secret" { + secret_id = aws_secretsmanager_secret.example_docdb_master_secret.id + secret_string = local.example_docdb_master_connection_string +} + # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/docdb_cluster resource "aws_docdb_cluster" "example" { cluster_identifier = var.name_prefix @@ -60,19 +73,14 @@ resource "aws_security_group" "example_docdb" { # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule resource "aws_vpc_security_group_ingress_rule" "example_docdb_mongo" { - for_each = { - for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : i => { - az = module.vpc.azs[i] - cidr_ipv4 = cidr_block - } - } + for_each = { for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block } security_group_id = aws_security_group.example_docdb.id ip_protocol = "tcp" - cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv4 = each.value from_port = local.example_docdb_port to_port = local.example_docdb_port tags = { - Name = "${var.name_prefix}-docdb-mongo-${each.value.az}" + Name = "${var.name_prefix}-docdb-mongo-${each.key}" } } diff --git a/example/go.mod b/example/go.mod index c173841..d872411 100644 --- a/example/go.mod +++ b/example/go.mod @@ -4,10 +4,24 @@ go 1.22.2 require ( github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6 go.mongodb.org/mongo-driver v1.15.0 ) require ( + github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/montanaflynn/stats v0.7.1 // indirect diff --git a/example/go.sum b/example/go.sum index b0d7994..ccf4b37 100644 --- a/example/go.sum +++ b/example/go.sum @@ -1,5 +1,33 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6 h1:TIOEjw0i2yyhmhRry3Oeu9YtiiHWISZ6j/irS1W3gX4= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6/go.mod h1:3Ba++UwWd154xtP4FRX5pUK3Gt4up5sDHCve6kVfE+g= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= diff --git a/example/main.go b/example/main.go index 956c10a..178e048 100644 --- a/example/main.go +++ b/example/main.go @@ -7,9 +7,12 @@ import ( "log" "net" "os" + "time" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -40,6 +43,8 @@ func dumpMachineNetworkAddresses() { } func handler(ctx context.Context, event events.APIGatewayV2HTTPRequest) (events.APIGatewayV2HTTPResponse, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() client, err := connectToMongoDB(ctx) if err != nil { return events.APIGatewayV2HTTPResponse{}, err @@ -69,9 +74,35 @@ func handler(ctx context.Context, event events.APIGatewayV2HTTPRequest) (events. // see Connecting Programmatically to Amazon DocumentDB at https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html#connect_programmatically-tls_enabled func connectToMongoDB(ctx context.Context) (*mongo.Client, error) { - connectionString := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING") + connectionString := "" + + connectionStringSecretID := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID") + if connectionStringSecretID != "" { + connectionStringSecretRegion := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION") + if connectionStringSecretRegion == "" { + return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION environment variable is not set") + } + + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(connectionStringSecretRegion)) + if err != nil { + return nil, fmt.Errorf("failed to load default config: %w", err) + } + client := secretsmanager.NewFromConfig(cfg) + secret, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: &connectionStringSecretID, + }) + if err != nil { + return nil, fmt.Errorf("failed to get secret value: %w", err) + } + connectionString = *secret.SecretString + } + + if connectionString == "" { + connectionString = os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING") + } + if connectionString == "" { - return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING environment variable is not set") + return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID or EXAMPLE_DOCDB_CONNECTION_STRING environment variable is not set") } client, err := mongo.Connect(ctx, options.Client().ApplyURI(connectionString)) diff --git a/lambda.tf b/lambda.tf index a26bda1..73fc79a 100644 --- a/lambda.tf +++ b/lambda.tf @@ -49,8 +49,21 @@ module "example_lambda_function" { } } + attach_policy_json = true + policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow", + Action = ["secretsmanager:GetSecretValue"] + Resource = aws_secretsmanager_secret.example_docdb_master_secret.arn + }, + ] + }) + environment_variables = { - EXAMPLE_DOCDB_CONNECTION_STRING = local.example_docdb_master_connection_string + EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION = var.region + EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID = aws_secretsmanager_secret.example_docdb_master_secret.name } } @@ -64,21 +77,26 @@ resource "aws_security_group" "example_lambda_function" { } } -# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule resource "aws_vpc_security_group_egress_rule" "example_lambda_function_mongo" { - for_each = { - for i, cidr_block in module.vpc.database_subnets_cidr_blocks : i => { - az = module.vpc.azs[i] - cidr_ipv4 = cidr_block - } - } + for_each = { for i, cidr_block in module.vpc.database_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block } security_group_id = aws_security_group.example_lambda_function.id ip_protocol = "tcp" - cidr_ipv4 = each.value.cidr_ipv4 + cidr_ipv4 = each.value from_port = local.example_docdb_port to_port = local.example_docdb_port tags = { - Name = "${var.name_prefix}-docdb-mongo-${each.value.az}" + Name = "${var.name_prefix}-docdb-mongo-${each.key}" + } +} + +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule +resource "aws_vpc_security_group_egress_rule" "example_lambda_function_vpc" { + security_group_id = aws_security_group.example_lambda_function.id + ip_protocol = "all" + cidr_ipv4 = module.vpc.vpc_cidr_block + tags = { + Name = "${var.name_prefix}-vpc" } } diff --git a/vpc.tf b/vpc.tf index 466752d..d79e60e 100644 --- a/vpc.tf +++ b/vpc.tf @@ -12,3 +12,47 @@ module "vpc" { database_subnets = ["10.0.30.0/24", "10.0.31.0/24", "10.0.32.0/24"] intra_subnets = ["10.0.40.0/24", "10.0.41.0/24", "10.0.42.0/24"] } + +# see https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest/submodules/vpc-endpoints +# see https://github.com/terraform-aws-modules/terraform-aws-vpc/blob/master/modules/vpc-endpoints +module "vpc_endpoints" { + source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" + version = "5.8.1" + + vpc_id = module.vpc.vpc_id + + endpoints = { + secretsmanager = { + service = "secretsmanager" + private_dns_enabled = true + security_group_ids = [aws_security_group.aws_secretsmanager.id] + subnet_ids = module.vpc.intra_subnets + tags = { + Name = "${var.name_prefix}-aws-secretsmanager" + } + } + } +} + +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group +resource "aws_security_group" "aws_secretsmanager" { + vpc_id = module.vpc.vpc_id + name = "aws-secretsmanager" + tags = { + Name = "${var.name_prefix}-aws-secretsmanager" + } +} + +# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule +resource "aws_vpc_security_group_ingress_rule" "aws_secretsmanager_intra" { + for_each = { for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block } + + security_group_id = aws_security_group.aws_secretsmanager.id + cidr_ipv4 = each.value + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + tags = { + Name = "${var.name_prefix}-aws-secretsmanager-intra-${each.key}" + } +}