get the documentdb connection string from secrets manager

README.md

@@ -13,12 +13,15 @@ This will:
     return its modified value.
   * Create the `counters` database.
   * Create the `hits` database collection.
+  * Get the database credentials from a Secret.
+    * The Secret is stored in [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/).
   * Upload it to the [Amazon ECR](https://aws.amazon.com/ecr/).
 * Create an [Amazon API Gateway](https://aws.amazon.com/api-gateway/).
   * Configure it to use the Go AWS Lambda Function.
 * Create a VPC and all the required plumbing required for the Go AWS Lambda
   Function to use an Amazon DocumentDB Database instance.
   * Make the Document DB Database instance available in a [VPC database subnet](https://docs.aws.amazon.com/documentdb/latest/developerguide/document-db-subnet-groups.html).
+  * Make the Secrets Manager service endpoint available as a [VPC Endpoint](https://docs.aws.amazon.com/whitepapers/latest/aws-privatelink/what-are-vpc-endpoints.html).
 
 # Usage (on a Ubuntu Desktop)
 
@@ -122,3 +125,9 @@ List this repository dependencies (and which have newer versions):
 ```bash
 GITHUB_COM_TOKEN='YOUR_GITHUB_PERSONAL_TOKEN' ./renovate.sh
 ```
+
+# Notes
+
+* There is no way to use an AWS IAM Role to authenticate as a DocumentDB User.
+  * This means we cannot use the Lambda Function IAM Role as a password-less
+    authentication mechanism. So, we must manage the DocumentDB User password.

documentdb.tf

@@ -20,6 +20,19 @@ resource "random_password" "example_docdb_master_password" {
   override_special = "!#$%&*()-_=+[]{}<>:?" # NB cannot contain /"@
 }
 
+# see https://repost.aws/knowledge-center/delete-secrets-manager-secret
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret
+resource "aws_secretsmanager_secret" "example_docdb_master_secret" {
+  name_prefix             = "${var.name_prefix}-docdb-master-"
+  recovery_window_in_days = 0
+}
+
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version
+resource "aws_secretsmanager_secret_version" "example_docdb_master_secret" {
+  secret_id     = aws_secretsmanager_secret.example_docdb_master_secret.id
+  secret_string = local.example_docdb_master_connection_string
+}
+
 # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/docdb_cluster
 resource "aws_docdb_cluster" "example" {
   cluster_identifier           = var.name_prefix
@@ -60,19 +73,14 @@ resource "aws_security_group" "example_docdb" {
 
 # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
 resource "aws_vpc_security_group_ingress_rule" "example_docdb_mongo" {
-  for_each = {
-    for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : i => {
-      az        = module.vpc.azs[i]
-      cidr_ipv4 = cidr_block
-    }
-  }
+  for_each = { for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block }
 
   security_group_id = aws_security_group.example_docdb.id
   ip_protocol       = "tcp"
-  cidr_ipv4         = each.value.cidr_ipv4
+  cidr_ipv4         = each.value
   from_port         = local.example_docdb_port
   to_port           = local.example_docdb_port
   tags = {
-    Name = "${var.name_prefix}-docdb-mongo-${each.value.az}"
+    Name = "${var.name_prefix}-docdb-mongo-${each.key}"
   }
 }

example/go.mod

@@ -4,10 +4,24 @@ go 1.22.2
 
 require (
 	github.com/aws/aws-lambda-go v1.47.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6
 	go.mongodb.org/mongo-driver v1.15.0
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect

example/go.sum

@@ -1,5 +1,33 @@
 github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
 github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
+github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
+github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6 h1:TIOEjw0i2yyhmhRry3Oeu9YtiiHWISZ6j/irS1W3gX4=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.28.6/go.mod h1:3Ba++UwWd154xtP4FRX5pUK3Gt4up5sDHCve6kVfE+g=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=

example/main.go

@@ -7,9 +7,12 @@ import (
 	"log"
 	"net"
 	"os"
+	"time"
 
 	"github.com/aws/aws-lambda-go/events"
 	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -40,6 +43,8 @@ func dumpMachineNetworkAddresses() {
 }
 
 func handler(ctx context.Context, event events.APIGatewayV2HTTPRequest) (events.APIGatewayV2HTTPResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
 	client, err := connectToMongoDB(ctx)
 	if err != nil {
 		return events.APIGatewayV2HTTPResponse{}, err
@@ -69,9 +74,35 @@ func handler(ctx context.Context, event events.APIGatewayV2HTTPRequest) (events.
 
 // see Connecting Programmatically to Amazon DocumentDB at https://docs.aws.amazon.com/documentdb/latest/developerguide/connect_programmatically.html#connect_programmatically-tls_enabled
 func connectToMongoDB(ctx context.Context) (*mongo.Client, error) {
-	connectionString := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING")
+	connectionString := ""
+
+	connectionStringSecretID := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID")
+	if connectionStringSecretID != "" {
+		connectionStringSecretRegion := os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION")
+		if connectionStringSecretRegion == "" {
+			return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION environment variable is not set")
+		}
+
+		cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(connectionStringSecretRegion))
+		if err != nil {
+			return nil, fmt.Errorf("failed to load default config: %w", err)
+		}
+		client := secretsmanager.NewFromConfig(cfg)
+		secret, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+			SecretId: &connectionStringSecretID,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get secret value: %w", err)
+		}
+		connectionString = *secret.SecretString
+	}
+
+	if connectionString == "" {
+		connectionString = os.Getenv("EXAMPLE_DOCDB_CONNECTION_STRING")
+	}
+
 	if connectionString == "" {
-		return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING environment variable is not set")
+		return nil, fmt.Errorf("the EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID or EXAMPLE_DOCDB_CONNECTION_STRING environment variable is not set")
 	}
 
 	client, err := mongo.Connect(ctx, options.Client().ApplyURI(connectionString))

lambda.tf

@@ -49,8 +49,21 @@ module "example_lambda_function" {
     }
   }
 
+  attach_policy_json = true
+  policy_json = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow",
+        Action   = ["secretsmanager:GetSecretValue"]
+        Resource = aws_secretsmanager_secret.example_docdb_master_secret.arn
+      },
+    ]
+  })
+
   environment_variables = {
-    EXAMPLE_DOCDB_CONNECTION_STRING = local.example_docdb_master_connection_string
+    EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_REGION = var.region
+    EXAMPLE_DOCDB_CONNECTION_STRING_SECRET_ID     = aws_secretsmanager_secret.example_docdb_master_secret.name
   }
 }
 
@@ -64,21 +77,26 @@ resource "aws_security_group" "example_lambda_function" {
   }
 }
 
-# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule
 resource "aws_vpc_security_group_egress_rule" "example_lambda_function_mongo" {
-  for_each = {
-    for i, cidr_block in module.vpc.database_subnets_cidr_blocks : i => {
-      az        = module.vpc.azs[i]
-      cidr_ipv4 = cidr_block
-    }
-  }
+  for_each = { for i, cidr_block in module.vpc.database_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block }
 
   security_group_id = aws_security_group.example_lambda_function.id
   ip_protocol       = "tcp"
-  cidr_ipv4         = each.value.cidr_ipv4
+  cidr_ipv4         = each.value
   from_port         = local.example_docdb_port
   to_port           = local.example_docdb_port
   tags = {
-    Name = "${var.name_prefix}-docdb-mongo-${each.value.az}"
+    Name = "${var.name_prefix}-docdb-mongo-${each.key}"
+  }
+}
+
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_egress_rule
+resource "aws_vpc_security_group_egress_rule" "example_lambda_function_vpc" {
+  security_group_id = aws_security_group.example_lambda_function.id
+  ip_protocol       = "all"
+  cidr_ipv4         = module.vpc.vpc_cidr_block
+  tags = {
+    Name = "${var.name_prefix}-vpc"
   }
 }

vpc.tf

@@ -12,3 +12,47 @@ module "vpc" {
   database_subnets = ["10.0.30.0/24", "10.0.31.0/24", "10.0.32.0/24"]
   intra_subnets    = ["10.0.40.0/24", "10.0.41.0/24", "10.0.42.0/24"]
 }
+
+# see https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest/submodules/vpc-endpoints
+# see https://github.com/terraform-aws-modules/terraform-aws-vpc/blob/master/modules/vpc-endpoints
+module "vpc_endpoints" {
+  source  = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
+  version = "5.8.1"
+
+  vpc_id = module.vpc.vpc_id
+
+  endpoints = {
+    secretsmanager = {
+      service             = "secretsmanager"
+      private_dns_enabled = true
+      security_group_ids  = [aws_security_group.aws_secretsmanager.id]
+      subnet_ids          = module.vpc.intra_subnets
+      tags = {
+        Name = "${var.name_prefix}-aws-secretsmanager"
+      }
+    }
+  }
+}
+
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group
+resource "aws_security_group" "aws_secretsmanager" {
+  vpc_id = module.vpc.vpc_id
+  name   = "aws-secretsmanager"
+  tags = {
+    Name = "${var.name_prefix}-aws-secretsmanager"
+  }
+}
+
+# see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule
+resource "aws_vpc_security_group_ingress_rule" "aws_secretsmanager_intra" {
+  for_each = { for i, cidr_block in module.vpc.intra_subnets_cidr_blocks : module.vpc.azs[i] => cidr_block }
+
+  security_group_id = aws_security_group.aws_secretsmanager.id
+  cidr_ipv4         = each.value
+  ip_protocol       = "tcp"
+  from_port         = 443
+  to_port           = 443
+  tags = {
+    Name = "${var.name_prefix}-aws-secretsmanager-intra-${each.key}"
+  }
+}