From ec50ceefac20b8a6567e4e187ea4b05d78cc7974 Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Thu, 21 Nov 2024 09:36:50 -0500
Subject: [PATCH 1/2] Add lifecycle rule for cleaning up noncurrent manifests

See https://github.com/dandi/dandi-infrastructure/issues/192 for
rationale behind one day rule.
---
 terraform/modules/dandiset_bucket/main.tf | 31 +++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf
index a485ce0..6ee2806 100644
--- a/terraform/modules/dandiset_bucket/main.tf
+++ b/terraform/modules/dandiset_bucket/main.tf
@@ -339,3 +339,34 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
     status = "Enabled"
   }
 }
+
+resource "aws_s3_bucket_lifecycle_configuration" "expire_noncurrent_manifest_files" {
+  # Must have bucket versioning enabled first
+  depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
+
+  count = var.versioning ? 1 : 0
+
+  bucket = aws_s3_bucket.dandiset_bucket.id
+
+  # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7
+  rule {
+    id = "ExpireOldManifestFileVersions"
+    filter {
+      # We only want to expire objects with the `dandisets/` prefix, i.e. manifest files.
+      # Other objects in this bucket are not subject to this lifecycle policy.
+      prefix = "dandisets/"
+    }
+
+    # Only keep 1 noncurrent version of manifest files
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = 1
+    }
+
+    # Also delete any delete markers associated with the expired object
+    expiration {
+      expired_object_delete_marker = true
+    }
+
+    status = "Enabled"
+  }
+}

From 744e744b4d18a05e17f1cba4c4a15e553aff3f1b Mon Sep 17 00:00:00 2001
From: Mike VanDenburgh <michael.vandenburgh@kitware.com>
Date: Thu, 21 Nov 2024 09:40:00 -0500
Subject: [PATCH 2/2] Temporarily disable manifest file GC in prod

---
 terraform/modules/dandiset_bucket/main.tf      | 2 +-
 terraform/modules/dandiset_bucket/variables.tf | 7 +++++++
 terraform/staging_bucket.tf                    | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/terraform/modules/dandiset_bucket/main.tf b/terraform/modules/dandiset_bucket/main.tf
index 6ee2806..38415f6 100644
--- a/terraform/modules/dandiset_bucket/main.tf
+++ b/terraform/modules/dandiset_bucket/main.tf
@@ -344,7 +344,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_noncurrent_manifest_fil
   # Must have bucket versioning enabled first
   depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
 
-  count = var.versioning ? 1 : 0
+  count = var.versioning && var.enable_manifest_file_expiration ? 1 : 0
 
   bucket = aws_s3_bucket.dandiset_bucket.id
 
diff --git a/terraform/modules/dandiset_bucket/variables.tf b/terraform/modules/dandiset_bucket/variables.tf
index 0eaf7db..f2f9525 100644
--- a/terraform/modules/dandiset_bucket/variables.tf
+++ b/terraform/modules/dandiset_bucket/variables.tf
@@ -34,3 +34,10 @@ variable "log_bucket_name" {
   type        = string
   description = "The name of the log bucket."
 }
+
+# TODO: remove this after it's ready to be enabled in production
+variable "enable_manifest_file_expiration" {
+  type        = bool
+  description = "Whether or not to enable expiration of manifest files."
+  default     = false
+}
diff --git a/terraform/staging_bucket.tf b/terraform/staging_bucket.tf
index c0481d6..11e798d 100644
--- a/terraform/staging_bucket.tf
+++ b/terraform/staging_bucket.tf
@@ -10,6 +10,7 @@ module "staging_dandiset_bucket" {
     aws         = aws
     aws.project = aws
   }
+  enable_manifest_file_expiration = true
 }
 
 module "staging_embargo_bucket" {