Merge pull request #199 from dandi/expire-noncurrent-manifest-files

Manifest file garbage collection

terraform/modules/dandiset_bucket/main.tf

@@ -339,3 +339,34 @@ resource "aws_s3_bucket_lifecycle_configuration" "expire_deleted_objects" {
     status = "Enabled"
   }
 }
+
+resource "aws_s3_bucket_lifecycle_configuration" "expire_noncurrent_manifest_files" {
+  # Must have bucket versioning enabled first
+  depends_on = [aws_s3_bucket_versioning.dandiset_bucket]
+
+  count = var.versioning && var.enable_manifest_file_expiration ? 1 : 0
+
+  bucket = aws_s3_bucket.dandiset_bucket.id
+
+  # Based on https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html#lifecycle-config-conceptual-ex7
+  rule {
+    id = "ExpireOldManifestFileVersions"
+    filter {
+      # We only want to expire objects with the `dandisets/` prefix, i.e. manifest files.
+      # Other objects in this bucket are not subject to this lifecycle policy.
+      prefix = "dandisets/"
+    }
+
+    # Only keep 1 noncurrent version of manifest files
+    noncurrent_version_expiration {
+      newer_noncurrent_versions = 1
+    }
+
+    # Also delete any delete markers associated with the expired object
+    expiration {
+      expired_object_delete_marker = true
+    }
+
+    status = "Enabled"
+  }
+}

terraform/modules/dandiset_bucket/variables.tf

@@ -34,3 +34,10 @@ variable "log_bucket_name" {
   type        = string
   description = "The name of the log bucket."
 }
+
+# TODO: remove this after it's ready to be enabled in production
+variable "enable_manifest_file_expiration" {
+  type        = bool
+  description = "Whether or not to enable expiration of manifest files."
+  default     = false
+}

terraform/staging_bucket.tf

@@ -10,6 +10,7 @@ module "staging_dandiset_bucket" {
     aws         = aws
     aws.project = aws
   }
+  enable_manifest_file_expiration = true
 }
 
 module "staging_embargo_bucket" {