Enable CNM-driven Ingestion

_templates/cnm-rule/new/cnm-rule.ejs.t

@@ -0,0 +1,27 @@
+---
+to: "app/stacks/cumulus/resources/rules/<%= collectionName %>/v<%= collectionVersion %>/<%= collectionName %>___<%= collectionVersion %>_CNM.json"
+message: >
+  hygen cnm-rule new
+  --provider <%= provider %>
+  --collection-name <%= collectionName %>
+  --collection-version <%= collectionVersion %>
+---
+{
+  "name": "<%= collectionName %>___<%= collectionVersion %>_CNM",
+  "state": "ENABLED",
+  "workflow": "CNMIngestAndPublishGranule",
+  "provider": "<%= provider %>",
+  "collection": {
+    "name": "<%= collectionName %>",
+    "version": "<%= collectionVersion %>"
+  },
+  "rule": {
+    "type": "sns"
+  },
+  "meta": {
+    "cnmResponseMethod": "sns"
+  },
+  "tags": [
+    "cnm"
+  ]
+}

_templates/cnm-rule/new/prompt.js

@@ -0,0 +1,20 @@
+// see types of prompts:
+// https://github.com/enquirer/enquirer/tree/master/examples
+//
+module.exports = [
+  {
+    type: 'input',
+    name: 'provider',
+    message: "Provider ID (example: maxar):"
+  },
+  {
+    type: 'input',
+    name: 'collectionName',
+    message: "Collection name (example: WV03_MSI_L1B):"
+  },
+  {
+    type: 'input',
+    name: 'collectionVersion',
+    message: "Collection version (example: 1):"
+  }
+]

app/stacks/cumulus/config/hooks/terraform.rb

@@ -24,6 +24,7 @@ def call(runner)
       "cumulus/tf-modules/ingest/message_template.tf",
       "discover_granules_workflow/tf-modules/workflow/main.tf",
       "ingest_and_publish_granule_workflow/tf-modules/workflow/main.tf",
+      "cnm_ingest_and_publish_granule_workflow/tf-modules/workflow/main.tf",
     ]
 
     filepaths.each do |filepath|

app/stacks/cumulus/iam.tf

@@ -130,6 +130,34 @@ resource "null_resource" "attach_system_bucket_policy" {
   }
 }
 
+#-------------------------------------------------------------------------------
+# Additional permissions to allow use of MCP customer-managed key
+#-------------------------------------------------------------------------------
+
+data "aws_iam_policy_document" "allow_use_mcp_key" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = ["arn:aws:kms:us-west-2:${data.ssm_parameters.mcp_account_id}:*"]
+  }
+}
+
+resource "aws_iam_policy" "allow_use_mcp_key" {
+  name   = "${var.prefix}-mcp-key-policy"
+  policy = data.aws_iam_policy_document.allow_use_mcp_key.json
+}
+
+resource "aws_iam_role_policy_attachment" "allow_use_mcp_key" {
+  role       = module.cumulus.lambda_processing_role_name
+  policy_arn = aws_iam_policy.allow_use_mcp_key.arn
+}
+
 #-------------------------------------------------------------------------------
 # Temporary workaround for dashboard permissions issue
 #-------------------------------------------------------------------------------

app/stacks/cumulus/main.tf

@@ -11,6 +11,12 @@ locals {
 
   cmr_provider = "CSDA"
 
+  cnm_to_cma_version  = "1.8.0"
+  cnm_to_cma_zip_name = "cnmToGranule-${local.cnm_to_cma_version}.zip"
+
+  cnm_response_version  = "2.2.0"
+  cnm_response_zip_name = "cnmResponse-${local.cnm_response_version}.zip"
+
   dynamo_tables = jsondecode("<%= json_output('data-persistence.dynamo_tables') %>")
 
   ecs_task_cpu                = 768
@@ -375,6 +381,115 @@ resource "aws_lambda_function" "record_workflow_failure" {
   }
 }
 
+resource "null_resource" "download_cnm_to_cma_zip_file" {
+  triggers = {
+    always_run = local.cnm_to_cma_version
+    bucket     = var.system_bucket
+  }
+
+  provisioner "local-exec" {
+    command = "curl -s -L -o ${local.cnm_to_cma_zip_name} https://github.com/podaac/cumulus-cnm-to-granule/releases/download/v${local.cnm_to_cma_version}/${local.cnm_to_cma_zip_name}"
+  }
+}
+
+resource "aws_s3_object" "cnm_to_cma_lambda_zip" {
+  depends_on = [null_resource.download_cnm_to_cma_zip_file]
+  bucket     = var.system_bucket
+  key        = "${var.prefix}/${local.cnm_to_cma_zip_name}"
+  source     = local.cnm_to_cma_zip_name
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "rm -f ${local.cnm_to_cma_zip_name}"
+  }
+}
+
+resource "aws_lambda_function" "cnm_to_cma" {
+  depends_on       = [aws_s3_object.cnm_to_cma_lambda_zip]
+  function_name    = "${var.prefix}-CNMToCMA"
+  s3_bucket        = var.system_bucket
+  s3_key           = aws_s3_object.cnm_to_cma_lambda_zip.id
+  handler          = "gov.nasa.cumulus.CnmToGranuleHandler::handleRequestStreams"
+  role             = module.cumulus.lambda_processing_role_arn
+  runtime          = "java11"
+  timeout          = 300
+  memory_size      = 128
+
+  source_code_hash = aws_s3_object.cnm_to_cma_lambda_zip.etag
+  layers           = [module.cma.lambda_layer_version_arn]
+
+  tags = local.tags
+
+  environment {
+    variables = {
+      stackName                   = var.prefix
+      CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
+    }
+  }
+
+  dynamic "vpc_config" {
+    for_each = length(module.vpc.subnets.ids) == 0 ? [] : [1]
+    content {
+      subnet_ids         = module.vpc.subnets.ids
+      security_group_ids = [aws_security_group.egress_only.id]
+    }
+  }
+}
+
+resource "null_resource" "download_cnm_response_zip_file" {
+  triggers = {
+    always_run = local.cnm_response_version
+    bucket     = var.system_bucket
+  }
+  provisioner "local-exec" {
+    command = "curl -s -L -o ${local.cnm_response_zip_name} https://github.com/podaac/cumulus-cnm-response-task/releases/download/v${local.cnm_response_version}/${local.cnm_response_zip_name}"
+  }
+}
+
+resource "aws_s3_object" "cnm_response_lambda_zip" {
+  depends_on = [null_resource.download_cnm_response_zip_file]
+  bucket     = var.system_bucket
+  key        = "${var.prefix}/${local.cnm_response_zip_name}"
+  source     = local.cnm_response_zip_name
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "rm -f ${local.cnm_response_zip_name}"
+  }
+}
+
+resource "aws_lambda_function" "cnm_response" {
+  depends_on       = [aws_s3_object.cnm_response_lambda_zip]
+  function_name    = "${var.prefix}-CnmResponse"
+  s3_bucket        = var.system_bucket
+  s3_key           = aws_s3_object.cnm_response_lambda_zip.id
+  handler          = "gov.nasa.cumulus.CNMResponse::handleRequestStreams"
+  role             = module.cumulus.lambda_processing_role_arn
+  runtime          = "java11"
+  timeout          = 300
+  memory_size      = 256
+
+  source_code_hash = aws_s3_object.cnm_response_lambda_zip.etag
+  layers           = [module.cma.lambda_layer_version_arn]
+
+  tags = local.tags
+
+  environment {
+    variables = {
+      stackName                   = var.prefix
+      CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
+    }
+  }
+
+  dynamic "vpc_config" {
+    for_each = length(module.vpc.subnets.ids) == 0 ? [] : [1]
+    content {
+      subnet_ids         = module.vpc.subnets.ids
+      security_group_ids = [aws_security_group.egress_only.id]
+    }
+  }
+}
+
 #-------------------------------------------------------------------------------
 # MODULES
 #-------------------------------------------------------------------------------
@@ -465,8 +580,6 @@ module "ingest_and_publish_granule_workflow" {
     sync_granule_task_arn : module.cumulus.sync_granule_task.task_arn,
     add_ummg_checksums_task_arn : aws_lambda_function.add_ummg_checksums.arn,
     add_missing_file_checksums_task_arn : module.cumulus.add_missing_file_checksums_task.task_arn,
-    fake_processing_task_arn : module.cumulus.fake_processing_task.task_arn,
-    files_to_granules_task_arn : module.cumulus.files_to_granules_task.task_arn,
     move_granules_task_arn : module.cumulus.move_granules_task.task_arn,
     update_granules_cmr_metadata_file_links_task_arn : module.cumulus.update_granules_cmr_metadata_file_links_task.task_arn,
     copy_to_archive_adapter_task_arn : module.cumulus.orca_copy_to_archive_adapter_task.task_arn,
@@ -475,6 +588,35 @@ module "ingest_and_publish_granule_workflow" {
   })
 }
 
+module "cnm_ingest_and_publish_granule_workflow" {
+  depends_on = [
+    aws_lambda_function.cnm_response,
+    aws_lambda_function.cnm_to_cma
+  ]
+
+  source = "https://github.com/nasa/cumulus/releases/download/<%= cumulus_version %>/terraform-aws-cumulus.zip//tf-modules/workflow"
+
+  prefix          = var.prefix
+  name            = "CNMIngestAndPublishGranule"
+  workflow_config = module.cumulus.workflow_config
+  system_bucket   = var.system_bucket
+  tags            = local.tags
+
+  state_machine_definition = templatefile("${path.module}/templates/cnm-ingest-and-publish-granule-workflow.asl.json", {
+    cnm_to_cma_task_arn: aws_lambda_function.cnm_to_cma.arn,
+    require_cmr_files_task_arn : aws_lambda_function.require_cmr_files.arn,
+    sync_granule_task_arn : module.cumulus.sync_granule_task.task_arn,
+    add_ummg_checksums_task_arn : aws_lambda_function.add_ummg_checksums.arn,
+    add_missing_file_checksums_task_arn : module.cumulus.add_missing_file_checksums_task.task_arn,
+    move_granules_task_arn : module.cumulus.move_granules_task.task_arn,
+    update_granules_cmr_metadata_file_links_task_arn : module.cumulus.update_granules_cmr_metadata_file_links_task.task_arn,
+    copy_to_archive_adapter_task_arn : module.cumulus.orca_copy_to_archive_adapter_task.task_arn,
+    post_to_cmr_task_arn : module.cumulus.post_to_cmr_task.task_arn,
+    cnm_response_task_arn: aws_lambda_function.cnm_response.arn,
+    record_workflow_failure_task_arn : aws_lambda_function.record_workflow_failure.arn,
+  })
+}
+
 module "cumulus" {
   source = "https://github.com/nasa/cumulus/releases/download/<%= cumulus_version %>/terraform-aws-cumulus.zip//tf-modules/cumulus"
 

app/stacks/cumulus/resources/rules/WV04_MSI_L1B/v1/WV04_MSI_L1B___1_CNM.json

@@ -0,0 +1,19 @@
+{
+  "name": "WV04_MSI_L1B___1_CNM",
+  "state": "ENABLED",
+  "workflow": "CNMIngestAndPublishGranule",
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV04_MSI_L1B",
+    "version": "1"
+  },
+  "rule": {
+    "type": "sns"
+  },
+  "meta": {
+    "cnmResponseMethod": "sns"
+  },
+  "tags": [
+    "cnm"
+  ]
+}

app/stacks/cumulus/resources/rules/WV04_Pan_L1B/v1/WV04_Pan_L1B___1_CNM.json

@@ -0,0 +1,19 @@
+{
+  "name": "WV04_Pan_L1B___1_CNM",
+  "state": "ENABLED",
+  "workflow": "CNMIngestAndPublishGranule",
+  "provider": "cumulus",
+  "collection": {
+    "name": "WV04_Pan_L1B",
+    "version": "1"
+  },
+  "rule": {
+    "type": "sns"
+  },
+  "meta": {
+    "cnmResponseMethod": "sns"
+  },
+  "tags": [
+    "cnm"
+  ]
+}

app/stacks/cumulus/ssm_parameters.tf

@@ -81,6 +81,12 @@ data "aws_ssm_parameter" "orca_s3_secret_key" {
   name = "/shared/cumulus/orca/dr/s3-secret-key"
 }
 
+# MCP Account ID
+
+data "aws_ssm_parameter" "mcp_account_id" {
+  name = "/shared/cumulus/mcp-account-id"
+}
+
 #-------------------------------------------------------------------------------
 # SSM Parameters required across ONLY non-sandbox (non-dev) environments
 #-------------------------------------------------------------------------------