Merge branch 'main' into update-for-mono-repo

.github/actions/cdk-deploy/action.yml

@@ -12,8 +12,6 @@ inputs:
     required: true
     type: string
 
-
-
 runs:
   using: "composite"
   steps:
@@ -58,9 +56,10 @@ runs:
         python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }}
         fi
 
+
     - name: Deploy
       id: deploy_auth_stack
       shell: bash
       working-directory: ${{ inputs.dir }}
       run: |
-        cdk deploy --all --require-approval never --outputs-file ${HOME}/cdk-outputs.json
+        cdk deploy --all --require-approval never --outputs-file ${HOME}/cdk-outputs.json

.github/workflows/cicd.yml

@@ -21,8 +21,10 @@ jobs:
         run: |
           if [ "${{ github.ref }}" = "refs/heads/main" ]; then
             echo "env_name=staging" >> $GITHUB_OUTPUT
+            echo "secret_name=veda-auth-staging" >> $GITHUB_OUTPUT
           elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then
             echo "env_name=development" >> $GITHUB_OUTPUT
+            echo "secret_name=veda-auth-dev" >> $GITHUB_OUTPUT
           elif [ "${{ github.ref }}" = "refs/heads/production" ]; then
             echo "env_name=production" >> $GITHUB_OUTPUT
           fi
@@ -47,8 +49,8 @@ jobs:
           lfs: "true"
           submodules: "recursive"
 
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+      - name: Configure awscli
+        uses: aws-actions/configure-aws-credentials@v3
         with:
           role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
           role-session-name: "ghgc-auth-github-${{ needs.define-environment.outputs.env_name }}-deployment"
@@ -57,4 +59,4 @@ jobs:
       - name: Run deployment
         uses: "./.github/actions/cdk-deploy"
         with:
-          env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }}
+          env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }}

.github/workflows/pr.yml

@@ -0,0 +1,50 @@
+name: Pull Request - Preview CDK Diff
+
+on: [pull_request]
+
+jobs:
+  predeploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 17
+
+      - name: Configure awscli
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+
+      - name: Install CDK
+        run: npm install -g aws-cdk@2
+
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key:  ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
+
+      - name: Install python dependencies
+        run: |
+            pip install -r requirements.txt
+
+      - name: Get environment configuration for target branch
+        run: |
+          ./scripts/get-env.sh "veda-auth-uah-env"
+      - name: Pre deployment CDK diff
+        run: |
+          echo $STAGE
+          cdk diff --outputs-file ${HOME}/cdk-outputs.json

.gitignore

@@ -9,3 +9,5 @@ __pycache__
 .cdk.staging
 cdk.out
 *.env
+.ipynb_checkpoints
+

app.py

@@ -1,39 +1,33 @@
 #!/usr/bin/env python3
 import subprocess
 
-import aws_cdk as cdk
+from aws_cdk import App, Tags, DefaultStackSynthesizer
 
 from infra.stack import AuthStack, BucketPermissions
 
 from config import app_settings
 
-git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()
-try:
-    git_tag = subprocess.check_output(["git", "describe", "--tags"]).decode().strip()
-except subprocess.CalledProcessError:
-    git_tag = "no-tag"
+app = App()
 
-tags = {
-    "Project": "veda",
-    "Owner": app_settings.owner,
-    "Client": "nasa-impact",
-    "Stack": app_settings.stage,
-    "GitCommit": git_sha,
-    "GitTag": git_tag,
-}
+stack = AuthStack(
+    app, 
+    f"{app_settings.app_name}-{app_settings.stage}", 
+    app_settings,
+    synthesizer=DefaultStackSynthesizer(
+        qualifier=app_settings.bootstrap_qualifier
+    )
+)
 
-app = cdk.App()
-stack = AuthStack(app, f"{app_settings.app_name}-{app_settings.stage}", app_settings)
-# Create Groups
-if app_settings.cognito_groups:
-    # Create a data managers group in user pool if data managers role is provided
-    if data_managers_role_arn := app_settings.data_managers_role_arn:
-        stack.add_cognito_group_with_existing_role(
-            "veda-data-store-managers",
-            "Authenticated users assume read write veda data access role",
-            role_arn=data_managers_role_arn,
-        )
+# Create an data managers group in user pool if data managers role is provided (legacy stack support)
+if app_settings.data_managers_group and app_settings.data_managers_role_arn:
+    stack.add_cognito_group_with_existing_role(
+        "veda-data-store-managers",
+        "Authenticated users assume read write veda data access role",
+        role_arn=app_settings.data_managers_role_arn,
+    )
 
+# Create Groups if Configured
+if app_settings.cognito_groups:
     stack.add_cognito_group(
         "veda-staging-writers",
         "Users that have read/write-access to the VEDA store and staging datastore",
@@ -115,7 +109,22 @@
 # Frontend Clients
 # stack.add_frontend_client('veda-dashboard')
 
+git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()
+try:
+    git_tag = subprocess.check_output(["git", "describe", "--tags"]).decode().strip()
+except subprocess.CalledProcessError:
+    git_tag = "no-tag"
+
+tags = {
+    "Project": "veda",
+    "Owner": app_settings.owner,
+    "Client": "nasa-impact",
+    "Stack": app_settings.stage,
+    "GitCommit": git_sha,
+    "GitTag": git_tag,
+}
+
 for key, value in tags.items():
-    cdk.Tags.of(stack).add(key, value)
+    Tags.of(stack).add(key, value)
 
 app.synth()

cdk.json

@@ -1,31 +1,3 @@
 {
-  "app": "python3 app.py",
-  "watch": {
-    "include": [
-      "**"
-    ],
-    "exclude": [
-      "README.md",
-      "cdk*.json",
-      "requirements*.txt",
-      "source.bat",
-      "**/*.pyc",
-      "**/*.tmp",
-      "**/__pycache__",
-      "tests",
-      "scripts"
-    ]
-  },
-  "context": {
-    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
-    "@aws-cdk/core:stackRelativeExports": true,
-    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
-    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
-    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
-    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
-    "@aws-cdk/core:target-partitions": [
-      "aws",
-      "aws-cn"
-    ]
-  }
+  "app": "python3 app.py"
 }

config.py

@@ -37,6 +37,11 @@ class Config(pydantic.BaseSettings):
         description="ARN of role to be assumed by authenticated users in data managers group.",
     )
 
+    data_managers_group: bool = pydantic.Field(
+        False,
+        description="When true create data managers group (mcp-deploy refactor now requires additional control setting to enable creating this group).",
+    )
+
     oidc_provider_url: Optional[str] = pydantic.Field(
         None,
         description="URL of OIDC provider to use for CI workers.",
@@ -54,7 +59,7 @@ class Config(pydantic.BaseSettings):
 
     # Since MCP doesn't allow creating identity pools, setting this as optional
     cognito_groups: Optional[bool] = pydantic.Field(
-        True,
+        False,
         description="whether to create cognito groups with bucket access permissions",
     )
 
@@ -67,5 +72,10 @@ class Config(pydantic.BaseSettings):
         "", description="The user pool id to use for user management"
     )
 
+    bootstrap_qualifier: Optional[str] = pydantic.Field(
+        None,
+        description="Custom bootstrap qualifier override if not using a default installation of AWS CDK Toolkit to synthesize app.",
+    )
+
 
 app_settings = Config(_env_file=os.environ.get("ENV_FILE", ".env"))

infra/stack.py

@@ -2,15 +2,14 @@
 from enum import Enum
 from typing import Any, Dict, Optional, Sequence
 
-from aws_cdk import CfnOutput, RemovalPolicy, SecretValue, Stack
+from aws_cdk import Aspects, CfnOutput, RemovalPolicy, SecretValue, Stack
 from aws_cdk import aws_cognito as cognito
 from aws_cdk import aws_cognito_identitypool_alpha as cognito_id_pool
 from aws_cdk import aws_iam as iam
 from aws_cdk import aws_s3 as s3
 from aws_cdk import aws_secretsmanager as secretsmanager
 from aws_cdk import custom_resources as cr
 from constructs import Construct
-from aws_cdk import Aspects
 
 from config import Config
 
@@ -51,7 +50,7 @@ def __init__(
             export_name=f"{stack_name}-userpool-id",
             value=self.userpool.user_pool_id,
         )
-        if app_settings.cognito_groups:
+        if app_settings.cognito_groups or app_settings.data_managers_group:
             self._group_precedence = 0
 
             if app_settings.identity_pool_arn:
@@ -63,22 +62,23 @@ def __init__(
                     "cognito-identity-pool-auth-provider",
                     name="Identity Pool Authentication Provider",
                 )
-                self.identitypool = self._create_identity_pool(
-                    userpool=self.userpool,
-                    auth_provider_client=auth_provider_client,
-                )
-            CfnOutput(
-                self,
-                "identitypool_id",
-                export_name=f"{stack_name}-identitypool-id",
-                value=self.identitypool.identity_pool_id,
-            )
-            CfnOutput(
-                self,
-                "identitypool_arn",
-                export_name=f"{stack_name}-identitypool-arn",
-                value=self.identitypool.identity_pool_arn,
-            )
+                if app_settings.data_managers_role_arn:
+                    self.identitypool = self._create_identity_pool(
+                        userpool=self.userpool,
+                        auth_provider_client=auth_provider_client,
+                    )
+                    CfnOutput(
+                        self,
+                        "identitypool_id",
+                        export_name=f"{stack_name}-identitypool-id",
+                        value=self.identitypool.identity_pool_id,
+                    )
+                    CfnOutput(
+                        self,
+                        "identitypool_arn",
+                        export_name=f"{stack_name}-identitypool-arn",
+                        value=self.identitypool.identity_pool_arn,
+                    )
             # CfnOutput(
             #     self,
             #     "identitypool_client_id",
@@ -328,15 +328,23 @@ def add_programmatic_client(
             user_pool_client_name=name or service_id,
             # disable_o_auth=True,
         )
-
-        self._create_secret(
+        cognito_sdk_secret = self._create_secret(
             service_id,
             {
                 "flow": "user_password",
                 "cognito_domain": self.domain.base_url(),
                 "client_id": client.user_pool_client_id,
+                "veda_client_id": client.user_pool_client_id,
+                "veda_userpool_id": self.userpool.user_pool_id,
             },
         )
+        stack_name = Stack.of(self).stack_name
+        CfnOutput(
+            self,
+            f"cognito-sdk-{service_id}-secret",
+            export_name=f"{stack_name}-cognito-sdk-secret",
+            value=cognito_sdk_secret.secret_name,
+        )
 
         return client
 
@@ -360,17 +368,29 @@ def add_service_client(
             user_pool_client_name=f"{service_id} Service Access",
             disable_o_auth=False,
         )
-
-        self._create_secret(
+        # temp: we are going provide client id, secret, and user pool id values twice in the secret (once with veda_ prefix)
+        service_client_secret = self._get_client_secret(client)
+        cognito_app_secret = self._create_secret(
             service_id,
             {
                 "flow": "client_credentials",
                 "cognito_domain": self.domain.base_url(),
                 "client_id": client.user_pool_client_id,
-                "client_secret": self._get_client_secret(client),
+                "client_secret": service_client_secret,
+                "userpool_id": self.userpool.user_pool_id,
+                "veda_client_id": client.user_pool_client_id,
+                "veda_client_secret": service_client_secret,
+                "veda_userpool_id": self.userpool.user_pool_id,
                 "scope": " ".join(scope.scope_name for scope in scopes),
             },
         )
+        stack_name = Stack.of(self).stack_name
+        CfnOutput(
+            self,
+            f"cognito-app-{service_id}-secret",
+            export_name=f"{stack_name}-cognito-app-secret",
+            value=cognito_app_secret.secret_name,
+        )
 
         return client
 

requirements.txt

@@ -7,6 +7,6 @@ boto3==1.24.15
 boto3-stubs[cognito-idp,cognito-identity]
 flake8==4.0.1
 click==8.1.3
-requests==2.28.0
+requests==2.31.0
 pre-commit==3.0.4
 pydantic[dotenv]

scripts/get-env.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Use this script to load environment variables for a deployment from AWS Secrets
+
+for s in $(aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
+    echo "$s" >> $GITHUB_ENV
+done