From 92ba2baef832ffd8c537740afa28553f11260771 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 12:46:18 -0700 Subject: [PATCH 01/17] ci: verbose cdk diff --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f8c2bfd..69f2dc8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -47,4 +47,4 @@ jobs: - name: Pre deployment CDK diff run: | echo $STAGE - cdk diff --outputs-file ${HOME}/cdk-outputs.json + cdk diff -v --outputs-file ${HOME}/cdk-outputs.json From 6a9cdc2e9aea12c889c10916fa3f9e61e513fc4a Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 12:58:18 -0700 Subject: [PATCH 02/17] ci: add latest aws sdk default setting --- cdk.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdk.json b/cdk.json index b4baa10..c95c7dd 100644 --- a/cdk.json +++ b/cdk.json @@ -1,3 +1,4 @@ { - "app": "python3 app.py" + "app": "python3 app.py", + "@aws-cdk/customresources:installLatestAwsSdkDefault": false } From 006703d48ec5932edf89f14dfcd7126242c79ab8 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 13:47:18 -0700 Subject: [PATCH 03/17] cdk context --- cdk.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdk.json b/cdk.json index c95c7dd..ffb8c78 100644 --- a/cdk.json +++ b/cdk.json @@ -1,4 +1,6 @@ { "app": "python3 app.py", - "@aws-cdk/customresources:installLatestAwsSdkDefault": false + "context": { + "@aws-cdk/customresources:installLatestAwsSdkDefault": false + } } From 3a8c58a77f71c2468263c69f9ef3929dd73f5330 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 14:10:16 -0700 Subject: [PATCH 04/17] ci: basic lower env deployment on push --- .github/workflows/cicd.yml | 56 +++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1c0b800..30dd68c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -9,7 +9,6 @@ on: branches: - main - dev - - production jobs: define-environment: @@ -21,18 +20,17 @@ jobs: run: | if [ "${{ github.ref }}" = "refs/heads/main" ]; then echo "env_name=staging" >> $GITHUB_OUTPUT - echo "secret_name=veda-auth-staging" >> $GITHUB_OUTPUT + echo "secret_name=veda-auth-uah-env" >> $GITHUB_OUTPUT elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then echo "env_name=development" >> $GITHUB_OUTPUT - echo "secret_name=veda-auth-dev" >> $GITHUB_OUTPUT - elif [ "${{ github.ref }}" = "refs/heads/production" ]; then - echo "env_name=production" >> $GITHUB_OUTPUT + echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" outputs: env_name: ${{ steps.define_environment.outputs.env_name }} + secret_name: ${{ steps.define_environment.outputs.secret_name }} deploy: name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀 @@ -43,20 +41,46 @@ jobs: concurrency: ${{ needs.define-environment.outputs.env_name }} steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 with: - lfs: "true" - submodules: "recursive" - + python-version: '3.9' + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 17 + - name: Configure awscli uses: aws-actions/configure-aws-credentials@v3 with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "veda-auth-github-${{ needs.define-environment.outputs.env_name }}-deployment" - aws-region: "us-west-2" + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - - name: Run deployment - uses: "./.github/actions/cdk-deploy" + - name: Install CDK + run: npm install -g aws-cdk@2 + + - uses: actions/cache@v3 with: - env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }} + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} + + - name: Install python dependencies + run: | + pip install -r requirements.txt + + - name: Get environment configuration for target branch + run: | + ./scripts/get-env.sh ${{ needs.define-environment.outputs.secret_name }} + + - name: Deploy + run: | + echo $STAGE + cdk deploy --require-approval never --outputs-file ${HOME}/cdk-outputs.json \ No newline at end of file From bd934061e496386c81f4567e85955a60eed5c89a Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 14:20:34 -0700 Subject: [PATCH 05/17] ci: less verbose --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 69f2dc8..f8c2bfd 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -47,4 +47,4 @@ jobs: - name: Pre deployment CDK diff run: | echo $STAGE - cdk diff -v --outputs-file ${HOME}/cdk-outputs.json + cdk diff --outputs-file ${HOME}/cdk-outputs.json From 376ddc473d2112317eacabb8deb158960093e84e Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 14:27:27 -0700 Subject: [PATCH 06/17] temp override to test workflow irl --- .github/workflows/cicd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 30dd68c..7445d0a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -9,6 +9,7 @@ on: branches: - main - dev + - fix/ci-for-dev jobs: define-environment: @@ -24,6 +25,9 @@ jobs: elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then echo "env_name=development" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/fix/ci-for-dev" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" From 90e987966125bb4a8d43c0a9009ccae147d0edfe Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 15:15:43 -0700 Subject: [PATCH 07/17] try deduplicate programmatic client export names --- infra/stack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/stack.py b/infra/stack.py index ffe2b44..5688cf2 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -59,7 +59,7 @@ def __init__( ) else: auth_provider_client = self.add_programmatic_client( - "cognito-identity-pool-auth-provider", + f"{stack_name}-identity-provider", name="Identity Pool Authentication Provider", ) if app_settings.data_managers_role_arn: @@ -333,8 +333,8 @@ def add_programmatic_client( stack_name = Stack.of(self).stack_name CfnOutput( self, - f"cognito-sdk-{service_id}-secret", - export_name=f"{stack_name}-cognito-sdk-secret", + f"{service_id}-secret", + export_name=f"{service_id}-secret", value=cognito_sdk_secret.secret_name, ) From 22e0c976064e968059948e6c40119dd9b6073f91 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 15:24:55 -0700 Subject: [PATCH 08/17] fix: cfn export programmatic client secret in app.py only --- infra/stack.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/infra/stack.py b/infra/stack.py index 5688cf2..fc5a4d3 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -330,13 +330,6 @@ def add_programmatic_client( "veda_userpool_id": self.userpool.user_pool_id, }, ) - stack_name = Stack.of(self).stack_name - CfnOutput( - self, - f"{service_id}-secret", - export_name=f"{service_id}-secret", - value=cognito_sdk_secret.secret_name, - ) return client From 541b8430cc6bced1aa0fd51b17eedf3a8146d94e Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 15:42:05 -0700 Subject: [PATCH 09/17] fix: stop creating the unused additional workflows service client secret with additional veda_ prefixes --- infra/stack.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/infra/stack.py b/infra/stack.py index fc5a4d3..568d08c 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -353,29 +353,6 @@ def add_service_client( user_pool_client_name=f"{service_id} Service Access", disable_o_auth=False, ) - # temp: we are going provide client id, secret, and user pool id values twice in the secret (once with veda_ prefix) - service_client_secret = self._get_client_secret(client) - cognito_app_secret = self._create_secret( - service_id, - { - "flow": "client_credentials", - "cognito_domain": self.domain.base_url(), - "client_id": client.user_pool_client_id, - "client_secret": service_client_secret, - "userpool_id": self.userpool.user_pool_id, - "veda_client_id": client.user_pool_client_id, - "veda_client_secret": service_client_secret, - "veda_userpool_id": self.userpool.user_pool_id, - "scope": " ".join(scope.scope_name for scope in scopes), - }, - ) - stack_name = Stack.of(self).stack_name - CfnOutput( - self, - f"cognito-app-{service_id}-secret", - export_name=f"{stack_name}-cognito-app-secret", - value=cognito_app_secret.secret_name, - ) return client From 3b756fea2c76e9dc62049f76fbc266be29ea915c Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 15:48:27 -0700 Subject: [PATCH 10/17] remove temporary cicd deploy on push to fix branch --- .github/workflows/cicd.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7445d0a..30dd68c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -9,7 +9,6 @@ on: branches: - main - dev - - fix/ci-for-dev jobs: define-environment: @@ -25,9 +24,6 @@ jobs: elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then echo "env_name=development" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT - elif [ "${{ github.ref }}" = "refs/heads/fix/ci-for-dev" ]; then - echo "env_name=development" >> $GITHUB_OUTPUT - echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" From 2a5b152c9e44323af11906d8d0e3adfdc01ff918 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 15:58:27 -0700 Subject: [PATCH 11/17] fix(ci): run cdk diff on staging for prs into main and on dev for prs into dev --- .github/workflows/pr.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f8c2bfd..b660022 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -43,7 +43,12 @@ jobs: - name: Get environment configuration for target branch run: | - ./scripts/get-env.sh "veda-auth-uah-env" + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + ./scripts/get-env.sh "veda-auth-uah-env" + elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then + ./scripts/get-env.sh "veda-auth-dev-env" + fi + - name: Pre deployment CDK diff run: | echo $STAGE From 685a94054bef703683e8928dd65a9781bd907c23 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 16:17:00 -0700 Subject: [PATCH 12/17] fix pre deploy check for branch based stack --- .github/workflows/pr.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b660022..cc80981 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -3,7 +3,30 @@ name: Pull Request - Preview CDK Diff on: [pull_request] jobs: + define-environment: + name: Set ✨ environment ✨ based on the branch 🌳 + runs-on: ubuntu-latest + steps: + - name: Set the environment + id: define_environment + run: | + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "env_name=staging" >> $GITHUB_OUTPUT + echo "secret_name=veda-auth-uah-env" >> $GITHUB_OUTPUT + elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then + echo "env_name=development" >> $GITHUB_OUTPUT + echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT + fi + - name: Print the environment + run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} + secret_name: ${{ steps.define_environment.outputs.secret_name }} + + predeploy: + name: Pre-deploy cdk diff for ${{ needs.define-environment.outputs.env_name }} 🚀 + needs: [define-environment] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -43,11 +66,7 @@ jobs: - name: Get environment configuration for target branch run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - ./scripts/get-env.sh "veda-auth-uah-env" - elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then - ./scripts/get-env.sh "veda-auth-dev-env" - fi + ./scripts/get-env.sh ${{ needs.define-environment.outputs.secret_name }} - name: Pre deployment CDK diff run: | From e9127fab263e22aaa6f78f0c686af9584330f86a Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 16:22:54 -0700 Subject: [PATCH 13/17] invalid yml --- .github/workflows/pr.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cc80981..327230d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -19,9 +19,10 @@ jobs: fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" - outputs: - env_name: ${{ steps.define_environment.outputs.env_name }} - secret_name: ${{ steps.define_environment.outputs.secret_name }} + + outputs: + env_name: ${{ steps.define_environment.outputs.env_name }} + secret_name: ${{ steps.define_environment.outputs.secret_name }} predeploy: From 9332671afb5ae2eaf81e1ae0884e0fa4888379c5 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 16:43:25 -0700 Subject: [PATCH 14/17] pr yml should use base_ref not ref; both pr and cicd updated to node 20 --- .github/workflows/cicd.yml | 4 ++-- .github/workflows/pr.yml | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 30dd68c..01c7c3f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -22,7 +22,7 @@ jobs: echo "env_name=staging" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-uah-env" >> $GITHUB_OUTPUT elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then - echo "env_name=development" >> $GITHUB_OUTPUT + echo "env_name=dev" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT fi - name: Print the environment @@ -50,7 +50,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 17 + node-version: 20 - name: Configure awscli uses: aws-actions/configure-aws-credentials@v3 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 327230d..65e8c32 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,5 +1,9 @@ name: Pull Request - Preview CDK Diff +permissions: + id-token: write + contents: read + on: [pull_request] jobs: @@ -10,16 +14,16 @@ jobs: - name: Set the environment id: define_environment run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then + if [ "${{ github.base_ref }}" == "main" ]; then echo "env_name=staging" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-uah-env" >> $GITHUB_OUTPUT - elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then - echo "env_name=development" >> $GITHUB_OUTPUT + elif [ "${{ github.base_ref }}" == "dev" ]; then + echo "env_name=dev" >> $GITHUB_OUTPUT echo "secret_name=veda-auth-dev-env" >> $GITHUB_OUTPUT fi - name: Print the environment run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}" - + outputs: env_name: ${{ steps.define_environment.outputs.env_name }} secret_name: ${{ steps.define_environment.outputs.secret_name }} @@ -39,7 +43,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 17 + node-version: 20 - name: Configure awscli uses: aws-actions/configure-aws-credentials@v3 From 7adb071dbb3021bc33d5f225603331feea008724 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 4 Mar 2024 17:37:57 -0700 Subject: [PATCH 15/17] try add back service client secret --- infra/stack.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/infra/stack.py b/infra/stack.py index 568d08c..92289e3 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -354,6 +354,26 @@ def add_service_client( disable_o_auth=False, ) + self._create_secret( + service_id, + { + "flow": "client_credentials", + "cognito_domain": self.domain.base_url(), + "client_id": client.user_pool_client_id, + "client_secret": self._get_client_secret(client), + "userpool_id": self.userpool.user_pool_id, + "scope": " ".join(scope.scope_name for scope in scopes), + }, + ) + + stack_name = Stack.of(self).stack_name + CfnOutput( + self, + f"cognito-app-{service_id}-secret", + export_name=f"{stack_name}-cognito-app-secret", + value=f"{stack_name}/{service_id}", + ) + return client @property From 40ab62899d8aca15322c7cc53f271384ba33d228 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 5 Mar 2024 08:13:20 -0700 Subject: [PATCH 16/17] export sdk secret name --- infra/stack.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/infra/stack.py b/infra/stack.py index 92289e3..decf08a 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -320,7 +320,7 @@ def add_programmatic_client( user_pool_client_name=name or service_id, # disable_o_auth=True, ) - cognito_sdk_secret = self._create_secret( + self._create_secret( service_id, { "flow": "user_password", @@ -330,6 +330,13 @@ def add_programmatic_client( "veda_userpool_id": self.userpool.user_pool_id, }, ) + stack_name = Stack.of(self).stack_name + CfnOutput( + self, + f"cognito-sdk-{service_id}-secret", + export_name=f"{stack_name}-cognito-sdk-secret", + value=f"{stack_name}/{service_id}", + ) return client From 90a12038463896e2152d74c7607fc9d6ce0052a9 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 5 Mar 2024 08:33:00 -0700 Subject: [PATCH 17/17] remove duplicate veda_ keys --- infra/stack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra/stack.py b/infra/stack.py index decf08a..64699e8 100644 --- a/infra/stack.py +++ b/infra/stack.py @@ -326,8 +326,7 @@ def add_programmatic_client( "flow": "user_password", "cognito_domain": self.domain.base_url(), "client_id": client.user_pool_client_id, - "veda_client_id": client.user_pool_client_id, - "veda_userpool_id": self.userpool.user_pool_id, + "userpool_id": self.userpool.user_pool_id, }, ) stack_name = Stack.of(self).stack_name