forward Elastalert emails first to country config's new /email endpoi…

…nt and forward from there
opencrvs · Jan 17, 2024 · 8eb1ae3 · 8eb1ae3
1 parent 210e7e6
commit 8eb1ae3
Show file tree

Hide file tree

Showing 22 changed files with 523 additions and 482 deletions.
diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
@@ -103,6 +103,7 @@ jobs:
           SMTP_HOST: ${{ secrets.SMTP_HOST }}
           SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+          SMTP_SECURE: ${{ secrets.SMTP_SECURE }}
           ALERT_EMAIL: ${{ secrets.ALERT_EMAIL }}
           DOCKERHUB_ACCOUNT: ${{ secrets.DOCKERHUB_ACCOUNT }}
           DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }}
@@ -115,7 +116,6 @@ jobs:
           ELASTICSEARCH_SUPERUSER_PASSWORD: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           MINIO_ROOT_USER: ${{ secrets.MINIO_ROOT_USER }}
           MINIO_ROOT_PASSWORD: ${{ secrets.MINIO_ROOT_PASSWORD }}
-          EMAIL_API_KEY: ${{ secrets.EMAIL_API_KEY }}
           INFOBIP_SENDER_ID: ${{ secrets.INFOBIP_SENDER_ID }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           INFOBIP_GATEWAY_ENDPOINT: ${{ secrets.INFOBIP_GATEWAY_ENDPOINT }}

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -107,6 +107,7 @@ jobs:
           SMTP_HOST: ${{ secrets.SMTP_HOST }}
           SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+          SMTP_SECURE: ${{ secrets.SMTP_SECURE }}
           ALERT_EMAIL: ${{ secrets.ALERT_EMAIL }}
           DOCKERHUB_ACCOUNT: ${{ secrets.DOCKERHUB_ACCOUNT }}
           DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }}
@@ -119,7 +120,6 @@ jobs:
           ELASTICSEARCH_SUPERUSER_PASSWORD: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           MINIO_ROOT_USER: ${{ secrets.MINIO_ROOT_USER }}
           MINIO_ROOT_PASSWORD: ${{ secrets.MINIO_ROOT_PASSWORD }}
-          EMAIL_API_KEY: ${{ secrets.EMAIL_API_KEY }}
           INFOBIP_SENDER_ID: ${{ secrets.INFOBIP_SENDER_ID }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           INFOBIP_GATEWAY_ENDPOINT: ${{ secrets.INFOBIP_GATEWAY_ENDPOINT }}
@@ -151,6 +151,7 @@ jobs:
           SMTP_HOST: ${{ secrets.SMTP_HOST }}
           SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
           SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+          SMTP_SECURE: ${{ secrets.SMTP_SECURE }}
           ALERT_EMAIL: ${{ secrets.ALERT_EMAIL }}
           DOCKERHUB_ACCOUNT: ${{ secrets.DOCKERHUB_ACCOUNT }}
           DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }}
@@ -163,7 +164,6 @@ jobs:
           ELASTICSEARCH_SUPERUSER_PASSWORD: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           MINIO_ROOT_USER: ${{ secrets.MINIO_ROOT_USER }}
           MINIO_ROOT_PASSWORD: ${{ secrets.MINIO_ROOT_PASSWORD }}
-          EMAIL_API_KEY: ${{ secrets.EMAIL_API_KEY }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           WIREGUARD_ADMIN_PASSWORD: ${{ secrets.WIREGUARD_ADMIN_PASSWORD }}
           INFOBIP_SENDER_ID: ${{ secrets.INFOBIP_SENDER_ID }}

diff --git a/.github/workflows/reset-2fa.yml b/.github/workflows/reset-2fa.yml
@@ -47,4 +47,4 @@ jobs:
 
       - name: Remove 2FA
         run: |
-          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -p ${{ secrets.SMTP_PORT }} ${{ vars.SSH_ARGS }} "rm /home/${{ github.event.inputs.user }}/.google_authenticator"
+          ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -p ${{ secrets.SSH_PORT }} ${{ vars.SSH_ARGS }} "rm /home/${{ github.event.inputs.user }}/.google_authenticator"
diff --git a/infrastructure/create-github-environment.js b/infrastructure/create-github-environment.js
@@ -51,9 +51,9 @@ const config = {
     SMTP_HOST: process.env.SMTP_HOST,
     SMTP_USERNAME: process.env.SMTP_USERNAME,
     SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+    SMTP_SECURE: process.env.SMTP_SECURE,
     SMTP_PORT: process.env.SMTP_PORT,
     SENDER_EMAIL_ADDRESS: process.env.SENDER_EMAIL_ADDRESS,
-    EMAIL_API_KEY: process.env.EMAIL_API_KEY,
     ALERT_EMAIL:
       'sentry-dev-aaaalrpiimoklruew7v7dgo2km@opencrvsworkspace.slack.com'
   },

diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh
@@ -139,26 +139,6 @@ validate_environment_variables() {
   # These ones are directly required by this script
   # and thus should be set in the environment variables even
   # if not required by compose files
-  if [ -z "$SMTP_HOST" ] ; then
-      echo 'Error: Missing environment variable SMTP_HOST.'
-      print_usage_and_exit
-  fi
-
-  if [ -z "$SMTP_PORT" ] ; then
-      echo 'Error: Missing environment variable SMTP_PORT.'
-      print_usage_and_exit
-  fi
-
-  if [ -z "$SMTP_USERNAME" ] ; then
-      echo 'Error: Missing environment variable SMTP_USERNAME.'
-      print_usage_and_exit
-  fi
-
-  if [ -z "$SMTP_PASSWORD" ] ; then
-      echo 'Error: Missing environment variable SMTP_PASSWORD.'
-      print_usage_and_exit
-  fi
-
   if [ -z "$ALERT_EMAIL" ] ; then
       echo 'Error: Missing environment variable ALERT_EMAIL.'
       print_usage_and_exit
@@ -371,14 +351,9 @@ EOF
 
 # Setup configuration files and compose file for the deployment domain
 configured_ssh "
-  SMTP_HOST=$SMTP_HOST
-  SMTP_PORT=$SMTP_PORT
-  SMTP_USERNAME=$SMTP_USERNAME
-  SMTP_PASSWORD=$SMTP_PASSWORD
-  ALERT_EMAIL=$ALERT_EMAIL
   MINIO_ROOT_USER=$MINIO_ROOT_USER
   MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
-  /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log"
+  /opt/opencrvs/infrastructure/setup-deploy-config.sh | tee -a $LOG_LOCATION/setup-deploy-config.log"
 
 rotate_secrets
 

diff --git a/infrastructure/docker-compose.deploy.yml b/infrastructure/docker-compose.deploy.yml
@@ -400,7 +400,6 @@ services:
       - ES_PASSWORD=${ELASTICSEARCH_SUPERUSER_PASSWORD}
     volumes:
       - '/opt/opencrvs/infrastructure/monitoring/elastalert/elastalert.yaml:/opt/elastalert/config.yaml'
-      - '/opt/opencrvs/infrastructure/monitoring/elastalert/auth.yaml:/opt/elastalert/auth.yaml'
       - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules'
     networks:
       - overlay_net
@@ -579,6 +578,11 @@ services:
         - 'traefik.http.middlewares.countryconfig.headers.stsseconds=31536000'
         - 'traefik.http.middlewares.countryconfig.headers.stsincludesubdomains=true'
         - 'traefik.http.middlewares.countryconfig.headers.stspreload=true'
+        # This is an invalid IP range, effectively blocking all IPs from accessing /email path.
+        # It's only meant to be accessed from the internal docker network.
+        - 'traefik.http.middlewares.block-all.ipwhitelist.sourcerange=255.255.255.255'
+        - 'traefik.http.routers.email-blocker.rule=Path(`/email`)'
+        - 'traefik.http.routers.email-blocker.middlewares=block-all'
       replicas: 1
     environment:
       - MONGO_URL=mongodb://mongo1/user-mgnt?replicaSet=rs0
@@ -587,7 +591,16 @@ services:
       - COUNTRY_CONFIG_URL=https://countryconfig.{{hostname}}
       - LOGIN_URL=https://login.{{hostname}}
       - CLIENT_APP_URL=https://register.{{hostname}}
-      - DOMAIN={{hostname}}
+      - NOTIFICATION_TRANSPORT=${NOTIFICATION_TRANSPORT}
+      - ALERT_EMAIL=${ALERT_EMAIL:-""}
+      - SMTP_HOST=${SMTP_HOST:-""}
+      - SMTP_PORT=${SMTP_PORT:-""}
+      - SMTP_USERNAME=${SMTP_USERNAME:-""}
+      - SMTP_PASSWORD=${SMTP_PASSWORD:-""}
+      - SMTP_SECURE=${SMTP_SECURE:-""}
+      - INFOBIP_GATEWAY_ENDPOINT=${INFOBIP_GATEWAY_ENDPOINT:-""}
+      - INFOBIP_API_KEY=${INFOBIP_API_KEY:-""}
+      - INFOBIP_SENDER_ID=${INFOBIP_SENDER_ID:-""}
     networks:
       - overlay_net
     logging:

diff --git a/infrastructure/docker-compose.production-deploy.yml b/infrastructure/docker-compose.production-deploy.yml
@@ -96,12 +96,14 @@ services:
       - OPENHIM_URL=http://openhim-core:5001/fhir
       - CONFIRM_REGISTRATION_URL=http://openhim-core:5001/confirm/registration
       - CHECK_INVALID_TOKEN=true
-      - EMAIL_API_KEY=${EMAIL_API_KEY}
-      - INFOBIP_GATEWAY_ENDPOINT=${INFOBIP_GATEWAY_ENDPOINT:-""}
-      - INFOBIP_API_KEY=${INFOBIP_API_KEY:-""}
-      - INFOBIP_SENDER_ID=${INFOBIP_SENDER_ID:-""}
-      - SENDER_EMAIL_ADDRESS=${SENDER_EMAIL_ADDRESS}
       - SENTRY_DSN=${SENTRY_DSN}
+      - SENDER_EMAIL_ADDRESS=${SENDER_EMAIL_ADDRESS}
+      - ALERT_EMAIL=${ALERT_EMAIL}
+      - SMTP_HOST=${SMTP_HOST}
+      - SMTP_PORT=${SMTP_PORT}
+      - SMTP_USERNAME=${SMTP_USERNAME}
+      - SMTP_PASSWORD=${SMTP_PASSWORD}
+      - SMTP_SECURE=${SMTP_SECURE}
     deploy:
       replicas: 2
 

diff --git a/infrastructure/docker-compose.staging-deploy.yml b/infrastructure/docker-compose.staging-deploy.yml
@@ -96,12 +96,14 @@ services:
       - OPENHIM_URL=http://openhim-core:5001/fhir
       - CONFIRM_REGISTRATION_URL=http://openhim-core:5001/confirm/registration
       - CHECK_INVALID_TOKEN=true
-      - EMAIL_API_KEY=${EMAIL_API_KEY}
-      - INFOBIP_GATEWAY_ENDPOINT=${INFOBIP_GATEWAY_ENDPOINT:-""}
-      - INFOBIP_API_KEY=${INFOBIP_API_KEY:-""}
-      - INFOBIP_SENDER_ID=${INFOBIP_SENDER_ID:-""}
-      - SENDER_EMAIL_ADDRESS=${SENDER_EMAIL_ADDRESS}
       - SENTRY_DSN=${SENTRY_DSN}
+      - SENDER_EMAIL_ADDRESS=${SENDER_EMAIL_ADDRESS}
+      - ALERT_EMAIL=${ALERT_EMAIL}
+      - SMTP_HOST=${SMTP_HOST}
+      - SMTP_PORT=${SMTP_PORT}
+      - SMTP_USERNAME=${SMTP_USERNAME}
+      - SMTP_PASSWORD=${SMTP_PASSWORD}
+      - SMTP_SECURE=${SMTP_SECURE}
     deploy:
       replicas: 1
 

diff --git a/infrastructure/monitoring/elastalert/auth.yaml b/infrastructure/monitoring/elastalert/auth.yaml
diff --git a/infrastructure/monitoring/elastalert/rules/alert.yaml b/infrastructure/monitoring/elastalert/rules/alert.yaml
@@ -21,13 +21,6 @@ timestamp_type: 'iso'
 use_strftime_index: false
 # https://github.com/Yelp/elastalert/issues/1260
 raw_count_keys: false
-alert_subject: '{{HOST}} {}: {} 🚨'
-alert_subject_args:
-  - 'kibana.alert.context.metrics__alert__inventory__threshold.alertState'
-  - 'rule.name'
-alert_text: 'Reason: {}. Login to https://kibana.{{HOST}} to view the alert.'
-alert_text_args:
-  - 'kibana.alert.context.metrics__alert__inventory__threshold.reason'
 filter:
   - bool:
       should:
@@ -42,12 +35,10 @@ filter:
               value: 'Low on available disk space'
       minimum_should_match: 1
 
-alert:
-  - 'email'
-email: '{{ALERT_EMAIL}}'
-from_addr: 'alert@opencrvs.org'
-smtp_host: '{{SMTP_HOST}}'
-# prettier-ignore
-smtp_port: {{SMTP_PORT}}
-smtp_ssl: false
-smtp_auth_file: '/opt/elastalert/auth.yaml'
+alert: post2
+http_post2_url: 'http://countryconfig:3040/email'
+http_post2_payload:
+  subject: '{% raw %}{{DOMAIN}}{% endraw %} {{kibana.alert.context.metrics__alert__inventory__threshold.alertState}}: {{rule.name}} 🚨'
+  html: 'Reason: {{kibana.alert.context.metrics__alert__inventory__threshold.reason}}. Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.'
+  from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}'
+  to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}'
diff --git a/infrastructure/monitoring/elastalert/rules/log-alert.yaml b/infrastructure/monitoring/elastalert/rules/log-alert.yaml
@@ -21,12 +21,6 @@ timestamp_type: 'iso'
 use_strftime_index: false
 # https://github.com/Yelp/elastalert/issues/1260
 raw_count_keys: false
-alert_subject: '{{HOST}}: {} 🚨'
-alert_subject_args:
-  - 'rule.name'
-alert_text: 'Reason: {}. Login to https://kibana.{{HOST}} to view the alert.'
-alert_text_args:
-  - 'kibana.alert.context.logs__alert__document__count.conditions'
 filter:
   - bool:
       should:
@@ -35,12 +29,10 @@ filter:
               value: 'Error in backup logs'
       minimum_should_match: 1
 
-alert:
-  - 'email'
-email: '{{ALERT_EMAIL}}'
-from_addr: 'alert@opencrvs.org'
-smtp_host: '{{SMTP_HOST}}'
-# prettier-ignore
-smtp_port: {{SMTP_PORT}}
-smtp_ssl: false
-smtp_auth_file: '/opt/elastalert/auth.yaml'
+alert: post2
+http_post2_url: 'http://countryconfig:3040/email'
+http_post2_payload:
+  subject: '{% raw %}{{DOMAIN}}{% endraw %}: {{rule.name}} 🚨'
+  html: 'Reason: {{kibana.alert.context.logs__alert__document__count.conditions}}. Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.'
+  from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}'
+  to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}'
diff --git a/infrastructure/monitoring/elastalert/rules/service-error.yaml b/infrastructure/monitoring/elastalert/rules/service-error.yaml
@@ -21,25 +21,16 @@ timestamp_type: 'iso'
 use_strftime_index: false
 # https://github.com/Yelp/elastalert/issues/1260
 raw_count_keys: false
-alert_subject: '{{HOST}} {}: {} {} 🚨'
-alert_subject_args:
-  - 'event.kind'
-  - 'rule.name'
-  - 'kibana.alert.context.apm__error_rate.serviceName'
-alert_text: 'Login to https://kibana.{{HOST}}/app/apm/services/{}/overview to view the alert.'
-alert_text_args:
-  - 'kibana.alert.context.apm__error_rate.serviceName'
 filter:
   - query:
       term:
         rule.name.keyword:
           value: 'Error in service'
-alert:
-  - 'email'
-email: '{{ALERT_EMAIL}}'
-from_addr: 'alert@opencrvs.org'
-smtp_host: '{{SMTP_HOST}}'
-# prettier-ignore
-smtp_port: {{SMTP_PORT}}
-smtp_ssl: false
-smtp_auth_file: '/opt/elastalert/auth.yaml'
+
+alert: post2
+http_post2_url: 'http://countryconfig:3040/email'
+http_post2_payload:
+  subject: '{% raw %}{{DOMAIN}}{% endraw %} {{event.kind}}: {{rule.name}} {{kibana.alert.context.apm__error_rate.serviceName}} 🚨'
+  html: 'Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %}/app/apm/services/{{kibana.alert.context.apm__error_rate.serviceName}}/overview to view the alert.'
+  from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}'
+  to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}'
diff --git a/infrastructure/monitoring/elastalert/rules/ssh-alert.yaml b/infrastructure/monitoring/elastalert/rules/ssh-alert.yaml
@@ -21,23 +21,16 @@ timestamp_type: 'iso'
 use_strftime_index: false
 # https://github.com/Yelp/elastalert/issues/1260
 raw_count_keys: false
-alert_subject: '{{HOST}}: Login to {} from {} 🚨'
-alert_subject_args:
-  - 'kibana.alert.context.__es-query.hits[0]_source.user.name'
-  - 'kibana.alert.context.__es-query.hits[0]_source.source.ip'
-alert_text: 'Login to https://kibana.{{HOST}} to view the alert.'
-
 filter:
   - query:
       term:
         rule.name.keyword:
           value: 'Succesful SSH login'
-alert:
-  - 'email'
-email: '{{ALERT_EMAIL}}'
-from_addr: 'alert@opencrvs.org'
-smtp_host: '{{SMTP_HOST}}'
-# prettier-ignore
-smtp_port: {{SMTP_PORT}}
-smtp_ssl: false
-smtp_auth_file: '/opt/elastalert/auth.yaml'
+
+alert: post2
+http_post2_url: 'http://countryconfig:3040/email'
+http_post2_payload:
+  subject: '{% raw %}{{DOMAIN}}{% endraw %}: Login to {{kibana.alert.context.__es-query.hits[0]_source.user.name}} from {{kibana.alert.context.__es-query.hits[0]_source.source.ip}} 🚨'
+  html: 'Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.'
+  from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}'
+  to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}'
diff --git a/infrastructure/setup-deploy-config.sh b/infrastructure/setup-deploy-config.sh
@@ -23,15 +23,6 @@ done
 KIBANA_ENCRYPTION_KEY=`uuidgen`
 sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml
 
-# Replace environment variables from all alert definition files
-for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do
-    sed -i -e "s%{{HOST}}%$1%" $file
-    sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file
-    sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file
-    sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file
-done
-sed -i -e "s%{{SMTP_USERNAME}}%$SMTP_USERNAME%" /opt/opencrvs/infrastructure/monitoring/elastalert/auth.yaml
-sed -i -e "s%{{SMTP_PASSWORD}}%$SMTP_PASSWORD%" /opt/opencrvs/infrastructure/monitoring/elastalert/auth.yaml
 sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json
 sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json
 

diff --git a/package.json b/package.json
@@ -38,8 +38,8 @@
     "@octokit/core": "4.2.1",
     "@types/bcryptjs": "^2.4.2",
     "@types/google-libphonenumber": "^7.4.23",
-    "@types/hapi__inert": "5.2.1",
     "@types/handlebars": "^4.1.0",
+    "@types/hapi__inert": "5.2.1",
     "@types/jsonwebtoken": "^8.5.8",
     "@types/libsodium-wrappers": "^0.7.10",
     "@types/react-intl": "^3.0.0",
@@ -69,7 +69,6 @@
     "@hapi/boom": "^9.1.1",
     "@hapi/hapi": "^20.0.1",
     "@hapi/inert": "^6.0.3",
-    "@sendgrid/mail": "^7.7.0",
     "@types/chalk": "^2.2.0",
     "@types/code": "^4.0.3",
     "@types/csv2json": "^1.4.0",
@@ -86,6 +85,7 @@
     "@types/mime-types": "^2.1.0",
     "@types/node": "^10.12.5",
     "@types/node-fetch": "^2.6.2",
+    "@types/nodemailer": "^6.4.14",
     "@types/pino": "^5.14.0",
     "@types/string-similarity": "^3.0.0",
     "@types/uuid": "^3.4.4",
@@ -110,6 +110,7 @@
     "lodash": "^4.17.21",
     "mongoose": "6.11.3",
     "node-fetch": "^2.6.1",
+    "nodemailer": "^6.9.8",
     "opener": "^1.5.1",
     "p-queue": "^6.6.2",
     "pino": "^5.14.0",