diff --git a/.github/workflows/get-secret-from-env.yml b/.github/workflows/get-secret-from-environment.yml similarity index 64% rename from .github/workflows/get-secret-from-env.yml rename to .github/workflows/get-secret-from-environment.yml index 2daea4c38..79fe4c73a 100644 --- a/.github/workflows/get-secret-from-env.yml +++ b/.github/workflows/get-secret-from-environment.yml @@ -9,11 +9,24 @@ on: env_name: required: true type: string + outputs: + secret_value: + description: 'Secret value, encrypted with the encryption key' + value: ${{ jobs.fetch-credentials.outputs.secret_value }} + environment_exists: + description: 'Whether the environment exists or not' + value: ${{ jobs.check-environment.outputs.environment_exists }} secrets: gh_token: required: true encryption_key: required: true + # All secrets that are we want to allow access to need + # to be defined in this list + BACKUP_ENCRYPTION_PASSPHRASE: + required: false + SSH_KEY: + required: false jobs: check-environment: @@ -46,17 +59,26 @@ jobs: fetch-credentials: name: Fetch Secret - needs: check-environment runs-on: ubuntu-22.04 + environment: ${{ inputs.env_name }} + needs: check-environment + # Without this Github actions will create the environment when it doesnt exist if: needs.check-environment.outputs.environment_exists == 'true' outputs: secret_value: ${{ steps.fetch-credentials.outputs.secret_value }} - environment_exists: ${{ needs.check-environment.outputs.environment_exists }} steps: - name: Fetch the secret id: fetch-credentials + env: + SECRET_NAME: ${{ inputs.secret_name }} run: | - SECRET_VALUE="${{ secrets[inputs.secret_name] }}" + SECRET_VALUE="${{ secrets[env.SECRET_NAME] }}" + if [ -z "$SECRET_VALUE" ]; then + echo "Secret ${{ inputs.secret_name }} is empty. Usually this means you have not explicitly stated the secrets" + echo "in both the workflow file get-secrets-from-environment and in the file you are using the reusable workflow from." + echo "Please make sure you have added the secret to the workflow files and retry." + exit 1 + fi echo -n "$SECRET_VALUE" | openssl enc -aes-256-cbc -pbkdf2 -salt -k "${{ secrets.encryption_key }}" -out encrypted_key.bin ENCODED_ENCRYPTED_SECRET=$(base64 < encrypted_key.bin) EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index eb15823ee..fd199b090 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -14,51 +14,70 @@ on: - qa - production - backup + - jump tag: type: choice description: Select group tag you want to execute default: all options: - all + - application - backups - checks - - updates - - application - - tools - - docker - - deployment - - users - crontab - - mongodb - data-partition - - swap - - ufw - - fail2ban - decrypt - - swarm + - deployment + - docker - elasticsearch + - fail2ban + - jump + - mongodb + - swap + - swarm + - tools - traefik + - ufw + - updates + - users debug: type: boolean description: Open SSH session to the runner after deployment default: false jobs: get-backup-ssh-key: - uses: ./.github/workflows/get-secret-from-env.yml + name: Get backup SSH key + uses: ./.github/workflows/get-secret-from-environment.yml with: secret_name: 'SSH_KEY' env_name: 'backup' secrets: gh_token: ${{ secrets.GH_TOKEN }} encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + SSH_KEY: ${{ secrets.SSH_KEY }} + + get-jump-ssh-key: + name: Get jump SSH key + uses: ./.github/workflows/get-secret-from-environment.yml + with: + secret_name: 'SSH_KEY' + env_name: 'jump' + secrets: + gh_token: ${{ secrets.GH_TOKEN }} + encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + SSH_KEY: ${{ secrets.SSH_KEY }} + get-production-encryption-key: - uses: ./.github/workflows/get-secret-from-env.yml + name: Get production backup encryption key + if: github.event.inputs.environment == 'staging' + uses: ./.github/workflows/get-secret-from-environment.yml with: secret_name: 'BACKUP_ENCRYPTION_PASSPHRASE' env_name: 'production' secrets: gh_token: ${{ secrets.GH_TOKEN }} encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + BACKUP_ENCRYPTION_PASSPHRASE: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }} provision: name: Provision ${{ github.event.inputs.environment }} @@ -124,10 +143,17 @@ jobs: - name: Write backup SSH key to file if: needs.get-backup-ssh-key.outputs.environment_exists == 'true' run: | - echo "${{ needs.get-production-encryption-key.outputs.backup-ssh-key }}" | base64 --decode | \ + echo "${{ needs.get-backup-ssh-key.outputs.secret_value }}" | base64 --decode | \ openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/backup_ssh_private_key chmod 600 /tmp/backup_ssh_private_key + - name: Write jump server SSH key to file + if: needs.get-jump-ssh-key.outputs.environment_exists == 'true' + run: | + echo "${{ needs.get-jump-ssh-key.outputs.secret_value }}" | base64 --decode | \ + openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/jump_ssh_private_key + chmod 600 /tmp/jump_ssh_private_key + - name: Check if backup environment if configured in inventory file if: needs.get-backup-ssh-key.outputs.environment_exists != 'true' run: | diff --git a/infrastructure/server-setup/inventory/jump.yml b/infrastructure/server-setup/inventory/jump.yml new file mode 100644 index 000000000..8b616f7f2 --- /dev/null +++ b/infrastructure/server-setup/inventory/jump.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + +all: + vars: + users: [] + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' diff --git a/infrastructure/server-setup/inventory/production.yml b/infrastructure/server-setup/inventory/production.yml index 44ef7bc9c..e21adb4d8 100644 --- a/infrastructure/server-setup/inventory/production.yml +++ b/infrastructure/server-setup/inventory/production.yml @@ -14,6 +14,8 @@ all: # SSH and other services should never be exposed to the public internet. only_allow_access_from_addresses: - 165.22.110.53 + - 159.89.14.13 + # Enable backups enable_backups: true backup_server_remote_target_directory: /home/backup/backups @@ -58,3 +60,8 @@ backups: # Written by provision pipeline. Assumes "backup" environment # exists in Github environments ansible_ssh_private_key_file: /tmp/backup_ssh_private_key + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' diff --git a/infrastructure/server-setup/inventory/staging.yml b/infrastructure/server-setup/inventory/staging.yml index 46b57692d..2d3a08dac 100644 --- a/infrastructure/server-setup/inventory/staging.yml +++ b/infrastructure/server-setup/inventory/staging.yml @@ -13,6 +13,7 @@ all: # SSH and other services should never be exposed to the public internet. only_allow_access_from_addresses: - 165.22.110.53 + - 159.89.14.13 # Enable backups but write them to a different location from where production writes them enable_backups: true backup_server_remote_target_directory: /home/backup/staging-backups @@ -77,3 +78,8 @@ backups: # Written by provision pipeline. Assumes "backup" environment # exists in Github environments ansible_ssh_private_key_file: /tmp/backup_ssh_private_key + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' diff --git a/infrastructure/server-setup/jump.yml b/infrastructure/server-setup/jump.yml new file mode 100644 index 000000000..870b0e844 --- /dev/null +++ b/infrastructure/server-setup/jump.yml @@ -0,0 +1,80 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. +--- +- hosts: docker-manager-first + become: yes + tasks: + - name: Fetch the public SSH key so it can be transferred to the jump machine + fetch: + src: '{{ provisioning_user }}/.ssh/authorized_keys' + dest: '/tmp/docker-manager-first_id_rsa.pub' + flat: yes + tags: + - jump + +- hosts: jump + become: yes + become_method: sudo + tasks: + - name: Ensure jump user is present + user: + name: 'jump' + state: present + create_home: true + home: '/home/jump' + shell: /bin/bash + tags: + - jump + + - name: Ensure application servers can login to jump server + blockinfile: + path: '/home/jump/.ssh/authorized_keys' + block: | + {{ lookup('file', '/tmp/docker-manager-first_id_rsa.pub') }} + marker: '# {mark} ANSIBLE MANAGED BLOCK docker-manager-first {{ manager_hostname }}' + create: yes + mode: 0600 + owner: 'jump' + + tags: + - jump + + - name: Only require public key from the user jump + blockinfile: + path: /etc/ssh/sshd_config + block: | + Match User jump + PasswordAuthentication no + AuthenticationMethods publickey + marker: '# {mark} ANSIBLE MANAGED BLOCK FOR USER jump' + become: yes + +- hosts: docker-manager-first + become: yes + tasks: + - name: Set destination server + set_fact: + destination_server: "{{ hostvars[groups['jump'][0]].ansible_host }}" + tags: + - jump + + - name: Check SSH connection to destination server + shell: ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 jump@{{ destination_server }} 'echo Connection successful' + remote_user: '{{ provisioning_user }}' + register: ssh_test + ignore_errors: yes + tags: + - jump + + - name: Fail if SSH connection test failed + fail: + msg: 'SSH connection to the jump server failed' + when: ssh_test.rc != 0 + tags: + - jump diff --git a/infrastructure/server-setup/playbook.yml b/infrastructure/server-setup/playbook.yml index 3ec5afdb1..093b98a6d 100644 --- a/infrastructure/server-setup/playbook.yml +++ b/infrastructure/server-setup/playbook.yml @@ -30,9 +30,6 @@ tags: - updates - - include_tasks: - file: tasks/backwards-compatibility.yml - - hosts: docker-manager-first, docker-workers become: yes become_method: sudo diff --git a/infrastructure/server-setup/tasks/backups/crontab.yml b/infrastructure/server-setup/tasks/backups/crontab.yml index d1b04496c..0af6d6d29 100644 --- a/infrastructure/server-setup/tasks/backups/crontab.yml +++ b/infrastructure/server-setup/tasks/backups/crontab.yml @@ -34,7 +34,7 @@ - name: Throw an error if periodic_restore_from_backup is true but backup_restore_encryption_passphrase is not defined fail: - msg: 'Error: backup_restore_encryption_passphrase is not defined. This usually means you have enabled periodic restore from production but you haven't set up a production environment yet. Please set up a production environment first.' + msg: "Error: backup_restore_encryption_passphrase is not defined. This usually means you have enabled periodic restore from production but you haven't set up a production environment yet. Please set up a production environment first." when: periodic_restore_from_backup and backup_restore_encryption_passphrase is not defined - name: 'Setup crontab to download a backup periodically the opencrvs data'