diff --git a/.github/workflows/get-secret-from-env.yml b/.github/workflows/get-secret-from-environment.yml similarity index 64% rename from .github/workflows/get-secret-from-env.yml rename to .github/workflows/get-secret-from-environment.yml index 2daea4c38..79fe4c73a 100644 --- a/.github/workflows/get-secret-from-env.yml +++ b/.github/workflows/get-secret-from-environment.yml @@ -9,11 +9,24 @@ on: env_name: required: true type: string + outputs: + secret_value: + description: 'Secret value, encrypted with the encryption key' + value: ${{ jobs.fetch-credentials.outputs.secret_value }} + environment_exists: + description: 'Whether the environment exists or not' + value: ${{ jobs.check-environment.outputs.environment_exists }} secrets: gh_token: required: true encryption_key: required: true + # All secrets that are we want to allow access to need + # to be defined in this list + BACKUP_ENCRYPTION_PASSPHRASE: + required: false + SSH_KEY: + required: false jobs: check-environment: @@ -46,17 +59,26 @@ jobs: fetch-credentials: name: Fetch Secret - needs: check-environment runs-on: ubuntu-22.04 + environment: ${{ inputs.env_name }} + needs: check-environment + # Without this Github actions will create the environment when it doesnt exist if: needs.check-environment.outputs.environment_exists == 'true' outputs: secret_value: ${{ steps.fetch-credentials.outputs.secret_value }} - environment_exists: ${{ needs.check-environment.outputs.environment_exists }} steps: - name: Fetch the secret id: fetch-credentials + env: + SECRET_NAME: ${{ inputs.secret_name }} run: | - SECRET_VALUE="${{ secrets[inputs.secret_name] }}" + SECRET_VALUE="${{ secrets[env.SECRET_NAME] }}" + if [ -z "$SECRET_VALUE" ]; then + echo "Secret ${{ inputs.secret_name }} is empty. Usually this means you have not explicitly stated the secrets" + echo "in both the workflow file get-secrets-from-environment and in the file you are using the reusable workflow from." + echo "Please make sure you have added the secret to the workflow files and retry." + exit 1 + fi echo -n "$SECRET_VALUE" | openssl enc -aes-256-cbc -pbkdf2 -salt -k "${{ secrets.encryption_key }}" -out encrypted_key.bin ENCODED_ENCRYPTED_SECRET=$(base64 < encrypted_key.bin) EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index eb15823ee..68bcd4ab7 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -14,56 +14,75 @@ on: - qa - production - backup + - jump tag: type: choice description: Select group tag you want to execute default: all options: - all + - application - backups - checks - - updates - - application - - tools - - docker - - deployment - - users - crontab - - mongodb - data-partition - - swap - - ufw - - fail2ban - decrypt - - swarm + - deployment + - docker - elasticsearch + - fail2ban + - jump + - mongodb + - swap + - swarm + - tools - traefik + - ufw + - updates + - users debug: type: boolean description: Open SSH session to the runner after deployment default: false jobs: get-backup-ssh-key: - uses: ./.github/workflows/get-secret-from-env.yml + name: Get backup SSH key + uses: ./.github/workflows/get-secret-from-environment.yml with: secret_name: 'SSH_KEY' env_name: 'backup' secrets: gh_token: ${{ secrets.GH_TOKEN }} encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + SSH_KEY: ${{ secrets.SSH_KEY }} + + get-jump-ssh-key: + name: Get jump SSH key + uses: ./.github/workflows/get-secret-from-environment.yml + with: + secret_name: 'SSH_KEY' + env_name: 'jump' + secrets: + gh_token: ${{ secrets.GH_TOKEN }} + encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + SSH_KEY: ${{ secrets.SSH_KEY }} + get-production-encryption-key: - uses: ./.github/workflows/get-secret-from-env.yml + name: Get production backup encryption key + if: github.event.inputs.environment == 'staging' + uses: ./.github/workflows/get-secret-from-environment.yml with: secret_name: 'BACKUP_ENCRYPTION_PASSPHRASE' env_name: 'production' secrets: gh_token: ${{ secrets.GH_TOKEN }} encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }} + BACKUP_ENCRYPTION_PASSPHRASE: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }} provision: name: Provision ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} - needs: [get-backup-ssh-key, get-production-encryption-key] + needs: [get-backup-ssh-key, get-jump-ssh-key, get-production-encryption-key] if: always() runs-on: ubuntu-22.04 outputs: @@ -124,10 +143,17 @@ jobs: - name: Write backup SSH key to file if: needs.get-backup-ssh-key.outputs.environment_exists == 'true' run: | - echo "${{ needs.get-production-encryption-key.outputs.backup-ssh-key }}" | base64 --decode | \ + echo "${{ needs.get-backup-ssh-key.outputs.secret_value }}" | base64 --decode | \ openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/backup_ssh_private_key chmod 600 /tmp/backup_ssh_private_key + - name: Write jump server SSH key to file + if: needs.get-jump-ssh-key.outputs.environment_exists == 'true' + run: | + echo "${{ needs.get-jump-ssh-key.outputs.secret_value }}" | base64 --decode | \ + openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/jump_ssh_private_key + chmod 600 /tmp/jump_ssh_private_key + - name: Check if backup environment if configured in inventory file if: needs.get-backup-ssh-key.outputs.environment_exists != 'true' run: | diff --git a/infrastructure/environments/setup-environment.ts b/infrastructure/environments/setup-environment.ts index 9d7d38a87..0f41050d9 100644 --- a/infrastructure/environments/setup-environment.ts +++ b/infrastructure/environments/setup-environment.ts @@ -19,7 +19,7 @@ import { } from './github' import editor from '@inquirer/editor' -import { readFileSync, writeFileSync } from 'fs' +import { writeFileSync } from 'fs' import { exec as callbackExec } from 'child_process' import { promisify } from 'util' import { join } from 'path' @@ -27,7 +27,6 @@ import { error, info, log, success, warn } from './logger' import { verifyConnection } from './ssh' const exec = promisify(callbackExec) -const dotenv = require('dotenv') const notEmpty = (value: string | number) => value.toString().trim().length > 0 ? true : 'Please enter a value' @@ -39,12 +38,8 @@ type Question = PromptObject & { valueLabel?: string } -type QuestionWithHiddenType = Omit, 'type'> & { - type: PromptObject['type'] | 'hidden' -} - type QuestionDescriptor = Omit, 'type'> & { - type: 'hidden' | 'disabled' | PromptObject['type'] + type: 'disabled' | PromptObject['type'] } type SecretAnswer = { @@ -86,7 +81,7 @@ if (!environment || typeof environment !== 'string') { } // Read users .env file based on the environment name they gave above, e.g. .env.production -dotenv.config({ +require('dotenv').config({ path: `${process.cwd()}/.env.${environment}` }) @@ -106,7 +101,7 @@ function findExistingValue( async function promptAndStoreAnswer( environment: string, - questions: Array>, + questions: Array>, existingValues: Array ) { log('') @@ -126,7 +121,7 @@ async function promptAndStoreAnswer( questionWithVariableLabel.scope, existingValues ) - if (existingVariable && questionWithVariableLabel.type !== 'hidden') { + if (existingVariable) { return [ { name: 'overWrite' + questionWithVariableLabel.name, @@ -148,10 +143,7 @@ async function promptAndStoreAnswer( } } - if ( - questionWithVariableLabel.valueType === 'SECRET' && - questionWithVariableLabel.type !== 'hidden' - ) { + if (questionWithVariableLabel.valueType === 'SECRET') { const existingSecret = findExistingValue( questionWithVariableLabel.valueLabel, 'SECRET', @@ -186,34 +178,14 @@ async function promptAndStoreAnswer( return questionWithVariableLabel }) - const promptQuestions = processedQuestions - .filter(({ type }) => type !== 'hidden') - .map(questionToPrompt) + const promptQuestions = processedQuestions.map(questionToPrompt) - const visibleQuestionResults = await prompts(promptQuestions, { + const result = await prompts(promptQuestions, { onCancel: () => { process.exit(1) } }) - const hiddenResults = Object.fromEntries( - processedQuestions - .filter(({ type }) => type === 'hidden') - .map((question) => { - if (question.valueType === 'VARIABLE') { - const existingVariable = findExistingValue( - question.valueLabel!, - 'VARIABLE', - question.scope, - existingValues - ) - return [question.name, existingVariable?.value || question.initial] - } - return undefined - }) - .filter((x): x is [string, string] => Boolean(x)) - ) - const result = { ...visibleQuestionResults, ...hiddenResults } ALL_ANSWERS.push(result) storeSecrets(environment, getAnswers(existingValues)) @@ -243,27 +215,9 @@ function generateLongPassword() { } function storeSecrets(environment: string, answers: Answers) { - let currentConfig: Record = {} - try { - currentConfig = dotenv.parse(readFileSync(`.env.${environment}`)) - } catch (error) { - /* empty */ - } - const allKnownKeys = Array.from( - new Set([...Object.keys(currentConfig), ...answers.map((a) => a.name)]) - ) - - const secretsFromAnswers = Object.fromEntries( - answers.map((update) => [update.name, update.value]) - ) - const secrets = allKnownKeys.map((key) => [ - key, - secretsFromAnswers[key] || currentConfig[key] - ]) - writeFileSync( `.env.${environment}`, - secrets.map(([name, value]) => `${name}="${value}"`).join('\n') + answers.map((update) => `${update.name}="${update.value}"`).join('\n') ) } @@ -775,6 +729,14 @@ ALL_QUESTIONS.push( ...sentryQuestions, ...derivedVariables ) + +/* + * These environment only need a subset of the environment variables + * as they are not used for application hosting + */ + +const SPECIAL_NON_APPLICATION_ENVIRONMENTS = ['jump', 'backup'] + ;(async () => { const { type } = await prompts( [ @@ -794,6 +756,7 @@ ALL_QUESTIONS.push( }, { title: 'Quality assurance (no PII data)', value: 'qa' }, { title: 'Backup', value: 'backup' }, + { title: 'Jump / Bastion', value: 'jump' }, { title: 'Other', value: 'development' } ] } @@ -883,14 +846,13 @@ ALL_QUESTIONS.push( existingValues ) - const sshKeyExists = existingValues.find( + const SSH_KEY_EXISTS = existingValues.find( (value) => value.name === 'SSH_KEY' && value.scope === 'ENVIRONMENT' ) - if (!sshKeyExists) { + if (!SSH_KEY_EXISTS) { const sshKey = await editor({ - message: `Paste the SSH private key for ${kleur.cyan('SSH_USER')} here:`, - default: process.env.SSH_KEY + message: `Paste the SSH private key for ${kleur.cyan('SSH_USER')} here:` }) const formattedSSHKey = sshKey.endsWith('\n') ? sshKey : sshKey + '\n' @@ -923,7 +885,7 @@ ALL_QUESTIONS.push( await promptAndStoreAnswer(environment, dockerhubQuestions, existingValues) - if (type === 'backup') { + if (SPECIAL_NON_APPLICATION_ENVIRONMENTS.includes(type)) { const { updateHosts } = await prompts( [ { @@ -1216,7 +1178,7 @@ ALL_QUESTIONS.push( } ] - if (type !== 'backup') { + if (!SPECIAL_NON_APPLICATION_ENVIRONMENTS.includes(type)) { derivedUpdates.push(...applicationServerUpdates) } diff --git a/infrastructure/server-setup/backups.yml b/infrastructure/server-setup/backups.yml index 5479bff35..84d3e3952 100644 --- a/infrastructure/server-setup/backups.yml +++ b/infrastructure/server-setup/backups.yml @@ -92,6 +92,11 @@ tags: - backups + - set_fact: + external_backup_server_user_home: '/home/{{ external_backup_server_user }}' + tags: + - backups + - name: 'Create backup directory' file: path: '{{ backup_server_remote_target_directory }}' @@ -120,21 +125,21 @@ tags: - backups -- hosts: backups - become: yes - become_method: sudo - vars: - manager_hostname: "{{ groups['docker-manager-first'][0] }}" - tasks: + - name: Get manager node hostname + set_fact: + manager_hostname: "{{ groups['docker-manager-first'][0] }}" + when: "{{ 'docker-manager-first' in groups }}" + - name: Ensure backup application servers can login to backup server blockinfile: - path: '{{ backup_server_user_home }}/.ssh/authorized_keys' + path: '{{ external_backup_server_user_home }}/.ssh/authorized_keys' block: | {{ lookup('file', '/tmp/docker-manager-first_id_rsa.pub') }} marker: '# {mark} ANSIBLE MANAGED BLOCK docker-manager-first {{ manager_hostname }}' create: yes - owner: '{{ backup_server_user }}' mode: 0600 + owner: '{{ external_backup_server_user }}' + when: "{{ 'docker-manager-first' in groups }}" tags: - backups diff --git a/infrastructure/server-setup/group_vars/all.yml b/infrastructure/server-setup/group_vars/all.yml index 3445cae5d..50603f70a 100644 --- a/infrastructure/server-setup/group_vars/all.yml +++ b/infrastructure/server-setup/group_vars/all.yml @@ -13,3 +13,4 @@ swap_file_size_mb: 8000 backup_server_user: 'backup' backup_server_user_home: '/home/backup' crontab_user: root +provisioning_user: provision diff --git a/infrastructure/server-setup/inventory/jump.yml b/infrastructure/server-setup/inventory/jump.yml new file mode 100644 index 000000000..8b616f7f2 --- /dev/null +++ b/infrastructure/server-setup/inventory/jump.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + +all: + vars: + users: [] + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' diff --git a/infrastructure/server-setup/inventory/production.yml b/infrastructure/server-setup/inventory/production.yml index 44ef7bc9c..3e940ecfc 100644 --- a/infrastructure/server-setup/inventory/production.yml +++ b/infrastructure/server-setup/inventory/production.yml @@ -14,6 +14,8 @@ all: # SSH and other services should never be exposed to the public internet. only_allow_access_from_addresses: - 165.22.110.53 + - 159.89.14.13 + # Enable backups enable_backups: true backup_server_remote_target_directory: /home/backup/backups @@ -58,3 +60,11 @@ backups: # Written by provision pipeline. Assumes "backup" environment # exists in Github environments ansible_ssh_private_key_file: /tmp/backup_ssh_private_key + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' + # Written by provision pipeline. Assumes "jump" environment + # exists in Github environments + ansible_ssh_private_key_file: /tmp/jump_ssh_private_key diff --git a/infrastructure/server-setup/inventory/staging.yml b/infrastructure/server-setup/inventory/staging.yml index 46b57692d..de33ecfa3 100644 --- a/infrastructure/server-setup/inventory/staging.yml +++ b/infrastructure/server-setup/inventory/staging.yml @@ -13,6 +13,7 @@ all: # SSH and other services should never be exposed to the public internet. only_allow_access_from_addresses: - 165.22.110.53 + - 159.89.14.13 # Enable backups but write them to a different location from where production writes them enable_backups: true backup_server_remote_target_directory: /home/backup/staging-backups @@ -77,3 +78,11 @@ backups: # Written by provision pipeline. Assumes "backup" environment # exists in Github environments ansible_ssh_private_key_file: /tmp/backup_ssh_private_key + +jump: + hosts: + opencrvs-bastion: + ansible_host: '159.89.14.13' + # Written by provision pipeline. Assumes "jump" environment + # exists in Github environments + ansible_ssh_private_key_file: /tmp/jump_ssh_private_key diff --git a/infrastructure/server-setup/jump.yml b/infrastructure/server-setup/jump.yml new file mode 100644 index 000000000..396fc0a07 --- /dev/null +++ b/infrastructure/server-setup/jump.yml @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. +--- +- hosts: docker-manager-first + become: yes + tasks: + - name: Fetch the public SSH key so it can be transferred to the jump machine + fetch: + src: '/home/{{ provisioning_user }}/.ssh/authorized_keys' + dest: '/tmp/docker-manager-first_id_rsa.pub' + flat: yes + tags: + - jump + +- hosts: jump + become: yes + become_method: sudo + tasks: + - name: Ensure jump user is present + user: + name: 'jump' + state: present + create_home: true + home: '/home/jump' + shell: /bin/bash + tags: + - jump + + - name: Only require public key from the user jump + blockinfile: + path: /etc/ssh/sshd_config + block: | + Match User jump + PasswordAuthentication no + AuthenticationMethods publickey + marker: '# {mark} ANSIBLE MANAGED BLOCK FOR USER jump' + become: yes + tags: + - jump + + - name: Get manager node hostname + set_fact: + manager_hostname: "{{ groups['docker-manager-first'][0] }}" + when: "{{ 'docker-manager-first' in groups }}" + + - name: Ensure application servers can login to jump server + blockinfile: + path: '/home/jump/.ssh/authorized_keys' + block: | + {{ lookup('file', '/tmp/docker-manager-first_id_rsa.pub') }} + marker: '# {mark} ANSIBLE MANAGED BLOCK docker-manager-first {{ manager_hostname }}' + create: yes + mode: 0600 + owner: 'jump' + when: "{{ 'docker-manager-first' in groups }}" + tags: + - jump + +- hosts: docker-manager-first + become: yes + tasks: + - name: Set destination server + set_fact: + destination_server: "{{ hostvars[groups['jump'][0]].ansible_host }}" + tags: + - jump + + - name: Check SSH connection to destination server + shell: ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 jump@{{ destination_server }} 'echo Connection successful' + remote_user: '{{ provisioning_user }}' + register: ssh_test + ignore_errors: yes + tags: + - jump + + - name: Fail if SSH connection test failed + fail: + msg: 'SSH connection to the jump server failed' + when: ssh_test.rc != 0 + tags: + - jump diff --git a/infrastructure/server-setup/playbook.yml b/infrastructure/server-setup/playbook.yml index 3ec5afdb1..87bea0507 100644 --- a/infrastructure/server-setup/playbook.yml +++ b/infrastructure/server-setup/playbook.yml @@ -30,9 +30,6 @@ tags: - updates - - include_tasks: - file: tasks/backwards-compatibility.yml - - hosts: docker-manager-first, docker-workers become: yes become_method: sudo @@ -189,3 +186,4 @@ when: hostvars[hostname]['data_label'] is defined - import_playbook: backups.yml +- import_playbook: jump.yml diff --git a/infrastructure/server-setup/tasks/backups/crontab.yml b/infrastructure/server-setup/tasks/backups/crontab.yml index d1b04496c..0af6d6d29 100644 --- a/infrastructure/server-setup/tasks/backups/crontab.yml +++ b/infrastructure/server-setup/tasks/backups/crontab.yml @@ -34,7 +34,7 @@ - name: Throw an error if periodic_restore_from_backup is true but backup_restore_encryption_passphrase is not defined fail: - msg: 'Error: backup_restore_encryption_passphrase is not defined. This usually means you have enabled periodic restore from production but you haven't set up a production environment yet. Please set up a production environment first.' + msg: "Error: backup_restore_encryption_passphrase is not defined. This usually means you have enabled periodic restore from production but you haven't set up a production environment yet. Please set up a production environment first." when: periodic_restore_from_backup and backup_restore_encryption_passphrase is not defined - name: 'Setup crontab to download a backup periodically the opencrvs data'