new commit

.github/workflows/get-secret-from-env.yml → .github/workflows/get-secret-from-environment.yml

@@ -9,11 +9,24 @@ on:
       env_name:
         required: true
         type: string
+    outputs:
+      secret_value:
+        description: 'Secret value, encrypted with the encryption key'
+        value: ${{ jobs.fetch-credentials.outputs.secret_value }}
+      environment_exists:
+        description: 'Whether the environment exists or not'
+        value: ${{ jobs.check-environment.outputs.environment_exists }}
     secrets:
       gh_token:
         required: true
       encryption_key:
         required: true
+      # All secrets that are we want to allow access to need
+      # to be defined in this list
+      BACKUP_ENCRYPTION_PASSPHRASE:
+        required: false
+      SSH_KEY:
+        required: false
 
 jobs:
   check-environment:
@@ -46,17 +59,26 @@ jobs:
 
   fetch-credentials:
     name: Fetch Secret
-    needs: check-environment
     runs-on: ubuntu-22.04
+    environment: ${{ inputs.env_name }}
+    needs: check-environment
+    # Without this Github actions will create the environment when it doesnt exist
     if: needs.check-environment.outputs.environment_exists == 'true'
     outputs:
       secret_value: ${{ steps.fetch-credentials.outputs.secret_value }}
-      environment_exists: ${{ needs.check-environment.outputs.environment_exists }}
     steps:
       - name: Fetch the secret
         id: fetch-credentials
+        env:
+          SECRET_NAME: ${{ inputs.secret_name }}
         run: |
-          SECRET_VALUE="${{ secrets[inputs.secret_name] }}"
+          SECRET_VALUE="${{ secrets[env.SECRET_NAME] }}"
+          if [ -z "$SECRET_VALUE" ]; then
+            echo "Secret ${{ inputs.secret_name }} is empty. Usually this means you have not explicitly stated the secrets"
+            echo "in both the workflow file get-secrets-from-environment and in the file you are using the reusable workflow from."
+            echo "Please make sure you have added the secret to the workflow files and retry."
+            exit 1
+          fi
           echo -n "$SECRET_VALUE" | openssl enc -aes-256-cbc -pbkdf2 -salt -k "${{ secrets.encryption_key }}" -out encrypted_key.bin
           ENCODED_ENCRYPTED_SECRET=$(base64 < encrypted_key.bin)
           EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)

.github/workflows/provision.yml

@@ -14,56 +14,75 @@ on:
           - qa
           - production
           - backup
+          - jump
       tag:
         type: choice
         description: Select group tag you want to execute
         default: all
         options:
           - all
+          - application
           - backups
           - checks
-          - updates
-          - application
-          - tools
-          - docker
-          - deployment
-          - users
           - crontab
-          - mongodb
           - data-partition
-          - swap
-          - ufw
-          - fail2ban
           - decrypt
-          - swarm
+          - deployment
+          - docker
           - elasticsearch
+          - fail2ban
+          - jump
+          - mongodb
+          - swap
+          - swarm
+          - tools
           - traefik
+          - ufw
+          - updates
+          - users
       debug:
         type: boolean
         description: Open SSH session to the runner after deployment
         default: false
 jobs:
   get-backup-ssh-key:
-    uses: ./.github/workflows/get-secret-from-env.yml
+    name: Get backup SSH key
+    uses: ./.github/workflows/get-secret-from-environment.yml
     with:
       secret_name: 'SSH_KEY'
       env_name: 'backup'
     secrets:
       gh_token: ${{ secrets.GH_TOKEN }}
       encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
+      SSH_KEY: ${{ secrets.SSH_KEY }}
+
+  get-jump-ssh-key:
+    name: Get jump SSH key
+    uses: ./.github/workflows/get-secret-from-environment.yml
+    with:
+      secret_name: 'SSH_KEY'
+      env_name: 'jump'
+    secrets:
+      gh_token: ${{ secrets.GH_TOKEN }}
+      encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
+      SSH_KEY: ${{ secrets.SSH_KEY }}
+
   get-production-encryption-key:
-    uses: ./.github/workflows/get-secret-from-env.yml
+    name: Get production backup encryption key
+    if: github.event.inputs.environment == 'staging'
+    uses: ./.github/workflows/get-secret-from-environment.yml
     with:
       secret_name: 'BACKUP_ENCRYPTION_PASSPHRASE'
       env_name: 'production'
     secrets:
       gh_token: ${{ secrets.GH_TOKEN }}
       encryption_key: ${{ secrets.GH_ENCRYPTION_PASSWORD }}
+      BACKUP_ENCRYPTION_PASSPHRASE: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }}
 
   provision:
     name: Provision ${{ github.event.inputs.environment }}
     environment: ${{ github.event.inputs.environment }}
-    needs: [get-backup-ssh-key, get-production-encryption-key]
+    needs: [get-backup-ssh-key, get-jump-ssh-key, get-production-encryption-key]
     if: always()
     runs-on: ubuntu-22.04
     outputs:
@@ -124,10 +143,17 @@ jobs:
       - name: Write backup SSH key to file
         if: needs.get-backup-ssh-key.outputs.environment_exists == 'true'
         run: |
-          echo "${{ needs.get-production-encryption-key.outputs.backup-ssh-key }}" | base64 --decode | \
+          echo "${{ needs.get-backup-ssh-key.outputs.secret_value }}" | base64 --decode | \
           openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/backup_ssh_private_key
           chmod 600 /tmp/backup_ssh_private_key
 
+      - name: Write jump server SSH key to file
+        if: needs.get-jump-ssh-key.outputs.environment_exists == 'true'
+        run: |
+          echo "${{ needs.get-jump-ssh-key.outputs.secret_value }}" | base64 --decode | \
+          openssl enc -aes-256-cbc -pbkdf2 -d -salt -k "${{ secrets.GH_ENCRYPTION_PASSWORD }}" -out /tmp/jump_ssh_private_key
+          chmod 600 /tmp/jump_ssh_private_key
+
       - name: Check if backup environment if configured in inventory file
         if: needs.get-backup-ssh-key.outputs.environment_exists != 'true'
         run: |

infrastructure/environments/setup-environment.ts

@@ -19,15 +19,14 @@ import {
 } from './github'
 
 import editor from '@inquirer/editor'
-import { readFileSync, writeFileSync } from 'fs'
+import { writeFileSync } from 'fs'
 import { exec as callbackExec } from 'child_process'
 import { promisify } from 'util'
 import { join } from 'path'
 import { error, info, log, success, warn } from './logger'
 import { verifyConnection } from './ssh'
 
 const exec = promisify(callbackExec)
-const dotenv = require('dotenv')
 
 const notEmpty = (value: string | number) =>
   value.toString().trim().length > 0 ? true : 'Please enter a value'
@@ -39,12 +38,8 @@ type Question<T extends string> = PromptObject<T> & {
   valueLabel?: string
 }
 
-type QuestionWithHiddenType<T extends string> = Omit<Question<T>, 'type'> & {
-  type: PromptObject<T>['type'] | 'hidden'
-}
-
 type QuestionDescriptor<T extends string> = Omit<Question<T>, 'type'> & {
-  type: 'hidden' | 'disabled' | PromptObject<T>['type']
+  type: 'disabled' | PromptObject<T>['type']
 }
 
 type SecretAnswer = {
@@ -86,7 +81,7 @@ if (!environment || typeof environment !== 'string') {
 }
 
 // Read users .env file based on the environment name they gave above, e.g. .env.production
-dotenv.config({
+require('dotenv').config({
   path: `${process.cwd()}/.env.${environment}`
 })
 
@@ -106,7 +101,7 @@ function findExistingValue<T extends string>(
 
 async function promptAndStoreAnswer(
   environment: string,
-  questions: Array<QuestionWithHiddenType<any>>,
+  questions: Array<QuestionDescriptor<any>>,
   existingValues: Array<Secret | Variable>
 ) {
   log('')
@@ -126,7 +121,7 @@ async function promptAndStoreAnswer(
         questionWithVariableLabel.scope,
         existingValues
       )
-      if (existingVariable && questionWithVariableLabel.type !== 'hidden') {
+      if (existingVariable) {
         return [
           {
             name: 'overWrite' + questionWithVariableLabel.name,
@@ -148,10 +143,7 @@ async function promptAndStoreAnswer(
       }
     }
 
-    if (
-      questionWithVariableLabel.valueType === 'SECRET' &&
-      questionWithVariableLabel.type !== 'hidden'
-    ) {
+    if (questionWithVariableLabel.valueType === 'SECRET') {
       const existingSecret = findExistingValue(
         questionWithVariableLabel.valueLabel,
         'SECRET',
@@ -186,34 +178,14 @@ async function promptAndStoreAnswer(
     return questionWithVariableLabel
   })
 
-  const promptQuestions = processedQuestions
-    .filter(({ type }) => type !== 'hidden')
-    .map(questionToPrompt)
+  const promptQuestions = processedQuestions.map(questionToPrompt)
 
-  const visibleQuestionResults = await prompts(promptQuestions, {
+  const result = await prompts(promptQuestions, {
     onCancel: () => {
       process.exit(1)
     }
   })
-  const hiddenResults = Object.fromEntries(
-    processedQuestions
-      .filter(({ type }) => type === 'hidden')
-      .map((question) => {
-        if (question.valueType === 'VARIABLE') {
-          const existingVariable = findExistingValue(
-            question.valueLabel!,
-            'VARIABLE',
-            question.scope,
-            existingValues
-          )
-          return [question.name, existingVariable?.value || question.initial]
-        }
-        return undefined
-      })
-      .filter((x): x is [string, string] => Boolean(x))
-  )
 
-  const result = { ...visibleQuestionResults, ...hiddenResults }
   ALL_ANSWERS.push(result)
   storeSecrets(environment, getAnswers(existingValues))
 
@@ -243,27 +215,9 @@ function generateLongPassword() {
 }
 
 function storeSecrets(environment: string, answers: Answers) {
-  let currentConfig: Record<string, string> = {}
-  try {
-    currentConfig = dotenv.parse(readFileSync(`.env.${environment}`))
-  } catch (error) {
-    /* empty */
-  }
-  const allKnownKeys = Array.from(
-    new Set([...Object.keys(currentConfig), ...answers.map((a) => a.name)])
-  )
-
-  const secretsFromAnswers = Object.fromEntries(
-    answers.map((update) => [update.name, update.value])
-  )
-  const secrets = allKnownKeys.map((key) => [
-    key,
-    secretsFromAnswers[key] || currentConfig[key]
-  ])
-
   writeFileSync(
     `.env.${environment}`,
-    secrets.map(([name, value]) => `${name}="${value}"`).join('\n')
+    answers.map((update) => `${update.name}="${update.value}"`).join('\n')
   )
 }
 
@@ -775,6 +729,14 @@ ALL_QUESTIONS.push(
   ...sentryQuestions,
   ...derivedVariables
 )
+
+/*
+ * These environment only need a subset of the environment variables
+ * as they are not used for application hosting
+ */
+
+const SPECIAL_NON_APPLICATION_ENVIRONMENTS = ['jump', 'backup']
+
 ;(async () => {
   const { type } = await prompts(
     [
@@ -794,6 +756,7 @@ ALL_QUESTIONS.push(
           },
           { title: 'Quality assurance (no PII data)', value: 'qa' },
           { title: 'Backup', value: 'backup' },
+          { title: 'Jump / Bastion', value: 'jump' },
           { title: 'Other', value: 'development' }
         ]
       }
@@ -883,14 +846,13 @@ ALL_QUESTIONS.push(
     existingValues
   )
 
-  const sshKeyExists = existingValues.find(
+  const SSH_KEY_EXISTS = existingValues.find(
     (value) => value.name === 'SSH_KEY' && value.scope === 'ENVIRONMENT'
   )
 
-  if (!sshKeyExists) {
+  if (!SSH_KEY_EXISTS) {
     const sshKey = await editor({
-      message: `Paste the SSH private key for ${kleur.cyan('SSH_USER')} here:`,
-      default: process.env.SSH_KEY
+      message: `Paste the SSH private key for ${kleur.cyan('SSH_USER')} here:`
     })
 
     const formattedSSHKey = sshKey.endsWith('\n') ? sshKey : sshKey + '\n'
@@ -923,7 +885,7 @@ ALL_QUESTIONS.push(
 
   await promptAndStoreAnswer(environment, dockerhubQuestions, existingValues)
 
-  if (type === 'backup') {
+  if (SPECIAL_NON_APPLICATION_ENVIRONMENTS.includes(type)) {
     const { updateHosts } = await prompts(
       [
         {
@@ -1216,7 +1178,7 @@ ALL_QUESTIONS.push(
     }
   ]
 
-  if (type !== 'backup') {
+  if (!SPECIAL_NON_APPLICATION_ENVIRONMENTS.includes(type)) {
     derivedUpdates.push(...applicationServerUpdates)
   }