Modularise playbook tasks, use only one playbook for all deployment s…

…izes (#798)

* split playbooks to different task modules, use only one playbook for all deployment sizes

* update provisioning pipeline

* try initialising the provision pipeline by adding a temporary push trigger

* setup ssh key before trying to provision

* add known hosts file

* do not try to mount cryptfs partition to /data if it's already mounted

.github/workflows/provision.yml

@@ -1,6 +1,9 @@
 name: Provision environment
 run-name: Provision ${{ github.event.inputs.environment }}
 on:
+  push:
+    branches:
+      - playbook-cleanup
   workflow_dispatch:
     inputs:
       environment:
@@ -12,10 +15,6 @@ on:
           - staging
           - qa
           - production
-      branch_name:
-        description: Branch to provision from
-        default: develop
-        required: true
 
 jobs:
   provision:
@@ -113,20 +112,19 @@ jobs:
           elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }}
           # ansible_sudo_pass: ${{ secrets.SUDO_PASSWORD }} in case your user is not root
 
-      # TODO: Iterate for 3 or 5 replicas
-      - name: Create ini file for 1 replica
-        id: ini-file
+      - name: Read known hosts
         run: |
-          touch ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
-          echo "[docker-manager-first]" > ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
-          echo "manager1 ansible_host=\"${{ secrets.SSH_HOST }}\" ansible_user=${{ secrets.SSH_USER }} ansible_ssh_private_key_file=/tmp/server.pem" >> ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
-          echo "" >> ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
-          echo "[all:vars]" >> ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
-          echo "data1_hostname=${{ vars.HOSTNAME }}" >> ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
+          cd ${{ github.event.repository.name }}
+          echo "KNOWN_HOSTS<<EOF" >> $GITHUB_ENV
+          sed -i -e '$a\' ./infrastructure/.known-hosts
+          cat ./infrastructure/.known-hosts >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
 
-      - name: Check ini content
-        run: |
-          cat ${{ github.event.repository.name }}/infrastructure/server-setup/replicas-1.ini
+      - name: Install SSH Key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_KEY }}
+          known_hosts: ${{ env.KNOWN_HOSTS }}
 
       - name: Run playbook on 1 replica in qa
         uses: dawidd6/action-ansible-playbook@v2
@@ -136,11 +134,15 @@ jobs:
           ANSIBLE_SSH_TIMEOUT: 30
           ANSIBLE_SSH_RETRIES: 20
         with:
-          playbook: playbook-1.yml
+          playbook: playbook.yml
           directory: ${{ github.event.repository.name }}/infrastructure/server-setup
+          inventory: |
+            [docker-manager-first]
+            ${{ vars.HOSTNAME }} ansible_host="${{ secrets.SSH_HOST }}" data_label=data1
+
+            [docker-workers]
           options: |
             --verbose
-            --inventory replicas-1.ini
             --extra-vars ""${{ steps.ansible-variables.outputs.EXTRA_VARS }}""
 
       - name: Run playbook on 1 replica in production
@@ -151,9 +153,13 @@ jobs:
           ANSIBLE_SSH_TIMEOUT: 30
           ANSIBLE_SSH_RETRIES: 20
         with:
-          playbook: playbook-1.yml
+          playbook: playbook.yml
           directory: ${{ github.event.repository.name }}/infrastructure/server-setup
+          inventory: |
+            [docker-manager-first]
+            ${{ vars.HOSTNAME }} ansible_host="${{ secrets.SSH_HOST }}" data_label=data1
+
+            [docker-workers]
           options: |
             --verbose
-            --inventory replicas-1.ini
             --extra-vars ""${{ steps.ansible-production-variables.outputs.EXTRA_VARS }}""

infrastructure/.known-hosts

@@ -0,0 +1,12 @@
+# Farajaland staging
+|1|QaneIg/kW2nT73307HQ/9Y9Bz5A=|RIaMnvGPGkJFWdEJFxWc8RLFs5E= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOKbJ1oRhgHaRxj4G8k9rkqIla59c4yWUkbfxX7yHPdWXmpwShOEaypF7SG9oXVP3+gWJG9aCLzv0F8GSFecB+w=
+|1|56R2lbUeZ1Ljt37oregbUT9t3Kg=|S3zskisluF1Z1OllInJr6P+x4/o= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCw3/MlAt3ENmyKxkRF2vuz5Kl+6BT/1XpyUwJTNt4TzD0WIJb+HNbxOdmLmrc3gQQ/17pshYWWjm3H08eEELHNpehbG8yUpYjbplIGvDKz+cmwII2Snz0heBnosD9FWsHOiuioUTtUslUM8HK9Qk/ysRXMvQB6VKbsymAj7PN0rZEdQJYDvKzQ8nY8VvfrMRW8EoglFB3arMFdkNbTja25OaMWSSJcpcAeUW6EREM/N6y35G8CwGba1yx0fW2kPGiUXPmYXefa333NI+6+yE7id5RZepPEV2qvcSyE6yhY0GWvBpMmJlmHPvYn4HRRA8ucAN8iMssb+hH9i1GqY8XHjJ7TTb0T4Qy3/RDtohXLUa6j2efw3iToYj88OKcx7kafiiTZigFhzj1cNZQf+jaWm/tfQZbietr+LUCjFToxEsa4uhvXOvoempkBXFw0pBdfv0fYkulyb+L+9Teu5BcMbjULbWNgvbm394J86/YpLZzyhAwxNBKYIVq8wA2geBc=
+|1|8qdU8nm5coaY6NvvJ9Hbg4fxmJw=|IJQksH0MG9lAsqn4eR9tolBwy5Q= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINtPEM0nhqrb+BC/mMF85xDodd0RnDZKyyCpqoePhUj5
+# Farajaland QA
+|1|z9A2rBv8YTjgWJ+0SFI5Mp2Lp+I=|MVFxVGBHPBUtJuQobkgxzzIBqhE= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDH9lVfT82JUM4YnJ0Le7mThJee+MTizFE6qfmPIxbd3WjN7Ak7O8UpViaNI77avUd2CKsO8gboKzFcU1eUCqm3qgVIEGmjaLl9V8MCIRPDHHdpRHIRuv8NFQxs9K8/W9KWKNVHLKwZmSvnre4X12QweX6ZUUmJXc+ECibHGJBrQQ9dgPETU+HnN1xw111zSY66Nbc0z8GA2FrnanUhtiKWz0r7/dOdXR05a0XOoi2KDDcoiiLBE2YYtLA4xgDBHTgeKJtV8W3DMVRg9KJ9bKfgPhwlrcVlUX4PPAYG3yIatvW8izoB51HLEeWBCxboZrzBtGJOPfy9AnTGZ8ROhDr+LpCxEHNZYnTXJti64C7pYKjn/qVYpev0CW+iqqjSR/Aksf+4j2rW/2tFAuICO3caYWlqjyySvXixwLe2ihOI3qlpt/uxjfdT07YTplzVvcu7z2I7AWe+u73Eo8STzLqzKmMSVIisN0boIXy2KkHWnIgW3/P53eVqxwng0tajI1M=
+|1|ypxkbArSjikXdEscQUXWyDMFlkM=|HV68rCR+h/IG9A3NsEhg1IqFt8Y= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDiEWSrjOQdi+r/L0W56994Zw+MtqRHgO1hVR5jkV3ayzJ0+m3auVWsk6Xs+HP2RauVRiO3idO86s8XGfwz/vxo=
+|1|8xUi4LJT0ArBoZMNZBzqdKna+X0=|ofos+azs78yCYodbGqkxPcvrzoc= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGSIBuxNbsqj+NYkQQcBxTQZP6hkrI5jSK79rHIajRlN
+# Farajaland production
+|1|mmWxyVhdNt+9vCZY8YSu/b5T6mI=|oWUySmyU/yK3gMAgrMpcfutjats= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEp49NpGjmoCmYAHnNbZF6dpo0G3L3Z3m++B4Pq3sVUTLwMuNv4WfoebiSJH20tcTq92XbFV5NfCmdnfoksr/1Y=
+|1|EWKHZMaMYdiCWDSqV8DsOmqFJ3c=|GQ0ApYEAoubxL2n9VFMY5cnN8Yc= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJkWafUYUyF1eOzj1WqhjwJc+TvMjKmz0bCqtci5BMNUsn+R+Z+CIOPvI8eqYEzXiJ7VGAZq1twRYGSnRTJyCja3eJfvQfxb8hGVz2fkf9rthgYABdoHSJoGMTt4EP2LtTduzvLPCBctiID6bXUFMkM6j2pmLQK/gZGEKNaJb86D7xt1HXQsqV5bAAKCaehgn6LAQ8zee9YZtoP8fwhoorTJEJ3kpsvsneEpV3kiuAPdyfB8zI7E3HHqXgD+ij0eFogK+NSUIMUexFyRZgAtKDBjnNRQLGMEzY1UzM7pojRr8Bb9vT4tGJGBpzfWvWn6WavIpBa3Ht6sXmXHGexGn2X8gyG5rHif2FAmTV7O4M+sBlpxqr7G906BJ8JwOl8qp8T9BnesWiExFdeDwzsRPS49KQOBpxOqfK5OC1ZQlEzVIR1SQpOAGjDgGM1XtR4jsBX6OBlR7hcYSw9F6wCWjrWFrmv6HVNAXZgYsnzXQmJPpDbRQAyHTvIHK9/DobQI8=
+|1|FuRTlvDs4p17HjsAIGQ7wQml0TM=|qDsBKG5gV6TDM2dw0lykGLS+11M= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO/ZvINySxP0MXtHBbaJ6FqDBaFUl/YVAyUItqfiyeez

infrastructure/server-setup/example-1.ini

@@ -8,11 +8,6 @@
 ; Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
 [docker-manager-first]
 ; Uncomment the line below
-; manager1 ansible_host="ENTER YOUR MANAGER HOST IP"
+ENTER_HOSTNAME_1 ansible_host="ENTER YOUR MANAGER HOST IP" data_label=data1
 
-; Below you can assign 1 node to be the data node, use the node's HOSTNAME in these variables.
-; These node will be used by databases to permanently store data.
-; Used for Mongo replica sets
-[all:vars]
-; Uncomment the line below
-; data1_hostname=ENTER_HOSTNAME_1
+[docker-workers]

infrastructure/server-setup/example-3.ini

@@ -8,19 +8,10 @@
 ; Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
 [docker-manager-first]
 ; Uncomment the line below
-; manager1 ansible_host="ENTER YOUR MANAGER HOST IP"
+ENTER_HOSTNAME_1 ansible_host="ENTER YOUR MANAGER HOST IP" data_label=data1
 
 [docker-workers]
 ; We recommend you add 2 workers for a usual production deployment
 ; Uncomment the lines below
-; worker1 ansible_host="ENTER YOUR WORKER 1 HOST IP"
-; worker2 ansible_host="ENTER YOUR WORKER 2 HOST IP"
-
-; Below you can assign 3 node to be data nodes, use the node's HOSTNAME in these variables.
-; These node will be used by databases to permanently store data.
-; Used for Mongo replica sets
-[all:vars]
-; Uncomment the lines below
-; data1_hostname=ENTER_HOSTNAME_1
-; data2_hostname=ENTER_HOSTNAME_2
-; data3_hostname=ENTER_HOSTNAME_3
+ENTER_HOSTNAME_2 ansible_host="ENTER YOUR WORKER HOST IP" data_label=data2
+ENTER_HOSTNAME_3 ansible_host="ENTER YOUR WORKER HOST IP" data_label=data3

infrastructure/server-setup/example-5.ini

@@ -8,23 +8,12 @@
 ; Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
 [docker-manager-first]
 ; Uncomment the line below
-; manager1 ansible_host="ENTER YOUR MANAGER HOST IP"
+ENTER_HOSTNAME_1 ansible_host="ENTER YOUR MANAGER HOST IP" data_label=data1
 
 [docker-workers]
 ; We recommend you add 4 workers for a scaled production deployment
 ; Uncomment the lines below
-; worker1 ansible_host="ENTER YOUR WORKER 1 HOST IP"
-; worker2 ansible_host="ENTER YOUR WORKER 2 HOST IP"
-; worker3 ansible_host="ENTER YOUR WORKER 3 HOST IP"
-; worker4 ansible_host="ENTER YOUR WORKER 4 HOST IP"
-
-; Below you can assign 5 node to be data nodes, use the node's HOSTNAME in these variables.
-; These node will be used by databases to permanently store data.
-; Used for Mongo replica sets
-[all:vars]
-; Uncomment the lines below
-; data1_hostname=ENTER_HOSTNAME_1
-; data2_hostname=ENTER_HOSTNAME_2
-; data3_hostname=ENTER_HOSTNAME_3
-; data4_hostname=ENTER_HOSTNAME_4
-; data5_hostname=ENTER_HOSTNAME_5
+ENTER_HOSTNAME_2 ansible_host="ENTER YOUR WORKER 1 HOST IP" data_label=data2
+ENTER_HOSTNAME_3 ansible_host="ENTER YOUR WORKER 2 HOST IP" data_label=data3
+ENTER_HOSTNAME_4 ansible_host="ENTER YOUR WORKER 3 HOST IP" data_label=data4
+ENTER_HOSTNAME_5 ansible_host="ENTER YOUR WORKER 4 HOST IP" data_label=data5