Merge branch 'develop' of github.com:opencrvs/opencrvs-countryconfig …

…into ocrvs-7584

infrastructure/logrotate.conf

@@ -72,6 +72,13 @@ include /etc/logrotate.d
     rotate 1
 }
 
+/var/log/opencrvs-backup.error.log {
+    missingok
+    monthly
+    create 0660 root application
+    rotate 1
+}
+
 /var/log/papertrail.log {
     missingok
     monthly

infrastructure/monitoring/elastalert/rules/log-error-alert.yaml

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+name: 'log-error-alert'
+type: 'frequency'
+index: 'kibana-alert-history-default'
+is_enabled: true
+num_events: 1
+realert:
+  minutes: 5
+terms_size: 50
+timeframe:
+  minutes: 5
+timestamp_field: '@timestamp'
+timestamp_type: 'iso'
+use_strftime_index: false
+# https://github.com/Yelp/elastalert/issues/1260
+raw_count_keys: false
+filter:
+  - bool:
+      should:
+        - term:
+            rule.name.keyword:
+              value: 'Error while backup  '
+
+
+alert: post2
+http_post2_url: 'http://countryconfig:3040/email'
+http_post2_payload:
+  subject: '{% raw %}{{DOMAIN}}{% endraw %}: {{rule.name}} 🚨'
+  html: 'Reason: {{kibana.alert.context.logs__alert__document__count.conditions}}. Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.'
+  from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}'
+  to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}'

infrastructure/monitoring/filebeat/filebeat.yml

@@ -11,6 +11,10 @@ filebeat.inputs:
     enabled: true
     paths:
       - /var/log/opencrvs-backup.log
+  - type: log
+    enabled: true
+    paths:
+      - /var/log/opencrvs-backup.error.log
   - type: log
     enabled: true
     paths:

infrastructure/monitoring/kibana/config.ndjson

@@ -1,6 +1,7 @@
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"metrics.inventory_threshold.fired","params":{"documents":["{\"@timestamp\":\"2024-08-06T07:57:35.644Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"metrics.alert.inventory.threshold","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-17T12:01:46.420Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-08-06T08:02:44.568Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.18"},"muteAll":false,"mutedInstanceIds":[],"name":"Available disk space in root file system","notifyWhen":"onActionGroupChange","params":{"alertOnNoData":true,"criteria":[{"comparator":">=","customMetric":{"aggregation":"max","field":"system.filesystem.used.pct","id":"alert-custom-metric","type":"custom"},"metric":"custom","threshold":[0.7],"timeSize":1,"timeUnit":"h","warningComparator":">=","warningThreshold":[0.5]}],"filterQuery":"{\"bool\":{\"should\":[{\"match_phrase\":{\"system.filesystem.mount_point\":\"/hostfs\"}}],\"minimum_should_match\":1}}","filterQueryText":"system.filesystem.mount_point : \"/hostfs\"","nodeType":"host","sourceId":"default"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":["infra","opencrvs-builtin"],"throttle":null,"updatedAt":"2024-08-06T08:01:56.542Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.18","id":"14778650-8541-11ee-9002-2f37fdc4e5d5","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1722931316554,643],"type":"alert","updated_at":"2024-08-06T08:01:56.554Z","version":"WzM5MywxXQ=="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"threshold_met","params":{"documents":["{\"@timestamp\":\"2022-04-18T07:05:33.819Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"apm.error_rate","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-06-01T11:30:27.033Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in service","notifyWhen":"onActionGroupChange","params":{"environment":"ENVIRONMENT_ALL","threshold":1,"windowSize":1,"windowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-05T03:00:20.633Z","updatedBy":"opencrvs-admin"},"coreMigrationVersion":"7.17.0","id":"3b6722e0-e19e-11ec-ba8e-51649755648d","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707273006619,214975],"type":"alert","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjAzNCwxOV0="}
 {"attributes":{"buildNum":46534,"defaultIndex":"metricbeat-*"},"coreMigrationVersion":"7.17.0","id":"7.17.0","migrationVersion":{"config":"7.13.0"},"references":[],"sort":[1707273006619,216009],"type":"config","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjQyOCwxOV0="}
+{"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","frequency":{"notifyWhen":"onActionGroupChange","summary":false,"throttle":null},"group":"logs.threshold.fired","params":{"documents":["{\"@timestamp\":\"2024-12-23T09:50:26.528Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{rule.params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]},"uuid":"1504b514-82e5-47de-a6ef-60795788f7e6"}],"alertTypeId":"logs.alert.document.count","apiKey":null,"apiKeyCreatedByUser":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2024-12-23T10:16:28.758Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-12-23T10:20:10.118Z","status":"pending","warning":null},"lastRun":{"alertsCount":{"active":1,"ignored":0,"new":1,"recovered":0},"outcome":"succeeded","outcomeMsg":null,"outcomeOrder":0,"warning":null},"legacyId":null,"meta":{"versionApiKeyLastmodified":"8.14.3"},"monitoring":{"run":{"calculated_metrics":{"p50":529,"p95":565,"p99":565,"success_ratio":1},"history":[{"duration":493,"success":true,"timestamp":1734948992101},{"duration":565,"success":true,"timestamp":1734949097318}],"last_run":{"metrics":{"duration":565,"gap_duration_s":null,"total_alerts_created":null,"total_alerts_detected":null,"total_indexing_duration_ms":null,"total_search_duration_ms":null},"timestamp":"2024-12-23T10:18:17.318Z"}}},"muteAll":false,"mutedInstanceIds":[],"name":"Error while backup  ","nextRun":"2024-12-23T11:18:17.247Z","notifyWhen":null,"params":{"count":{"comparator":"more than or equals","value":1},"criteria":[{"comparator":"equals","field":"log.file.path","value":"/var/log/opencrvs-backup.error.log"}],"logView":{"logViewId":"log-view-reference-0","type":"log-view-reference"},"timeSize":1,"timeUnit":"h"},"revision":1,"running":false,"schedule":{"interval":"1h"},"scheduledTaskId":null,"snoozeSchedule":[],"tags":[],"throttle":null,"updatedAt":"2024-12-23T10:17:24.571Z","updatedBy":"opencrvs-admin"},"coreMigrationVersion":"8.8.0","created_at":"2024-12-23T10:17:24.572Z","id":"8caf3676-e78a-4e7c-ba85-c150df55901b","managed":false,"references":[{"id":"default","name":"param:log-view-reference-0","type":"infrastructure-monitoring-log-view"}],"sort":[1734949097887,115],"type":"alert","typeMigrationVersion":"10.1.0","updated_at":"2024-12-23T10:18:17.887Z","version":"WzgyNDExMyw0MDRd"}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"logs.threshold.fired","params":{"documents":["{\"@timestamp\":\"2023-11-22T08:25:47.329Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"logs.alert.document.count","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-22T08:32:38.272Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in backup logs","notifyWhen":"onActionGroupChange","params":{"count":{"comparator":"more than or equals","value":1},"criteria":[{"comparator":"matches","field":"message","value":"error"},{"comparator":"equals","field":"log.file.path","value":"/var/log/opencrvs-backup.log"}],"timeSize":1,"timeUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:22.558Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"b166fcb0-8911-11ee-8111-2f3be9e93efc","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294436464,232766],"type":"alert","updated_at":"2024-02-07T08:27:16.464Z","version":"WzQ5NTQzNCwxOV0="}
 {"attributes":{"anomalyThreshold":50,"description":"","fields":{"container":"container.id","host":"host.name","message":["message","@message"],"pod":"kubernetes.pod.uid","tiebreaker":"_doc","timestamp":"@timestamp"},"inventoryDefaultView":"0","logColumns":[{"timestampColumn":{"id":"5e7f964a-be8a-40d8-88d2-fbcfbdca0e2f"}},{"fieldColumn":{"field":"event.dataset","id":" eb9777a8-fcd3-420e-ba7d-172fff6da7a2"}},{"messageColumn":{"id":"b645d6da-824b-4723-9a2a-e8cece1645c0"}}],"logIndices":{"indexName":"logs-*,filebeat-*,kibana_sample_data_logs*,logstash*,ecs-logstash*","type":"index_name"},"metricAlias":"metrics-*,metricbeat-*","metricsExplorerDefaultView":"0","name":"Default"},"coreMigrationVersion":"7.17.0","id":"default","migrationVersion":{"infrastructure-ui-source":"7.16.2"},"references":[],"sort":[1707273006619,217714],"type":"infrastructure-ui-source","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MzAyNywxOV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"query matched","params":{"documents":["{\"@timestamp\":\"2023-11-20T10:19:30.521Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-20T09:12:19.237Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Successful SSH login","notifyWhen":"onActionGroupChange","params":{"esQuery":"{ \"query\": { \"bool\": { \"must\": [ \n  { \"term\": { \"log.file.path\": \"/var/log/auth.log\" } },\n  { \"term\": { \"event.outcome\": \"success\" }}\n  ] } } }","index":["filebeat-*"],"size":100,"threshold":[1],"thresholdComparator":">=","timeField":"@timestamp","timeWindowSize":1,"timeWindowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:19.537Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"e79aaa90-8784-11ee-b9ba-89bbe73df7ff","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294457367,232778],"type":"alert","updated_at":"2024-02-07T08:27:37.367Z","version":"WzQ5NTQ0MCwxOV0="}

infrastructure/server-setup/tasks/application.yml

@@ -49,6 +49,14 @@
     state: touch
     mode: 'u+rwX,g+rwX,o-rwx'
 
+- name: Create backup logfile for error
+  ansible.builtin.file:
+    path: /var/log/opencrvs-backup.error.log
+    owner: '{{ ansible_user }}'
+    group: 'application'
+    state: touch
+    mode: 'u+rwX,g+rwX,o-rwx'
+
 - name: Create restore logfile
   ansible.builtin.file:
     path: /var/log/opencrvs-restore.log

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -20,7 +20,7 @@
     name: 'backup opencrvs'
     minute: '0'
     hour: '0'
-    job: 'bash {{ crontab_user_home }}/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ backup_server_user }} --ssh_host={{ backup_hostname }} --ssh_port={{ backup_port }} --remote_dir={{ backup_server_remote_target_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>&1'
+    job: 'bash {{ crontab_user_home }}/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ backup_server_user }} --ssh_host={{ backup_hostname }} --ssh_port={{ backup_port }} --remote_dir={{ backup_server_remote_target_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>> /var/log/opencrvs-backup.error.log'
     state: "{{ 'present' if (backup_hostname is defined and backup_encryption_passphrase and (enable_backups | default(false))) else 'absent' }}"
 
 ##
@@ -44,7 +44,7 @@
     name: 'download opencrvs backup'
     minute: '30'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_restore_encryption_passphrase }} --ssh_user={{ backup_server_user }} --ssh_host={{ backup_hostname }} --ssh_port={{ backup_port }} --remote_dir={{ backup_server_remote_source_directory }} >> /var/log/opencrvs-restore.log 2>&1'
+    job: 'bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_restore_encryption_passphrase }} --ssh_user={{ backup_server_user }} --ssh_host={{ backup_hostname }} --ssh_port={{ backup_port }} --remote_dir={{ backup_server_remote_source_directory }} >> /var/log/opencrvs-restore.log 2>&1'
     state: "{{ 'present' if (backup_hostname is defined and backup_restore_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"
 
 - name: 'Setup crontab to restore the opencrvs data'
@@ -54,5 +54,5 @@
     name: 'restore opencrvs'
     minute: '0'
     hour: '1'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1'
+    job: 'bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1'
     state: "{{ 'present' if (backup_hostname is defined and backup_restore_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}"