ARTESCA-9658 Fix retry ETCd backup

CHANGELOG.md

@@ -7,6 +7,11 @@
 - Remove `nodes-darwin` MacOS related grafana dashboard
   (PR[4178](https://github.com/scality/metalk8s/pull/4178))
 
+### Bug fixes
+
+- Fix a bug in retry logic for ETCd backup
+  (PR[4197](https://github.com/scality/metalk8s/pull/4197))
+
 ## Release 125.0.6
 
 ### Enhancements

scripts/backup.sh.in

@@ -181,7 +181,7 @@ replicate_archives() {
 
 run "Backing up MetalK8s configurations" backup_metalk8s_conf
 run "Backing up CAs certificates and keys" backup_cas
-run_with_retry 12 5 "Backing up etcd data" backup_etcd
+run "Backing up etcd data" with_retry 20 15 backup_etcd # try for 5 minutes (every 15 seconds, 20 times)
 run "Creating backup archive '$BACKUP_ARCHIVE'" create_archive
 
 if (( REPLICATION )); then

scripts/common.sh.in

@@ -218,24 +218,29 @@ run() {
     fi
 }
 
-run_with_retry() {
-    local retries=$1
-    local delay=$2
+with_retry() {
+    local retries=$1 delay=$2
     shift 2
 
-    local -i i=0
-    while true; do
-        if [[ $(( ++i )) -gt $retries ]]; then
-            die "Failed to run '$*' after ${retries} retries."
-        fi
+    for ((i = 1; i <= retries; i++)); do
+        echo "> Attempt ${i}/${retries}"
 
-        echo -n "> ${i}/${retries} "
-        if run "${@}"; then
-            break
+        local rctemp rc
+        rctemp=$(mktemp)
+        echo "$(set +e; "${@}"; echo -n $? > "$rctemp")"
+        rc=$(cat "$rctemp")
+        rm -f "$rctemp"
+
+        if [ "$rc" -eq 0 ]; then
+            echo "Succeed"
+            return 0
         fi
 
+        echo "Failed"
         sleep "${delay}"
     done
+
+    die "Failed to run '$*' after ${retries} retries."
 }
 
 check_package_manager_yum() {