Skip to content

Commit

Permalink
ARTESCA-9658 fix retry ETCd backoup
Browse files Browse the repository at this point in the history
  • Loading branch information
aprucolimartins committed Nov 14, 2023
1 parent b710792 commit 695afd8
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 10 deletions.
2 changes: 1 addition & 1 deletion scripts/backup.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ replicate_archives() {

run "Backing up MetalK8s configurations" backup_metalk8s_conf
run "Backing up CAs certificates and keys" backup_cas
run_with_retry 12 5 "Backing up etcd data" backup_etcd
run "Backing up etcd data" with_retry 20 15 backup_etcd # try for 5 minutes (every 15 seconds, 20 times)
run "Creating backup archive '$BACKUP_ARCHIVE'" create_archive

if (( REPLICATION )); then
Expand Down
23 changes: 14 additions & 9 deletions scripts/common.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -218,24 +218,29 @@ run() {
fi
}

run_with_retry() {
with_retry() {
local retries=$1
local delay=$2
shift 2

local -i i=0
while true; do
if [[ $(( ++i )) -gt $retries ]]; then
die "Failed to run '$*' after ${retries} retries."
fi
for ((i = 1; i <= retries; i++)); do
echo "> Attempt ${i}/${retries}"

echo -n "> ${i}/${retries} "
if run "${@}"; then
break
local rctemp=$(mktemp)
echo $(set +e; "${@}"; echo -n $? > "$rctemp")
local rc=$(cat "$rctemp")
rm -f "$rctemp"

if [ $rc -eq 0 ]; then
echo "Succeed"
return 0
fi

echo "Failed"
sleep "${delay}"
done

die "Failed to run '$*' after ${retries} retries."
}

check_package_manager_yum() {
Expand Down

0 comments on commit 695afd8

Please sign in to comment.