65b3abbae9
ISO signing via a remote server may fail due to network hiccups. Retry on errors. TESTS ====================================== 1) Run build-iso Jenkins job normally and make sure it works 2) Run build-iso with SIGNING_SERVER set to an invalid host name and make sure it retries Story: 2010226 Task: 48064 Signed-off-by: Davlet Panech <davlet.panech@windriver.com> Change-Id: Icaa8e07827ddfcc2583f875e5a57247ce7bf8613
97 lines
2.4 KiB
Bash
97 lines
2.4 KiB
Bash
# bash
|
|
|
|
#
|
|
# Copyright (c) 2022 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
#
|
|
# Function to call a command, with support for retries
|
|
#
|
|
# with_retries [<options>] <retries> <cmd> [<cmd_args>...]
|
|
#
|
|
# Options:
|
|
# -d <secs> | --delay <secs>
|
|
# Wait given number of seconds between retries
|
|
# -t <secs> | --timeout <secs>
|
|
# Each iteration of the command runs under a timeout
|
|
# -k <secs> | --kill-timeout <secs>
|
|
# Each iteration of the command is killed violently
|
|
# if it doesn't exit voluntarily within the set time
|
|
# after the initial timeout signal.
|
|
#
|
|
function with_retries {
|
|
local delay=5
|
|
local max_time=0
|
|
local kill_time=0
|
|
local to_cmd=""
|
|
|
|
while [ $1 != "" ]; do
|
|
case "$1" in
|
|
-d | --delay)
|
|
delay=$2
|
|
shift 2
|
|
;;
|
|
-t | --timeout)
|
|
max_time=$2
|
|
shift 2
|
|
;;
|
|
-k | --kill-timeout)
|
|
kill_time=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
break
|
|
;;
|
|
esac
|
|
done
|
|
|
|
local max_attempts=$1
|
|
local cmd=$2
|
|
shift 2
|
|
|
|
if [ ${max_time} -gt 0 ]; then
|
|
to_cmd="timeout "
|
|
if [ ${kill_time} -gt 0 ]; then
|
|
to_cmd+="--kill-after=${kill_time} "
|
|
fi
|
|
to_cmd+="${max_time} "
|
|
fi
|
|
|
|
# Pop the first two arguments off the list,
|
|
# so we can pass additional args to the command safely
|
|
|
|
local -i attempt=0
|
|
local rc=0
|
|
|
|
while :; do
|
|
let attempt++
|
|
|
|
echo "Running: ${cmd} $@" >&2
|
|
${to_cmd} ${cmd} "$@"
|
|
rc=$?
|
|
if [ $rc -eq 0 ]; then
|
|
return 0
|
|
fi
|
|
|
|
if [ $rc -eq 124 ]; then
|
|
echo "Command (${cmd}) timed out, attempt ${attempt} of ${max_attempts}." >&2
|
|
elif [ $rc -eq 137 ]; then
|
|
echo "Command (${cmd}) timed out and killed, attempt ${attempt} of ${max_attempts}." >&2
|
|
else
|
|
echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." >&2
|
|
fi
|
|
|
|
if [ ${attempt} -lt ${max_attempts} ]; then
|
|
echo "Waiting ${delay} seconds before retrying..." >&2
|
|
sleep ${delay}
|
|
continue
|
|
else
|
|
echo "Max command attempts reached. Aborting..." >&2
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|