Merge "Retries for certain wheel and image build commands"

build-tools/build-docker-images/build-stx-base.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# This utility builds the StarlingX wheel tarball
+# This utility builds the StarlingX base image
 #
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -31,6 +33,7 @@ CLEAN=no
 TAG_LATEST=no
 LATEST_TAG=latest
 HOST=${HOSTNAME}
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -52,11 +55,12 @@ Options:
     --registry:   Docker registry
     --clean:      Remove image(s) from local registry
     --hostname:   build repo host
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -132,6 +136,10 @@ while true; do
             HOST=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -223,6 +231,9 @@ done
 docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
 BASE_IMAGE_PRESENT=$?
 
+# Pull the image anyway, to ensure it is up to date
+docker pull ${OS}:${OS_VERSION}
+
 # Build the image
 IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
 IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
@@ -238,7 +249,7 @@ fi
 BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
 
 # Build base image
-docker build "${BUILD_ARGS[@]}"
+with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
 if [ $? -ne 0 ]; then
     echo "Failed running docker build command" >&2
     exit 1

build-tools/build-docker-images/build-stx-images.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-# This utility builds the StarlingX wheel tarball
+# This utility builds the StarlingX container images
 #
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -35,6 +37,7 @@ TAG_LIST_FILE=
 TAG_LIST_LATEST_FILE=
 declare -a ONLY
 declare -a SKIP
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -61,6 +64,7 @@ Options:
     --skip <image> : Skip building the specified image(s). Multiple images
                      can be specified with a comma-separated list, or with
                      multiple --skip arguments.
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 
 EOF
@@ -276,7 +280,7 @@ function build_image_loci {
 
     local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}"
 
-    docker build ${WORKDIR}/loci --no-cache \
+    with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \
         "${BUILD_ARGS[@]}" \
         --tag ${build_image_name}  2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
@@ -361,7 +365,7 @@ function build_image_docker {
         BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
     fi
     BASE_BUILD_ARGS+=(--tag ${build_image_name})
-    docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
+    with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log
 
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to build ${LABEL}... Aborting"
@@ -395,7 +399,7 @@ function build_image {
     esac
 }
 
-OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -477,6 +481,10 @@ while true; do
             TAG_LATEST=yes
             shift
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -555,6 +563,9 @@ fi
 docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$"
 BASE_IMAGE_PRESENT=$?
 
+# Pull the image anyway, to ensure it's up to date
+docker pull ${BASE}
+
 # Download loci, if needed.
 get_loci
 if [ $? -ne 0 ]; then
@@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_
     done
 done
 
-if [ "${CLEAN}" = "yes" ]; then
+if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
     # Delete the images
     echo "Deleting images"
     docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]}

build-tools/build-wheels/build-base-wheels.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,6 +8,10 @@
 # for a set of upstream python modules.
 #
 
+MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
+
+source ${MY_SCRIPT_DIR}/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -21,6 +25,7 @@ OS=centos
 OS_VERSION=7.5.1804
 BUILD_STREAM=stable
 PROXY=""
+declare -i MAX_ATTEMPTS=1
 
 function usage {
     cat >&2 <<EOF
@@ -34,11 +39,12 @@ Options:
     --keep-container: Skip deletion of container used for the build
     --proxy:          Set proxy <URL>:<PORT>
     --stream:         Build stream, stable or dev (default: stable)
+    --attempts:       Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -81,6 +87,10 @@ while true; do
             PROXY=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
 BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH})
 
 # Build image
-docker build "${BUILD_ARGS[@]}"
+with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
 if [ $? -ne 0 ]; then
     echo "Failed to create build image in docker" >&2
     exit 1
@@ -230,7 +240,7 @@ fi
 RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
 
 # Run container to build wheels
-docker run ${RUN_ARGS[@]}
+with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
 
 if [ "${KEEP_IMAGE}" = "no" ]; then
     # Delete the builder image

build-tools/build-wheels/build-wheel-tarball.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -9,6 +9,8 @@
 
 MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
 
+source ${MY_SCRIPT_DIR}/utils.sh
+
 # Required env vars
 if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
     echo "Environment not setup for builds" >&2
@@ -25,6 +27,7 @@ PUSH=no
 PROXY=""
 CLEAN=no
 DOCKER_USER=${USER}
+declare -i MAX_ATTEMPTS=1
 
 # List of top-level services for images, which should not be listed in upper-constraints.txt
 SKIP_CONSTRAINTS=(
@@ -56,11 +59,12 @@ Options:
     --proxy:      Set proxy <URL>:<PORT>
     --user:       Docker repo userid
     --version:    Version for pushed image (if used with --push)
+    --attempts:   Max attempts, in case of failure (default: 1)
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -111,6 +115,10 @@ while true; do
             VERSION=$2
             shift 2
             ;;
+        --attempts)
+            MAX_ATTEMPTS=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then
     BUILD_BASE_WL_ARGS+=(--proxy ${PROXY})
 fi
 
-${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]}
+${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
 if [ $? -ne 0 ]; then
     echo "Failure running build-base-wheels.sh" >&2
     exit 1
@@ -177,13 +185,13 @@ else
     OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
 fi
 
-wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
+with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download global-requirements.txt" >&2
     exit 1
 fi
 
-wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
+with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download upper-constraints.txt" >&2
     exit 1

build-tools/build-wheels/docker/docker-build-wheel.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2018 Wind River Systems, Inc.
+# Copyright (c) 2018-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -11,6 +11,7 @@
 CFGFILE=/wheels.cfg
 OUTPUTDIR=/wheels
 FAILED_LOG=$OUTPUTDIR/failed.lst
+declare -i MAX_ATTEMPTS=5
 
 #
 # Function to log the start of a build
@@ -102,6 +103,41 @@ function fix_setup {
 
 }
 
+#
+# Function to call a command, with support for retries
+#
+function with_retries {
+    local max_attempts=$1
+    local cmd=$2
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+    shift 2
+
+    local -i attempt=0
+
+    while :; do
+        let -i attempt++
+
+        echo "Running: ${cmd} $@"
+        ${cmd} "$@"
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+
+        echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            local delay=5
+            echo "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}
+
 #
 # Function to use git to clone the module source and build a wheel.
 #
@@ -176,9 +212,8 @@ function from_tar {
             taropts="xf"
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi
@@ -228,9 +263,8 @@ function from_zip {
             continue
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi
@@ -280,9 +314,8 @@ function from_pypi {
             continue
         fi
 
-        wget $wgetsrc
+        with_retries ${MAX_ATTEMPTS} wget $wgetsrc
         if [ $? -ne 0 ]; then
-            echo "Failure running: wget $wgetsrc"
             echo $wheelname >> $FAILED_LOG
             continue
         fi

build-tools/build-wheels/utils.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Image and wheel build utility functions
+#
+
+#
+# Function to call a command, with support for retries
+#
+function with_retries {
+    local max_attempts=$1
+    local cmd=$2
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+    shift 2
+
+    local -i attempt=0
+
+    while :; do
+        let -i attempt++
+
+        echo "Running: ${cmd} $@"
+        ${cmd} "$@"
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+
+        echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            local delay=5
+            echo "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}
+