Retry failed docker login

Sometimes docker login fails, retry the operation to compensate. In
particular, this seems to happen when a previous Jenkins build was not
stopped properly.

TESTS
=========================
Run docker-login & host-docker-login Jenkins jobs with valid and invalid
passwords.

Story: 2010055
Task: 51129

Change-Id: Id1998da7ba1de7539349b0a8b3937524e614aa4d
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>

scripts/docker-login.sh

@@ -87,5 +87,5 @@ rm -f $old_docker_config_file $new_docker_config_file
 
 notice "logging in to remote repos"
 for reg in "${login_repos[@]}" ; do
-    docker_login "$reg"
+    QUIET=true docker_login "$reg"
 done

scripts/host-docker-login.sh

@@ -11,11 +11,33 @@ SCRIPTS_DIR="$(dirname "$0")"
 source "$SCRIPTS_DIR/lib/job_utils.sh"
 load_build_config
 declare_job_env DOCKER_REGISTRY_LOGIN_LIST
+declare_job_env DOCKER_LOGIN_ATTEMPTS 3
 
 for spec in $DOCKER_REGISTRY_LOGIN_LIST ; do
     if [[ "$(parse_docker_registry "$spec" | awk '{print $1}')" == "docker.io" ]] ; then
         info "logging in to Docker Hub DOCKER_CONFIG=$DOCKER_CONFIG"
-        docker login </dev/null
+        if ! docker login </dev/null ; then
+            docker logout || true
+            login_ok=0
+            attempt=2
+            while true ; do
+                info "logging in to Docker Hub DOCKER_CONFIG=$DOCKER_CONFIG [attempt $attempt]"
+                if docker login </dev/null ; then
+                    login_ok=1
+                    break
+                fi
+                docker logout || true
+                let attempt++
+                if [[ $attempt -gt $DOCKER_LOGIN_ATTEMPTS ]] ; then
+                    break
+                fi
+                sleep 10
+            done
+            if [[ $login_ok -ne 1 ]] ; then
+                die "failed to login to Docker Hub"
+                exit 1
+            fi
+        fi # if ! docker login
         break
     fi
 done

scripts/lib/job_utils.sh

@@ -311,12 +311,48 @@ stx_docker_cmd() {
 # Usage: docker_login REGISTRY
 # Login to docker in builder pod
 docker_login() {
-    local reg="$1"
+    local docker_reg_label="$1"
-    local login_arg
+    local docker_reg
-    if [[ "$reg" != "docker.io" ]] ; then
+    local docker_login_attempts="${DOCKER_LOGIN_ATTEMPTS:-3}"
-        login_arg="$reg"
+    if [[ "$docker_reg_label" != "docker.io" ]] ; then
+        docker_reg="$docker_reg_label"
     fi
-    stx_docker_cmd "docker login $login_arg </dev/null"
+    local script
+    {
+        read -r -d '' script <<'_END'
+            DOCKER_REG_LABEL='@@DOCKER_REG_LABEL@@'
+            DOCKER_REG='@@DOCKER_REG@@'
+            DOCKER_LOGIN_ATTEMPTS='@@DOCKER_LOGIN_ATTEMPTS@@'
+            echo "logging in to docker registry $DOCKER_REG_LABEL" >&2
+            if ! docker login $DOCKER_REG </dev/null ; then
+                docker logout || true
+                login_ok=0
+                attempt=2
+                while true ; do
+                    echo "logging in to docker registry $DOCKER_REG_LABEL [attempt $attempt]" >&2
+                    if docker login $DOCKER_REG </dev/null ; then
+                        login_ok=1
+                        break
+                    fi
+                    docker logout $DOCKER_REG || true
+                    let attempt++
+                    if [[ $attempt -gt $DOCKER_LOGIN_ATTEMPTS ]] ; then
+                        break
+                    fi
+                    sleep 10
+                done
+                if [[ $login_ok -ne 1 ]] ; then
+                    echo "ERROR: failed to login to docker registry $DOCKER_REG_LABEL" >&2
+                    exit 1
+                fi
+            fi
+_END
+    } || true  # read -d '' always returns non-zero (bash bug?)
+    script="$(echo "$script" | sed \
+        -e "s/@@DOCKER_REG_LABEL@@/$docker_reg_label/g" \
+        -e "s/@@DOCKER_REG@@/$docker_reg/g" \
+        -e "s/@@DOCKER_LOGIN_ATTEMPTS@@/$docker_login_attempts/g")"
+    stx_docker_cmd "$script"
 }
 
 #