From bb472a9f37b5acb5c8bf1544f854584f90b5dd88 Mon Sep 17 00:00:00 2001 From: "Liu, ZhipengS" Date: Mon, 25 May 2020 15:16:43 +0800 Subject: [PATCH] Add proxy setting for container image build scripts Users in some places like China need to use proxy during building image. So, it is better to let user be able to use proxy by adding these proxy arguments. Closes-bug: 1890383 Change-Id: If9ef7d09a5f53148252b4ab62c40ab0f143e1c3f Signed-off-by: Liu, ZhipengS --- .../build-docker-images/build-stx-images.sh | 64 +++++++++++++++---- .../build-docker-images/update-stx-image.sh | 22 +++++-- build-tools/build-wheels/build-base-wheels.sh | 48 ++++++++++---- .../build-wheels/build-wheel-tarball.sh | 34 ++++++++-- 4 files changed, 134 insertions(+), 34 deletions(-) diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 7c82d4c1..f13d6509 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -26,8 +26,10 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PREFIX=dev LATEST_PREFIX="" PUSH=no -PROXY="" CONFIG_FILE="" +HTTP_PROXY="" +HTTPS_PROXY="" +NO_PROXY="" DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= @@ -57,7 +59,9 @@ Options: --wheels: Specify path to wheels tarball or image, URL or docker tag (required option) --wheels-alternate: Specify path to alternate wheels tarball or image, URL or docker tag --push: Push to docker repo - --proxy: Set proxy : + --http_proxy: Set proxy :, urls splitted with "," + --https_proxy: Set proxy :, urls splitted with "," + --no_proxy: Set proxy , urls splitted with "," --user: Docker repo userid --registry: Docker registry --prefix: Prefix on the image tag (default: dev) @@ -287,7 +291,20 @@ function post_build { if [ -n "${CUSTOMIZATION}" ]; then - docker run --entrypoint /bin/bash --name ${USER}_update_img ${build_image_name} -c "${CUSTOMIZATION}" + local -a PROXY_ARGS= + if [ ! -z "$HTTP_PROXY" ]; then + PROXY_ARGS+=(--env http_proxy=$HTTP_PROXY) + fi + + if [ ! -z "$HTTPS_PROXY" ]; then + PROXY_ARGS+=(--env https_proxy=$HTTPS_PROXY) + fi + + if [ ! -z "$NO_PROXY" ]; then + PROXY_ARGS+=(--env no_proxy=$NO_PROXY) + fi + + docker run "${PROXY_ARGS[@]}" --entrypoint /bin/bash --name ${USER}_update_img ${build_image_name} -c "${CUSTOMIZATION}" if [ $? -ne 0 ]; then echo "Failed to add customization for ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) @@ -402,8 +419,16 @@ function build_image_loci { BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) fi - if [ ! -z "$PROXY" ]; then - BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + if [ ! -z "$HTTP_PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY) + fi + + if [ ! -z "$HTTPS_PROXY" ]; then + BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY) + fi + + if [ ! -z "$NO_PROXY" ]; then + BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY) fi if [ -n "${PROJECT_REF}" ]; then @@ -554,9 +579,18 @@ function build_image_docker { BASE_BUILD_ARGS+=(${real_docker_context} --no-cache) BASE_BUILD_ARGS+=(--file ${real_docker_file}) BASE_BUILD_ARGS+=(--build-arg "BASE=${BASE}") - if [ ! -z "$PROXY" ]; then - BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + if [ ! -z "$HTTP_PROXY" ]; then + BASE_BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY) fi + + if [ ! -z "$HTTPS_PROXY" ]; then + BASE_BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY) + fi + + if [ ! -z "$NO_PROXY" ]; then + BASE_BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY) + fi + BASE_BUILD_ARGS+=(--tag ${build_image_name}) with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log @@ -639,7 +673,7 @@ function build_image_script { cp $(dirname ${image_build_file})/${SCRIPT} ${SCRIPT} local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log + with_retries ${MAX_ATTEMPTS} ${COMMAND} ${SCRIPT} ${ARGS} ${build_image_name} $HTTP_PROXY $HTTPS_PROXY $NO_PROXY 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" @@ -681,7 +715,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -736,8 +770,16 @@ while true; do PUSH=yes shift ;; - --proxy) - PROXY=$2 + --http_proxy) + HTTP_PROXY=$2 + shift 2 + ;; + --https_proxy) + HTTPS_PROXY=$2 + shift 2 + ;; + --no_proxy) + NO_PROXY=$2 shift 2 ;; --user) diff --git a/build-tools/build-docker-images/update-stx-image.sh b/build-tools/build-docker-images/update-stx-image.sh index 3146f278..c6f476ed 100755 --- a/build-tools/build-docker-images/update-stx-image.sh +++ b/build-tools/build-docker-images/update-stx-image.sh @@ -18,7 +18,9 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then fi PUSH=no -PROXY="" +HTTP_PROXY="" +HTTPS_PROXY="" +NO_PROXY="" DOCKER_USER= DOCKER_REGISTRY= FILE_BASEDIR=${PWD} @@ -59,7 +61,9 @@ Options: --customize: Customization script --extra: Extra file (to be accessible to customization script) --push: Push to docker repo - --proxy: Set proxy : + --http_proxy: Set http proxy :, urls splitted with "," + --https_proxy: Set https proxy :, urls splitted with "," + --no_proxy: set bypass list for proxy urls splitted with "," --user: Docker repo userid --registry: Docker registry --clean: Remove image(s) from local registry @@ -184,7 +188,7 @@ function read_params_from_file { FILE_BASEDIR=$(dirname ${FILE}) } -OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,proxy:,user:,registry:,clean,attempts:,update-id: -- "$@") +OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,clean,attempts:,update-id: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -235,8 +239,16 @@ while true; do PUSH=yes shift ;; - --proxy) - PROXY=$2 + --http_proxy) + HTTP_PROXY=$2 + shift 2 + ;; + --https_proxy) + HTTPS_PROXY=$2 + shift 2 + ;; + --no_proxy) + NO_PROXY=$2 shift 2 ;; --user) diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index b8425fe6..4ab913c1 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -24,7 +24,9 @@ KEEP_CONTAINER=no OS=centos OS_VERSION=7.5.1804 BUILD_STREAM=stable -PROXY="" +HTTP_PROXY="" +HTTPS_PROXY="" +NO_PROXY="" declare -i MAX_ATTEMPTS=1 function usage { @@ -37,14 +39,16 @@ Options: --os-version: Specify OS version --keep-image: Skip deletion of the wheel build image in docker --keep-container: Skip deletion of container used for the build - --proxy: Set proxy : + --http_proxy: Set http proxy :, urls splitted by "," + --https_proxy: Set https proxy :, urls splitted by "," + --no_proxy: Set bypass list for proxy , urls splitted by "," --stream: Build stream, stable or dev (default: stable) --attempts: Max attempts, in case of failure (default: 1) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -83,8 +87,16 @@ while true; do BUILD_STREAM=$2 shift 2 ;; - --proxy) - PROXY=$2 + --http_proxy) + HTTP_PROXY=$2 + shift 2 + ;; + --https_proxy) + HTTPS_PROXY=$2 + shift 2 + ;; + --no_proxy) + NO_PROXY=$2 shift 2 ;; --attempts) @@ -212,10 +224,18 @@ BASE_IMAGE_PRESENT=$? declare -a BUILD_ARGS BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION}) BUILD_ARGS+=(--build-arg BUILD_STREAM=${BUILD_STREAM}) -if [ ! -z "$PROXY" ]; then - BUILD_ARGS+=(--build-arg http_proxy=$PROXY) - BUILD_ARGS+=(--build-arg https_proxy=$PROXY) +if [ ! -z "$HTTP_PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$HTTP_PROXY) fi + +if [ ! -z "$HTTPS_PROXY" ]; then + BUILD_ARGS+=(--build-arg https_proxy=$HTTPS_PROXY) +fi + +if [ ! -z "$NO_PROXY" ]; then + BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY) +fi + BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}) @@ -233,10 +253,16 @@ if [ "${KEEP_CONTAINER}" = "no" ]; then fi declare -a RUN_ARGS -if [ ! -z "$PROXY" ]; then - RUN_ARGS+=(--env http_proxy=$PROXY) - RUN_ARGS+=(--env https_proxy=$PROXY) +if [ ! -z "$HTTP_PROXY" ]; then + RUN_ARGS+=(--env http_proxy=$HTTP_PROXY) fi +if [ ! -z "$HTTPS_PROXY" ]; then + RUN_ARGS+=(--env https_proxy=$HTTPS_PROXY) +fi +if [ ! -z "$NO_PROXY" ]; then + RUN_ARGS+=(--env no_proxy=$NO_PROXY) +fi + RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) # Run container to build wheels diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index c0909fe7..3625f05b 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -24,7 +24,9 @@ BUILD_STREAM=stable CURRENT_STABLE_OPENSTACK=ussuri VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PUSH=no -PROXY="" +HTTP_PROXY="" +HTTPS_PROXY="" +NO_PROXY="" CLEAN=no DOCKER_USER=${USER} declare -i MAX_ATTEMPTS=1 @@ -56,7 +58,9 @@ Options: --os-version: Specify OS version --stream: Build stream, stable or dev (default: stable) --push: Push to docker repo - --proxy: Set proxy : + --http_proxy: Set http proxy :, urls splitted by "," + --https_proxy: Set https proxy :, urls splitted by "," + --no_proxy: Set bypass list for proxy , urls splitted by "," --user: Docker repo userid --version: Version for pushed image (if used with --push) --attempts: Max attempts, in case of failure (default: 1) @@ -64,7 +68,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -107,8 +111,16 @@ while true; do BUILD_STREAM=$2 shift 2 ;; - --proxy) - PROXY=$2 + --http_proxy) + HTTP_PROXY=$2 + shift 2 + ;; + --https_proxy) + HTTPS_PROXY=$2 + shift 2 + ;; + --no_proxy) + NO_PROXY=$2 shift 2 ;; --version) @@ -147,8 +159,16 @@ fi # Build the base wheels and retrieve the StarlingX wheels declare -a BUILD_BASE_WL_ARGS BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --stream ${BUILD_STREAM}) -if [ ! -z "$PROXY" ]; then - BUILD_BASE_WL_ARGS+=(--proxy ${PROXY}) +if [ ! -z "$HTTP_PROXY" ]; then + BUILD_BASE_WL_ARGS+=(--http_proxy ${HTTP_PROXY}) +fi + +if [ ! -z "$HTTPS_PROXY" ]; then + BUILD_BASE_WL_ARGS+=(--https_proxy ${HTTPS_PROXY}) +fi + +if [ ! -z "$NO_PROXY" ]; then + BUILD_BASE_WL_ARGS+=(--no_proxy ${NO_PROXY}) fi ${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}