Merge "Update to build tools to support Build layering."

build-tools/build-iso

@@ -40,7 +40,8 @@ usage () {
 MY_YUM_CONF=""
 STD_REPO_ID="local-std"
 RT_REPO_ID="local-rt"
-DISTRO_LAYER_REPO_ID="StxCentos7Distro"
+LOWER_LAYER_STD_REPO_ID="StxCentos7Distro"
+LOWER_LAYER_RT_REPO_ID="StxCentos7Distro-rt"
 
 NPROCS=$(nproc)
 
@@ -294,6 +295,7 @@ function init_output_dir {
    echo "  Preparing package lists"
    image_inc_list iso std centos > "${PKGLIST_STX}"
    image_inc_list iso dev centos > "${PKGLIST_DEV}"
+   image_inc_list iso layer centos ${LAYER} > "${PKGLIST_THIS_LAYER}"
 
    echo "  Copying base files"
 
@@ -345,6 +347,7 @@ function init_output_dir {
            export MY_BUILD_DIR=$MY_BUILD_DIR; \
            export MY_WORKSPACE=$MY_WORKSPACE; \
            export MY_REPO=$MY_REPO; \
+           export LAYER=$LAYER; \
            export MY_BUILD_CFG=$MY_BUILD_CFG; \
            export MY_MOCK_ROOT=$MY_MOCK_ROOT; \
            export PATH=$MY_REPO/build-tools:\$PATH; \
@@ -438,10 +441,15 @@ function final_touches {
       EXTRA_ARGS="--pkglist '${PKGLIST_DEV}'"
    fi
 
+   for PKGLIST_LOWER_LAYER in ${PKGLIST_LOWER_LAYER_LIST}; do
+       EXTRA_ARGS+=" --pkglist ${PKGLIST_LOWER_LAYER}"
+   done
+
    python "$BSP_FILES_PATH/platform_comps.py" \
       --groups "$OUTPUT_DIST_DIR/comps.xml" \
       --pkglist "${PKGLIST_MINIMAL}" \
       --pkglist "${PKGLIST_STX}" \
+      --pkglist "${PKGLIST_THIS_LAYER}" \
       ${EXTRA_ARGS}
    if [ $? -ne 0 ]; then
       echo "Failed to update comps.xml"
@@ -511,9 +519,11 @@ function extract_installer_files {
 
    \cd kickstart.work
 
-   extract_pkg_from_local_repo platform-kickstarts ${MY_YUM_CONF} ${STD_REPO_ID}
+   echo "MY_YUM_CONF=${MY_YUM_CONF}"
-   extract_pkg_from_local_repo platform-kickstarts-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID}
+   cat ${MY_YUM_CONF}
-   extract_pkg_from_local_repo platform-kickstarts-extracfgs ${MY_YUM_CONF} ${STD_REPO_ID}
+   extract_pkg_from_local_repo platform-kickstarts ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
+   extract_pkg_from_local_repo platform-kickstarts-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
+   extract_pkg_from_local_repo platform-kickstarts-extracfgs ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
 
    \cp --preserve=all www/pages/feed/rel-*/*.cfg pxeboot/*.cfg ../kickstarts/ &&
    \cp --preserve=all extra_cfgs/*.cfg ../extra_cfgs/
@@ -569,9 +579,9 @@ EOM
    \mkdir $WORKDIR
    \cd $WORKDIR
 
-   extract_pkg_from_local_repo pxe-network-installer ${MY_YUM_CONF} ${STD_REPO_ID}
+   extract_pkg_from_local_repo pxe-network-installer ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
-   extract_pkg_from_local_repo grub2-efi-x64-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID} ${DISTRO_LAYER_REPO_ID}
+   extract_pkg_from_local_repo grub2-efi-x64-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
-   extract_pkg_from_local_repo grub2-efi-x64-modules ${MY_YUM_CONF} ${STD_REPO_ID} ${DISTRO_LAYER_REPO_ID}
+   extract_pkg_from_local_repo grub2-efi-x64-modules ${MY_YUM_CONF} ${STD_REPO_ID} ${LOWER_LAYER_STD_REPO_ID}
 
    \mkdir -p $OUTPUT_DIST_DIR/isolinux/pxeboot/EFI/centos/x86_64-efi
 
@@ -700,10 +710,17 @@ printf   "*************************\n\n"
 # Init variables
 init_vars
 check_vars
+DISTRO="centos"
 PKGLIST_MINIMAL="${INTERNAL_REPO_ROOT}/build-tools/build_iso/minimal_rpm_list.txt"
 PKGLIST_STX="${OUTPUT_DIR}/image.inc"
 PKGLIST_DEV="${OUTPUT_DIR}/image-dev.inc"
-DISTRO="centos"
+PKGLIST_THIS_LAYER="${OUTPUT_DIR}/image-layer.inc"
+PKGLIST_LOWER_LAYER_DIR="${MY_REPO}/cgcs-centos-repo/layer_image_inc"
+
+PKGLIST_LOWER_LAYER_LIST=""
+if [ -d ${PKGLIST_LOWER_LAYER_DIR} ]; then
+    PKGLIST_LOWER_LAYER_LIST="$(find ${PKGLIST_LOWER_LAYER_DIR} -name '*image.inc')"
+fi
 
 # Create skeleton build dir
 init_output_dir
@@ -724,6 +741,15 @@ if [ $? -eq 2 ]; then
     exit 1
 fi
 
+
+for PKGLIST_LOWER_LAYER in $PKGLIST_LOWER_LAYER_LIST; do
+    install_pkg_list "${PKGLIST_LOWER_LAYER}"
+    if [ $? -eq 2 ]; then
+        echo "Error: Failed to install packages from ${PKGLIST_LOWER_LAYER}"
+        exit 1
+    fi
+done
+
 if [ "x${RELEASE_BUILD}" == "x" ]; then
     echo "Installing StarlingX developer packages"
     install_pkg_list "${PKGLIST_DEV}"
@@ -731,28 +757,15 @@ if [ "x${RELEASE_BUILD}" == "x" ]; then
         echo "Error: Failed to install packages from ${PKGLIST_DEV}"
         exit 1
     fi
-fi
 
-
+    for PKGLIST_LOWER_LAYER in $PKGLIST_LOWER_LAYER_LIST; do
-required_layer_cfg_name="${DISTRO}_required_build_layer.cfg"
+        install_pkg_list "${PKGLIST_LOWER_LAYER}"
-layer_cfg_name="${DISTRO}_build_layer.cfg"
-layer_cfgs=$(find $(for x in $GIT_LIST; do echo $x/; done) -maxdepth 1 -name ${layer_cfg_name})
-for line in $(grep -v '^#' ${BUILD_ISO_DIR}/build_iso/${required_layer_cfg_name}); do
-    layer=${line%%,*}
-    url=${line##*,}
-    cat $layer_cfgs | grep -q "^${layer}$"
-    if [ $? -ne 0 ]; then
-        echo "need ${layer} from ${url}"
-        # layer_inc=${MY_REPO}/cgcs-centos-repo/$(basename ${url})
-        layer_inc=${MY_WORKSPACE}/export/$(basename ${url})
-        curl -s ${url} > ${layer_inc}
-        install_pkg_list ${layer_inc}
         if [ $? -eq 2 ]; then
-            echo "Error: Failed to install packages from ${layer_inc}"
+            echo "Error: Failed to install packages from ${PKGLIST_LOWER_LAYER}"
             exit 1
         fi
-    fi
     done
+fi
 
 \cd $OUTPUT_DIST_DIR
 chmod -R 644 isolinux/Packages/*
@@ -778,7 +791,7 @@ final_touches
 # Sign the ISO
 sign_iso
 
-make_report "${PKGLIST_MINIMAL}" "${PKGLIST_STX}"
+make_report "${PKGLIST_MINIMAL}" "${PKGLIST_STX}" "${PKGLIST_THIS_LAYER}" ${PKGLIST_LOWER_LAYER_LIST}
 
 # Check sanity
 FILESIZE=$(wc -c <"$OUTPUT_FILE")

build-tools/build-pkgs-parallel

@@ -28,7 +28,7 @@ usage () {
     echo "   after a clone/pull, nor the final build prior to creating an iso or patch,"
     echo "   but can be used for intermediate builds."
     echo "   i.e. while debugging compilation failures."
-    echo "      build-pkgs-parallel [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
+    echo "      build-pkgs-parallel [--layer] [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
     echo ""
     echo "   Delete source rpms, and the directories associated with it's creation:"
     echo "   Note: does not clean an edit environment"
@@ -62,7 +62,7 @@ export BUILD_AVOIDANCE_URL=""
 
 
 # read the options
-TEMP=$(getopt -o h --long parallel,rt,std,installer,containers,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@")
+TEMP=$(getopt -o h --long parallel,rt,std,installer,containers,layer:,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -91,6 +91,7 @@ while true ; do
         --no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;;
         --formal)         EXTRA_ARGS_COMMON+=" --formal" ; shift ;;
         --careful)        EXTRA_ARGS_RPM+=" --careful" ; shift ;;
+        --layer)          EXTRA_ARGS_COMMON+=" --layer=$2"; shift 2 ;;
         --no-required)    EXTRA_ARGS_RPM+=" --no-required" ; shift ;;
         --no-build-info)  EXTRA_ARGS_COMMON+=" --no-build-info" ; shift ;;
         --no-autoclean)   EXTRA_ARGS_RPM+=" --no-autoclean" ; shift ;;

build-tools/build-pkgs-serial

@@ -62,7 +62,7 @@ export BUILD_AVOIDANCE_URL=""
 
 
 # read the options
-TEMP=$(getopt -o h --long serial,rt,std,installer,containers,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@")
+TEMP=$(getopt -o h --long serial,rt,std,installer,containers,layer:,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -91,6 +91,7 @@ while true ; do
         --no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;;
         --formal)         EXTRA_ARGS_COMMON+=" --formal" ; shift ;;
         --careful)        EXTRA_ARGS_RPM+=" --careful" ; shift ;;
+        --layer)          EXTRA_ARGS_COMMON+=" --layer=$2"; shift 2 ;;
         --no-required)    EXTRA_ARGS_RPM+=" --no-required" ; shift ;;
         --no-build-info)  EXTRA_ARGS_COMMON+=" --no-build-info" ; shift ;;
         --no-autoclean)   EXTRA_ARGS_RPM+=" --no-autoclean" ; shift ;;

build-tools/build-rpms-parallel

@@ -279,6 +279,20 @@ compute_resources () {
 }
 
 
+#
+# Create a list of rpms in the directory
+#
+create_lst () {
+   local DIR=${1}
+
+       (cd $DIR
+        [ -f rpm.lst ] && \rm -rf rpm.lst
+        [ -f srpm.lst ] && \rm -rf srpm.lst
+        find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
+        find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
+       )
+}
+
 #
 # Delete old repodata and reate a new one
 #
@@ -1208,11 +1222,14 @@ update_cgcs_repo () {
           echo ""
           echo "Need to recreate $REPO_NAME/$d/repodata"
           mkdir -p $d
+
           if [ -d $d/repodata ]; then
              update_repodata "$d"
           else
              recreate_repodata "$d"
           fi
+
+          create_lst "$d"
        fi
     done
     echo "$CURR_HEAD" > $LAST_HEAD_FILE
@@ -1406,7 +1423,7 @@ FORMAL_FLAG=0
 CAREFUL=0
    
 # read the options
-TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
+TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,layer: -n "$ME" -- "$@")
 
 if [ $? -ne 0 ]; then
     usage
@@ -1421,12 +1438,6 @@ trap my_exit EXIT
 # extract options and their arguments into variables.
 while true ; do
     case "$1" in
-        -a|--arga)
-            case "$2" in
-                "") ARG_A='some default value' ; shift 2 ;;
-                *) ARG_A=$2 ; shift 2 ;;
-            esac ;;
-        -b|--argb) ARG_B=1 ; shift ;;
         --careful) CAREFUL=1 ; shift ;;
         --no-descendants) NO_DESCENDANTS=1 ; shift ;;
         --no-required) NO_REQUIRED=1 ; shift ;;
@@ -1440,12 +1451,8 @@ while true ; do
         -h|--help) HELP=1 ; shift ;;
         --clean) CLEAN_FLAG=1 ; shift ;;
         --tmpfs-clean) if [ -n "$MY_WORKSPACE" ]; then export MY_WORKSPACE=$MY_WORKSPACE/$BUILD_TYPE; exit 0; fi ;;
-        -c|--argc)
-            case "$2" in
-                "") shift 2 ;;
-                *) ARG_C=$2 ; shift 2 ;;
-            esac ;;
         --parallel) shift ;;
+        --layer) export LAYER=$2 ; shift ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error!" ; exit 1 ;;
     esac
@@ -1473,7 +1480,10 @@ export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
 export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
 
 IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
-image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}"
+image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}"
+
+DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc"
+image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}"
 
 LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
 
@@ -1536,6 +1546,9 @@ export SOURCES_DIR="$RPM_BUILD_BASE/SOURCES"
 export PLATFORM_RELEASE
 
 if [ ! -d $BUILD_BASE ]; then
+   if [ $CLEAN_FLAG -eq 1 ]; then
+       exit 0
+   fi
    echo "ERROR: expected to find directory at '$BUILD_BASE'"
    exit 1
 fi
@@ -1721,6 +1734,7 @@ clean_list () {
    echo "Cleaning repodata"
    for d in $(find -L  $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results   -type d -name repodata); do
       recreate_repodata $(dirname $d)
+      create_lst $(dirname $d)
    done
 
    echo ""
@@ -2338,6 +2352,7 @@ fi
 echo "Recreate repodata"
 for d in $(find -L $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results  -type d -name repodata); do
    update_repodata $(dirname "$d")
+   create_lst $(dirname "$d")
 done
 
 

build-tools/build-rpms-serial

@@ -137,6 +137,20 @@ str_lst_contains() {
 }
 
 
+#
+# Create a list of rpms in the directory
+#
+create_lst () {
+   local DIR=${1}
+
+       (cd $DIR
+        [ -f rpm.lst ] && \rm -rf rpm.lst
+        [ -f srpm.lst ] && \rm -rf srpm.lst
+        find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
+        find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
+       )
+}
+
 #
 # Delete old repodata and reate a new one
 #
@@ -990,11 +1004,14 @@ update_cgcs_repo () {
           echo ""
           echo "Need to recreate $REPO_NAME/$d/repodata"
           mkdir -p $d
+
           if [ -d $d/repodata ]; then
              update_repodata "$d"
           else
              recreate_repodata "$d"
           fi
+
+          create_lst "$d"
        fi
     done
     echo "$CURR_HEAD" > $LAST_HEAD_FILE
@@ -1175,7 +1192,7 @@ FORMAL_FLAG=0
 CAREFUL=0
    
 # read the options
-TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
+TEMP=$(getopt -o h --long serial,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,formal,careful,help,layer: -n "$ME" -- "$@")
 
 if [ $? -ne 0 ]; then
     usage
@@ -1190,12 +1207,6 @@ trap my_exit EXIT
 # extract options and their arguments into variables.
 while true ; do
     case "$1" in
-        -a|--arga)
-            case "$2" in
-                "") ARG_A='some default value' ; shift 2 ;;
-                *) ARG_A=$2 ; shift 2 ;;
-            esac ;;
-        -b|--argb) ARG_B=1 ; shift ;;
         --careful) CAREFUL=1 ; shift ;;
         --no-descendants) NO_DESCENDANTS=1 ; shift ;;
         --no-required) NO_REQUIRED=1 ; shift ;;
@@ -1208,12 +1219,8 @@ while true ; do
         --containers) BUILD_TYPE=containers; shift ;;
         -h|--help) HELP=1 ; shift ;;
         --clean) CLEAN_FLAG=1 ; shift ;;
-        -c|--argc)
-            case "$2" in
-                "") shift 2 ;;
-                *) ARG_C=$2 ; shift 2 ;;
-            esac ;;
         --serial) shift ;;
+        --layer) export LAYER=$2 ; shift ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error!" ; exit 1 ;;
     esac
@@ -1241,7 +1248,10 @@ export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
 export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
 
 IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
-image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}"
+image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}"
+
+DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc"
+image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}"
 
 LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
 
@@ -1305,6 +1315,9 @@ export SOURCES_DIR="$RPM_BUILD_BASE/SOURCES"
 export PLATFORM_RELEASE
 
 if [ ! -d $BUILD_BASE ]; then
+   if [ $CLEAN_FLAG -eq 1 ]; then
+       exit 0
+   fi
    echo "ERROR: expected to find directory at '$BUILD_BASE'"
    exit 1
 fi
@@ -1481,6 +1494,7 @@ clean_list () {
    echo "Cleaning repodata"
    for d in $(find -L  $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results   -type d -name repodata); do
       recreate_repodata $(dirname $d)
+      create_lst $(dirname $d)
    done
 
    echo ""
@@ -2060,6 +2074,7 @@ fi
 echo "Recreate repodata"
 for d in $(find -L $MY_WORKSPACE/rpmbuild $MY_WORKSPACE/results  -type d -name repodata); do
    update_repodata $(dirname "$d")
+   create_lst $(dirname "$d")
 done
 
 

build-tools/build-srpms-common.sh

@@ -12,6 +12,21 @@ SRC_BUILD_TYPE_SRPM="srpm"
 SRC_BUILD_TYPE_SPEC="spec"
 SRC_BUILD_TYPES="$SRC_BUILD_TYPE_SRPM $SRC_BUILD_TYPE_SPEC"
 
+set_build_info () {
+    local info_file="$MY_WORKSPACE/BUILD_INFO"
+    local layer_prefix="${LAYER^^}_"
+    if [ "${LAYER}" == "" ]; then
+        layer_prefix=""
+    fi
+    mkdir -p "$(dirname ${info_file})"
+    echo "${layer_prefix}OS=\"centos\"" > "${info_file}"
+    echo "${layer_prefix}JOB=\"n/a\"" >> "${info_file}"
+    echo "${layer_prefix}BUILD_BY=\"${USER}\"" >> "${info_file}"
+    echo "${layer_prefix}BUILD_NUMBER=\"n/a\"" >> "${info_file}"
+    echo "${layer_prefix}BUILD_HOST=\"$(hostname)\"" >> "${info_file}"
+    echo "${layer_prefix}BUILD_DATE=\"$(date '+%Y-%m-%d %H:%M:%S %z')\"" >> "${info_file}"
+}
+
 
 str_lst_contains() {
     TARGET="$1"

build-tools/build-srpms-parallel

@@ -71,11 +71,26 @@ if [ $? -ne 0 ]; then
    CREATEREPO="createrepo"
 fi
 
+#
+# Create a list of rpms in the directory
+#
+create_lst () {
+   local DIR=${1}
+
+       (cd $DIR
+        [ -f rpm.lst ] && \rm -rf rpm.lst
+        [ -f srpm.lst ] && \rm -rf srpm.lst
+        find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
+        find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
+       )
+}
+
+
 usage () {
     echo ""
     echo "Usage: "
     echo "   Create source rpms:"
-    echo "      $ME [--rt | --std | --installer | --containers] [--no-descendants] [--formal] [ list of package names ]"
+    echo "      $ME [--rt | --std | --installer | --containers] [--layer=<layer>] [--no-descendants] [--formal] [ list of package names ]"
     echo ""
     echo "   Delete source rpms, and the directories associated with it's creation:"
     echo "   Note: does not clean an edit environment"
@@ -161,7 +176,7 @@ EDIT_FLAG=0
 NO_META_PATCH_FLAG=0
 
 # read the options
-TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
+TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,layer: -n "$ME" -- "$@")
 
 if [ $? -ne 0 ]; then
     usage
@@ -175,20 +190,9 @@ export BUILD_TYPE=std
 # extract options and their arguments into variables.
 while true ; do
     case "$1" in
-        -a|--arga)
-            case "$2" in
-                "") ARG_A='some default value' ; shift 2 ;;
-                *) ARG_A=$2 ; shift 2 ;;
-            esac ;;
-        -b|--argb) ARG_B=1 ; shift ;;
         --no-descendants) NO_DESCENDANTS=1 ; shift ;;
         --no-build-info) NO_BUILD_INFO=1 ; shift ;;
         -h|--help) HELP=1 ; shift ;;
-        -c|--argc)
-            case "$2" in
-                "") shift 2 ;;
-                *) ARG_C=$2 ; shift 2 ;;
-            esac ;;
         --clean) CLEAN_FLAG=1 ; shift ;;
         --formal) FORMAL_FLAG=1 ; shift ;;
         --std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;;
@@ -198,6 +202,7 @@ while true ; do
         --edit) EDIT_FLAG=1 ; shift ;;
         --no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;;
         --parallel) shift ;;
+        --layer) export LAYER=$2 ; shift ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error!" ; exit 1 ;;
     esac
@@ -292,6 +297,9 @@ if [ ! -d $CGCS_BASE ]; then
 fi
 
 if [ ! -d $BUILD_BASE ]; then
+   if [ $CLEAN_FLAG -eq 1 ]; then
+       exit 0
+   fi
    echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'"
    exit 1
 fi
@@ -1340,6 +1348,11 @@ for i in $(seq 0 $((max_workers-1))); do
    \cp $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i
 done
 
+# create a build info
+if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ] && [ $NO_BUILD_INFO -eq 0 ]; then
+    set_build_info
+fi
+
 # Build src.rpm's
 LOG_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-log-XXXXXX)
 if [ "x$LOG_DIR" == "x" ]; then
@@ -1352,6 +1365,11 @@ for GIT_ROOT in $GIT_LIST; do
    if [ $STOP_SCHEDULING -eq 1 ]; then
        break;
    fi
+
+   if echo "$GIT_ROOT" | grep do-not-build; then
+       continue
+   fi
+
    for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
       if [ $STOP_SCHEDULING -eq 1 ]; then
           break;
@@ -1442,6 +1460,10 @@ if [ $ALL -eq 1 ]; then
     AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX)
     if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then
     for GIT_ROOT in $GIT_LIST; do
+        if echo "$GIT_ROOT" | grep -q do-not-build; then
+            continue
+        fi
+
         for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
             (
             src_dir="$GIT_ROOT/$p"
@@ -1546,6 +1568,7 @@ if [ $EDIT_FLAG -ne 1 ]; then
       (cd $d/..
        \rm -rf repodata
        $CREATEREPO $(pwd)
+       create_lst $(pwd)
       )
    done
    echo "==== Update repodata complete ====="

build-tools/build-srpms-serial

@@ -58,11 +58,26 @@ if [ $? -ne 0 ]; then
    CREATEREPO="createrepo"
 fi
 
+#
+# Create a list of rpms in the directory
+#
+create_lst () {
+   local DIR=${1}
+
+       (cd $DIR
+        [ -f rpm.lst ] && \rm -rf rpm.lst
+        [ -f srpm.lst ] && \rm -rf srpm.lst
+        find . -name '*.rpm' -and -not -name '*.src.rpm' | sed 's#^[.][/]##' | sort > rpm.lst
+        find . -name '*.src.rpm' | sed 's#^[.][/]##' | sort > srpm.lst
+       )
+}
+
+
 usage () {
     echo ""
     echo "Usage: "
     echo "   Create source rpms:"
-    echo "      $ME [--rt | --std | --installer | --containers] [--no-descendants] [--formal] [ list of package names ]"
+    echo "      $ME [--rt | --std | --installer | --containers] [--layer=<layer>] [--no-descendants] [--formal] [ list of package names ]"
     echo ""
     echo "   Delete source rpms, and the directories associated with it's creation:"
     echo "   Note: does not clean an edit environment"
@@ -147,7 +162,7 @@ EDIT_FLAG=0
 NO_META_PATCH_FLAG=0
 
 # read the options
-TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
+TEMP=$(getopt -o h --long serial,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,layer: -n "$ME" -- "$@")
 
 if [ $? -ne 0 ]; then
     usage
@@ -161,20 +176,9 @@ export BUILD_TYPE=std
 # extract options and their arguments into variables.
 while true ; do
     case "$1" in
-        -a|--arga)
-            case "$2" in
-                "") ARG_A='some default value' ; shift 2 ;;
-                *) ARG_A=$2 ; shift 2 ;;
-            esac ;;
-        -b|--argb) ARG_B=1 ; shift ;;
         --no-descendants) NO_DESCENDANTS=1 ; shift ;;
         --no-build-info) NO_BUILD_INFO=1 ; shift ;;
         -h|--help) HELP=1 ; shift ;;
-        -c|--argc)
-            case "$2" in
-                "") shift 2 ;;
-                *) ARG_C=$2 ; shift 2 ;;
-            esac ;;
         --clean) CLEAN_FLAG=1 ; shift ;;
         --formal) FORMAL_FLAG=1 ; shift ;;
         --std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;;
@@ -184,6 +188,7 @@ while true ; do
         --edit) EDIT_FLAG=1 ; shift ;;
         --no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;;
         --serial) shift ;;
+        --layer) export LAYER=$2 ; shift ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error!" ; exit 1 ;;
     esac
@@ -278,6 +283,9 @@ if [ ! -d $CGCS_BASE ]; then
 fi
 
 if [ ! -d $BUILD_BASE ]; then
+   if [ $CLEAN_FLAG -eq 1 ]; then
+       exit 0
+   fi
    echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'"
    exit 1
 fi
@@ -1214,8 +1222,17 @@ if [ $EDIT_FLAG -eq 0 ]; then
     fi
 fi
 
+# create a build info
+if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ] && [ $NO_BUILD_INFO -eq 0 ]; then
+    set_build_info
+fi
+
 for GIT_ROOT in $GIT_LIST; do
    export GIT_BASE="$GIT_ROOT"
+   if echo "$GIT_ROOT" | grep -q do-not-build; then
+       continue
+   fi
+
    for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
       src_dir="$GIT_ROOT/$p"
       if [ -d $src_dir ]; then
@@ -1269,6 +1286,10 @@ if [ $ALL -eq 1 ]; then
     echo
     echo "Auditing for obsolete srpms"
     for GIT_ROOT in $GIT_LIST; do
+        if echo "$GIT_ROOT" | grep do-not-build; then
+            continue
+        fi
+
         for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do
             src_dir="$GIT_ROOT/$p"
             if [ -d $src_dir ]; then
@@ -1365,6 +1386,7 @@ if [ $EDIT_FLAG -ne 1 ]; then
       (cd $d/..
        \rm -rf repodata
        $CREATEREPO $(pwd)
+       create_lst $(pwd)
       )
    done
    echo "==== Update repodata complete ====="

build-tools/build_iso/cgts_deps.sh

@@ -202,6 +202,8 @@ function check_all_explicit_deps_installed {
         fi
     done < $TMPFILE
 
+    # Strip leading spaces.  Don't want isomething like ' ' to trigger a failure
+    PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^[ ]*//"`
     if [ -z "$PKGS_TO_CHECK" ]; then
         >&2 echo "All explicitly specified packages resolved!"
     else

build-tools/create-yum-conf

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+#
+# usage: create-yum-conf [<layer>]
+#
+
+LAYER=${1:-$LAYER}
+
 if [ "$MY_WORKSPACE" == "" ]; then
    echo "ERROR: MY_WORKSPACE not defined"
    exit 1;
@@ -21,10 +27,17 @@ if [ "$MY_BUILD_DIR" == "" ]; then
 fi
 
 MY_YUM_CONF="$MY_WORKSPACE/yum.conf"
-MOCK_CFG_PROTO="$MY_REPO/cgcs-centos-repo/mock.cfg.proto"
 YUM_DIR="$MY_WORKSPACE/yum"
 YUM_CACHE="$YUM_DIR/cache"
 
+# Try to find a layer specific mock.cfg.proto
+MOCK_CFG_PROTO="$MY_REPO/cgcs-centos-repo/mock.cfg.${LAYER}.proto"
+if [ ! -f "$MOCK_CFG_PROTO" ]; then
+    # Not present, Use default mock.cfg.proto
+    MOCK_CFG_PROTO="$MY_REPO/cgcs-centos-repo/mock.cfg.proto"
+fi
+
+
 if [ -f "$MOCK_CFG_PROTO" ]; then
     if [ -f "$MY_YUM_CONF" ]; then
         N=$(find $MOCK_CFG_PROTO $MY_REPO/build-tools/create-yum-conf -cnewer $MY_YUM_CONF | wc -l) 

build-tools/default_build_srpm

@@ -164,7 +164,7 @@ if [ "x$STAGING" != "x" ]; then
    
 fi
 
-TRANSFORM=`echo "$SRC_DIR" | sed 's/^\./\\./' | sed 's:^/::'`
+TRANSFORM=`echo "$SRC_DIR" | sed 's/^\./\\./' | sed 's:^/::' | sed 's#^.*/\.\./##'`
 
 if [ "x$STAGING" != "x" ]; then
 	pushd $STAGING
@@ -191,6 +191,7 @@ fi
 
 if [ $TAR_NEEDED -gt 0 ]; then
     echo "Creating tar file: $TAR_PATH ..."
+    echo "tar czf  $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude='$SRC_DIR/$DISTRO' --exclude='pbr-*.egg' --transform 's,^$TRANSFORM,$TAR_NAME-$VERSION,'"
     tar czf  $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION,"
     if [ $? -ne 0 ]; then
 		if [ "x$STAGING" != "x" ]; then

build-tools/image-utils.sh

@@ -16,8 +16,8 @@ get_release_info () {
     local dir=""
     local path=""
 
-    for dir in utilities integ stx-utilities stx-integ; do
+    for dir in $GIT_LIST; do
-        path="$MY_REPO/stx/$dir/utilities/build-info/release-info.inc"
+        path="$dir/utilities/build-info/release-info.inc"
         if [ -f "$path" ]; then
             echo "$path"
             return 0
@@ -32,8 +32,8 @@ get_bsp_dir () {
     local dir=""
     local path=""
 
-    for dir in stx-metal metal; do
+    for dir in $GIT_LIST; do
-        path="$MY_REPO/stx/$dir/bsp-files"
+        path="$dir/bsp-files"
         if [ -d "$path" ]; then
             echo "$path"
             return 0
@@ -45,12 +45,14 @@ get_bsp_dir () {
 }
 
 #
-# image_inc_list <build_target> <build_type> <distro>
+# image_inc_list <build_target> <list_type> <distro> [<layer>]
 #
 # Parameters:
 #    build_target: One of 'iso', 'guest' ...
-#    build_type:   One of 'std', 'rt', 'dev' ...
+#    list_type:    One of 'std', 'dev', 'layer'
 #    distro:       One of 'centos', ...
+#    layer:        One of 'compiler', 'distro', 'flock', ...
+#                  Only required if list_type == layer
 #
 # Returns: A list of unique package that must be included for
 #          the desired distro's build target and build type.
@@ -59,28 +61,52 @@ get_bsp_dir () {
 
 image_inc_list () {
     local build_target=$1
-    local build_type=$2
+    local list_type=$2
     local distro=$3
+    local layer=$4
 
+    if [ "${list_type}" = "layer" ]; then
+        local required_layer_cfg_name="required_layer_${build_target}_inc.cfg"
+        local layer_cfg_name="${distro}_build_layer.cfg"
+        local root_dir="${MY_REPO}/../stx-tools/centos-mirror-tools/config/${distro}/${layer}"
+        local layer_cfgs=""
+
+        layer_cfgs=$(find $(for x in $GIT_LIST; do echo $x/; done) -maxdepth 1 -name ${layer_cfg_name})
+
+        if [ -f ${root_dir}/${required_layer_cfg_name} ]; then
+            for line in $(grep -v '^#' ${root_dir}/${required_layer_cfg_name}); do
+                local lower_layer=${line%%,*}
+                local url=${line##*,}
+                grep -q "^${lower_layer}$" $layer_cfgs
+                if [ $? -ne 0 ]; then
+                    curl ${url}
+                fi
+            done | sort --unique
+        fi
+    else
+        local root_dir=""
         local root_file=""
-    local build_type_extension=""
+        local list_type_extension=""
+        local list_type_extension_bt=""
         local search_target=""
 
-    if [ "${build_type}" != "std" ]; then
+        if [ "${list_type}" != "std" ]; then
-        build_type_extension="_${build_type}"
+            list_type_extension="_${list_type}"
-        build_type_extension_bt="-${build_type}"
+            list_type_extension_bt="-${list_type}"
         fi
 
         root_dir="${MY_REPO}/build-tools/build_${build_target}"
-    root_file="${root_dir}/image${build_type_extension_bt}.inc"
+        root_file="${root_dir}/image${list_type_extension_bt}.inc"
-    search_target=${distro}_${build_target}_image${build_type_extension}.inc
+        search_target=${distro}_${build_target}_image${list_type_extension}.inc
 
         (
         if [ -f ${root_file} ]; then
             grep '^[^#]' ${root_file}
         fi
+
         for d in $GIT_LIST; do
             find $d/ -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} +
         done
         ) | sort --unique
+    fi
 }

build-tools/mockchain-parallel-1.3.4

@@ -329,6 +329,19 @@ def do_build(opts, cfg, pkg):
                    '--resultdir', resdir,
                    '--uniqueext', opts.uniqueext,
                    '-r', cfg, ]
+
+    # Ensure repo is up-to-date.
+    # Note: Merely adding --update to mockcmd failed to update
+    mockcmd_update=mockcmd
+    mockcmd_update.append('--update')
+    cmd = subprocess.Popen(
+        mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = cmd.communicate()
+    if cmd.returncode != 0:
+        if (isinstance(err, bytes)):
+            err = err.decode("utf-8")
+        sys.stderr.write(err)
+
     # heuristic here, if user pass for mock "-d foo", but we must be care to leave
     # "-d'foo bar'" or "--define='foo bar'" as is
     compiled_re_1 = re.compile(r'^(-\S)\s+(.+)')

build-tools/mockchain-parallel-1.4.16

@@ -331,6 +331,19 @@ def do_build(opts, cfg, pkg):
                    '--resultdir', resdir,
                    '--uniqueext', opts.uniqueext,
                    '-r', cfg, ]
+
+    # Ensure repo is up-to-date.
+    # Note: Merely adding --update to mockcmd failed to update
+    mockcmd_update=mockcmd
+    mockcmd_update.append('--update')
+    cmd = subprocess.Popen(
+        mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = cmd.communicate()
+    if cmd.returncode != 0:
+        if (isinstance(err, bytes)):
+            err = err.decode("utf-8")
+        sys.stderr.write(err)
+
     # heuristic here, if user pass for mock "-d foo", but we must be care to leave
     # "-d'foo bar'" or "--define='foo bar'" as is
     compiled_re_1 = re.compile(r'^(-\S)\s+(.+)')

build-tools/modify-build-cfg

@@ -8,10 +8,20 @@
 # passed through to the mock environment, so this scripts sets the variables
 # to literal values.
 #
-# usage: modify-build-cfg [file.cfg]
+# usage: modify-build-cfg [file.cfg] [<layer>]
 #
 
+LAYER=${2:-$LAYER}
+
+# Try to find a layer specific mock.cfg.proto
+MOCK_CFG_PROTO="$MY_REPO/cgcs-centos-repo/mock.cfg.${LAYER}.proto"
+echo "==== Try MOCK_CFG_PROTO=$MOCK_CFG_PROTO ===="
+if [ ! -f "$MOCK_CFG_PROTO" ]; then
+    # Not present, Use default mock.cfg.proto
     MOCK_CFG_PROTO="$MY_REPO/cgcs-centos-repo/mock.cfg.proto"
+fi
+
+echo "==== Use MOCK_CFG_PROTO=$MOCK_CFG_PROTO ===="
 if [ ! -f "$MOCK_CFG_PROTO" ]; then
    echo "ERROR: Couldn't find mock config prototype at '$MOCK_CFG_PROTO'"
    exit 1
@@ -56,6 +66,7 @@ if [ ! -f $FILE ]; then
       if [ "$bt" != "$BUILD_TYPE" ]; then
          # Use the range of lines starting with pattern [local-$bt] until the next line starting with []
          sed -i "/^\[local-$bt\]/,/^\[/ s/enabled=1/enabled=0/" $FILE
+         sed -i "/^\[StxCentos7Distro-$bt\]/,/^\[/ s/enabled=1/enabled=0/" $FILE
       fi
    done
 fi

build-tools/repo_files/mock.cfg.all.proto

@@ -0,0 +1,58 @@
+config_opts['root'] = 'BUILD_ENV/mock'
+config_opts['target_arch'] = 'x86_64'
+config_opts['legal_host_arches'] = ('x86_64',)
+config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
+config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
+config_opts['releasever'] = '7'
+config_opts['rpmbuild_networking'] = False
+
+
+config_opts['yum.conf'] = """
+[main]
+keepcache=1
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+syslog_ident=mock
+syslog_device=
+
+# repos
+[local-std]
+name=local-std
+baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[local-rt]
+name=local-rt
+baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[local-installer]
+name=local-installer
+baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[StxCentos7Distro]
+name=Stx-Centos-7-Distro
+enabled=1
+baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/Binary
+failovermethod=priority
+exclude=kernel-devel libvirt-devel
+
+[StxCentos7Distro-rt]
+name=Stx-Centos-7-Distro-rt
+enabled=1
+baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/rt/Binary
+failovermethod=priority
+
+"""

build-tools/repo_files/mock.cfg.distro.proto

@@ -0,0 +1,58 @@
+config_opts['root'] = 'BUILD_ENV/mock'
+config_opts['target_arch'] = 'x86_64'
+config_opts['legal_host_arches'] = ('x86_64',)
+config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
+config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
+config_opts['releasever'] = '7'
+config_opts['rpmbuild_networking'] = False
+
+
+config_opts['yum.conf'] = """
+[main]
+keepcache=1
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+syslog_ident=mock
+syslog_device=
+
+# repos
+[local-std]
+name=local-std
+baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[local-rt]
+name=local-rt
+baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[local-installer]
+name=local-installer
+baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS
+enabled=1
+skip_if_unavailable=1
+metadata_expire=0
+
+[StxCentos7Distro]
+name=Stx-Centos-7-Distro
+enabled=1
+baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/Binary
+failovermethod=priority
+exclude=kernel-devel libvirt-devel
+
+[StxCentos7Distro-rt]
+name=Stx-Centos-7-Distro-rt
+enabled=1
+baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/rt/Binary
+failovermethod=priority
+
+"""

build-tools/repo_files/mock.cfg.proto

@@ -6,6 +6,7 @@ config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
 config_opts['releasever'] = '7'
 config_opts['rpmbuild_networking'] = False
 
+
 config_opts['yum.conf'] = """
 [main]
 keepcache=1
@@ -41,12 +42,16 @@ enabled=1
 skip_if_unavailable=1
 metadata_expire=0
 
-[TisCentos7Distro]
+[StxCentos7Distro]
-name=Tis-Centos-7-Distro
+name=Stx-Centos-7-Distro
 enabled=1
 baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/Binary
 failovermethod=priority
-exclude=kernel-devel libvirt-devel
 
+[StxCentos7Distro-rt]
+name=Stx-Centos-7-Distro-rt
+enabled=1
+baseurl=LOCAL_BASE/MY_REPO_DIR/cgcs-centos-repo/rt/Binary
+failovermethod=priority
 
 """

build-tools/update-pxe-network-installer

@@ -3,6 +3,30 @@
 ## based on RPMs generated by "build-pkgs" and "build-iso"
 ## created by Yong Hu (yong.hu@intel.com), 05/24/2018
 
+find_and_copy_rpm () {
+    local name="${1}"
+    local pattern="${2}"
+    local build_type="${3}"
+    local dest_dir="${4}"
+
+    echo " --> find ${name} rpm"
+    found=$(find $MY_BUILD_DIR/${build_type}/rpmbuild/RPMS -type f -name "${pattern}" | head -n 1)
+    if [ ! -n "${found}" ];then
+        if [ "${build_type}" != "rt" ]; then
+            found=$(find $MY_REPO/cgcs-centos-repo/Binary -type l -name "${pattern}" | head -n 1)
+        else
+            found=$(find $MY_REPO/cgcs-centos-repo/${build_type}/Binary -type l -name "${pattern}" | head -n 1)
+        fi
+    fi
+
+    if [ -n "${found}" ] && [ -f "${found}" ];then
+        \cp -f "${found}" "${dest_dir}/"
+    else
+        echo "ERROR: failed to find ${name} RPM!"
+        exit -1
+    fi
+}
+
 
 echo "Start to update pxe-network-installer images .... "
 timestamp=$(date +%F_%H%M)
@@ -58,59 +82,12 @@ fi
 mkdir -p $kernel_rpms_rt
 
 echo " -------- start to search standard kernel rpm and related kernel modules --------"
-echo " --> find standard kernel rpm"
+find_and_copy_rpm 'standard kernel'        'kernel-[0-9]*.x86_64.rpm'                     std "$kernel_rpms_std"
-std_kernel=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages/ -type f -name "kernel-[0-9]*.x86_64.rpm")
+find_and_copy_rpm 'e1000e kernel module'   'kmod-e1000e-[0-9]*.x86_64.rpm'                std "$kernel_rpms_std"
-if [ -n $std_kernel ] && [ -f $std_kernel ];then
+find_and_copy_rpm 'i40e kernel module'     'kmod-i40e-[0-9]*.x86_64.rpm'                  std "$kernel_rpms_std"
-    cp -f $std_kernel $kernel_rpms_std/.
+find_and_copy_rpm 'ixgbe kernel module'    'kmod-ixgbe-[0-9]*.x86_64.rpm'                 std "$kernel_rpms_std"
-else
+find_and_copy_rpm 'tpm kernel module'      'kmod-tpm-[0-9]*.x86_64.rpm'                   std "$kernel_rpms_std"
-    echo "ERROR: failed to find kernel RPM!"
+find_and_copy_rpm 'mlnx-ofa kernel module' 'mlnx-ofa_kernel-modules-[0-9]*.x86_64.rpm'    std "$kernel_rpms_std"
-    exit -1
-fi
-
-echo "--> find e1000e kernel module"
-e1000e_module=$(find $MY_BUILD_DIR/std/rpmbuild/RPMS -type f -name "kmod-e1000e-[0-9]*.x86_64.rpm")
-if [ -n $e1000e_module ] && [ -f $e1000e_module ];then
-    cp -f $e1000e_module $kernel_rpms_std/.
-else
-    echo "ERROR: failed to find e1000e kernel module RPM!"
-    exit -1
-fi
-
-echo "--> find i40e kernel module"
-i40e_module=$(find $MY_BUILD_DIR/std/rpmbuild/RPMS -type f -name "kmod-i40e-[0-9]*.x86_64.rpm")
-if [ -n $i40e_module ] && [ -f $i40e_module ];then
-    cp -f $i40e_module $kernel_rpms_std/.
-else
-    echo "ERROR: failed to find i40e kernel module RPM!"
-    exit -1
-fi
-
-echo "--> find ixgbe kernel module"
-ixgbe_module=$(find $MY_BUILD_DIR/std/rpmbuild/RPMS -type f -name "kmod-ixgbe-[0-9]*.x86_64.rpm")
-if [ -n $ixgbe_module ] && [ -f $ixgbe_module ];then
-    cp -f $ixgbe_module $kernel_rpms_std/.
-else
-    echo "ERROR: failed to find ixgbe kernel module RPM!"
-    exit -1
-fi
-
-echo "--> find tpm kernel module"
-tpm_module=$(find $MY_BUILD_DIR/std/rpmbuild/RPMS -type f -name "kmod-tpm-[0-9]*.x86_64.rpm")
-if [ -n $tpm_module ] && [ -f $tpm_module ];then
-    cp -f $tpm_module $kernel_rpms_std/.
-else
-    echo "ERROR: failed to find tpm kernel module RPM!"
-    exit -1
-fi
-
-echo "--> find mlnx-ofa_kernel_modules"
-mlnx_ofa_kernel_modules=$(find $MY_BUILD_DIR/std/rpmbuild/RPMS -type f -name "mlnx-ofa_kernel-modules-[0-9]*.x86_64.rpm")
-if [ -n $mlnx_ofa_kernel_modules ] && [ -f $mlnx_ofa_kernel_modules ];then
-    cp -f $mlnx_ofa_kernel_modules $kernel_rpms_std/.
-else
-    echo "ERROR: failed to find mlnx-ofa_kernel-modules RPM!"
-    exit -1
-fi
 echo " -------- successfully found standard kernel rpm and related kernel modules --------"
 echo ""
 
@@ -121,149 +98,23 @@ fi
 mkdir -p $rootfs_rpms
 
 echo " step 3:  start to search rpms for rootfs"
-echo "--> find anaconda rpm"
+find_and_copy_rpm 'anaconda'                   'anaconda-[0-9]*.x86_64.rpm'                   installer "$rootfs_rpms/."
-anaconda=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "anaconda-[0-9]*.x86_64.rpm")
+find_and_copy_rpm 'anaconda-core'              'anaconda-core-[0-9]*.x86_64.rpm'              installer "$rootfs_rpms/."
-if [ -n $anaconda ] && [ -f $anaconda ];then
+find_and_copy_rpm 'anaconda-tui'               'anaconda-tui-[0-9]*.x86_64.rpm'               installer "$rootfs_rpms/."
-    cp -f $anaconda $rootfs_rpms/.
+find_and_copy_rpm 'anaconda-widgets'           'anaconda-widgets-[0-9]*.x86_64.rpm'           installer "$rootfs_rpms/."
-else
+find_and_copy_rpm 'rpm'                        'rpm-[0-9]*.x86_64.rpm'                        installer "$rootfs_rpms/."
-    echo "ERROR: failed to find anaconda RPM!"
+find_and_copy_rpm 'rpm-build'                  'rpm-build-[0-9]*.x86_64.rpm'                  installer "$rootfs_rpms/."
-    exit -1
+find_and_copy_rpm 'rpm-build-libs'             'rpm-build-libs-[0-9]*.x86_64.rpm'             installer "$rootfs_rpms/."
-fi
+find_and_copy_rpm 'rpm-libs'                   'rpm-libs-[0-9]*.x86_64.rpm'                   installer "$rootfs_rpms/."
+find_and_copy_rpm 'rpm-plugin-systemd-inhibit' 'rpm-plugin-systemd-inhibit-[0-9]*.x86_64.rpm' installer "$rootfs_rpms/."
+find_and_copy_rpm 'rpm-python'                 'rpm-python-[0-9]*.x86_64.rpm'                 installer "$rootfs_rpms/."
 
-echo "--> find anaconda-core rpm"
+find_and_copy_rpm 'systemd'       'systemd-[0-9]*.x86_64.rpm'       std "$rootfs_rpms/."
-anaconda_core=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "anaconda-core-[0-9]*.x86_64.rpm")
+find_and_copy_rpm 'systemd-libs'  'systemd-libs-[0-9]*.x86_64.rpm'  std "$rootfs_rpms/."
-if [ -n $anaconda_core ] && [ -f $anaconda_core ];then
+find_and_copy_rpm 'systemd-sysv'  'systemd-sysv-[0-9]*.x86_64.rpm'  std "$rootfs_rpms/."
-    cp -f $anaconda_core $rootfs_rpms/.
+find_and_copy_rpm 'lz4'           'lz4-[0-9]*.x86_64.rpm'           std "$rootfs_rpms/."
-else
+find_and_copy_rpm 'bind-utils'    'bind-utils-[0-9]*.x86_64.rpm'    std "$rootfs_rpms/."
-    echo "ERROR: failed to find anaconda-core RPM!"
+find_and_copy_rpm 'ima-evm-utils' 'ima-evm-utils-[0-9]*.x86_64.rpm' std "$rootfs_rpms/."
-    exit -1
-fi
-
-echo "--> find anaconda-tui rpm"
-anaconda_tui=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "anaconda-tui-[0-9]*.x86_64.rpm")
-if [ -n $anaconda_tui ] && [ -f $anaconda_tui ];then
-    cp -f $anaconda_tui $rootfs_rpms/.
-else
-    echo "ERROR: failed to find anaconda-tui RPM!"
-    exit -1
-fi
-
-echo "--> find anaconda-widgets rpm"
-anaconda_widgets=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "anaconda-widgets-[0-9]*.x86_64.rpm")
-if [ -n $anaconda_widgets ] && [ -f $anaconda_widgets ];then
-    cp -f $anaconda_widgets $rootfs_rpms/.
-else
-    echo "ERROR: failed to find anaconda-widgets RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-xx.x86_64 rpm"
-rpm_rpm=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-[0-9]*.x86_64.rpm")
-if [ -n $rpm_rpm ] && [ -f $rpm_rpm ];then
-    cp -f $rpm_rpm $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-xx.x86_64 RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-build rpm"
-rpm_build=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-build-[0-9]*.x86_64.rpm")
-if [ -n $rpm_build ] && [ -f $rpm_build ];then
-    cp -f $rpm_build $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-build RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-build-libs rpm"
-rpm_build_libs=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-build-libs-[0-9]*.x86_64.rpm")
-if [ -n $rpm_build_libs ] && [ -f $rpm_build_libs ];then
-    cp -f $rpm_build_libs $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-build-libs RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-libs rpm"
-rpm_libs=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-libs-[0-9]*.x86_64.rpm")
-if [ -n $rpm_libs ] && [ -f $rpm_libs ];then
-    cp -f $rpm_libs $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-libs RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-plugin-systemd-inhibit rpm"
-rpm_plugin_systemd_inhibit=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-plugin-systemd-inhibit-[0-9]*.x86_64.rpm")
-if [ -n $rpm_plugin_systemd_inhibit ] && [ -f $rpm_plugin_systemd_inhibit ];then
-    cp -f $rpm_plugin_systemd_inhibit $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-plugin-systemd-inhibit RPM!"
-    exit -1
-fi
-
-echo "--> find rpm-python rpm"
-rpm_python=$(find $MY_BUILD_DIR/installer/rpmbuild/RPMS -type f -name "rpm-python-[0-9]*.x86_64.rpm")
-if [ -n $rpm_python ] && [ -f $rpm_python ];then
-    cp -f $rpm_python $rootfs_rpms/.
-else
-    echo "ERROR: failed to find rpm-python RPM!"
-    exit -1
-fi
-
-echo "--> find systemd rpm"
-systemd=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "systemd-[0-9]*.x86_64.rpm")
-if [ -n $systemd ] && [ -f $systemd ];then
-    cp -f $systemd $rootfs_rpms/.
-else
-    echo "ERROR: failed to find systemd RPM!"
-    exit -1
-fi
-
-echo "--> find systemd-libs rpm"
-systemd_libs=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "systemd-libs-[0-9]*.x86_64.rpm")
-if [ -n $systemd_libs ] && [ -f $systemd_libs ];then
-    cp -f $systemd_libs $rootfs_rpms/.
-else
-    echo "ERROR: failed to find systemd-libs RPM!"
-    exit -1
-fi
-
-echo "--> find systemd-sysv rpm"
-systemd_sysv=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "systemd-sysv-[0-9]*.x86_64.rpm")
-if [ -n $systemd_sysv ] && [ -f $systemd_sysv ];then
-    cp -f $systemd_sysv $rootfs_rpms/.
-else
-    echo "ERROR: failed to find systemd-sysv RPM!"
-    exit -1
-fi
-
-echo "--> find lz4 rpm"
-lz4_rpm=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "lz4-[0-9]*.x86_64.rpm")
-if [ -n $lz4_rpm ] && [ -f $lz4_rpm ];then
-    cp -f $lz4_rpm $rootfs_rpms/.
-else
-    echo "ERROR: failed to find lz4_rpm RPM!"
-    exit -1
-fi
-
-echo "--> find bind-utils rpm"
-bind_utils=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "bind-utils-[0-9]*.x86_64.rpm")
-if [ -n $bind_utils ] && [ -f $bind_utils ];then
-    cp -f $bind_utils $rootfs_rpms/.
-else
-    echo "ERROR: failed to find bind-utils RPM!"
-    exit -1
-fi
-
-echo "--> find ima-evm-utils rpm"
-ima_evm_utils=$(find $MY_BUILD_DIR/export/dist/isolinux/Packages -type f -name "ima-evm-utils-[0-9]*.x86_64.rpm")
-if [ -n $ima_evm_utils ] && [ -f $ima_evm_utils ];then
-    cp -f $ima_evm_utils $rootfs_rpms/.
-else
-    echo "ERROR: failed to find ima-evm-utils RPM!"
-    exit -1
-fi
 echo " ---------------- successfully found rpms for rootfs --------------------------------"
 
 echo "step 4: make installer images in this work dir"