diff --git a/scripts/archive-misc.sh b/scripts/archive-misc.sh index 9c3e32e..10cb41c 100755 --- a/scripts/archive-misc.sh +++ b/scripts/archive-misc.sh @@ -13,17 +13,21 @@ # set -e -source $(dirname "$0")/lib/job_utils.sh +THIS_DIR="$(readlink -f "$(dirname "$0")")" +source "$THIS_DIR"/lib/job_utils.sh +source "$THIS_DIR"/lib/publish_utils.sh load_build_env +notice "archiving misc files" + #VERBOSE_ARG="--verbose" exclude_args=() exclude_args+=(--exclude "/localdisk/designer/**") # symlink inside -exclude_args+=(--exclude "/aptly") # symlink -exclude_args+=(--exclude "/mirrors") # symlink -exclude_args+=(--exclude "/docker") # symlink +exclude_args+=(--exclude "/aptly") # see below +exclude_args+=(--exclude "/mirrors") # see below +exclude_args+=(--exclude "/docker") # see below exclude_args+=(--exclude "/workspace") # symlink exclude_args+=(--exclude "/repo") # symlink exclude_args+=(--exclude "/localdisk/workdir/**") # ostree temp files @@ -35,3 +39,130 @@ safe_copy_dir $DRY_RUN_ARG $VERBOSE_ARG \ "${exclude_args[@]}" \ "$BUILD_HOME/" "$BUILD_OUTPUT_HOME/" + +print_regfile_name_if_exists() { + if [[ -f "$1" ]] ; then + echo "$1" + fi +} + +find_old_archive_dirs() { + find "$BUILD_OUTPUT_ROOT" -mindepth 1 -maxdepth 1 -type d \! -name "$TIMESTAMP" \ + -regextype posix-extended -regex '.*/[0-9]{4,}[^/]*$' +} + +find_old_checksum_files__mirrors() { + local archive_dir package_dir + find_old_archive_dirs | while read archive_dir ; do + print_regfile_name_if_exists "$archive_dir/mirrors/$CHECKSUMS_FILENAME" + print_regfile_name_if_exists "$archive_dir/aptly/$CHECKSUMS_FILENAME" + done + check_pipe_status +} + +find_old_checksum_files__aptly() { + find_old_checksum_files__mirrors +} + +find_old_checksum_files__docker() { + local archive_dir + find_old_archive_dirs | while read archive_dir ; do + print_regfile_name_if_exists "$archive_dir/docker/$CHECKSUMS_FILENAME" + done + check_pipe_status +} + +# Usage: do_archive_dir DIR_ID [EXTRA_CHECKSUMS_FILE...] +# +# DIR_ID is "mirrors" "docker" or "aptly" +# +# Example: +# +# # archive mirrors/ +# do_archive_dir "mirrors" +# +# # archive aptly/ , but also consider files archived under "mirrors" by the +# # the previous line for hardlinking +# do_archive_dir "aptly" "$BUILD_OUTPUT_HOME/mirrors/StxChecksums" +# +do_archive_dir() { + local id="$1" ; shift || : + local dir="$id" + local spec + local spec_id spec_metod + + notice "archiving $id" + + # ARCHIVE_BIG_DIRS contains a space-separated list of "method" + # or "dir:method" pairs, eg: + # "top-symlink aptly:shecksum-hardlink", + spec_method="checksum-hardlink" + for spec in $ARCHIVE_BIG_DIRS ; do + if [[ "$spec" =~ : ]] ; then + spec_id="${spec%%:*}" + if [[ "$spec_id" == "$id" ]] ; then + spec_method="${spec#*:}" + fi + continue + fi + spec_method="$spec" + done + + info "dir=$dir method=$spec_method" + + case "$spec_method" in + top-symlink) + if [[ -e "$BUILD_HOME/$dir" ]] ; then + if [[ -e "$BUILD_OUTPUT_HOME/$dir" && -d "$BUILD_OUTPUT_HOME/$dir" ]] ; then + safe_rm $DRY_RUN_ARG "$BUILD_OUTPUT_HOME/$dir" + fi + maybe_run ln -sfn "$BUILD_HOME/$dir" "$BUILD_OUTPUT_HOME/$dir" + fi + ;; + checksum-hardlink|checksum-copy) + if [[ -e "$BUILD_HOME/$dir" ]] ; then + + if [[ -e "$BUILD_OUTPUT_HOME/$dir" ]] ; then + safe_rm "$BUILD_OUTPUT_HOME/$dir" + fi + tmp_dir="$BUILD_HOME/tmp/archive-misc" + mkdir -p "$tmp_dir/$id" + cp -a "$THIS_DIR/helpers/archive-dir.sh" "$tmp_dir/" + local archive_args=() + if [[ "$spec_method" == "checksum-hardlink" ]] ; then + local old_checksums_file_list="$tmp_dir/$id/old_checksums_file.list" + local find_func=find_old_checksum_files__$id + $find_func >"$old_checksums_file_list" + archive_args+=("--checksum-hardlink" "$old_checksums_file_list") + local extra_checksums_file + for extra_checksums_file in "$@" ; do + print_regfile_name_if_exists "$extra_checksums_file" + done >>"$old_checksums_file_list" + fi + + #local egid + #egid=$(id -g) + #archive_args+=(--owner "$EUID" --group "$egid") + + local src_dir="$BUILD_HOME/$dir" + local dst_dir="$BUILD_OUTPUT_HOME/$dir" + maybe_run mkdir -p "$dst_dir" + safe_docker_run $DRY_RUN_ARG --writeable-archive-root --rm "$COREUTILS_DOCKER_IMG" "$tmp_dir/archive-dir.sh" \ + "${archive_args[@]}" \ + -j ${BUILD_PACKAGES_PARALLEL_JOBS:-1} \ + --output-checksums "$BUILD_OUTPUT_HOME/$dir/$CHECKSUMS_FILENAME" \ + "$src_dir" \ + "$dst_dir" \ + "$tmp_dir/$id" + + fi + ;; + *) + die "ARCHIVE_BIG_DIRS: invalid copy method \"$spec_method\": expecting \"top_symlink\", \"checksum-hardlink\" or \"checksum-copy\"" + ;; + esac +} + +do_archive_dir "mirrors" +do_archive_dir "aptly" "$BUILD_OUTPUT_HOME/mirrors/$CHECKSUMS_FILENAME" +do_archive_dir "docker" diff --git a/scripts/helpers/archive-dir.sh b/scripts/helpers/archive-dir.sh new file mode 100755 index 0000000..1bc0784 --- /dev/null +++ b/scripts/helpers/archive-dir.sh @@ -0,0 +1,359 @@ +#!/bin/bash + +PROGNAME="${BASH_SOURCE[0]##*/}" +SRC_DIR= +DST_DIR= +CHECKSUM_FILES_LIST_FILE= +DST_CHECKSUMS_FILE= +CHANGE_OWNER= +CHANGE_GROUP= +JOBS=1 + +usage() { + echo -n "\ +Usage: $0 [OPTIONS...] SRC_DIR DST_DIR TMP_DIR + +Archive SRC_DIR in DST_DIR, using TMP_DIR for temporary files. + + -j,--jobs=N calculate checksums in parallel (default: 1) + --owner=OWNER set copied file's owner as specified + --group=GROUP set copied file's group as specified + + --output-checksums=CK_FILE + save StxChecksums to this file; by default print it to + STDOUT + + --checksum-hardlink=CK_LIST_FILE + Hardlink destination files if possible. CK_LIST_FILE + must contain a list of existing StxChecksums file names + from previously-archived directories, one per line. + We will use the files with matching properties & checksums + to create hard links in DST_DIR. + +If executed by root, we will preserve owners/groups of the copied files, +unless they are overridden on the command line. + +If this script is called by non-root, it will create all files with the +calling user's effective user & group ownership. + +" + exit 0 +} + +cmdline_error() { + if [[ "$#" -gt 0 ]] ; then + echo "ERROR:" "$@" >&2; + fi + echo "Type \`$0 --help' for more info" >&2 + exit 1 +} + +check_pipe_status() { + local -a pipestatus=(${PIPESTATUS[*]}) + local -i i + for ((i=0; i<${#pipestatus[*]}; ++i)) ; do + [[ "${pipestatus[$i]}" -eq 0 ]] || return 1 + done + return 0 +} + +# Process command line +temp=$(getopt -o h,j: --long help,jobs:,owner:,group:,output-checksums:,checksum-hardlink: -n "$PROGNAME" -- "$@") || cmdline_error +eval set -- "$temp" +while [[ "$#" -gt 0 ]] ; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -j|--jobs) + JOBS="$2" + if [[ ! "$JOBS" =~ ^[0-9]{1,2}$ || "$JOBS" -le 0 || "$JOBS" -ge 99 ]] ; then + cmdline_error "$1 must be an integer [1.99]" + fi + shift 2 + ;; + --owner) + CHANGE_OWNER="$2" + shift 2 + ;; + --group) + CHANGE_GROUP="$2" + shift 2 + ;; + --checksum-hardlink) + CHECKSUM_FILES_LIST_FILE="$2" + shift 2 + ;; + --output-checksums) + DST_CHECKSUMS_FILE="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + cmdline_error + ;; + esac +done +[[ "$#" -ge 3 ]] || cmdline_error "not enough arguments" +[[ "$#" -le 3 ]] || cmdline_error "too many arguments" +SRC_DIR="$1" +DST_DIR="$2" +TMP_DIR="$3" + +if [[ ! "$EGID" ]] ; then + EGID="$(id -g)" || exit 1 +fi + +set -e + +# +# Combine checksum list files into one +# +if [[ "$CHECKSUM_FILES_LIST_FILE" ]] ; then + echo $'\n## Combining checksum lists into one' >&2 + combined_checksums_file="$TMP_DIR/combined_checksums.list" + while read checksums_file ; do + # skip empty lines and comments + if echo "$checksums_file" | grep -E '^\s*(#.*)$' ; then + continue + fi + # skip missing files + [[ -f "$checksums_file" ]] || continue + # add file path to the second token (file name) + checksums_dir="$(dirname "$checksums_file")" + awk -v "DIR=$checksums_dir/" '{ if (match($0, /^[[:space:]]*[^[:space:]]+[[:space:]]+/) >= 0) print substr($0, 1, RLENGTH) DIR substr($0, RLENGTH+1) }' \ + "$checksums_file" + done <"$CHECKSUM_FILES_LIST_FILE" | sort >"$combined_checksums_file" + check_pipe_status +fi + +# +# Create source file lists +# + +# Cretate a list file with each source file or dir + their stat properties +echo $'\n## Compiling file list: '"$SRC_DIR" >&2 +full_list_file="$TMP_DIR/full.list" +( cd "$SRC_DIR" && find -printf 'type=%y owner=%U group=%G mode=%#m size=%s mtime=%T@ checksum= name=%p\n' ) \ + | sed 's#name=[.]/#name=#' \ + | sed 's#\(mtime=[0-9]\+\)[.][0-9]\+#\1#g' \ + >"${full_list_file}" +check_pipe_status + +# Create another list file that contains only regular files, and fill in the +# "checksum=" field. +# Use "flock" when printing in xarg's sub-jobs, to avoid interleaved output. +echo $'\n## Calculating checksums: '"$SRC_DIR" >&2 +regfile_list_file="$TMP_DIR/regfile.list" +if [[ "$JOBS" -eq 1 ]] ; then + let xargs_max_args=256 +else + let xargs_max_args="8" # calculate checksums in chunks of 8 files in parallel +fi +export SRC_DIR +\grep '^type=f' "$full_list_file" | xargs -r -d '\n' -n $xargs_max_args --process-slot-var=OUT_SUFFIX -P $JOBS bash -c ' + for line in "$@" ; do + name="${line##*name=}" + flock -s "$SRC_DIR" echo " SHA256 $name" >&2 + checksum="$(sha256sum "$SRC_DIR/$name" | awk "{print \$1}")" + [[ -n "$checksum" ]] || exit 1 + output_line="${line/ checksum= / checksum=$checksum }" + flock -s "$SRC_DIR" echo "$output_line" + done +' unused_arg | sort -k 8 >"$regfile_list_file" || exit 1 # sort by the last field "name=..." +[[ "${PIPESTATUS[1]}" -eq 0 ]] || exit 1 + +# Create a list file that contains only directories +# Sort by the last field "name=..." +dir_list_file="$TMP_DIR/dir.list" +\grep '^type=d' "$full_list_file" | sort -k 8 >"$dir_list_file" + +# Create a list file that contains all other entries (non-dirs & non-files) +other_list_file="$TMP_DIR/other.list" +\grep '^type=[^df]' "$full_list_file" | sort -k 8 >"$other_list_file" + +# +# create directories +# +echo $'\n## Creating directories: '"$DST_DIR" >&2 +while read line ; do + [[ -n "$line" ]] || continue + name="${line##*name=}" + mode="$(echo "$line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')" + install_args=() + if [[ "$CHANGE_OWNER" ]] ; then + install_args+=("--owner" "$CHANGE_OWNER") + elif [[ $EUID -eq 0 ]] ; then + owner="$(echo "$line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')" + install_args+=("--owner" "$owner") + fi + if [[ "$CHANGE_GROUP" ]] ; then + install_args+=("--group" "$CHANGE_GROUP") + elif [[ $EUID -eq 0 ]] ; then + group="$(echo "$line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')" + install_args+=("--group" "$group") + fi + echo " MKDIR $name" >&2 + if [[ -e "$DST_DIR/$name" && ! -d "$DST_DIR/$name" ]] ; then + \rm "$DST_DIR/$name" || exit 1 + fi + install -d "${install_args[@]}" "$DST_DIR/$name" +done <"$dir_list_file" + +# +# Copy or hardlink regular files +# +echo $'\n## Copying regular files: '"$SRC_DIR" >&2 +if [[ "$DST_CHECKSUMS_FILE" ]] ; then + DST_CHECKSUMS_FD=5 + exec 5<>"$DST_CHECKSUMS_FILE" || exit 1 +else + DST_CHECKSUMS_FD=1 +fi +# read the list of regular files +while read line ; do + [[ -n "$line" ]] || continue + + # source file name relative to SRC_DIR + name="${line##*name=}" + + # source checksum + checksum="$(echo "$line" | sed -n -r 's#.* checksum=([^[:space:]]+).*#\1#p')" + [[ -n "$name" && -n "$checksum" ]] || continue + + # source owner; or a user-provided override + install_args=() + if [[ "$CHANGE_OWNER" ]] ; then + owner="$CHANGE_OWNER" + install_args+=("--owner" "$owner") + elif [[ $EUID -eq 0 ]] ; then + owner="$(echo "$line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')" + install_args+=("--owner" "$owner") + else + owner=$EUID + fi + + # source group; or a user-provided override + if [[ "$CHANGE_GROUP" ]] ; then + group="$CHANGE_GROUP" + install_args+=("--group" "$group") + elif [[ $EGID -eq 0 ]] ; then + group="$(echo "$line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')" + install_args+=("--group" "$group") + else + group=$EGID + fi + + # source file's mode/permissions + mode="$(echo "$line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')" + + # Search for the checksum in an older StxChecksums file + if [[ "$CHECKSUM_FILES_LIST_FILE" ]] ; then + matching_checksums_file="$TMP_DIR/matching_checksums.list" + if \grep "^$checksum " "$combined_checksums_file" >"$matching_checksums_file" ; then + ( + # As we read previosuly-archived files properties from StxChecksums, + # make sure they have not changed compared to the actual files on disk. + while read ref_checksum ref_name ref_size ref_mtime ref_dev ref_inode ref_path x_rest ; do + [[ -f "$ref_path" ]] || continue + # read on-disk file properties + ref_stat=($(stat -c '%s %Y %u %g %#04a' "$ref_path" || true)) + [[ "${#ref_stat[@]}" -eq 5 ]] || continue + + # on-disk size does not match StxChecksums + ref_ondisk_size="${ref_stat[0]}" + [[ "$ref_size" == "$ref_ondisk_size" ]] || continue + + # on-disk mtime does not match StxChecksums + ref_ondisk_mtime="${ref_stat[1]}" + [[ "${ref_mtime}" == "$ref_ondisk_mtime" ]] || continue + + # on-disk owner does not match requested owner + ref_ondisk_owner="${ref_stat[2]}" + [[ "${owner}" == "$ref_ondisk_owner" ]] || continue + + # on-disk group does not match requested group + ref_ondisk_group="${ref_stat[3]}" + [[ "${group}" == "$ref_ondisk_group" ]] || continue + + # on-disk mode does not match the mode of the source file + ref_ondisk_mode="${ref_stat[4]}" + [[ "${mode}" == "$ref_ondisk_mode" ]] || continue + + # At this point checksum, size, mtime, mode, owner, group and checksums of the + # exsiting file match with the file we are trying to copy. + # Use that file to create a hardlink. + echo " LINK $name (from $ref_name)" >&2 + if ln -f "$ref_name" "${DST_DIR}/$name" ; then + echo "$checksum $name $ref_size $ref_mtime $ref_dev $ref_inode $DST_DIR/$name" + exit 0 + fi + done <"$matching_checksums_file" + # checksum not found in older archives + exit 1 + ) && continue || true + fi + fi + + # No matching files found: really copy it + + if [[ -e "$DST_DIR/$name" ]] ; then + \rm "$DST_DIR/$name" || exit 1 + fi + + # source file's size & mtime + size="$(echo "$line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')" + mtime="$(echo "$line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')" + + # copy it to $DST_DIR + echo " COPY $name" >&2 + rm -f "$DST_DIR/$name" + install --preserve-timestamps "${install_args[@]}" --mode="$mode" -T "$SRC_DIR/$name" "$DST_DIR/$name" || exit 1 + + # check destination file properties + dst_stat=($(stat -c '%s %d %i' "$DST_DIR/$name")) || exit 1 + dst_size="${dst_stat[0]}" + dst_dev="${dst_stat[1]}" + dst_ino="${dst_stat[2]}" + + # file changed while copying + if [[ "$dst_size" != "$size" ]] ; then + echo "ERROR: $SRC_DIR/$name changed while copying!" >&2 + exit 1 + fi + + # print out a line for StxChecksums using source file properties (preserved + # during copying), but with destination file's dev & ino. + echo "$checksum $name $size $mtime $dst_dev $dst_ino $DST_DIR/$name" +done <"$regfile_list_file" >&$DST_CHECKSUMS_FD + +# +# copy special files +# +echo $'\n## Copying special files: '"$DST_DIR" >&2 +while read line ; do + [[ -n "$line" ]] || continue + name="${line##*name=}" + type="$(echo "$line" | sed 's#^type=\(.\) .*#\1#g')" + [[ -n "$name" && -n "$type" ]] || continue + echo " CREATE type=$type $name" >&2 + if [[ -e "$DST_DIR/$name" ]] ; then + rm "$DST_DIR/$name" || exit 1 + fi + cp -a --no-dereference "$SRC_DIR/$name" "$DST_DIR/$name" || exit 1 + if [[ "$CHANGE_OWNER" || "$CHANGE_GROUP" ]] ; then + chown_arg= + if [[ "$CHANGE_OWNER" ]] ; then + chown_arg="$CHANGE_OWNER" + fi + if [[ "$CHANGE_GROUP" ]] ; then + chown_arg+=":$CHANGE_GROUP" + fi + chown --no-dereference "$chown_arg" "$DST_DIR/$name" || exit 1 + fi +done <"$other_list_file" + diff --git a/scripts/lib/job_utils.sh b/scripts/lib/job_utils.sh index b77f8d2..6938552 100644 --- a/scripts/lib/job_utils.sh +++ b/scripts/lib/job_utils.sh @@ -309,12 +309,23 @@ parse_docker_registry() { # /localdisk/designer/$USER ro # read-only # /localdisk/loadbuild/$USER ro # read-only # /localdisk/designer/$USER/$PROJECT # read/write ie BUILD_HOME -# /localdisk/loadbuild/$USER/$PROJECT/$TIMESTAMP # read/write ie BUILD_OUTPUT_ROOT +# /localdisk/loadbuild/$USER/$PROJECT/$TIMESTAMP # read/write ie BUILD_OUTPUT_HOME +# +# With "--writeable-archive-root" the last entry above is replaced with +# /localdisk/loadbuild/$USER/$PROJECT # read/write ie BUILD_OUTPUT_ROOT +# +# This is required in order to create hardlinks between files under +# different $TIMESTAMP's # __get_safe_dirs() { require_env TIMESTAMP require_env USER local root norm_root + local writeable_archive_root="no" + + if [[ "$1" == "--writeable-archive-root" ]] ; then + writeable_archive_root="yes" + fi # designer & loadbuild roots for root in ${DESIGNER_ROOTS/:/ } ${LOADBUILD_ROOTS/:/ } ; do @@ -374,12 +385,16 @@ __get_safe_dirs() { error -i --dump-stack "invalid BUILD_OUTPUT_ROOT" return 1 fi - echo "$out_root/$TIMESTAMP" + if [[ "$writeable_archive_root" == "yes" ]] ; then + echo "$out_root" + else + echo "$out_root/$TIMESTAMP" + fi ) || return 1 } # -# Usage: __ensure_host_path_readable_in_priv_container PATHS... +# Usage: __ensure_host_path_readable_in_priv_container [--writeable-archive-root] PATHS... # # Make sure each host PATH can be read in a privileged container, # ie anything under @@ -389,7 +404,12 @@ __get_safe_dirs() { __ensure_host_path_readable_in_priv_container() { # safe roots local safe_roots_str - safe_roots_str="$(__get_safe_dirs | sed -r 's/\s+ro$//' ; check_pipe_status)" || return 1 + local safe_dirs_args=() + if [[ "$1" == "--writeable-archive-root" ]] ; then + safe_dirs_args+=("$1") + shift + fi + safe_roots_str="$(__get_safe_dirs "${safe_dirs_args[@]}" | sed -r 's/\s+ro$//' ; check_pipe_status)" || return 1 local -a safe_roots readarray -t safe_roots <<<"$safe_roots_str" || return 1 @@ -418,7 +438,7 @@ __ensure_host_path_readable_in_priv_container() { } # -# Usage: __ensure_host_path_writable_in_priv_container PATHS... +# Usage: __ensure_host_path_writable_in_priv_container [--writeable-archive-root] PATHS... # # Make sure a host path is OK to write in a privileged container, # ie any path under BUILD_OUTPUT_ROOT @@ -426,7 +446,12 @@ __ensure_host_path_readable_in_priv_container() { __ensure_host_path_writable_in_priv_container() { # safe roots that don't end with " ro" local safe_roots_str - safe_roots_str="$(__get_safe_dirs | grep -v -E '\s+ro$' ; check_pipe_status)" || return 1 + local safe_dirs_args=() + if [[ "$1" == "--writeable-archive-root" ]] ; then + safe_dirs_args+=("$1") + shift + fi + safe_roots_str="$(__get_safe_dirs "${safe_dirs_args[@]}" | grep -v -E '\s+ro$' ; check_pipe_status)" || return 1 local -a safe_roots readarray -t safe_roots <<<"$safe_roots_str" || return 1 @@ -455,21 +480,31 @@ __ensure_host_path_writable_in_priv_container() { } # -# Usage: __safe_docker_run [--dry-run] +# Usage: __safe_docker_run [--dry-run] [--writeable-archive-root] # safe_docker_run() { local dry_run=0 local dry_run_prefix - if [[ "$1" == "--dry-run" ]] ; then - dry_run=1 - dry_run_prefix="(dry_run) " - shift || true - fi + while [[ "$#" -gt 0 ]] ; do + if [[ "$1" == "--dry-run" ]] ; then + dry_run=1 + dry_run_prefix="(dry_run) " + shift || true + continue + fi + if [[ "$1" == "--writeable-archive-root" ]] ; then + safe_dirs_args+=("$1") + shift || true + continue + fi + break + done # construct mount options local -a mount_opts local safe_dirs_str - safe_dirs_str="$(__get_safe_dirs)" || return 1 + safe_dirs_str="$(__get_safe_dirs "${safe_dirs_args[@]}")" || return 1 + local dir flags while read dir flags ; do [[ -d "$dir" ]] || continue local mount_str="type=bind,src=$dir,dst=$dir" @@ -506,6 +541,7 @@ safe_docker_run() { # [--include PATTERN] # [--delete] # [--chown USER:GROUP] +# [--writeable-archive-root] # [--dry-run] # [-v | --verbose] # SRC_DIR... DST_DIR @@ -517,9 +553,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR # parse command line local opts local -a rsync_opts + local -a safe_dirs_args local user_group local dry_run_arg= - opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l exclude:,include:,delete,chown:,dry-run,verbose -- "$@") + opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l exclude:,include:,delete,chown:,--writeable-archive-root,dry-run,verbose -- "$@") [[ $? -eq 0 ]] || return 1 eval set -- "${opts}" while true ; do @@ -544,6 +581,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR user_group="$2" shift 2 ;; + --writeable-archive-root) + safe_dirs_args+=("$1") + shift + ;; -v | --verbose) rsync_opts+=("--verbose") shift @@ -580,11 +621,11 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR done # make sure all dirs are readable - __ensure_host_path_readable_in_priv_container "$@" || return 1 + __ensure_host_path_readable_in_priv_container "${safe_dirs_args[@]}" "$@" || return 1 # if dst_dir exists, it must be writable if [[ -d "${dst_dir}" ]] ; then - __ensure_host_path_writable_in_priv_container "$dst_dir" || return 1 + __ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$dst_dir" || return 1 # dst_dir doesn't exist, but there are multiple sources elif [[ "${#src_dirs[@]}" -gt 1 ]] ; then error "$dst_dir: does not exist or not a directory" @@ -593,7 +634,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR # parent, but rename it to basename(dst_dir). This is how "cp" behaves. else src_dirs=("${src_dirs[0]%/}/") - __ensure_host_path_writable_in_priv_container "$dst_dir" || return 1 + __ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$dst_dir" || return 1 fi # --chown: resolve USER:GROUP to UID:GID @@ -636,18 +677,24 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR safe_rm() { local usage_msg=" Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS... + --writeable-archive-root --dry-run -v,--verbose " # parse command line local opts + local -a safe_dirs_args local -a rm_opts local -a rm_cmd=("rm") - opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l dry-run,verbose -- "$@") + opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l writeable-archive-root,dry-run,verbose -- "$@") [[ $? -eq 0 ]] || return 1 eval set -- "${opts}" while true ; do case "$1" in + --writeable-archive-root) + safe_dirs_args+=("$1") + shift + ;; --dry-run) rm_cmd=("echo" "(dry run)" "rm") shift @@ -675,7 +722,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS... fi # make sure all paths are writeable - __ensure_host_path_writable_in_priv_container "$@" + __ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$@" # run rsync in docker rm_opts+=(--one-file-system --preserve-root --recursive --force) @@ -692,6 +739,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS... safe_chown() { local usage_msg=" Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS... + --writeable-archive-root --dry-run -v,--verbose -R,--recursive @@ -699,8 +747,9 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS... # parse command line local cmd_args local dry_run_arg + local -a safe_dirs_args local -a cmd=("chown") - opts=$(getopt -n "${FUNCNAME[0]}" -o "vR" -l dry-run,verbose,recursive -- "$@") + opts=$(getopt -n "${FUNCNAME[0]}" -o "vR" -l dry-run,verbose,recursive,writeable-archive-root -- "$@") [[ $? -eq 0 ]] || return 1 eval set -- "${opts}" while true ; do @@ -717,6 +766,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS... cmd_args+=("--recursive") shift ;; + --writeable-archive-root) + safe_dirs_args+=("$1") + shift + ;; --) shift break @@ -736,7 +789,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS... fi local user_group="$1" ; shift - __ensure_host_path_writable_in_priv_container "$@" + __ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$@" # resolve USER:GROUP to UID:GID local uid_gid diff --git a/scripts/lib/publish_utils.sh b/scripts/lib/publish_utils.sh index 7e54a45..04d9e54 100644 --- a/scripts/lib/publish_utils.sh +++ b/scripts/lib/publish_utils.sh @@ -121,6 +121,7 @@ find_publish_dirs() { } find_checksum_files() { + local dir subdir find_publish_dirs | while read dir ; do for subdir in "$@" ; do if [[ -d "$dir/$subdir" ]] ; then diff --git a/scripts/templates/build.conf.example.in b/scripts/templates/build.conf.example.in index f926ed0..4681421 100644 --- a/scripts/templates/build.conf.example.in +++ b/scripts/templates/build.conf.example.in @@ -97,6 +97,44 @@ PUBLISH_ROOT_URL="http://$(hostname -f):8088${PUBLISH_ROOT}" PUBLISH_SUBDIR="export" # may be empty PUBLISH_LATEST_LINK=false # create latest symlink? +# +# Archiving of some of the following directories is handled specially: +# aptly +# docker +# mirrors +# +# This parameter determines how to archive them: +# +# checksum-hardlink +# Look for identical files in older builds' outputs and link +# them. Create a single StxChecksums file at the top level of the +# destination directory, containing each file's checksum and stat +# properties. Search for link candidates in older builds' outputs +# by looking for StxChecksums files there. +# This option is the default. +# +# checksum-copy +# Copy the files and create a single StxChecksums file at top level +# of the destination directory +# +# top-symlink +# Create a symlink in $BUILD_OUTPUT_HOME that points +# back to $BUILD_HOME. +# +# This parameter may also contain a list of directory:method pairs to +# use a different archiving method for each directory. +# +# Examples: +# ======== +# +# # hardlink all dirs +# ARCHIVE_BIG_DIRS="checksum-hardlink" # same method for all dirs +# +# # hardlink all dirs, but symlink "mirrors" +# ARCHIVE_BIG_DIRS="checksum-hardlink mirrors:top-symlink" +# +#ARCHIVE_BIG_DIRS="checksum-hardlink" + ################################################## # Docker configuration ##################################################