build.conf: new parameter ARCHIVE_BIG_DIRS
New configuration ARCHIVE_BIG_DIRS to hardlink, copy or symlink the
bigger directories, "mirrors", "aptly" and "docker", when archiving
them to /localdisk/loadbuild.
This option can take on these values:
* checksum-hardlink (default): hardlink files with identical files
from previous builds' archive directories. Look for StxChecksums
files in older builds' directories to determine whether the files
are identical.
* checksum-copy: really copy the files
* top-symlink: create a top-level symlink, such as
$BUILD_OUTPUT_HOME/mirrors that points to $BUILD_HOME/mirrors
This option also accepts a space-separated list of "dir:method"
pairs, eg:
checksum-hardlink mirrors:top-symlink
means "use the hardlink method by default, but for mirros/ use the
top-symlink method". The last specification in the list takes
precedence in case of multiple matches.
The default behavior, when this option is empty or missing in
build.conf, is "checksum-hardlink".
New script: helpers/archive-dir.sh, executed as root within a docker
container during the build.
TESTS
=====================================
* Call "archive-misc.sh" manually in various ways and check the output
* Jenkins build + ARCHIVE_BIG_DIRS="checksum-hardlink"
- Run a full build and make sure most files are copied, but the
files common between mirrors and aptly (4rd-party DEBs) are
hardlinked
- Run a second build and make sure all files are hardlinked
- Add a dummy change to "build-info", rebuild it, then make sure
that package is copied, but all other files are hardlinked
* Jenkins build + ARCHIVE_BIG_DIRS commented out
- Make sure Jenkins creates the hardlinks
Story: 2010226
Task: 47860
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I604f840007b79461b5a48dcc4c21fc6316de9a52
This commit is contained in:
Davlet Panech
parent
adc5bb8806
commit
0a297aae9d
@ -13,17 +13,21 @@
|
||||
#
|
||||
|
||||
set -e
|
||||
source $(dirname "$0")/lib/job_utils.sh
|
||||
THIS_DIR="$(readlink -f "$(dirname "$0")")"
|
||||
source "$THIS_DIR"/lib/job_utils.sh
|
||||
source "$THIS_DIR"/lib/publish_utils.sh
|
||||
|
||||
load_build_env
|
||||
|
||||
notice "archiving misc files"
|
||||
|
||||
#VERBOSE_ARG="--verbose"
|
||||
|
||||
exclude_args=()
|
||||
exclude_args+=(--exclude "/localdisk/designer/**") # symlink inside
|
||||
exclude_args+=(--exclude "/aptly") # symlink
|
||||
exclude_args+=(--exclude "/mirrors") # symlink
|
||||
exclude_args+=(--exclude "/docker") # symlink
|
||||
exclude_args+=(--exclude "/aptly") # see below
|
||||
exclude_args+=(--exclude "/mirrors") # see below
|
||||
exclude_args+=(--exclude "/docker") # see below
|
||||
exclude_args+=(--exclude "/workspace") # symlink
|
||||
exclude_args+=(--exclude "/repo") # symlink
|
||||
exclude_args+=(--exclude "/localdisk/workdir/**") # ostree temp files
|
||||
@ -35,3 +39,130 @@ safe_copy_dir $DRY_RUN_ARG $VERBOSE_ARG \
|
||||
"${exclude_args[@]}" \
|
||||
"$BUILD_HOME/" "$BUILD_OUTPUT_HOME/"
|
||||
|
||||
|
||||
print_regfile_name_if_exists() {
|
||||
if [[ -f "$1" ]] ; then
|
||||
echo "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
find_old_archive_dirs() {
|
||||
find "$BUILD_OUTPUT_ROOT" -mindepth 1 -maxdepth 1 -type d \! -name "$TIMESTAMP" \
|
||||
-regextype posix-extended -regex '.*/[0-9]{4,}[^/]*$'
|
||||
}
|
||||
|
||||
find_old_checksum_files__mirrors() {
|
||||
local archive_dir package_dir
|
||||
find_old_archive_dirs | while read archive_dir ; do
|
||||
print_regfile_name_if_exists "$archive_dir/mirrors/$CHECKSUMS_FILENAME"
|
||||
print_regfile_name_if_exists "$archive_dir/aptly/$CHECKSUMS_FILENAME"
|
||||
done
|
||||
check_pipe_status
|
||||
}
|
||||
|
||||
find_old_checksum_files__aptly() {
|
||||
find_old_checksum_files__mirrors
|
||||
}
|
||||
|
||||
find_old_checksum_files__docker() {
|
||||
local archive_dir
|
||||
find_old_archive_dirs | while read archive_dir ; do
|
||||
print_regfile_name_if_exists "$archive_dir/docker/$CHECKSUMS_FILENAME"
|
||||
done
|
||||
check_pipe_status
|
||||
}
|
||||
|
||||
# Usage: do_archive_dir DIR_ID [EXTRA_CHECKSUMS_FILE...]
|
||||
#
|
||||
# DIR_ID is "mirrors" "docker" or "aptly"
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# # archive mirrors/
|
||||
# do_archive_dir "mirrors"
|
||||
#
|
||||
# # archive aptly/ , but also consider files archived under "mirrors" by the
|
||||
# # the previous line for hardlinking
|
||||
# do_archive_dir "aptly" "$BUILD_OUTPUT_HOME/mirrors/StxChecksums"
|
||||
#
|
||||
do_archive_dir() {
|
||||
local id="$1" ; shift || :
|
||||
local dir="$id"
|
||||
local spec
|
||||
local spec_id spec_metod
|
||||
|
||||
notice "archiving $id"
|
||||
|
||||
# ARCHIVE_BIG_DIRS contains a space-separated list of "method"
|
||||
# or "dir:method" pairs, eg:
|
||||
# "top-symlink aptly:shecksum-hardlink",
|
||||
spec_method="checksum-hardlink"
|
||||
for spec in $ARCHIVE_BIG_DIRS ; do
|
||||
if [[ "$spec" =~ : ]] ; then
|
||||
spec_id="${spec%%:*}"
|
||||
if [[ "$spec_id" == "$id" ]] ; then
|
||||
spec_method="${spec#*:}"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
spec_method="$spec"
|
||||
done
|
||||
|
||||
info "dir=$dir method=$spec_method"
|
||||
|
||||
case "$spec_method" in
|
||||
top-symlink)
|
||||
if [[ -e "$BUILD_HOME/$dir" ]] ; then
|
||||
if [[ -e "$BUILD_OUTPUT_HOME/$dir" && -d "$BUILD_OUTPUT_HOME/$dir" ]] ; then
|
||||
safe_rm $DRY_RUN_ARG "$BUILD_OUTPUT_HOME/$dir"
|
||||
fi
|
||||
maybe_run ln -sfn "$BUILD_HOME/$dir" "$BUILD_OUTPUT_HOME/$dir"
|
||||
fi
|
||||
;;
|
||||
checksum-hardlink|checksum-copy)
|
||||
if [[ -e "$BUILD_HOME/$dir" ]] ; then
|
||||
|
||||
if [[ -e "$BUILD_OUTPUT_HOME/$dir" ]] ; then
|
||||
safe_rm "$BUILD_OUTPUT_HOME/$dir"
|
||||
fi
|
||||
tmp_dir="$BUILD_HOME/tmp/archive-misc"
|
||||
mkdir -p "$tmp_dir/$id"
|
||||
cp -a "$THIS_DIR/helpers/archive-dir.sh" "$tmp_dir/"
|
||||
local archive_args=()
|
||||
if [[ "$spec_method" == "checksum-hardlink" ]] ; then
|
||||
local old_checksums_file_list="$tmp_dir/$id/old_checksums_file.list"
|
||||
local find_func=find_old_checksum_files__$id
|
||||
$find_func >"$old_checksums_file_list"
|
||||
archive_args+=("--checksum-hardlink" "$old_checksums_file_list")
|
||||
local extra_checksums_file
|
||||
for extra_checksums_file in "$@" ; do
|
||||
print_regfile_name_if_exists "$extra_checksums_file"
|
||||
done >>"$old_checksums_file_list"
|
||||
fi
|
||||
|
||||
#local egid
|
||||
#egid=$(id -g)
|
||||
#archive_args+=(--owner "$EUID" --group "$egid")
|
||||
|
||||
local src_dir="$BUILD_HOME/$dir"
|
||||
local dst_dir="$BUILD_OUTPUT_HOME/$dir"
|
||||
maybe_run mkdir -p "$dst_dir"
|
||||
safe_docker_run $DRY_RUN_ARG --writeable-archive-root --rm "$COREUTILS_DOCKER_IMG" "$tmp_dir/archive-dir.sh" \
|
||||
"${archive_args[@]}" \
|
||||
-j ${BUILD_PACKAGES_PARALLEL_JOBS:-1} \
|
||||
--output-checksums "$BUILD_OUTPUT_HOME/$dir/$CHECKSUMS_FILENAME" \
|
||||
"$src_dir" \
|
||||
"$dst_dir" \
|
||||
"$tmp_dir/$id"
|
||||
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
die "ARCHIVE_BIG_DIRS: invalid copy method \"$spec_method\": expecting \"top_symlink\", \"checksum-hardlink\" or \"checksum-copy\""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
do_archive_dir "mirrors"
|
||||
do_archive_dir "aptly" "$BUILD_OUTPUT_HOME/mirrors/$CHECKSUMS_FILENAME"
|
||||
do_archive_dir "docker"
|
||||
|
||||
359
scripts/helpers/archive-dir.sh
Executable file
359
scripts/helpers/archive-dir.sh
Executable file
@ -0,0 +1,359 @@
|
||||
#!/bin/bash
|
||||
|
||||
PROGNAME="${BASH_SOURCE[0]##*/}"
|
||||
SRC_DIR=
|
||||
DST_DIR=
|
||||
CHECKSUM_FILES_LIST_FILE=
|
||||
DST_CHECKSUMS_FILE=
|
||||
CHANGE_OWNER=
|
||||
CHANGE_GROUP=
|
||||
JOBS=1
|
||||
|
||||
usage() {
|
||||
echo -n "\
|
||||
Usage: $0 [OPTIONS...] SRC_DIR DST_DIR TMP_DIR
|
||||
|
||||
Archive SRC_DIR in DST_DIR, using TMP_DIR for temporary files.
|
||||
|
||||
-j,--jobs=N calculate checksums in parallel (default: 1)
|
||||
--owner=OWNER set copied file's owner as specified
|
||||
--group=GROUP set copied file's group as specified
|
||||
|
||||
--output-checksums=CK_FILE
|
||||
save StxChecksums to this file; by default print it to
|
||||
STDOUT
|
||||
|
||||
--checksum-hardlink=CK_LIST_FILE
|
||||
Hardlink destination files if possible. CK_LIST_FILE
|
||||
must contain a list of existing StxChecksums file names
|
||||
from previously-archived directories, one per line.
|
||||
We will use the files with matching properties & checksums
|
||||
to create hard links in DST_DIR.
|
||||
|
||||
If executed by root, we will preserve owners/groups of the copied files,
|
||||
unless they are overridden on the command line.
|
||||
|
||||
If this script is called by non-root, it will create all files with the
|
||||
calling user's effective user & group ownership.
|
||||
|
||||
"
|
||||
exit 0
|
||||
}
|
||||
|
||||
cmdline_error() {
|
||||
if [[ "$#" -gt 0 ]] ; then
|
||||
echo "ERROR:" "$@" >&2;
|
||||
fi
|
||||
echo "Type \`$0 --help' for more info" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
check_pipe_status() {
|
||||
local -a pipestatus=(${PIPESTATUS[*]})
|
||||
local -i i
|
||||
for ((i=0; i<${#pipestatus[*]}; ++i)) ; do
|
||||
[[ "${pipestatus[$i]}" -eq 0 ]] || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# Process command line
|
||||
temp=$(getopt -o h,j: --long help,jobs:,owner:,group:,output-checksums:,checksum-hardlink: -n "$PROGNAME" -- "$@") || cmdline_error
|
||||
eval set -- "$temp"
|
||||
while [[ "$#" -gt 0 ]] ; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-j|--jobs)
|
||||
JOBS="$2"
|
||||
if [[ ! "$JOBS" =~ ^[0-9]{1,2}$ || "$JOBS" -le 0 || "$JOBS" -ge 99 ]] ; then
|
||||
cmdline_error "$1 must be an integer [1.99]"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
--owner)
|
||||
CHANGE_OWNER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--group)
|
||||
CHANGE_GROUP="$2"
|
||||
shift 2
|
||||
;;
|
||||
--checksum-hardlink)
|
||||
CHECKSUM_FILES_LIST_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--output-checksums)
|
||||
DST_CHECKSUMS_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
cmdline_error
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ "$#" -ge 3 ]] || cmdline_error "not enough arguments"
|
||||
[[ "$#" -le 3 ]] || cmdline_error "too many arguments"
|
||||
SRC_DIR="$1"
|
||||
DST_DIR="$2"
|
||||
TMP_DIR="$3"
|
||||
|
||||
if [[ ! "$EGID" ]] ; then
|
||||
EGID="$(id -g)" || exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
#
|
||||
# Combine checksum list files into one
|
||||
#
|
||||
if [[ "$CHECKSUM_FILES_LIST_FILE" ]] ; then
|
||||
echo $'\n## Combining checksum lists into one' >&2
|
||||
combined_checksums_file="$TMP_DIR/combined_checksums.list"
|
||||
while read checksums_file ; do
|
||||
# skip empty lines and comments
|
||||
if echo "$checksums_file" | grep -E '^\s*(#.*)$' ; then
|
||||
continue
|
||||
fi
|
||||
# skip missing files
|
||||
[[ -f "$checksums_file" ]] || continue
|
||||
# add file path to the second token (file name)
|
||||
checksums_dir="$(dirname "$checksums_file")"
|
||||
awk -v "DIR=$checksums_dir/" '{ if (match($0, /^[[:space:]]*[^[:space:]]+[[:space:]]+/) >= 0) print substr($0, 1, RLENGTH) DIR substr($0, RLENGTH+1) }' \
|
||||
"$checksums_file"
|
||||
done <"$CHECKSUM_FILES_LIST_FILE" | sort >"$combined_checksums_file"
|
||||
check_pipe_status
|
||||
fi
|
||||
|
||||
#
|
||||
# Create source file lists
|
||||
#
|
||||
|
||||
# Cretate a list file with each source file or dir + their stat properties
|
||||
echo $'\n## Compiling file list: '"$SRC_DIR" >&2
|
||||
full_list_file="$TMP_DIR/full.list"
|
||||
( cd "$SRC_DIR" && find -printf 'type=%y owner=%U group=%G mode=%#m size=%s mtime=%T@ checksum= name=%p\n' ) \
|
||||
| sed 's#name=[.]/#name=#' \
|
||||
| sed 's#\(mtime=[0-9]\+\)[.][0-9]\+#\1#g' \
|
||||
>"${full_list_file}"
|
||||
check_pipe_status
|
||||
|
||||
# Create another list file that contains only regular files, and fill in the
|
||||
# "checksum=" field.
|
||||
# Use "flock" when printing in xarg's sub-jobs, to avoid interleaved output.
|
||||
echo $'\n## Calculating checksums: '"$SRC_DIR" >&2
|
||||
regfile_list_file="$TMP_DIR/regfile.list"
|
||||
if [[ "$JOBS" -eq 1 ]] ; then
|
||||
let xargs_max_args=256
|
||||
else
|
||||
let xargs_max_args="8" # calculate checksums in chunks of 8 files in parallel
|
||||
fi
|
||||
export SRC_DIR
|
||||
\grep '^type=f' "$full_list_file" | xargs -r -d '\n' -n $xargs_max_args --process-slot-var=OUT_SUFFIX -P $JOBS bash -c '
|
||||
for line in "$@" ; do
|
||||
name="${line##*name=}"
|
||||
flock -s "$SRC_DIR" echo " SHA256 $name" >&2
|
||||
checksum="$(sha256sum "$SRC_DIR/$name" | awk "{print \$1}")"
|
||||
[[ -n "$checksum" ]] || exit 1
|
||||
output_line="${line/ checksum= / checksum=$checksum }"
|
||||
flock -s "$SRC_DIR" echo "$output_line"
|
||||
done
|
||||
' unused_arg | sort -k 8 >"$regfile_list_file" || exit 1 # sort by the last field "name=..."
|
||||
[[ "${PIPESTATUS[1]}" -eq 0 ]] || exit 1
|
||||
|
||||
# Create a list file that contains only directories
|
||||
# Sort by the last field "name=..."
|
||||
dir_list_file="$TMP_DIR/dir.list"
|
||||
\grep '^type=d' "$full_list_file" | sort -k 8 >"$dir_list_file"
|
||||
|
||||
# Create a list file that contains all other entries (non-dirs & non-files)
|
||||
other_list_file="$TMP_DIR/other.list"
|
||||
\grep '^type=[^df]' "$full_list_file" | sort -k 8 >"$other_list_file"
|
||||
|
||||
#
|
||||
# create directories
|
||||
#
|
||||
echo $'\n## Creating directories: '"$DST_DIR" >&2
|
||||
while read line ; do
|
||||
[[ -n "$line" ]] || continue
|
||||
name="${line##*name=}"
|
||||
mode="$(echo "$line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')"
|
||||
install_args=()
|
||||
if [[ "$CHANGE_OWNER" ]] ; then
|
||||
install_args+=("--owner" "$CHANGE_OWNER")
|
||||
elif [[ $EUID -eq 0 ]] ; then
|
||||
owner="$(echo "$line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')"
|
||||
install_args+=("--owner" "$owner")
|
||||
fi
|
||||
if [[ "$CHANGE_GROUP" ]] ; then
|
||||
install_args+=("--group" "$CHANGE_GROUP")
|
||||
elif [[ $EUID -eq 0 ]] ; then
|
||||
group="$(echo "$line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')"
|
||||
install_args+=("--group" "$group")
|
||||
fi
|
||||
echo " MKDIR $name" >&2
|
||||
if [[ -e "$DST_DIR/$name" && ! -d "$DST_DIR/$name" ]] ; then
|
||||
\rm "$DST_DIR/$name" || exit 1
|
||||
fi
|
||||
install -d "${install_args[@]}" "$DST_DIR/$name"
|
||||
done <"$dir_list_file"
|
||||
|
||||
#
|
||||
# Copy or hardlink regular files
|
||||
#
|
||||
echo $'\n## Copying regular files: '"$SRC_DIR" >&2
|
||||
if [[ "$DST_CHECKSUMS_FILE" ]] ; then
|
||||
DST_CHECKSUMS_FD=5
|
||||
exec 5<>"$DST_CHECKSUMS_FILE" || exit 1
|
||||
else
|
||||
DST_CHECKSUMS_FD=1
|
||||
fi
|
||||
# read the list of regular files
|
||||
while read line ; do
|
||||
[[ -n "$line" ]] || continue
|
||||
|
||||
# source file name relative to SRC_DIR
|
||||
name="${line##*name=}"
|
||||
|
||||
# source checksum
|
||||
checksum="$(echo "$line" | sed -n -r 's#.* checksum=([^[:space:]]+).*#\1#p')"
|
||||
[[ -n "$name" && -n "$checksum" ]] || continue
|
||||
|
||||
# source owner; or a user-provided override
|
||||
install_args=()
|
||||
if [[ "$CHANGE_OWNER" ]] ; then
|
||||
owner="$CHANGE_OWNER"
|
||||
install_args+=("--owner" "$owner")
|
||||
elif [[ $EUID -eq 0 ]] ; then
|
||||
owner="$(echo "$line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')"
|
||||
install_args+=("--owner" "$owner")
|
||||
else
|
||||
owner=$EUID
|
||||
fi
|
||||
|
||||
# source group; or a user-provided override
|
||||
if [[ "$CHANGE_GROUP" ]] ; then
|
||||
group="$CHANGE_GROUP"
|
||||
install_args+=("--group" "$group")
|
||||
elif [[ $EGID -eq 0 ]] ; then
|
||||
group="$(echo "$line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')"
|
||||
install_args+=("--group" "$group")
|
||||
else
|
||||
group=$EGID
|
||||
fi
|
||||
|
||||
# source file's mode/permissions
|
||||
mode="$(echo "$line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')"
|
||||
|
||||
# Search for the checksum in an older StxChecksums file
|
||||
if [[ "$CHECKSUM_FILES_LIST_FILE" ]] ; then
|
||||
matching_checksums_file="$TMP_DIR/matching_checksums.list"
|
||||
if \grep "^$checksum " "$combined_checksums_file" >"$matching_checksums_file" ; then
|
||||
(
|
||||
# As we read previosuly-archived files properties from StxChecksums,
|
||||
# make sure they have not changed compared to the actual files on disk.
|
||||
while read ref_checksum ref_name ref_size ref_mtime ref_dev ref_inode ref_path x_rest ; do
|
||||
[[ -f "$ref_path" ]] || continue
|
||||
# read on-disk file properties
|
||||
ref_stat=($(stat -c '%s %Y %u %g %#04a' "$ref_path" || true))
|
||||
[[ "${#ref_stat[@]}" -eq 5 ]] || continue
|
||||
|
||||
# on-disk size does not match StxChecksums
|
||||
ref_ondisk_size="${ref_stat[0]}"
|
||||
[[ "$ref_size" == "$ref_ondisk_size" ]] || continue
|
||||
|
||||
# on-disk mtime does not match StxChecksums
|
||||
ref_ondisk_mtime="${ref_stat[1]}"
|
||||
[[ "${ref_mtime}" == "$ref_ondisk_mtime" ]] || continue
|
||||
|
||||
# on-disk owner does not match requested owner
|
||||
ref_ondisk_owner="${ref_stat[2]}"
|
||||
[[ "${owner}" == "$ref_ondisk_owner" ]] || continue
|
||||
|
||||
# on-disk group does not match requested group
|
||||
ref_ondisk_group="${ref_stat[3]}"
|
||||
[[ "${group}" == "$ref_ondisk_group" ]] || continue
|
||||
|
||||
# on-disk mode does not match the mode of the source file
|
||||
ref_ondisk_mode="${ref_stat[4]}"
|
||||
[[ "${mode}" == "$ref_ondisk_mode" ]] || continue
|
||||
|
||||
# At this point checksum, size, mtime, mode, owner, group and checksums of the
|
||||
# exsiting file match with the file we are trying to copy.
|
||||
# Use that file to create a hardlink.
|
||||
echo " LINK $name (from $ref_name)" >&2
|
||||
if ln -f "$ref_name" "${DST_DIR}/$name" ; then
|
||||
echo "$checksum $name $ref_size $ref_mtime $ref_dev $ref_inode $DST_DIR/$name"
|
||||
exit 0
|
||||
fi
|
||||
done <"$matching_checksums_file"
|
||||
# checksum not found in older archives
|
||||
exit 1
|
||||
) && continue || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# No matching files found: really copy it
|
||||
|
||||
if [[ -e "$DST_DIR/$name" ]] ; then
|
||||
\rm "$DST_DIR/$name" || exit 1
|
||||
fi
|
||||
|
||||
# source file's size & mtime
|
||||
size="$(echo "$line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')"
|
||||
mtime="$(echo "$line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')"
|
||||
|
||||
# copy it to $DST_DIR
|
||||
echo " COPY $name" >&2
|
||||
rm -f "$DST_DIR/$name"
|
||||
install --preserve-timestamps "${install_args[@]}" --mode="$mode" -T "$SRC_DIR/$name" "$DST_DIR/$name" || exit 1
|
||||
|
||||
# check destination file properties
|
||||
dst_stat=($(stat -c '%s %d %i' "$DST_DIR/$name")) || exit 1
|
||||
dst_size="${dst_stat[0]}"
|
||||
dst_dev="${dst_stat[1]}"
|
||||
dst_ino="${dst_stat[2]}"
|
||||
|
||||
# file changed while copying
|
||||
if [[ "$dst_size" != "$size" ]] ; then
|
||||
echo "ERROR: $SRC_DIR/$name changed while copying!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# print out a line for StxChecksums using source file properties (preserved
|
||||
# during copying), but with destination file's dev & ino.
|
||||
echo "$checksum $name $size $mtime $dst_dev $dst_ino $DST_DIR/$name"
|
||||
done <"$regfile_list_file" >&$DST_CHECKSUMS_FD
|
||||
|
||||
#
|
||||
# copy special files
|
||||
#
|
||||
echo $'\n## Copying special files: '"$DST_DIR" >&2
|
||||
while read line ; do
|
||||
[[ -n "$line" ]] || continue
|
||||
name="${line##*name=}"
|
||||
type="$(echo "$line" | sed 's#^type=\(.\) .*#\1#g')"
|
||||
[[ -n "$name" && -n "$type" ]] || continue
|
||||
echo " CREATE type=$type $name" >&2
|
||||
if [[ -e "$DST_DIR/$name" ]] ; then
|
||||
rm "$DST_DIR/$name" || exit 1
|
||||
fi
|
||||
cp -a --no-dereference "$SRC_DIR/$name" "$DST_DIR/$name" || exit 1
|
||||
if [[ "$CHANGE_OWNER" || "$CHANGE_GROUP" ]] ; then
|
||||
chown_arg=
|
||||
if [[ "$CHANGE_OWNER" ]] ; then
|
||||
chown_arg="$CHANGE_OWNER"
|
||||
fi
|
||||
if [[ "$CHANGE_GROUP" ]] ; then
|
||||
chown_arg+=":$CHANGE_GROUP"
|
||||
fi
|
||||
chown --no-dereference "$chown_arg" "$DST_DIR/$name" || exit 1
|
||||
fi
|
||||
done <"$other_list_file"
|
||||
|
||||
@ -309,12 +309,23 @@ parse_docker_registry() {
|
||||
# /localdisk/designer/$USER ro # read-only
|
||||
# /localdisk/loadbuild/$USER ro # read-only
|
||||
# /localdisk/designer/$USER/$PROJECT # read/write ie BUILD_HOME
|
||||
# /localdisk/loadbuild/$USER/$PROJECT/$TIMESTAMP # read/write ie BUILD_OUTPUT_ROOT
|
||||
# /localdisk/loadbuild/$USER/$PROJECT/$TIMESTAMP # read/write ie BUILD_OUTPUT_HOME
|
||||
#
|
||||
# With "--writeable-archive-root" the last entry above is replaced with
|
||||
# /localdisk/loadbuild/$USER/$PROJECT # read/write ie BUILD_OUTPUT_ROOT
|
||||
#
|
||||
# This is required in order to create hardlinks between files under
|
||||
# different $TIMESTAMP's
|
||||
#
|
||||
__get_safe_dirs() {
|
||||
require_env TIMESTAMP
|
||||
require_env USER
|
||||
local root norm_root
|
||||
local writeable_archive_root="no"
|
||||
|
||||
if [[ "$1" == "--writeable-archive-root" ]] ; then
|
||||
writeable_archive_root="yes"
|
||||
fi
|
||||
|
||||
# designer & loadbuild roots
|
||||
for root in ${DESIGNER_ROOTS/:/ } ${LOADBUILD_ROOTS/:/ } ; do
|
||||
@ -374,12 +385,16 @@ __get_safe_dirs() {
|
||||
error -i --dump-stack "invalid BUILD_OUTPUT_ROOT"
|
||||
return 1
|
||||
fi
|
||||
echo "$out_root/$TIMESTAMP"
|
||||
if [[ "$writeable_archive_root" == "yes" ]] ; then
|
||||
echo "$out_root"
|
||||
else
|
||||
echo "$out_root/$TIMESTAMP"
|
||||
fi
|
||||
) || return 1
|
||||
}
|
||||
|
||||
#
|
||||
# Usage: __ensure_host_path_readable_in_priv_container PATHS...
|
||||
# Usage: __ensure_host_path_readable_in_priv_container [--writeable-archive-root] PATHS...
|
||||
#
|
||||
# Make sure each host PATH can be read in a privileged container,
|
||||
# ie anything under
|
||||
@ -389,7 +404,12 @@ __get_safe_dirs() {
|
||||
__ensure_host_path_readable_in_priv_container() {
|
||||
# safe roots
|
||||
local safe_roots_str
|
||||
safe_roots_str="$(__get_safe_dirs | sed -r 's/\s+ro$//' ; check_pipe_status)" || return 1
|
||||
local safe_dirs_args=()
|
||||
if [[ "$1" == "--writeable-archive-root" ]] ; then
|
||||
safe_dirs_args+=("$1")
|
||||
shift
|
||||
fi
|
||||
safe_roots_str="$(__get_safe_dirs "${safe_dirs_args[@]}" | sed -r 's/\s+ro$//' ; check_pipe_status)" || return 1
|
||||
local -a safe_roots
|
||||
readarray -t safe_roots <<<"$safe_roots_str" || return 1
|
||||
|
||||
@ -418,7 +438,7 @@ __ensure_host_path_readable_in_priv_container() {
|
||||
}
|
||||
|
||||
#
|
||||
# Usage: __ensure_host_path_writable_in_priv_container PATHS...
|
||||
# Usage: __ensure_host_path_writable_in_priv_container [--writeable-archive-root] PATHS...
|
||||
#
|
||||
# Make sure a host path is OK to write in a privileged container,
|
||||
# ie any path under BUILD_OUTPUT_ROOT
|
||||
@ -426,7 +446,12 @@ __ensure_host_path_readable_in_priv_container() {
|
||||
__ensure_host_path_writable_in_priv_container() {
|
||||
# safe roots that don't end with " ro"
|
||||
local safe_roots_str
|
||||
safe_roots_str="$(__get_safe_dirs | grep -v -E '\s+ro$' ; check_pipe_status)" || return 1
|
||||
local safe_dirs_args=()
|
||||
if [[ "$1" == "--writeable-archive-root" ]] ; then
|
||||
safe_dirs_args+=("$1")
|
||||
shift
|
||||
fi
|
||||
safe_roots_str="$(__get_safe_dirs "${safe_dirs_args[@]}" | grep -v -E '\s+ro$' ; check_pipe_status)" || return 1
|
||||
local -a safe_roots
|
||||
readarray -t safe_roots <<<"$safe_roots_str" || return 1
|
||||
|
||||
@ -455,21 +480,31 @@ __ensure_host_path_writable_in_priv_container() {
|
||||
}
|
||||
|
||||
#
|
||||
# Usage: __safe_docker_run [--dry-run] <DOCKER RUN OPTIONS>
|
||||
# Usage: __safe_docker_run [--dry-run] [--writeable-archive-root] <DOCKER RUN OPTIONS>
|
||||
#
|
||||
safe_docker_run() {
|
||||
local dry_run=0
|
||||
local dry_run_prefix
|
||||
if [[ "$1" == "--dry-run" ]] ; then
|
||||
dry_run=1
|
||||
dry_run_prefix="(dry_run) "
|
||||
shift || true
|
||||
fi
|
||||
while [[ "$#" -gt 0 ]] ; do
|
||||
if [[ "$1" == "--dry-run" ]] ; then
|
||||
dry_run=1
|
||||
dry_run_prefix="(dry_run) "
|
||||
shift || true
|
||||
continue
|
||||
fi
|
||||
if [[ "$1" == "--writeable-archive-root" ]] ; then
|
||||
safe_dirs_args+=("$1")
|
||||
shift || true
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
|
||||
# construct mount options
|
||||
local -a mount_opts
|
||||
local safe_dirs_str
|
||||
safe_dirs_str="$(__get_safe_dirs)" || return 1
|
||||
safe_dirs_str="$(__get_safe_dirs "${safe_dirs_args[@]}")" || return 1
|
||||
local dir flags
|
||||
while read dir flags ; do
|
||||
[[ -d "$dir" ]] || continue
|
||||
local mount_str="type=bind,src=$dir,dst=$dir"
|
||||
@ -506,6 +541,7 @@ safe_docker_run() {
|
||||
# [--include PATTERN]
|
||||
# [--delete]
|
||||
# [--chown USER:GROUP]
|
||||
# [--writeable-archive-root]
|
||||
# [--dry-run]
|
||||
# [-v | --verbose]
|
||||
# SRC_DIR... DST_DIR
|
||||
@ -517,9 +553,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR
|
||||
# parse command line
|
||||
local opts
|
||||
local -a rsync_opts
|
||||
local -a safe_dirs_args
|
||||
local user_group
|
||||
local dry_run_arg=
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l exclude:,include:,delete,chown:,dry-run,verbose -- "$@")
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l exclude:,include:,delete,chown:,--writeable-archive-root,dry-run,verbose -- "$@")
|
||||
[[ $? -eq 0 ]] || return 1
|
||||
eval set -- "${opts}"
|
||||
while true ; do
|
||||
@ -544,6 +581,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR
|
||||
user_group="$2"
|
||||
shift 2
|
||||
;;
|
||||
--writeable-archive-root)
|
||||
safe_dirs_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
-v | --verbose)
|
||||
rsync_opts+=("--verbose")
|
||||
shift
|
||||
@ -580,11 +621,11 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR
|
||||
done
|
||||
|
||||
# make sure all dirs are readable
|
||||
__ensure_host_path_readable_in_priv_container "$@" || return 1
|
||||
__ensure_host_path_readable_in_priv_container "${safe_dirs_args[@]}" "$@" || return 1
|
||||
|
||||
# if dst_dir exists, it must be writable
|
||||
if [[ -d "${dst_dir}" ]] ; then
|
||||
__ensure_host_path_writable_in_priv_container "$dst_dir" || return 1
|
||||
__ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$dst_dir" || return 1
|
||||
# dst_dir doesn't exist, but there are multiple sources
|
||||
elif [[ "${#src_dirs[@]}" -gt 1 ]] ; then
|
||||
error "$dst_dir: does not exist or not a directory"
|
||||
@ -593,7 +634,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR
|
||||
# parent, but rename it to basename(dst_dir). This is how "cp" behaves.
|
||||
else
|
||||
src_dirs=("${src_dirs[0]%/}/")
|
||||
__ensure_host_path_writable_in_priv_container "$dst_dir" || return 1
|
||||
__ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$dst_dir" || return 1
|
||||
fi
|
||||
|
||||
# --chown: resolve USER:GROUP to UID:GID
|
||||
@ -636,18 +677,24 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] SRC_DIR... DST_DIR
|
||||
safe_rm() {
|
||||
local usage_msg="
|
||||
Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS...
|
||||
--writeable-archive-root
|
||||
--dry-run
|
||||
-v,--verbose
|
||||
"
|
||||
# parse command line
|
||||
local opts
|
||||
local -a safe_dirs_args
|
||||
local -a rm_opts
|
||||
local -a rm_cmd=("rm")
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l dry-run,verbose -- "$@")
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "v" -l writeable-archive-root,dry-run,verbose -- "$@")
|
||||
[[ $? -eq 0 ]] || return 1
|
||||
eval set -- "${opts}"
|
||||
while true ; do
|
||||
case "$1" in
|
||||
--writeable-archive-root)
|
||||
safe_dirs_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
rm_cmd=("echo" "(dry run)" "rm")
|
||||
shift
|
||||
@ -675,7 +722,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS...
|
||||
fi
|
||||
|
||||
# make sure all paths are writeable
|
||||
__ensure_host_path_writable_in_priv_container "$@"
|
||||
__ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$@"
|
||||
|
||||
# run rsync in docker
|
||||
rm_opts+=(--one-file-system --preserve-root --recursive --force)
|
||||
@ -692,6 +739,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] PATHS...
|
||||
safe_chown() {
|
||||
local usage_msg="
|
||||
Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS...
|
||||
--writeable-archive-root
|
||||
--dry-run
|
||||
-v,--verbose
|
||||
-R,--recursive
|
||||
@ -699,8 +747,9 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS...
|
||||
# parse command line
|
||||
local cmd_args
|
||||
local dry_run_arg
|
||||
local -a safe_dirs_args
|
||||
local -a cmd=("chown")
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "vR" -l dry-run,verbose,recursive -- "$@")
|
||||
opts=$(getopt -n "${FUNCNAME[0]}" -o "vR" -l dry-run,verbose,recursive,writeable-archive-root -- "$@")
|
||||
[[ $? -eq 0 ]] || return 1
|
||||
eval set -- "${opts}"
|
||||
while true ; do
|
||||
@ -717,6 +766,10 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS...
|
||||
cmd_args+=("--recursive")
|
||||
shift
|
||||
;;
|
||||
--writeable-archive-root)
|
||||
safe_dirs_args+=("$1")
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
@ -736,7 +789,7 @@ Usage: ${FUNCNAME[0]} [OPTIONS...] USER[:GROUP] PATHS...
|
||||
fi
|
||||
local user_group="$1" ; shift
|
||||
|
||||
__ensure_host_path_writable_in_priv_container "$@"
|
||||
__ensure_host_path_writable_in_priv_container "${safe_dirs_args[@]}" "$@"
|
||||
|
||||
# resolve USER:GROUP to UID:GID
|
||||
local uid_gid
|
||||
|
||||
@ -121,6 +121,7 @@ find_publish_dirs() {
|
||||
}
|
||||
|
||||
find_checksum_files() {
|
||||
local dir subdir
|
||||
find_publish_dirs | while read dir ; do
|
||||
for subdir in "$@" ; do
|
||||
if [[ -d "$dir/$subdir" ]] ; then
|
||||
|
||||
@ -97,6 +97,44 @@ PUBLISH_ROOT_URL="http://$(hostname -f):8088${PUBLISH_ROOT}"
|
||||
PUBLISH_SUBDIR="export" # may be empty
|
||||
PUBLISH_LATEST_LINK=false # create latest symlink?
|
||||
|
||||
#
|
||||
# Archiving of some of the following directories is handled specially:
|
||||
# aptly
|
||||
# docker
|
||||
# mirrors
|
||||
#
|
||||
# This parameter determines how to archive them:
|
||||
#
|
||||
# checksum-hardlink
|
||||
# Look for identical files in older builds' outputs and link
|
||||
# them. Create a single StxChecksums file at the top level of the
|
||||
# destination directory, containing each file's checksum and stat
|
||||
# properties. Search for link candidates in older builds' outputs
|
||||
# by looking for StxChecksums files there.
|
||||
# This option is the default.
|
||||
#
|
||||
# checksum-copy
|
||||
# Copy the files and create a single StxChecksums file at top level
|
||||
# of the destination directory
|
||||
#
|
||||
# top-symlink
|
||||
# Create a symlink in $BUILD_OUTPUT_HOME that points
|
||||
# back to $BUILD_HOME.
|
||||
#
|
||||
# This parameter may also contain a list of directory:method pairs to
|
||||
# use a different archiving method for each directory.
|
||||
#
|
||||
# Examples:
|
||||
# ========
|
||||
#
|
||||
# # hardlink all dirs
|
||||
# ARCHIVE_BIG_DIRS="checksum-hardlink" # same method for all dirs
|
||||
#
|
||||
# # hardlink all dirs, but symlink "mirrors"
|
||||
# ARCHIVE_BIG_DIRS="checksum-hardlink mirrors:top-symlink"
|
||||
#
|
||||
#ARCHIVE_BIG_DIRS="checksum-hardlink"
|
||||
|
||||
##################################################
|
||||
# Docker configuration
|
||||
##################################################
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user