From bd803643a1799fb13dfad35b78a4ac7f8e20e110 Mon Sep 17 00:00:00 2001 From: Davlet Panech Date: Tue, 15 Oct 2024 13:20:35 -0400 Subject: [PATCH] archive-dir.sh: handle unusual file names archive-dir.sh fails on files that contain substrings similar to "name=...". This happens when we parse the list of file names and attributes, one per line, of the form: type=f owner=myuser ... name=some/file/name.ext Problem observed with some docker FS layers containing files whose names include a "name=..." token, eg: type=f owner=myuser ... name=aaa,name=bbb,ccc The parsing algorithm picks up the last "name=bbb,ccc", incorrectly. SOLUTION =================== * use greedy wildcard when looking for "name=" (ie, look for 1st occurance) * cut off "name=..." before attempting to parse out the other attributes, to make sure we don't confuse them with parts of the file name * ignore errors from "look" utility; eg when the list file is empty TESTS =================== * Set ARCHIVE_BIG_DIRS="top-symlink docker:checksum-hardlink" in build.conf * Create a file named "aaa,name=bbb,ccc" in $STX_BUILD_HOME/docker/ * Run Jenkins build that triggers the"archive-misc step, but skips package, ISO & docker image builds. * Make sure the file gets created at destination w/o errors Closes-Bug: 2084570 Signed-off-by: Davlet Panech Change-Id: Ib455c1b381e0b5ddaa40714c27340b2ec3f5676a --- scripts/helpers/archive-dir.sh | 36 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/helpers/archive-dir.sh b/scripts/helpers/archive-dir.sh index 2b84263..6d19506 100755 --- a/scripts/helpers/archive-dir.sh +++ b/scripts/helpers/archive-dir.sh @@ -242,19 +242,21 @@ process_lines() { echo $'\n## Creating directories: '"$DST_DIR" >&2 while read -r line ; do [[ -n "$line" ]] || continue - name="${line##*name=}" - mode="$(echo "$line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')" + name="${line#* name=}" + [[ -n "$name" ]] || continue + attr_line="${line% name=*}" + mode="$(echo "$attr_line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')" install_args=() if [[ "$CHANGE_OWNER" ]] ; then install_args+=("--owner" "$CHANGE_OWNER") elif [[ $EUID -eq 0 ]] ; then - owner="$(echo "$line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')" + owner="$(echo "$attr_line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')" install_args+=("--owner" "$owner") fi if [[ "$CHANGE_GROUP" ]] ; then install_args+=("--group" "$CHANGE_GROUP") elif [[ $EUID -eq 0 ]] ; then - group="$(echo "$line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')" + group="$(echo "$attr_line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')" install_args+=("--group" "$group") fi echo " MKDIR $name" >&2 @@ -290,14 +292,17 @@ process_regfiles() { local matching_checksums_file matching_checksums_file="$TMP_DIR/matching_checksums-$$.list" - local line + local line attr_line for line in "$@" ; do # source file name relative to SRC_DIR local name - name="${line##*name=}" + name="${line#* name=}" [[ "$name" ]] || continue + # all attributes leading up to name= + attr_line="${line% name=*}" + # source checksum local checksum #flock -s "$DST_DIR" echo " SHA256 $name" >&2 @@ -314,7 +319,7 @@ process_regfiles() { owner="$CHANGE_OWNER" install_args+=("--owner" "$owner") elif [[ $EUID -eq 0 ]] ; then - owner="$(echo "$line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')" + owner="$(echo "$attr_line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')" install_args+=("--owner" "$owner") else owner=$EUID @@ -326,7 +331,7 @@ process_regfiles() { group="$CHANGE_GROUP" install_args+=("--group" "$group") elif [[ $EGID -eq 0 ]] ; then - group="$(echo "$line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')" + group="$(echo "$attr_line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')" install_args+=("--group" "$group") else group=$EGID @@ -334,11 +339,11 @@ process_regfiles() { # source file's mode/permissions local mode - mode="$(echo "$line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')" + mode="$(echo "$attr_line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')" # Search for the checksum in an older StxChecksums file if [[ "$combined_checksums_file" ]] ; then - if look "$checksum " "$combined_checksums_file" >"$matching_checksums_file" ; then + if look "$checksum " "$combined_checksums_file" >"$matching_checksums_file" 2>/dev/null ; then ( # As we read previosuly-archived files properties from StxChecksums, # make sure they have not changed compared to the actual files on disk. @@ -396,8 +401,8 @@ process_regfiles() { # source file's size & mtime local size mtime - size="$(echo "$line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')" - mtime="$(echo "$line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')" + size="$(echo "$attr_line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')" + mtime="$(echo "$attr_line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')" # copy it to $DST_DIR flock -s "$DST_DIR" echo " COPY $name" >&2 @@ -467,14 +472,15 @@ process_other() { if [[ $XTRACE -eq 1 ]] ; then set -x fi - local line + local line attr_line for line in "$@" ; do local name - name="${line##*name=}" + name="${line#* name=}" [[ -n "$name" ]] || continue + attr_line="${line% name=*}" local type - type="$(echo "$line" | sed 's#^type=\(.\) .*#\1#g')" + type="$(echo "$attr_line" | sed 's#^type=\(.\) .*#\1#g')" [[ -n "$type" ]] || continue flock -s "$DST_DIR" echo " CREATE type=$type $name" >&2