starlingx/jenkins-pipelines
Code Issues Proposed changes

archive-dir.sh: handle unusual file names

Browse Source 
archive-dir.sh fails on files that contain substrings similar to
"name=...". This happens when we parse the list of file names and
attributes, one per line, of the form:

  type=f owner=myuser ... name=some/file/name.ext

Problem observed with some docker FS layers containing files whose
names include a "name=..." token, eg:

  type=f owner=myuser ... name=aaa,name=bbb,ccc

The parsing algorithm picks up the last "name=bbb,ccc", incorrectly.

SOLUTION
===================
* use greedy wildcard when looking for "name=" (ie, look for 1st
  occurance)
* cut off "name=..." before attempting to parse out the other
  attributes, to make sure we don't confuse them with parts of the file
  name
* ignore errors from "look" utility; eg when the list file is empty

TESTS
===================
* Set ARCHIVE_BIG_DIRS="top-symlink docker:checksum-hardlink" in
  build.conf
* Create a file named "aaa,name=bbb,ccc" in $STX_BUILD_HOME/docker/
* Run Jenkins build that triggers the"archive-misc step, but skips
  package, ISO & docker image builds.
* Make sure the file gets created at destination w/o errors

Closes-Bug: 2084570
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: Ib455c1b381e0b5ddaa40714c27340b2ec3f5676a
This commit is contained in:
Davlet Panech 2024-10-15 13:20:35 -04:00
parent 79271f8ffe
commit bd803643a1
1 changed files with 21 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
scripts/helpers/archive-dir.sh
View File

@ -242,19 +242,21 @@ process_lines() {
echo $'\n## Creating directories: '"$DST_DIR" >&2
while read -r line ; do
[[ -n "$line" ]] || continue
name="${line##*name=}"
mode="$(echo "$line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')"
name="${line#* name=}"
[[ -n "$name" ]] || continue
attr_line="${line% name=*}"
mode="$(echo "$attr_line" | sed -n -r 's#.*mode=([0-9]+).*#\1#p')"
install_args=()
if [[ "$CHANGE_OWNER" ]] ; then
install_args+=("--owner" "$CHANGE_OWNER")
elif [[ $EUID -eq 0 ]] ; then
owner="$(echo "$line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')"
owner="$(echo "$attr_line" | sed -n -r 's#.*owner=([0-9]+).*#\1#p')"
install_args+=("--owner" "$owner")
fi
if [[ "$CHANGE_GROUP" ]] ; then
install_args+=("--group" "$CHANGE_GROUP")
elif [[ $EUID -eq 0 ]] ; then
group="$(echo "$line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')"
group="$(echo "$attr_line" | sed -n -r 's#.*group=([0-9]+).*#\1#p')"
install_args+=("--group" "$group")
fi
echo " MKDIR $name" >&2
@ -290,14 +292,17 @@ process_regfiles() {
local matching_checksums_file
matching_checksums_file="$TMP_DIR/matching_checksums-$$.list"
local line
local line attr_line
for line in "$@" ; do
# source file name relative to SRC_DIR
local name
name="${line##*name=}"
name="${line#* name=}"
[[ "$name" ]] || continue
# all attributes leading up to name=
attr_line="${line% name=*}"
# source checksum
local checksum
#flock -s "$DST_DIR" echo " SHA256 $name" >&2
@ -314,7 +319,7 @@ process_regfiles() {
owner="$CHANGE_OWNER"
install_args+=("--owner" "$owner")
elif [[ $EUID -eq 0 ]] ; then
owner="$(echo "$line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')"
owner="$(echo "$attr_line" | sed -n -r 's#.* owner=([0-9]+).*#\1#p')"
install_args+=("--owner" "$owner")
else
owner=$EUID
@ -326,7 +331,7 @@ process_regfiles() {
group="$CHANGE_GROUP"
install_args+=("--group" "$group")
elif [[ $EGID -eq 0 ]] ; then
group="$(echo "$line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')"
group="$(echo "$attr_line" | sed -n -r 's#.* group=([0-9]+).*#\1#p')"
install_args+=("--group" "$group")
else
group=$EGID
@ -334,11 +339,11 @@ process_regfiles() {
# source file's mode/permissions
local mode
mode="$(echo "$line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')"
mode="$(echo "$attr_line" | sed -n -r 's#.* mode=([^[:space:]]+).*#\1#p')"
# Search for the checksum in an older StxChecksums file
if [[ "$combined_checksums_file" ]] ; then
if look "$checksum " "$combined_checksums_file" >"$matching_checksums_file" ; then
if look "$checksum " "$combined_checksums_file" >"$matching_checksums_file" 2>/dev/null ; then
(
# As we read previosuly-archived files properties from StxChecksums,
# make sure they have not changed compared to the actual files on disk.
@ -396,8 +401,8 @@ process_regfiles() {
# source file's size & mtime
local size mtime
size="$(echo "$line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')"
mtime="$(echo "$line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')"
size="$(echo "$attr_line" | sed -n -r 's#.* size=([^[:space:]]+).*#\1#p')"
mtime="$(echo "$attr_line" | sed -n -r 's#.* mtime=([^[:space:]]+).*#\1#p')"
# copy it to $DST_DIR
flock -s "$DST_DIR" echo " COPY $name" >&2
@ -467,14 +472,15 @@ process_other() {
if [[ $XTRACE -eq 1 ]] ; then
set -x
fi
local line
local line attr_line
for line in "$@" ; do
local name
name="${line##*name=}"
name="${line#* name=}"
[[ -n "$name" ]] || continue
attr_line="${line% name=*}"
local type
type="$(echo "$line" | sed 's#^type=\(.\) .*#\1#g')"
type="$(echo "$attr_line" | sed 's#^type=\(.\) .*#\1#g')"
[[ -n "$type" ]] || continue
flock -s "$DST_DIR" echo " CREATE type=$type $name" >&2