be58140638
Story: 2010226 Task: 46011 Change-Id: I721f1509637b910c05e4151eeb7ad7e1cb8db119 Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
366 lines
13 KiB
Bash
366 lines
13 KiB
Bash
# bash
|
|
|
|
#
|
|
# Copyright (c) 2022 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
source $(dirname "${BASH_SOURCE[0]}")/utils.sh || exit 1
|
|
source $(dirname "${BASH_SOURCE[0]}")/glob_utils.sh || exit 1
|
|
|
|
# used by need_build() below
|
|
NEED_BUILD_PATTERNS=(
|
|
|
|
"! cgcs-root/stx/docs/*"
|
|
"! cgcs-root/stx/test/*"
|
|
"! cgcs-root/local-build-data/*"
|
|
"! cgcs-root/wrs/docs/*"
|
|
|
|
" cgcs-root/wrs/titanium-tools/docker-images/*" # used by *_SDK_Build
|
|
" cgcs-root/wrs/titanium-tools/lab/*"
|
|
"! cgcs-root/wrs/titanium-tools/*"
|
|
|
|
# " stx-tools/centos-mirror-tools/yum.conf.sample"
|
|
# " stx-tools/centos-mirror-tools/config/centos/$LAYER"
|
|
# "! stx-tools/*"
|
|
|
|
" cgcs-root/build-tools/build_iso/*.cfg"
|
|
" cgcs-root/build-tools/build_iso/minimal_rpm_list.txt"
|
|
" cgcs-root/build-tools/signing/*"
|
|
" cgcs-root/build-tools/certificats/*"
|
|
" cgcs-root/build-tools/build-docker-images/*"
|
|
" cgcs-root/build-tools/build-wheels/*"
|
|
"! cgcs-root/build-tools/*"
|
|
)
|
|
|
|
# Usage: ( cd $MY_REPO_ROOT_DIR && print_last_commits ; ) >LAST_COMMITS
|
|
print_last_commits() {
|
|
local dir padded_dir
|
|
for dir in ./.repo/manifests $( find . \( -path './.repo' -prune \) -o \( -xtype d -name .git -printf '%h\n' \) | sort ) ; do
|
|
pushd "$dir" >/dev/null || exit 1
|
|
padded_dir="$(printf "%-52s" "$dir")"
|
|
git log --pretty=tformat:"$padded_dir %H" -n 1 || exit 1
|
|
popd >/dev/null || exit 1
|
|
done
|
|
}
|
|
|
|
#
|
|
# Usage: diff_last_commits MY_WORKSPACE/../LAST_COMMITS [PATTERNS...]
|
|
#
|
|
# Diff git HEADs with a LAST_COMMITS file generated by another build
|
|
# Return true (0) if there are no differences
|
|
#
|
|
# PATTERNS may be used to filter out changes in some files when comparing
|
|
#
|
|
# If any file changed since LAST_COMMITs matches a PATTERN, we return
|
|
# false (ie changes detected & a rebuild is required). If it matches a
|
|
# "negative" pattern that begins with "!", we continue to the next file
|
|
# instead. Patterns are matched in order until a match is found. A
|
|
# combination of positive and negative patterns may be used to skip parts
|
|
# of the source tree.
|
|
#
|
|
# Patterns are similar to shell glob patterns, except "*" and "?" match
|
|
# any character including "/".
|
|
# Leading and trailing whitespace as well as leading "./" in patterns
|
|
# are not significant.
|
|
# Changes in files that didn't match any patterns are treated as positive
|
|
# matches.
|
|
#
|
|
# EXAMPLE:
|
|
#
|
|
# diff_last_commits $MY_WORKSPACE/../LAST_COMMITS \
|
|
# # detect changes in this file \
|
|
# "stx-tools/centos-mirror-tools/yum.conf.sample" \
|
|
# # ignore other changes under centos-mirror-tools \
|
|
# "!stx-tools/centos/mirror-tools/*" \
|
|
# # detect changes everywhere else (implied)
|
|
# "*"
|
|
#
|
|
|
|
diff_last_commits() {
|
|
local last_commits_file="$1" ; shift || :
|
|
local debug=2
|
|
local dir dir_regex
|
|
local last_commit
|
|
local commit_files_str
|
|
local -a commit_files
|
|
local file
|
|
local match pattern_expr pattern regex
|
|
|
|
# no previous builds: assume builds different
|
|
if [[ ! -f "$last_commits_file" ]] ; then
|
|
[[ "$debug" == 0 ]] || echo "## file \`$last_commits_file' doesn't exist: return false" >&2
|
|
return 1
|
|
fi
|
|
|
|
# find all gits
|
|
[[ "$debug" == 0 ]] || echo "## looking for diffs between \`$PWD' and \`$last_commits_file'" >&2
|
|
for dir in ./.repo/manifests $( find . \( -path './.repo' -prune \) -o \( -xtype d -name .git -printf '%h\n' \) | sort ) ; do
|
|
[[ "$debug" == 0 ]] || echo "## checking \`$dir'" >&2
|
|
|
|
if [[ "$dir" == "." ]] ; then
|
|
dir_prefix=""
|
|
else
|
|
dir_prefix="${dir#./}"/
|
|
fi
|
|
|
|
# find last commit for this dir
|
|
# create a regex LAST_COMMITS, eg: ./cgcs-root/stx/config => ^[.][/]cgcs-root[/]stx[/]config[^a-zA-Z0-9/_-]
|
|
dir_regex="$(echo "$dir" | sed \
|
|
-e 's:/:[/]:g' \
|
|
-e 's:$:[^a-zA-Z0-9/_-]:' \
|
|
-e 's:^[.][.]:^[.][.]:' \
|
|
-e 's:^[.]:^[.]:' \
|
|
)"
|
|
last_commit=$(grep "$dir_regex" "$last_commits_file" | awk ' { print $2 } ')
|
|
|
|
# it didn't exist in previous buid: assume builds different
|
|
if [[ -z "$last_commit" ]] ; then
|
|
[[ "$debug" == 0 ]] || echo "## $dir: not present in \`$last_commits_file': return false" >&2
|
|
return 1
|
|
fi
|
|
|
|
# get all files changed since last_commit
|
|
commit_files_str="$(cd "$dir" && git diff-tree --no-commit-id --name-only -r $last_commit..HEAD)" || exit 1
|
|
readarray -t commit_files < <(echo -n "$commit_files_str")
|
|
|
|
# check each file against PATTERNs
|
|
for file in "${commit_files[@]}" ; do
|
|
match=0
|
|
for pattern_expr in "$@" ; do
|
|
# convert glob pattern to regex
|
|
pattern="$(echo "$pattern_expr" | sed -r -e 's/^\s*[!]?\s*//' -e 's/\s*$//')"
|
|
regex="$(glob_to_basic_regex "$pattern")"
|
|
[[ "$debug" -lt 2 ]] || echo "## trying to match pattern \`$pattern' / regex \`$regex'" >&2
|
|
# check if the file matches
|
|
if echo "${dir_prefix}$file" | grep -q -E "$regex" >/dev/null || \
|
|
echo "$dir/$file" | grep -q -E "$regex" >/dev/null ; then
|
|
# pattern doesn't begin with "!": assume builds different
|
|
if ! echo "$pattern_expr" | grep -q -E '^\s*[!]' ; then
|
|
[[ "$debug" == 0 ]] || echo "## file \`${dir_prefix}$file' matched positive pattern \`$pattern_expr': return false" >&2
|
|
return 1
|
|
fi
|
|
# "!" pattern: continue to next file
|
|
[[ "$debug" == 0 ]] || echo "## file \`${dir_prefix}$file' matched negative pattern \`$pattern_expr': continue to next file" >&2
|
|
match=1
|
|
break
|
|
fi
|
|
done # for pattern_expr ...
|
|
if [[ $match == 0 ]] ; then
|
|
[[ "$debug" == 0 ]] || echo "## file \`${dir_prefix}$file' didn't match any negative patterns: return false" >&2
|
|
return 1
|
|
fi
|
|
done # for file ...
|
|
|
|
done # for dir ...
|
|
|
|
[[ "$debug" == 0 ]] || echo "## no diffs found: return true" >&2
|
|
return 0
|
|
|
|
}
|
|
|
|
#
|
|
# Usage: print_changelog LAST_COMMITS_FILE [DEFAULT_FROM_TIMESTAMP]
|
|
#
|
|
# Print out the change log since LAST_COMMITS_FILE.
|
|
#
|
|
# DEFAULT_FROM_TIMESTAMP will be used for repos missing from LAST_COMMITS_FILE
|
|
# and must be a date or date/time in ISO format. Defaults to the value of
|
|
# BUILD_TIMESTAMP global variable minus 1 day at midnight, or yesterday's midnight.
|
|
#
|
|
# LAST_COMMITS_FILE need not exist.
|
|
#
|
|
print_changelog() {
|
|
local last_commits_file="$1"
|
|
|
|
local default_from_timestamp
|
|
if [[ -n "$2" ]] ; then
|
|
default_from_timestamp="$2"
|
|
else
|
|
local build_date
|
|
build_date="${BUILD_TIMESTAMP:0:10}"
|
|
[[ -n "$build_date" ]] || build_date=$(date '+%Y-%m-%d') || return 1
|
|
default_from_timestamp="$(date --date="$build_date - 1 day" '+%Y-%m-%d 00:00:00')" || return 1
|
|
fi
|
|
|
|
local dir
|
|
for dir in ./.repo/manifests $( find . \( -path './.repo' -prune \) -o \( -xtype d -name .git -printf '%h\n' \) | sort ) ; do (
|
|
set -e
|
|
padded_dir="$(printf "%-52s" "$dir")"
|
|
commit=
|
|
if [[ -f "$last_commits_file" ]] ; then
|
|
# create a regex LAST_COMMITS, eg: ./cgcs-root/stx/config => ^[.][/]cgcs-root[/]stx[/]config[^a-zA-Z0-9/_-]
|
|
regex="$(echo "$dir" | sed \
|
|
-e 's:/:[/]:g' \
|
|
-e 's:$:[^a-zA-Z0-9/_-]:' \
|
|
-e 's:^[.][.]:^[.][.]:' \
|
|
-e 's:^[.]:^[.]:' \
|
|
)"
|
|
commit=$(grep "$regex" "$last_commits_file" | awk ' { print $2 } ')
|
|
fi
|
|
if [[ -n "$commit" ]] ; then
|
|
git_log_args=("$commit..")
|
|
else
|
|
git_log_args=(--after "$default_from_timestamp")
|
|
fi
|
|
pushd "$dir" >/dev/null
|
|
git log --date=iso --pretty=tformat:"$padded_dir %H %cd%x09%cn%x09%s" "${git_log_args[@]}"
|
|
popd >/dev/null
|
|
) ; done
|
|
}
|
|
|
|
#
|
|
# Usage: print_changelog_since TIMESTAMP
|
|
#
|
|
print_changelog_since() {
|
|
print_changelog "" "$1"
|
|
}
|
|
|
|
# Usage: create_standard_changelogs
|
|
#
|
|
# Create changelog files in $MY_WORKSPACE:
|
|
#
|
|
# CHANGELOG
|
|
# changes since LAST_COMMITS left by most recent successful build,
|
|
# used for rebuild calculations
|
|
#
|
|
# CHANGELOG.OLD
|
|
# changes since midnight of previous day (24-48 hours)
|
|
#
|
|
# CHANGELOG.IMG_DEV
|
|
# changes since LAST_COMMITS left by most recent dev images build
|
|
#
|
|
# CHANGELOG.IMG_STABLE
|
|
# changes since LAST_COMMITS left by most recent stable images build
|
|
#
|
|
# LAST_COMMITS
|
|
# SHA's of each git's HEAD
|
|
#
|
|
create_standard_changelogs() {
|
|
require_env "MY_REPO_ROOT_DIR" "MY_WORKSPACE"
|
|
local deploy_dir="${DEPLOY_DIR:-$MY_WORKSPACE/..}"
|
|
local changelog_file="$MY_WORKSPACE/CHANGELOG"
|
|
|
|
local build_date
|
|
build_date="${BUILD_TIMESTAMP:0:10}"
|
|
[[ -n "$build_date" ]] || build_date=$(date '+%Y-%m-%d') || return 1
|
|
|
|
local default_from_timestamp
|
|
default_from_timestamp="$(date --date="$build_date - 1 day" '+%Y-%m-%d 00:00:00')" || return 1
|
|
|
|
# CHANGELOG
|
|
echo "## creating $changelog_file (since last iso build)"
|
|
(
|
|
set -e
|
|
cd "$MY_REPO_ROOT_DIR"
|
|
print_changelog "$deploy_dir/LAST_COMMITS" "$default_from_timestamp"
|
|
) >"$changelog_file" || exit 1
|
|
|
|
# CHANGELOG.OLD
|
|
echo "## creating $changelog_file.OLD (since yesterday midnight)"
|
|
(
|
|
set -e
|
|
cd "$MY_REPO_ROOT_DIR"
|
|
print_changelog "" "$default_from_timestamp"
|
|
) >"$changelog_file.OLD" || exit 1
|
|
|
|
# CHNAGELOG.IMG_DEV
|
|
echo "## creating $changelog_file.IMG_DEV (since last dev images build)"
|
|
(
|
|
set -e
|
|
cd "$MY_REPO_ROOT_DIR"
|
|
print_changelog "$deploy_dir/LAST_COMMITS_IMG_DEV" "$default_from_timestamp"
|
|
) >"$changelog_file.IMG_DEV" || exit 1
|
|
|
|
# CHNAGELOG.IMG_STABLE
|
|
echo "## creating $changelog_file.IMG_STABLE (since last stable images build)"
|
|
(
|
|
set -e
|
|
cd "$MY_REPO_ROOT_DIR"
|
|
print_changelog "$deploy_dir/LAST_COMMITS_IMG_STABLE" "$default_from_timestamp"
|
|
) >"$changelog_file.IMG_STABLE" || exit 1
|
|
|
|
# LAST_COMMITS
|
|
(
|
|
set -e
|
|
cd "$MY_REPO_ROOT_DIR"
|
|
print_last_commits
|
|
) >"$MY_WORKSPACE/LAST_COMMITS" || exit 1
|
|
|
|
echo "## LAST_COMMITS" >&2
|
|
cat "$MY_WORKSPACE/LAST_COMMITS" >&2
|
|
echo "## END LAST_COMMITS" >&2
|
|
|
|
echo "## CHANGELOG" >&2
|
|
cat "$changelog_file" >&2
|
|
echo "## END CHANGELOG" >&2
|
|
}
|
|
|
|
# Usage: need_build [BUILD_DATE]
|
|
#
|
|
# Return true if build is required. BUILD_DATE defaults
|
|
# to BUILD_TIMESTAMP global var, or today's date.
|
|
#
|
|
# This will create either a NEED_BUID or NO_BUILD_REQUIRED file
|
|
# in MY_WORKSPACE.
|
|
#
|
|
# If any of these job parameters are set to true, this function returns true:
|
|
# FORCE_BUILD
|
|
# BUILD_DOCKER_IMAGES_DEV
|
|
# BUILD_DOCKER_IMAGES_STABLE
|
|
#
|
|
# These job parameters/env vars must be set to the week days when
|
|
# images should be built, eg:
|
|
#
|
|
# BUILD_DOCKER_IMAGES_DAYS_DEV=""
|
|
# BUILD_DOCKER_IMAGES_DAYS_STABLE="mon tue"
|
|
# FORCE_BUILD_DAYS="sat"
|
|
#
|
|
need_build() {
|
|
local build_date build_weekday build_reason
|
|
build_date="$1"
|
|
[[ -n "$build_date" ]] || build_date="${BUILD_TIMESTAMP:0:10}"
|
|
[[ -n "$build_date" ]] || build_date=$(date '+%Y-%m-%d') || return 1
|
|
build_weekday=$(get_weekday "$build_date") || exit 1
|
|
local deploy_dir="${DEPLOY_DIR:-$MY_WORKSPACE/..}"
|
|
|
|
require_env MY_WORKSPACE MY_REPO_ROOT_DIR FORCE_BUILD BUILD_DOCKER_IMAGES_DEV BUILD_DOCKER_IMAGES_STABLE
|
|
rm -f "$MY_WORKSPACE/NO_BUILD_REQUIRED" "$MY_WORKSPACE/NEED_BUILD" || exit 1
|
|
if $FORCE_BUILD ; then
|
|
build_reason="forced"
|
|
elif in_list "$build_weekday" $(normalize_weekdays $FORCE_BUILD_DAYS) ; then
|
|
build_reason="forced:schedule"
|
|
elif $BUILD_DOCKER_IMAGES_DEV ; then
|
|
build_reason="dev_images_forced"
|
|
elif $BUILD_DOCKER_IMAGES_STABLE ; then
|
|
build_reason="stable_images_forced"
|
|
elif ! diff_last_commits "$deploy_dir/LAST_COMMITS" "${NEED_BUILD_PATTERNS[@]}" ; then
|
|
build_reason="changes_detected"
|
|
elif in_list "$build_weekday" $(normalize_weekdays $BUILD_DOCKER_IMAGES_DAYS_DEV) && \
|
|
! diff_last_commits "$deploy_dir/LAST_COMMITS_IMG_DEV" "${NEED_BUILD_PATTERNS[@]}" ; then
|
|
build_reason="dev_images_changes_detected"
|
|
elif in_list "$build_weekday" $(normalize_weekdays $BUILD_DOCKER_IMAGES_DAYS_STABLE) && \
|
|
! diff_last_commits "$deploy_dir/LAST_COMMITS_IMG_STABLE" "${NEED_BUILD_PATTERNS[@]}" ; then
|
|
build_reason="stable_images_changes_detected"
|
|
else
|
|
touch "$MY_WORKSPACE/NO_BUILD_REQUIRED" || exit 1
|
|
echo "## No new content. Build not required."
|
|
return 1
|
|
fi
|
|
echo "REASON=$build_reason" >"$MY_WORKSPACE/NEED_BUILD" || exit 1
|
|
echo "## Build required ($build_reason)"
|
|
return 0
|
|
}
|
|
|
|
#
|
|
# Return true if the build is being forced:
|
|
# FORCE_BUILD is "true" -- OR
|
|
# FORCE_BUILD_DAYS matches the build time stamp
|
|
#
|
|
is_build_forced() {
|
|
[[ -f "$MY_WORKSPACE/NEED_BUILD" ]] || return 1
|
|
grep -E -q "^\\s*REASON=['\"]?forced:?" "$MY_WORKSPACE/NEED_BUILD" >/dev/null
|
|
}
|
|
|