From 3f23f278a57066ad6970b5b2468db4a032e8b5e4 Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Tue, 15 Sep 2020 20:22:43 -0500 Subject: [PATCH] Add gate runner script with bash-based execution Added ability to run deployment scripts directly (not via ansible), to be able to observe realtime output from commands. Change-Id: I736b075d6ecf0d7badc1e7703f9238339ab09592 Signed-off-by: Ruslan Aliev --- tools/deployment/22_test_configs.sh | 2 +- tools/export_sops | 16 +++++ tools/gate/00_setup.sh | 2 +- tools/gate/20_run_gate_runner.sh | 98 ++++++++++++++++++++++++++--- 4 files changed, 109 insertions(+), 9 deletions(-) create mode 100755 tools/export_sops diff --git a/tools/deployment/22_test_configs.sh b/tools/deployment/22_test_configs.sh index 7872f95b5..1acfd2045 100755 --- a/tools/deployment/22_test_configs.sh +++ b/tools/deployment/22_test_configs.sh @@ -42,7 +42,7 @@ export AIRSHIP_CONFIG_MANIFEST_DIRECTORY=${AIRSHIP_CONFIG_MANIFEST_DIRECTORY:-"/ export EXTERNAL_KUBECONFIG=${EXTERNAL_KUBECONFIG:-""} # Remove the contents of the .airship folder, preserving the kustomize plugin directory -rm -rf $HOME/.airship/config +rm -rf $HOME/.airship/*config* mkdir -p $HOME/.airship echo "Generate ~/.airship/config and ~/.airship/kubeconfig" diff --git a/tools/export_sops b/tools/export_sops new file mode 100755 index 000000000..7d2fbf9a4 --- /dev/null +++ b/tools/export_sops @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export SOPS_IMPORT_PGP="$(curl -fsSL https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc)" +export SOPS_PGP_FP="FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4" diff --git a/tools/gate/00_setup.sh b/tools/gate/00_setup.sh index 54b4125a7..90fc46927 100755 --- a/tools/gate/00_setup.sh +++ b/tools/gate/00_setup.sh @@ -36,7 +36,7 @@ sudo apt update sudo DEBIAN_FRONTEND=noninteractive apt -y install software-properties-common python3-pip curl wget ca-certificates sudo DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install docker.io make -PACKAGES="ansible netaddr" +PACKAGES="ansible netaddr yq" if [[ -z "${http_proxy}" ]]; then sudo pip3 install $PACKAGES else diff --git a/tools/gate/20_run_gate_runner.sh b/tools/gate/20_run_gate_runner.sh index 62105dde8..462146aef 100755 --- a/tools/gate/20_run_gate_runner.sh +++ b/tools/gate/20_run_gate_runner.sh @@ -17,14 +17,98 @@ # specific language governing permissions and limitations # under the License. -set -xe +set -xeo pipefail + +source tools/export_sops -TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"} -ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"} -PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"} export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD} export AIRSHIP_CONFIG_PHASE_REPO_URL=${AIRSHIP_CONFIG_PHASE_REPO_URL:-$PWD} -sudo -E --preserve-env=AIRSHIPCTL_WS ANSIBLE_STDOUT_CALLBACK=debug ansible-playbook -i "$ANSIBLE_HOSTS" \ - playbooks/airshipctl-gate-runner.yaml \ - -e @"$PLAYBOOK_CONFIG" -v +ZUUL_JOBS_PATH=zuul.d/jobs.yaml +GATE_RUNNER_YAML_PATH=playbooks/airshipctl-gate-runner.yaml +OUTPUT_DIR="" +STOP_SCRIPT="" +SKIP_LIST="" +MUTE=0 + +show_help() { +cat << EOF +Usage: $0 [options] +Run set of deployments scripts for airshipctl + +-h, --help Display help +-s, --stop-at NUMBER Specify script number where to stop execution +-p --pass LIST Comma separated list of script numbers to skip +-o, --output-dir DIRNAME The output of each script will be saved in the specified directory in a separate file +-m, --mute Mute the output from scripts + +EOF +} + +# read the options +options=$(getopt -o hmo:p:s: --long help,mute,output-dir:,pass:,stop-at: -- "$@") +if [ $? != 0 ] ; then echo "Failed to parse options...exiting." >&2 ; exit 1 ; fi +eval set -- "$options" + +while true; do + case "$1" in + -s | --stop-at) + STOP_SCRIPT="$2" + shift 2 + ;; + -p | --pass) + SKIP_LIST="$2" + shift 2 + ;; + -o | --output-dir) + OUTPUT_DIR="$2" + mkdir -p $OUTPUT_DIR + shift 2 + ;; + -m | --mute ) + MUTE=1 + shift + ;; + -h | --help ) + show_help + exit 0 + ;; + -- ) + shift + break + ;; + esac +done + +SCRIPT_LIST=$(cat $ZUUL_JOBS_PATH | yq '.[9].job.vars.gate_scripts[]' -c -r) +if [[ ! $SCRIPT_LIST ]]; then + SCRIPT_LIST=$(cat $GATE_RUNNER_YAML_PATH | yq '.[0].tasks[0].set_fact.deploy_test_site_scripts_default[]' -c -r) +fi + +SKIP_LIST=$(echo ${SKIP_LIST//,/ }) + +for script in $SCRIPT_LIST; do + SCRIPT_NAME=$(awk -F "/" "{ print \$NF }" <<<$script) + if [[ $SCRIPT_NAME =~ ([0-9]+) ]]; then + SCRIPT_NUM="${BASH_REMATCH[1]}" + fi + if [[ " ${SKIP_LIST[@]} " =~ " ${SCRIPT_NUM} " ]]; then + if [[ $STOP_SCRIPT ]] && [[ $SCRIPT_NAME =~ "${STOP_SCRIPT}_"* ]]; then + break + fi + continue + fi + + echo -e "\033[0;32m[ *** Run script $script *** ] \033[0m " + cmd="sudo --preserve-env=AIRSHIPCTL_WS,AIRSHIP_CONFIG_PHASE_REPO_URL,SOPS_IMPORT_PGP,SOPS_PGP_FP $script" + if [[ $OUTPUT_DIR ]]; then + $cmd > ${OUTPUT_DIR}/${SCRIPT_NAME}.out 2>&1 + elif [[ "$MUTE" -eq "1" ]]; then + $cmd > /dev/null 2>&1 + else + $cmd + fi + if [[ $STOP_SCRIPT ]] && [[ $SCRIPT_NAME =~ "${STOP_SCRIPT}_"* ]]; then + break + fi +done