x/sahara-ci-config
Code Issues Proposed changes

Divide jobs log to different files

Browse Source 
Divide logs for each job for Sahara by service

Change-Id: I1472fc30a467218f4aff79c553163968638dad01
This commit is contained in:
Denis Egorenko 2015-02-24 18:47:47 +04:00
parent b8885ccdec
commit d564f6b2f5
6 changed files with 54 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
jenkins_job_builder/diskimage.yaml
View File

@ -12,13 +12,9 @@
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-nova'
- job-template:
@ -35,13 +31,9 @@
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-{network}'
- job-template:
@ -58,13 +50,9 @@
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-nova'
- job-template:
@ -81,13 +69,9 @@
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-neutron'
- project:

20
jenkins_job_builder/macros.yaml
View File

@ -154,6 +154,16 @@
- target: 'tarballs/{project}/'
source: 'dist/*.tar.gz'
- publisher:
name: sahara-logs
publishers:
- scp:
site: 'localhost'
files:
- target: 'logs/$LOG_PATH'
source: '**/logs/*.txt'
copy-after-failure: true
- publisher:
name: console-log
publishers:
@ -164,6 +174,16 @@
copy-console: true
copy-after-failure: true
- publisher:
name: trigger-cleanup
publishers:
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- publisher:
name: console-log-periodic
publishers:

48
jenkins_job_builder/sahara.yaml
View File

@ -18,13 +18,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: trusty-neutron || trusty-nova
- job:
@ -47,13 +43,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: trusty-neutron || trusty-nova
- job-template:
@ -73,13 +65,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-neutron'
- job-template:
@ -99,13 +87,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME\
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-nova'
- job-template:
@ -125,13 +109,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-neutron'
- job-template:
@ -151,13 +131,9 @@
fail: true
- timestamps
publishers:
- sahara-logs
- console-log
- trigger-parameterized-builds:
- project: "integration-cleanup"
predefined-parameters:
"PREV_BUILD=$BUILD_NUMBER-$ZUUL_CHANGE-$ZUUL_PATCHSET\
\nPREV_JOB=$JOB_NAME
\nHOST_NAME=$NODE_NAME"
- trigger-cleanup
node: 'trusty-nova'
- project:

4
slave-scripts/dib.sh
View File

@ -235,7 +235,9 @@ write_tests_conf sahara/tests/integration/configs/itest.conf
run_tests
cat_logs /tmp/sahara
print_python_env /tmp/sahara
mv /tmp/sahara/logs $WORKSPACE
if [ "$FAILURE" != 0 ]; then
exit 1

37
slave-scripts/functions.sh
View File

@ -99,6 +99,7 @@ admin_tenant_name=ci" >> $conf_path
start_sahara() {
conf_path=$1
conf_dir=$(dirname $1)
mkdir logs
if [ "$ZUUL_BRANCH" == "stable/icehouse" ]
then
sahara_bin=sahara-api
@ -113,12 +114,12 @@ start_sahara() {
exit 1
fi
if [ "$ZUUL_BRANCH" == "master" -a \( "$PLUGIN_TYPE" == "vanilla2" -a "$hadoop_version" == "2-4" -o "$PLUGIN_TYPE" == "hdp2" -o "$PLUGIN_TYPE" == " transient" \) ]; then
screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file log-api.txt"
screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file logs/sahara-log-api.txt"
sleep 2
screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file log-engine-1.txt"
screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file log-engine-2.txt"
screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file logs/sahara-log-engine-1.txt"
screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file logs/sahara-log-engine-2.txt"
else
screen -dmS $sahara_bin /bin/bash -c "PYTHONUNBUFFERED=1 $sahara_bin --config-dir $conf_dir -d --log-file log.txt"
screen -dmS $sahara_bin /bin/bash -c "PYTHONUNBUFFERED=1 $sahara_bin --config-dir $conf_dir -d --log-file logs/sahara-log.txt"
fi
api_responding_timeout=30
@ -270,27 +271,9 @@ run_tests() {
fi
}
cat_logs() {
log_path=$1
echo "-----------Python integration env-----------"
cd $log_path && .tox/integration/bin/pip freeze
echo "-----------Python sahara env-----------"
cd $log_path && pip freeze
if [ "$ZUUL_BRANCH" == "master" -a \( "$PLUGIN_TYPE" == "vanilla2" -a "$hadoop_version" == "2-4" -o "$PLUGIN_TYPE" == "hdp2" \) ]; then
echo "-----------Sahara API Log------------"
cat $log_path/log-api.txt
echo "-------------------------------------"
echo "-----------Sahara Engine Log---------"
echo "-----------Engine-1 Log---------"
cat $log_path/log-engine-1.txt
echo "-------------------------------------"
echo "-----------Engine-2 Log---------"
cat $log_path/log-engine-2.txt
echo "-------------------------------------"
else
echo "-----------Sahara Log------------"
cat $log_path/log.txt
fi
print_python_env() {
sahara_workspace=$1
cd $sahara_workspace
.tox/integration/bin/pip freeze > logs/python-integration-env.txt
pip freeze > logs/python-system-env.txt
}

2
slave-scripts/gate-sahara.sh
View File

@ -114,7 +114,7 @@ write_tests_conf $WORKSPACE/sahara/tests/integration/configs/itest.conf
run_tests
cat_logs $WORKSPACE
print_python_env $WORKSPACE
if [ "$FAILURE" != 0 ]; then
exit 1