From 0a5838fa0992b265d5fcab7a5539c906d45e67ab Mon Sep 17 00:00:00 2001
From: Denis Egorenko <degorenko@mirantis.com>
Date: Wed, 3 Jun 2015 18:35:36 +0300
Subject: [PATCH] Rename all plugins to full plugin version
Rename short plugin names to full plugin names and add
OS version to job name.
Also sort by alphabetical functions-* files.
Change-Id: I5155613715fd2ced99034807f14f495487d2d8ac
---
config/devstack/local.sh | 30 ++---
config/{sahara => }/pip.conf | 0
config/zuul/layout.yaml | 70 +++++------
jenkins_job_builder/diskimage.yaml | 101 +++++----------
jenkins_job_builder/sahara.yaml | 121 +++++++-----------
slave-scripts/dib.sh | 112 +++++++----------
slave-scripts/functions-common.sh | 196 ++++++++++++++++-------------
slave-scripts/functions-dib.sh | 136 ++++++++------------
slave-scripts/gate-sahara.sh | 75 +++++------
slave-scripts/gate-ui-tests.sh | 4 +-
10 files changed, 370 insertions(+), 475 deletions(-)
rename config/{sahara => }/pip.conf (100%)
diff --git a/config/devstack/local.sh b/config/devstack/local.sh
index c2e4a5b0..4d2c18a7 100755
--- a/config/devstack/local.sh
+++ b/config/devstack/local.sh
@@ -24,13 +24,14 @@ else
USE_NEUTRON=false
fi
-VANILLA26_IMAGE_PATH=/home/ubuntu/images/sahara-vanilla-2.6.0-ubuntu-14.04.qcow2
-HDP2_IMAGE_PATH=/home/ubuntu/images/centos_6-6_hdp-2.qcow2
-CENTOS_CDH_IMAGE_PATH=/home/ubuntu/images/centos_sahara_cloudera_latest.qcow2
-UBUNTU_CDH_IMAGE_PATH=/home/ubuntu/images/ubuntu_sahara_cloudera_latest.qcow2
-SPARK_IMAGE_PATH=/home/ubuntu/images/sahara_spark_latest.qcow2
-MAPR_IMAGE_PATH=/home/ubuntu/images/ubuntu_mapr_latest.qcow2
-NATIVE_UBUNTU_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img
+VANILLA_2_6_0_IMAGE_PATH=/home/ubuntu/images/vanilla_2.6.0_u14.qcow2
+HDP_2_0_6_IMAGE_PATH=/home/ubuntu/images/hdp_2.0.6_c6.6.qcow2
+CENTOS_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_c6.6.qcow2
+UBUNTU_CDH_5_3_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.3.0_u12.qcow2
+UBUNTU_CDH_5_4_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.4.0_u12.qcow2
+SPARK_1_0_0_IMAGE_PATH=/home/ubuntu/images/spark_1.0.0_u14.qcow2
+MAPR_4_0_2_IMAGE_PATH=/home/ubuntu/images/mapr_4.0.2_u14.qcow2
+UBUNTU_12_04_IMAGE_PATH=/home/ubuntu/images/ubuntu-12.04-server-cloudimg-amd64-disk1.img
# setup ci tenant and ci users
CI_TENANT_ID=$(keystone tenant-create --name ci --description 'CI tenant' | grep -w id | get_field 2)
@@ -69,13 +70,14 @@ nova flavor-create --is-public true m1.small 2 1024 20 1
source $ADMIN_RCFILE ci-user ci
# add images for tests
-glance image-create --name ubuntu_vanilla_2.6_latest --file $VANILLA26_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu'
-glance image-create --name sahara_hdp_2_latest --file $HDP2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user'
-glance image-create --name centos_cdh_latest --file $CENTOS_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user"
-glance image-create --name ubuntu_cdh_latest --file $UBUNTU_CDH_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu"
-glance image-create --name sahara_spark_latest --file $SPARK_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu"
-glance image-create --name ubuntu_mapr_latest --file $MAPR_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu"
-glance image-create --name ubuntu-test-image --file $NATIVE_UBUNTU_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true
+glance image-create --name $(basename -s .qcow2 $VANILLA_2_6_0_IMAGE_PATH) --file $VANILLA_2_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.6.0'='True' --property '_sahara_tag_vanilla'='True' --property '_sahara_username'='ubuntu'
+glance image-create --name $(basename -s .qcow2 $HDP_2_0_6_IMAGE_PATH) --file $HDP_2_0_6_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_2.0.6'='True' --property '_sahara_tag_hdp'='True' --property '_sahara_username'='cloud-user'
+glance image-create --name $(basename -s .qcow2 $CENTOS_CDH_5_3_0_IMAGE_PATH) --file $CENTOS_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="cloud-user"
+glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_3_0_IMAGE_PATH) --file $UBUNTU_CDH_5_3_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.3.0'='True' --property '_sahara_tag_5'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu"
+glance image-create --name $(basename -s .qcow2 $UBUNTU_CDH_5_4_0_IMAGE_PATH) --file $UBUNTU_CDH_5_4_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.4.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu"
+glance image-create --name $(basename -s .qcow2 $SPARK_1_0_0_IMAGE_PATH) --file $SPARK_1_0_0_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.0.0'='True' --property '_sahara_username'="ubuntu"
+glance image-create --name $(basename -s .qcow2 $MAPR_4_0_2_IMAGE_PATH) --file $MAPR_4_0_2_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_4.0.2.mrv2'='True' --property '_sahara_username'="ubuntu"
+glance image-create --name ubuntu-test-image --file $UBUNTU_12_04_IMAGE_PATH --disk-format qcow2 --container-format bare --is-public=true
glance image-update --name ubuntu-12.04 --property '_sahara_tag_ci'='True' ubuntu-12.04-server-cloudimg-amd64-disk1
glance image-update --name ubuntu-14.04 trusty-server-cloudimg-amd64-disk1
diff --git a/config/sahara/pip.conf b/config/pip.conf
similarity index 100%
rename from config/sahara/pip.conf
rename to config/pip.conf
diff --git a/config/zuul/layout.yaml b/config/zuul/layout.yaml
index 6fd01918..f6aed1e6 100644
--- a/config/zuul/layout.yaml
+++ b/config/zuul/layout.yaml
@@ -27,51 +27,51 @@ pipelines:
projects:
- name: openstack/sahara
check:
- - gate-sahara-neutron-heat-vanilla_2.6
- - gate-sahara-neutron-direct-spark-aio
- - gate-sahara-neutron-direct-transient
- - gate-sahara-neutron-heat-transient
- - gate-sahara-neutron-heat-mapr
- - gate-sahara-nova-direct-cdh_5.3.0_ubuntu-aio
- - gate-sahara-nova-direct-cdh_5.3.0_centos-aio
- - gate-sahara-nova-direct-cdh_5.4.0-aio
- - gate-sahara-nova-heat-hdp_2
+ - gate-sahara-neutron-heat-vanilla_2.6.0-u14
+ - gate-sahara-neutron-direct-spark_1.0.0-u14-aio
+ - gate-sahara-neutron-direct-transient-u14
+ - gate-sahara-neutron-heat-transient-u14
+ - gate-sahara-neutron-heat-mapr_4.0.2.mrv2-u14
+ - gate-sahara-nova-direct-cdh_5.3.0-u12-aio
+ - gate-sahara-nova-direct-cdh_5.3.0-c6.6-aio
+ - gate-sahara-nova-direct-cdh_5.4.0-u12-aio
+ - gate-sahara-nova-heat-hdp_2.0.6-c6.6
- gate-sahara-pep8-trunk
# - tempest-sahara-tests
- name: openstack/python-saharaclient
check:
- - gate-saharaclient-neutron-heat-vanilla_2.6
- - gate-saharaclient-neutron-direct-spark-aio
- - gate-saharaclient-neutron-direct-transient
- - gate-saharaclient-neutron-heat-transient
- - gate-saharaclient-neutron-heat-mapr
- - gate-saharaclient-nova-direct-cdh_5.3.0_ubuntu-aio
- - gate-saharaclient-nova-direct-cdh_5.3.0_centos-aio
- - gate-saharaclient-nova-direct-cdh_5.4.0-aio
- - gate-saharaclient-nova-heat-hdp_2
+ - gate-saharaclient-neutron-heat-vanilla_2.6.0-u14
+ - gate-saharaclient-neutron-direct-spark_1.0.0-u14-aio
+ - gate-saharaclient-neutron-direct-transient-u14
+ - gate-saharaclient-neutron-heat-transient-u14
+ - gate-saharaclient-neutron-heat-mapr_4.0.2.mrv2-u14
+ - gate-saharaclient-nova-direct-cdh_5.3.0-u12-aio
+ - gate-saharaclient-nova-direct-cdh_5.3.0-c6.6-aio
+ - gate-saharaclient-nova-direct-cdh_5.4.0-u12-aio
+ - gate-saharaclient-nova-heat-hdp_2.0.6-c6.6
- gate-sahara-pep8-trunk
# - tempest-saharaclient-tests
- name: openstack/sahara-image-elements
check:
- - dib-neutron-heat-vanilla_2.6-ubuntu
- - dib-neutron-heat-vanilla_2.6-centos
- - dib-neutron-heat-mapr
- - dib-nova-direct-spark-aio
- - dib-nova-heat-hdp_2
- - dib-nova-direct-cdh_5.3.0-ubuntu-aio
- - dib-nova-direct-cdh_5.3.0-centos-aio
- - dib-nova-direct-cdh_5.4.0-ubuntu-aio
+ - dib-neutron-heat-vanilla_2.6.0-u14
+ - dib-neutron-heat-vanilla_2.6.0-c6.6
+ - dib-neutron-heat-mapr_4.0.2.mrv2-u14
+ - dib-neutron-direct-spark_1.0.0-u14-aio
+ - dib-nova-heat-hdp_2.0.6-c6.6
+ - dib-nova-direct-cdh_5.3.0-u12-aio
+ - dib-nova-direct-cdh_5.3.0-c6.6-aio
+ - dib-nova-direct-cdh_5.4.0-u12-aio
post:
- - dib-neutron-heat-vanilla_2.6-ubuntu
- - dib-neutron-heat-vanilla_2.6-centos
- - dib-neutron-heat-mapr
- - dib-nova-direct-spark-aio
- - dib-nova-heat-hdp_2
- - dib-nova-direct-cdh_5.3.0-ubuntu-aio
- - dib-nova-direct-cdh_5.3.0-centos-aio
- - dib-nova-direct-cdh_5.4.0-ubuntu-aio
+ - dib-neutron-heat-vanilla_2.6.0-u14
+ - dib-neutron-heat-vanilla_2.6.0-c6.6
+ - dib-neutron-heat-mapr_4.0.2.mrv2-u14
+ - dib-neutron-direct-spark_1.0.0-u14-aio
+ - dib-nova-heat-hdp_2.0.6-c6.6
+ - dib-nova-direct-cdh_5.3.0-u12-aio
+ - dib-nova-direct-cdh_5.3.0-c6.6-aio
+ - dib-nova-direct-cdh_5.4.0-u12-aio
- name: stackforge/sahara-ci-config
check:
@@ -91,7 +91,7 @@ jobs:
- all-files-match-any:
- ^doc/.*$
- ^.*\.rst$
- - name: ^.*-neutron-heat-mapr$
+ - name: ^.*-mapr_4.0.2.mrv2-u14$
branch: master
voting: false
- name: ^.*-cdh_5.4.0-.*$
diff --git a/jenkins_job_builder/diskimage.yaml b/jenkins_job_builder/diskimage.yaml
index 77a910f3..c4102f0b 100644
--- a/jenkins_job_builder/diskimage.yaml
+++ b/jenkins_job_builder/diskimage.yaml
@@ -1,101 +1,62 @@
-- job:
- name: 'dib-nova-heat-hdp_2'
- defaults: global
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh hdp_2"
-
- properties:
- - zeromq-event
-
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: 'trusty-nova'
-
-- job:
- name: 'dib-neutron-heat-mapr'
- defaults: global
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh mapr"
-
- properties:
- - zeromq-event
-
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: 'trusty-neutron'
-
- job-template:
- name: 'dib-nova-direct-{plugin}-{os}-aio'
+ name: 'dib-{network}-{engine}-{plugin}-{os}-aio'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}"
-
properties:
- zeromq-event
-
publishers:
- sahara-logs
- console-log
- trigger-cleanup
- node: 'trusty-nova'
-
-- job:
- name: 'dib-nova-direct-spark-aio'
- defaults: global
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin}"
-
- properties:
- - zeromq-event
-
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: 'trusty-nova'
+ node: 'trusty-{network}'
- job-template:
- name: 'dib-neutron-heat-vanilla_2.6-{os}'
+ name: 'dib-{network}-{engine}-{plugin}-{os}'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/dib.sh {plugin} {os}"
-
properties:
- zeromq-event
-
publishers:
- sahara-logs
- console-log
- trigger-cleanup
- node: 'trusty-neutron'
+ node: 'trusty-{network}'
- project:
name: sahara-image-elements
github-org: openstack
- plugin: cdh_5.3.0
+ network: neutron
+ engine: heat
+ plugin: vanilla_2.6.0
os:
- - ubuntu
- - centos
+ - u14
+ - c6.6
+
jobs:
- - 'dib-neutron-heat-vanilla_2.6-{os}'
- - 'dib-nova-heat-hdp_2'
- - 'dib-neutron-heat-mapr'
- - 'dib-nova-direct-spark-aio'
- - 'dib-nova-direct-{plugin}-{os}-aio'
- - 'dib-nova-direct-{plugin}-{os}-aio':
- plugin: cdh_5.4.0
- os: ubuntu
+ - 'dib-{network}-{engine}-{plugin}-{os}-aio'
+ - 'dib-{network}-{engine}-{plugin}-{os}-aio':
+ network: nova
+ engine: direct
+ plugin:
+ - cdh_5.3.0
+ - cdh_5.4.0
+ os:
+ - u12
+ - c6.6
+ - 'dib-{network}-{engine}-{plugin}-{os}-aio':
+ engine: direct
+ plugin: spark_1.0.0
+ os: u14
+ - 'dib-{network}-{engine}-{plugin}-{os}':
+ plugin: mapr_4.0.2.mrv2
+ os: u14
+ - 'dib-{network}-{engine}-{plugin}-{os}':
+ network: nova
+ plugin: hdp_2.0.6
+ os: c6.6
diff --git a/jenkins_job_builder/sahara.yaml b/jenkins_job_builder/sahara.yaml
index 6fb849bc..4465c26d 100644
--- a/jenkins_job_builder/sahara.yaml
+++ b/jenkins_job_builder/sahara.yaml
@@ -1,7 +1,6 @@
-- job:
- name: 'tempest-saharaclient-tests'
+- job-template:
+ name: 'tempest-{project_name}-tests'
defaults: global
- concurrent: false
builders:
- gerrit-git-prep
- ci-config-prep
@@ -10,26 +9,7 @@
- zeromq-event
- build-blocker:
blocking-jobs:
- - "tempest-sahara-tests"
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: trusty-neutron || trusty-nova
-
-- job:
- name: 'tempest-sahara-tests'
- defaults: global
- concurrent: false
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/tempest.sh"
- properties:
- - zeromq-event
- - build-blocker:
- blocking-jobs:
- - "tempest-saharaclient-tests"
+ - "tempest-.*-tests"
publishers:
- sahara-logs
- console-log
@@ -37,84 +17,72 @@
node: trusty-neutron || trusty-nova
- job-template:
- name: 'gate-saharaclient-neutron-{plugin-neutron}'
+ name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh"
+ - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
- node: 'trusty-neutron'
+ node: 'trusty-{network}'
- job-template:
- name: 'gate-saharaclient-nova-{plugin-nova_network}'
+ name: 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio'
defaults: global
builders:
- gerrit-git-prep
- ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-saharaclient.sh"
+ - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh"
properties:
- zeromq-event
publishers:
- sahara-logs
- console-log
- trigger-cleanup
- node: 'trusty-nova'
-
-- job-template:
- name: 'gate-sahara-neutron-{plugin-neutron}'
- defaults: global
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh"
- properties:
- - zeromq-event
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: 'trusty-neutron'
-
-- job-template:
- name: 'gate-sahara-nova-{plugin-nova_network}'
- defaults: global
- builders:
- - gerrit-git-prep
- - ci-config-prep
- - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara.sh"
- properties:
- - zeromq-event
- publishers:
- - sahara-logs
- - console-log
- - trigger-cleanup
- node: 'trusty-nova'
+ node: 'trusty-{network}'
- project:
- name: sahara
- github-org: openstack
- plugin-neutron:
- - heat-vanilla_2.6
- - direct-spark-aio
- - direct-transient
- - heat-transient
- - heat-mapr
- plugin-nova_network:
- - direct-cdh_5.3.0_ubuntu-aio
- - direct-cdh_5.3.0_centos-aio
- - direct-cdh_5.4.0-aio
- - heat-hdp_2
+ name: main
+ project_name:
+ - sahara
+ - saharaclient
+ network: nova
+ engine: direct
+ plugin:
+ - cdh_5.3.0
+ - cdh_5.4.0
+ os:
+ - u12
+ - c6.6
jobs:
- - 'gate-sahara-neutron-{plugin-neutron}'
- - 'gate-sahara-nova-{plugin-nova_network}'
- - 'gate-saharaclient-neutron-{plugin-neutron}'
- - 'gate-saharaclient-nova-{plugin-nova_network}'
+ - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio'
+ - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
+ engine: heat
+ plugin:
+ - vanilla_2.6.0
+ - transient
+ - mapr_4.0.2
+ os: u14
+ network: neutron
+ - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
+ plugin: transient
+ os: u14
+ network: neutron
+ - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}-aio':
+ plugin: spark_1.0.0
+ os: u14
+ network: neutron
+ - 'gate-{project_name}-{network}-{engine}-{plugin}-{os}':
+ network: nova
+ engine: heat
+ plugin: hdp_2.0.6
+ os: c6.6
+ - 'tempest-{project_name}-tests'
- job:
name: gate-sahara-pep8-trunk
@@ -125,7 +93,6 @@
\n<p>This job runs pep8 check using trunk version of hacking"
disabled: false
node: trusty-neutron || trusty-nova
-
builders:
- gerrit-git-prep
- ci-config-prep
diff --git a/slave-scripts/dib.sh b/slave-scripts/dib.sh
index ea45f74c..6c57dae6 100755
--- a/slave-scripts/dib.sh
+++ b/slave-scripts/dib.sh
@@ -10,20 +10,17 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM}
cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH"
SAHARA_PATH="/tmp/sahara"
-sahara_conf_path="$SAHARA_PATH/etc/sahara/sahara.conf"
+sahara_conf_file="$SAHARA_PATH/etc/sahara/sahara.conf"
sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci
engine=$(echo $JOB_NAME | awk -F '-' '{ print $3 }')
-job_type="$1"
-image_type=${2:-ubuntu}
-# Image names
-vanilla_two_six_image=$HOST-sahara-vanilla-${image_type}-${ZUUL_CHANGE}-hadoop_2.6
-hdp_two_image=$HOST-sahara-hdp-centos-${ZUUL_CHANGE}-hadoop_2
-spark_image=$HOST-sahara-spark-ubuntu-${ZUUL_CHANGE}
-cdh_image=$HOST-${image_type}-cdh-${ZUUL_CHANGE}
-cdh_5_4_0_image=$HOST-${image_type}-cdh_5.4.0-${ZUUL_CHANGE}
-mapr_402mrv2_image=$HOST-${image_type}-mapr-${ZUUL_CHANGE}
+plugin="$1"
+os="$2"
+image_name=${HOST}_${plugin}_${os}_${ZUUL_CHANGE}
+eval ${plugin//./_}_image=$image_name
+mode="aio"
+sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ')
# Clone Sahara
git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BRANCH
@@ -31,79 +28,66 @@ git clone https://review.openstack.org/openstack/sahara $SAHARA_PATH -b $ZUUL_BR
# make verbose the scripts execution of disk-image-create
export DIB_DEBUG_TRACE=1
-case $job_type in
- vanilla_2.6)
- if [ "${image_type}" == 'centos' ]; then
- username='cloud-user'
- else
- username='ubuntu'
- fi
+if [ "${os}" == 'c6.6' ]; then
+ username="cloud-user"
+ os_type="centos"
+else
+ username="ubuntu"
+ os_type="ubuntu"
+fi
- env ${image_type}_vanilla_hadoop_2_6_image_name=${vanilla_two_six_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $image_type -v 2.6
- check_error_code $? ${vanilla_two_six_image}.qcow2
- upload_image "vanilla-2.6" "${username}" ${vanilla_two_six_image}
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml"
- insert_scenario_value $tests_config_file vanilla_two_six_image
+case $plugin in
+ vanilla_2.6.0)
+ env ${os_type}_vanilla_hadoop_2_6_image_name=${vanilla_2_6_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p vanilla -i $os_type -v 2.6
+ check_error_code $? ${vanilla_2_6_0_image}.qcow2
+ upload_image "${plugin}" "${username}" ${vanilla_2_6_0_image}
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml"
;;
- spark)
- env ubuntu_spark_image_name=${spark_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark
- check_error_code $? ${spark_image}.qcow2
- upload_image "spark" "ubuntu" ${spark_image}
- tests_config_file="$sahara_templates_path/spark-1.0.0.yaml"
- insert_scenario_value $tests_config_file spark_image
- insert_config_value $sahara_conf_path DEFAULT plugins spark
+ spark_1.0.0)
+ env ubuntu_spark_image_name=${spark_1_0_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p spark
+ check_error_code $? ${spark_1_0_0_image}.qcow2
+ upload_image "${plugin}" "${username}" ${spark_1_0_0_image}
+ scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml"
;;
- hdp_2)
- env centos_hdp_hadoop_2_image_name=${hdp_two_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2
- check_error_code $? ${hdp_two_image}.qcow2
- upload_image "hdp2" "cloud-user" ${hdp_two_image}
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml"
- insert_scenario_value $tests_config_file hdp_two_image
+ hdp_2.0.6)
+ env centos_hdp_hadoop_2_image_name=${hdp_2_0_6_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p hdp -v 2
+ check_error_code $? ${hdp_2_0_6_image}.qcow2
+ upload_image "${plugin}" "${username}" ${hdp_2_0_6_image}
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml"
;;
cdh_5.3.0)
- if [ "${image_type}" == 'centos' ]; then
- username='cloud-user'
- else
- username='ubuntu'
- fi
- env cloudera_5_3_${image_type}_image_name=${cdh_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $image_type -v 5.3
- check_error_code $? ${cdh_image}.qcow2
- upload_image "cdh_5.3.0" ${username} ${cdh_image}
- tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml"
- insert_config_value $sahara_conf_path DEFAULT plugins cdh
- insert_scenario_value $tests_config_file cdh_image
+ env cloudera_5_3_${os_type}_image_name=${cdh_5_3_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $os_type -v 5.3
+ check_error_code $? ${cdh_5_3_0_image}.qcow2
+ upload_image "${plugin}" "${username}" ${cdh_5_3_0_image}
+ scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml"
;;
cdh_5.4.0)
env cloudera_5_4_ubuntu_image_name=${cdh_5_4_0_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i ubuntu -v 5.4
check_error_code $? ${cdh_5_4_0_image}.qcow2
- upload_image "cdh_5.4.0" "ubuntu" ${cdh_5_4_0_image}
- tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml"
- insert_config_value $sahara_conf_path DEFAULT plugins cdh
- insert_scenario_value $tests_config_file cdh_5_4_0_image
+ upload_image "${plugin}" "${username}" ${cdh_5_4_0_image}
+ scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml"
;;
- mapr)
- env mapr_ubuntu_image_name=${mapr_402mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu
- check_error_code $? ${mapr_402mrv2_image}.qcow2
- upload_image "mapr" "ubuntu" ${mapr_402mrv2_image}
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
- insert_config_value $sahara_conf_path DEFAULT plugins mapr
- insert_scenario_value $tests_config_file mapr_402mrv2_image
+ mapr_4.0.2.mrv2)
+ env mapr_ubuntu_image_name=${mapr_4_0_2_mrv2_image} SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p mapr -i ubuntu
+ check_error_code $? ${mapr_4_0_2_mrv2_image}.qcow2
+ upload_image "${plugin}" "${username}" ${mapr_4_0_2_mrv2_image}
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
;;
esac
cd $SAHARA_PATH
sudo pip install . --no-cache-dir
enable_pypi
-write_sahara_main_conf "$sahara_conf_path" "$engine"
-write_tests_conf "$tests_config_file" "$cluster_name"
-start_sahara "$sahara_conf_path" && run_tests "$tests_config_file"
+write_sahara_main_conf "$sahara_conf_file" "$engine" "$sahara_plugin"
+write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name"
+start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file"
print_python_env
-cleanup_image "$job_type" "$image_type"
+cleanup_image "$plugin" "$os"
diff --git a/slave-scripts/functions-common.sh b/slave-scripts/functions-common.sh
index 97b93931..4c99358c 100755
--- a/slave-scripts/functions-common.sh
+++ b/slave-scripts/functions-common.sh
@@ -1,15 +1,9 @@
#!/bin/bash -xe
-sahara_configs_path=$WORKSPACE/sahara-ci-config/config/sahara
-ci_flavor_id=\'20\'
-medium_flavor_id=\'3\'
-large_flavor_id=\'4\'
-
-enable_pypi() {
- mkdir -p ~/.pip
- export PIP_USE_MIRRORS=True
- cp $sahara_configs_path/pip.conf ~/.pip/pip.conf
-}
+configs_path=$WORKSPACE/sahara-ci-config/config
+eval ci_flavor_id="\'20\'"
+eval medium_flavor_id="\'3\'"
+eval large_flavor_id="\'4\'"
conf_has_option() {
local file=$1
@@ -21,6 +15,19 @@ conf_has_option() {
[ -n "$line" ]
}
+enable_pypi() {
+ mkdir -p ~/.pip
+ export PIP_USE_MIRRORS=True
+ cp $configs_path/pip.conf ~/.pip/pip.conf
+}
+
+failure() {
+ local reason=$1
+ print_python_env
+ echo "$reason"
+ exit 1
+}
+
insert_config_value() {
local file=$1
local section=$2
@@ -46,30 +53,21 @@ $option = $value
}
insert_scenario_value() {
- local config=$1
- local value=$2
- sed -i "s/%${value}%/${!value}/g" $config
-}
+ local file=$1
+ local main_key=$2
+ local stop_key=$3
+ local sub_key=$4
+ local value=$5
+ local old_value=$6
-write_sahara_main_conf() {
- local conf_path=$1
- local engine=$2
- insert_config_value $conf_path DEFAULT infrastructure_engine $engine
- insert_config_value $conf_path DEFAULT api_workers 4
- insert_config_value $conf_path DEFAULT use_identity_api_v3 true
- insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON
- insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30
- insert_config_value $conf_path DEFAULT node_domain ci
- insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
- insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/
- insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/
- insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME
- insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD
- insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME
+ [[ -z $main_key || -z $sub_key ]] && return
- echo "----------- sahara.conf -----------"
- cat $conf_path
- echo "--------------- end ---------------"
+ if ! scenario_has_option "$file" "$main_key" "$stop_key" "$sub_key"; then
+ echo "No such keys: $main_key -> $sub_key in scenario $file file. Skip setting value $value"
+ else
+ local sep=$(echo -ne "\x01")
+ sed -i -e '/'${main_key}':/,/'${stop_key}'/ s'${sep}'\([ \t]'${sub_key}':[ \t]\).*'${old_value}'.*$'${sep}'\1'${value}${sep} $file
+ fi
}
print_python_env() {
@@ -78,58 +76,6 @@ print_python_env() {
pip freeze > $WORKSPACE/logs/python-system-env.txt
}
-failure() {
- local reason=$1
- print_python_env
- echo "$reason"
- exit 1
-}
-
-start_sahara() {
- local conf_path=$1
- local conf_dir=$(dirname $1)
- mkdir $WORKSPACE/logs
- sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed"
- if [ "$DISTRIBUTE_MODE" == "True" ]; then
- screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt"
- sleep 2
- screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt"
- screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt"
- else
- screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt"
- fi
-
- api_responding_timeout=30
- if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then
- failure "Sahara API failed to respond within ${api_responding_timeout} seconds"
- fi
-}
-
-write_tests_conf() {
- local test_conf=$1
- local cluster_name=$2
- local addr=$(ifconfig eth0| awk -F ' *|:' '/inet addr/{print $4}')
- if [ "$USE_NEUTRON" == "true" ]; then
- NETWORK="neutron"
- else
- NETWORK="nova-network"
- fi
- local test_scenario_credentials=$(dirname $1)/credentials.yaml
- insert_scenario_value $test_scenario_credentials OS_USERNAME
- insert_scenario_value $test_scenario_credentials OS_PASSWORD
- insert_scenario_value $test_scenario_credentials OS_TENANT_NAME
- insert_scenario_value $test_scenario_credentials OPENSTACK_HOST
- insert_scenario_value $test_scenario_credentials NETWORK
- insert_scenario_value $test_conf cluster_name
- insert_scenario_value $test_conf ci_flavor_id
- insert_scenario_value $test_conf medium_flavor_id
- insert_scenario_value $test_conf large_flavor_id
-
- echo "----------- tests config -----------"
- cat $test_conf
- echo "---------------- end ---------------"
-}
-
run_tests() {
local config=$1
local concurrency=${2:-"1"}
@@ -143,3 +89,85 @@ run_tests() {
STATUS=$(grep "\ -\ Failed" tox.log | awk '{print $3}')
if [ "$STATUS" != "0" ]; then failure "Integration tests have failed"; fi
}
+
+scenario_has_option() {
+ local file=$1
+ local main_key=$2
+ local stop_key=$3
+ local sub_key=$4
+ local line
+
+ line=$(sed -ne "/$main_key\:/,/$stop_key/ { /[ \t]$sub_key:/ p; }" "$file")
+ [ -n "$line" ]
+}
+
+start_sahara() {
+ local conf_path=$1
+ local conf_dir=$(dirname $1)
+ local mode=$2
+ mkdir $WORKSPACE/logs
+ sahara-db-manage --config-file $conf_path upgrade head || failure "Command 'sahara-db-manage' failed"
+ if [ "$mode" == "distribute" ]; then
+ screen -dmS sahara-api /bin/bash -c "PYTHONUNBUFFERED=1 sahara-api --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-api.txt"
+ sleep 2
+ screen -dmS sahara-engine_1 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-1.txt"
+ screen -dmS sahara-engine_2 /bin/bash -c "PYTHONUNBUFFERED=1 sahara-engine --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log-engine-2.txt"
+ else
+ screen -dmS sahara-all /bin/bash -c "PYTHONUNBUFFERED=1 sahara-all --config-dir $conf_dir -d --log-file $WORKSPACE/logs/sahara-log.txt"
+ fi
+
+ api_responding_timeout=30
+ if ! timeout ${api_responding_timeout} sh -c "while ! curl -s http://127.0.0.1:8386/v1.1/ 2>/dev/null | grep -q 'Authentication required' ; do sleep 1; done"; then
+ failure "Sahara API failed to respond within ${api_responding_timeout} seconds"
+ fi
+}
+
+write_sahara_main_conf() {
+ local conf_path=$1
+ local engine=$2
+ local plugin=$3
+ insert_config_value $conf_path DEFAULT infrastructure_engine $engine
+ insert_config_value $conf_path DEFAULT api_workers 4
+ insert_config_value $conf_path DEFAULT use_identity_api_v3 true
+ insert_config_value $conf_path DEFAULT use_neutron $USE_NEUTRON
+ insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30
+ insert_config_value $conf_path DEFAULT node_domain ci
+ insert_config_value $conf_path DEFAULT plugins $plugin
+ insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
+ insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/
+ insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/
+ insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME
+ insert_config_value $conf_path keystone_authtoken admin_password $OS_PASSWORD
+ insert_config_value $conf_path keystone_authtoken admin_tenant_name $OS_TENANT_NAME
+
+ echo "----------- sahara.conf -----------"
+ cat $conf_path
+ echo "--------------- end ---------------"
+}
+
+write_tests_conf() {
+ local test_conf=$1
+ local cluster_name=$2
+ local image_name=$3
+ local os_auth_url="http://$OPENSTACK_HOST:5000/v2.0/"
+ if [ "$USE_NEUTRON" == "true" ]; then
+ NETWORK="neutron"
+ else
+ NETWORK="nova-network"
+ fi
+ local test_scenario_credentials=$(dirname $1)/credentials.yaml
+ insert_scenario_value $test_scenario_credentials credentials "" os_username $OS_USERNAME
+ insert_scenario_value $test_scenario_credentials credentials "" os_password $OS_PASSWORD
+ insert_scenario_value $test_scenario_credentials credentials "" os_tenant $OS_TENANT_NAME
+ insert_scenario_value $test_scenario_credentials credentials "" os_auth_url $os_auth_url
+ insert_scenario_value $test_scenario_credentials network "" "type" $NETWORK
+ insert_scenario_value $test_conf clusters node_group_templates image $image_name
+ insert_scenario_value $test_conf cluster "" name $cluster_name
+ insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $ci_flavor_id ci_flavor_id
+ insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $medium_flavor_id medium_flavor_id
+ insert_scenario_value $test_conf node_group_templates "\\$" flavor_id $large_flavor_id large_flavor_id
+
+ echo "----------- tests config -----------"
+ cat $test_conf
+ echo "---------------- end ---------------"
+}
diff --git a/slave-scripts/functions-dib.sh b/slave-scripts/functions-dib.sh
index 8cc7fc66..7c43e180 100755
--- a/slave-scripts/functions-dib.sh
+++ b/slave-scripts/functions-dib.sh
@@ -2,55 +2,6 @@
CUR_IMAGE=none
-register_new_image() {
- local image_name=$1
- local image_properties=$2
- glance image-create --name $1 --file $1.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties
-}
-
-delete_image() {
- # "|| true" here, to avoid error code producing in case of missing image
- glance image-delete $1 || true
-}
-
-upload_image() {
- local plugin=$1
- local username=$2
- local image=$3
- delete_image "$image"
-
- case "$plugin" in
- vanilla-2.4)
- image_properties="--property _sahara_tag_2.4.1=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
- ;;
- vanilla-2.6)
- image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
- ;;
- hdp2)
- image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}"
- ;;
- cdh_5.3.0)
- image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
- ;;
- cdh_5.4.0)
- image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
- ;;
- spark)
- image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}"
- ;;
- mapr)
- image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}"
- ;;
- esac
- register_new_image "$image" "$image_properties"
- CUR_IMAGE="$image"
-}
-
-rename_image() {
- # 1 - source image, 2 - target image
- glance image-update $1 --name $2
-}
-
check_error_code() {
if [ "$1" != "0" -o ! -f "$2" ]; then
echo "$2 image isn't build"
@@ -58,6 +9,22 @@ check_error_code() {
fi
}
+cleanup_image() {
+ local plugin=$1
+ local os=$2
+ if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then
+ delete_image "$CUR_IMAGE"
+ else
+ delete_image ${plugin}_${os}
+ rename_image "$CUR_IMAGE" ${plugin}_${os}
+ fi
+}
+
+delete_image() {
+ # "|| true" here, to avoid error code producing in case of missing image
+ glance image-delete $1 || true
+}
+
failure() {
local reason=$1
echo "$reason"
@@ -66,37 +33,42 @@ failure() {
exit 1
}
-cleanup_image() {
- local job_type=$1
- local os=$2
- if [ "$ZUUL_PIPELINE" == "check" -o "$ZUUL_BRANCH" != "master" ]; then
- delete_image "$CUR_IMAGE"
- else
- case $job_type in
- vanilla_2.6)
- delete_image ${os}_vanilla_2.6_latest
- rename_image "$CUR_IMAGE" ${os}_vanilla_2.6_latest
- ;;
- hdp_2)
- delete_image sahara_hdp_2_latest
- rename_image "$CUR_IMAGE" sahara_hdp_2_latest
- ;;
- cdh_5.3.0)
- delete_image ${os}_cdh_latest
- rename_image "$CUR_IMAGE" ${os}_cdh_latest
- ;;
- cdh_5.4.0)
- delete_image ${os}_cdh_5.4.0_latest
- rename_image "$CUR_IMAGE" ${os}_cdh_5.4.0_latest
- ;;
- spark)
- delete_image sahara_spark_latest
- rename_image "$CUR_IMAGE" sahara_spark_latest
- ;;
- mapr)
- delete_image ubuntu_mapr_latest
- rename_image "$CUR_IMAGE" ubuntu_mapr_latest
- ;;
- esac
- fi
+register_new_image() {
+ local image_name=$1
+ local image_properties=$2
+ glance image-create --name $image_name --file $image_name.qcow2 --disk-format qcow2 --container-format bare --is-public=true --property '_sahara_tag_ci'='True' $image_properties
+}
+
+rename_image() {
+ # 1 - source image, 2 - target image
+ glance image-update $1 --name $2
+}
+
+upload_image() {
+ local plugin=$1
+ local username=$2
+ local image=$3
+ delete_image "$image"
+ case "$plugin" in
+ vanilla_2.6.0)
+ image_properties="--property _sahara_tag_2.6.0=True --property _sahara_tag_vanilla=True --property _sahara_username=${username}"
+ ;;
+ hdp_2.0.6)
+ image_properties="--property _sahara_tag_2.0.6=True --property _sahara_tag_hdp=True --property _sahara_username=${username}"
+ ;;
+ cdh_5.3.0)
+ image_properties="--property _sahara_tag_5.3.0=True --property _sahara_tag_5=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
+ ;;
+ cdh_5.4.0)
+ image_properties="--property _sahara_tag_5.4.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}"
+ ;;
+ spark_1.0.0)
+ image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.0.0=True --property _sahara_username=${username}"
+ ;;
+ mapr_4.0.2.mrv2)
+ image_properties="--property _sahara_tag_mapr=True --property _sahara_tag_4.0.2.mrv2=True --property _sahara_username=${username}"
+ ;;
+ esac
+ register_new_image "$image" "$image_properties"
+ CUR_IMAGE="$image"
}
diff --git a/slave-scripts/gate-sahara.sh b/slave-scripts/gate-sahara.sh
index 452fbd6e..4558ad12 100755
--- a/slave-scripts/gate-sahara.sh
+++ b/slave-scripts/gate-sahara.sh
@@ -9,69 +9,50 @@ CLUSTER_HASH=${CLUSTER_HASH:-$RANDOM}
cluster_name="$HOST-$ZUUL_CHANGE-$CLUSTER_HASH"
SAHARA_PATH=${1:-$WORKSPACE}
-sahara_conf_path=$SAHARA_PATH/etc/sahara/sahara.conf
+sahara_conf_file=$SAHARA_PATH/etc/sahara/sahara.conf
sahara_templates_path=$SAHARA_PATH/etc/scenario/sahara-ci
-job_type=$(echo $JOB_NAME | awk -F '-' '{ print $5 }')
engine_type=$(echo $JOB_NAME | awk -F '-' '{ print $4 }')
+plugin=$(echo $JOB_NAME | awk -F '-' '{ print $5 }')
+os=$(echo $JOB_NAME | awk -F '-' '{ print $6 }')
+image_name=${plugin}_${os}
+mode="aio"
+sahara_plugin=$(echo $plugin | awk -F '_' '{ print $1 } ')
-# Image names
-hdp_two_image=sahara_hdp_2_latest
-vanilla_two_six_image=ubuntu_vanilla_2.6_latest
-spark_image=sahara_spark_latest
-cdh_5_3_0_centos_image=centos_cdh_latest
-cdh_5_3_0_ubuntu_image=ubuntu_cdh_latest
-cdh_5_4_0_image=ubuntu_cdh_5.4.0_latest
-mapr_402mrv2_image=ubuntu_mapr_latest
-
-case $job_type in
- hdp_2)
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/hdp-2.0.6.yaml"
- insert_scenario_value $tests_config_file hdp_two_image
+case $plugin in
+ hdp_2.0.6)
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/hdp-2.0.6.yaml"
;;
- vanilla_2.6)
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/vanilla-2.6.0.yaml"
- insert_scenario_value $tests_config_file vanilla_two_six_image
+ vanilla_2.6.0)
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/vanilla-2.6.0.yaml"
;;
transient)
+ # transient is using image with latest vanilla version
+ image_name=vanilla_2.6.0_u14
concurrency=3
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/transient.yaml"
- insert_scenario_value $tests_config_file vanilla_two_six_image
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/transient.yaml"
;;
- cdh_5.3.0*)
- insert_config_value $sahara_conf_path DEFAULT plugins cdh
- if [[ "$job_type" =~ centos ]]; then
- cdh_image=$cdh_5_3_0_centos_image
- else
- cdh_image=$cdh_5_3_0_ubuntu_image
- fi
- tests_config_file="$sahara_templates_path/cdh-5.3.0.yaml"
- insert_scenario_value $tests_config_file cdh_image
+ cdh_5.3.0)
+ scenario_conf_file="$sahara_templates_path/cdh-5.3.0.yaml"
;;
cdh_5.4.0)
- insert_config_value $sahara_conf_path DEFAULT plugins cdh
- tests_config_file="$sahara_templates_path/cdh-5.4.0.yaml"
- insert_scenario_value $tests_config_file cdh_5_4_0_image
+ scenario_conf_file="$sahara_templates_path/cdh-5.4.0.yaml"
;;
- spark)
- insert_config_value $sahara_conf_path DEFAULT plugins spark
- tests_config_file="$sahara_templates_path/spark-1.0.0.yaml"
- insert_scenario_value $tests_config_file spark_image
+ spark_1.0.0)
+ scenario_conf_file="$sahara_templates_path/spark-1.0.0.yaml"
;;
- mapr)
- insert_config_value $sahara_conf_path DEFAULT plugins mapr
- DISTRIBUTE_MODE=True
- tests_config_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
- insert_scenario_value $tests_config_file mapr_402mrv2_image
+ mapr_4.0.2.mrv2)
+ mode=distribute
+ scenario_conf_file="$sahara_templates_path/mapr-4.0.2.mrv2.yaml"
;;
esac
sudo pip install . --no-cache-dir
enable_pypi
-write_sahara_main_conf "$sahara_conf_path" "$engine_type"
-write_tests_conf "$tests_config_file" "$cluster_name"
-start_sahara "$sahara_conf_path" && run_tests "$tests_config_file" "$concurrency"
+write_sahara_main_conf "$sahara_conf_file" "$engine_type" "$sahara_plugin"
+write_tests_conf "$scenario_conf_file" "$cluster_name" "$image_name"
+start_sahara "$sahara_conf_file" "$mode" && run_tests "$scenario_conf_file" "$concurrency"
print_python_env
diff --git a/slave-scripts/gate-ui-tests.sh b/slave-scripts/gate-ui-tests.sh
index 8d36f9de..4167441e 100755
--- a/slave-scripts/gate-ui-tests.sh
+++ b/slave-scripts/gate-ui-tests.sh
@@ -49,12 +49,12 @@ floating_ip_pool = 'public'
keystone_url = 'http://$OPENSTACK_HOST:5000/v2.0'
await_element = 120
image_name_for_register = '$TEST_IMAGE'
-image_name_for_edit = "sahara-itests-ci-vanilla-image"
+image_name_for_edit = 'sahara-itests-ci-vanilla-image'
security_groups = default
[vanilla]
skip_plugin_tests = False
skip_edp_test = False
-base_image = "sahara-itests-ci-vanilla-image"
+base_image = 'sahara-itests-ci-vanilla-image'
[hdp]
skip_plugin_tests = False
hadoop_version = '1.3.2'