Support dependent changes in gerrit-git-prep.

Also added full hostname support to gerrit-git-prep -- 'review.openstack.org'
instead of 'openstack'.  This means it can be used with 'review-dev.openstack.org'
for instance.

Updated the jenkins job filler to use a 'review_site' parameter to facilitate that.

Updated the jenknis jobs filler to use slightly different variables
  review_site
  github_org
  publisher_site
which are more explicit about their purpose than 'site' and 'host'.

Make the docs/tarball publishers require a site param.

Change-Id: If24be8f7a74c09b215f94d371d5f09a695e2c098
This commit is contained in:
James E. Blair 2012-05-24 16:42:34 +00:00
parent 735715899c
commit 3c660d675e
19 changed files with 187 additions and 89 deletions

View File

@ -21,28 +21,33 @@ import hashlib
import yaml
import sys
import xml.etree.ElementTree as XML
from xml.dom import minidom
import pycurl
import jenkins
import ConfigParser
from StringIO import StringIO
import re
parser = argparse.ArgumentParser()
subparser = parser.add_subparsers(help='update or delete job', dest='command')
subparser = parser.add_subparsers(help='update, test or delete job', dest='command')
parser_update = subparser.add_parser('update')
parser_update.add_argument('file', help='YAML file for update', type=file)
parser_update = subparser.add_parser('test')
parser_update.add_argument('file', help='YAML file for test', type=file)
parser_delete = subparser.add_parser('delete')
parser_delete.add_argument('name', help='name of job')
parser.add_argument('--conf', dest='conf', help='Configuration file')
options = parser.parse_args()
if options.conf:
conf = options.conf
conf = options.conf
else:
conf = 'jenkins_jobs.ini'
conf = 'jenkins_jobs.ini'
conffp = open(conf, 'r')
config = ConfigParser.ConfigParser()
config.readfp(conffp)
if not options.command == 'test':
conffp = open(conf, 'r')
config = ConfigParser.ConfigParser()
config.readfp(conffp)
class YamlParser(object):
def __init__(self, yfile):
@ -115,8 +120,13 @@ In modules/jenkins_jobs"
def md5(self):
return hashlib.md5(self.output()).hexdigest()
# Pretty printing ideas from http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python
pretty_text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
def output(self):
return XML.tostring(self.xml)
out = minidom.parseString(XML.tostring(self.xml)).toprettyxml(indent=' ')
return self.pretty_text_re.sub('>\g<1></', out)
class CacheStorage(object):
def __init__(self):
@ -170,14 +180,18 @@ def delete_job():
remote_jenkins = Jenkins(config.get('jenkins','url'), config.get('jenkins','user'), config.get('jenkins','password'))
remote_jenkins.delete_job(options.name)
def update_job():
def update_job(test = False):
yparse = YamlParser(options.file)
cache = CacheStorage()
remote_jenkins = Jenkins(config.get('jenkins','url'), config.get('jenkins','user'), config.get('jenkins','password'))
if not test:
remote_jenkins = Jenkins(config.get('jenkins','url'), config.get('jenkins','user'), config.get('jenkins','password'))
while True:
try:
xml = yparse.get_next_xml()
job = yparse.get_name()
if test:
print xml.output()
continue
md5 = xml.md5()
if remote_jenkins.is_job(job) and not cache.is_cached(job):
old_md5 = remote_jenkins.get_job_md5(job)
@ -193,4 +207,6 @@ if options.command == 'delete':
delete_job()
elif options.command == 'update':
update_job()
elif options.command == 'test':
update_job(True)

View File

@ -41,11 +41,7 @@ class builders(object):
self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-docs.sh')
def _gerrit_git_prep(self, xml_parent):
if self.data['main'].has_key('host'):
site = self.data['main']['host']
else:
site = self.data['main']['site']
self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/gerrit-git-prep.sh {site}'.format(site=site))
self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/gerrit-git-prep.sh {site}'.format(site=self.data['main']['review_site']))
def _pep8(self, xml_parent):
self._add_script(xml_parent, 'tox -v -epep8 | tee pep8.txt')

View File

@ -27,7 +27,7 @@ class properties(object):
properties = XML.SubElement(xml_parent, 'properties')
github = XML.SubElement(properties, 'com.coravy.hudson.plugins.github.GithubProjectProperty')
github_url = XML.SubElement(github, 'projectUrl')
github_url.text = "https://github.com/{site}/{project}".format(site=main['site'], project=main['project'])
github_url.text = "https://github.com/{org}/{project}".format(org=main['github_org'], project=main['project'])
throttle = XML.SubElement(properties, 'hudson.plugins.throttleconcurrents.ThrottleJobProperty')
XML.SubElement(throttle, 'maxConcurrentPerNode').text = '0'
XML.SubElement(throttle, 'maxConcurrentTotal').text = '0'

View File

@ -14,7 +14,8 @@
# under the License.
# Jenkins Job module for docs publishers
# No additional YAML needed
# publish:
# site: 'glance.openstack.org'
import xml.etree.ElementTree as XML
@ -24,9 +25,10 @@ class publisher_docs(object):
def gen_xml(self, xml_parent):
main = self.data['main']
publish = self.data['publisher']
publishers = XML.SubElement(xml_parent, 'publishers')
scp = XML.SubElement(publishers, 'be.certipost.hudson.plugin.SCPRepositoryPublisher')
XML.SubElement(scp, 'siteName').text = '{proj}.{site}.org'.format(proj=main['project'], site=main['site'])
XML.SubElement(scp, 'siteName').text = publish['site']
entries = XML.SubElement(scp, 'entries')
entry = XML.SubElement(entries, 'be.certipost.hudson.plugin.Entry')
XML.SubElement(entry, 'filePath').text = 'docs/{proj}'.format(proj=main['project'])

View File

@ -14,11 +14,9 @@
# under the License.
# Jenkins Job module for tarball publishers
# To use you can optionally add the following into your YAML:
# publisher:
# uploadProject: 'glance'
#
# If you do not add this it will default to the project specified in the YAML
# To use you add the following into your YAML:
# publish:
# site: 'glance.openstack.org'
import xml.etree.ElementTree as XML
@ -27,19 +25,16 @@ class publisher_tarball(object):
self.data = data
def gen_xml(self, xml_parent):
if self.data.has_key('publisher') and self.data['publisher'].has_key('upload_project'):
project = self.data['publisher']['upload_project']
else:
project = self.data['main']['project']
site = self.data['publisher']['site']
publishers = XML.SubElement(xml_parent, 'publishers')
archiver = XML.SubElement(publishers, 'hudson.tasks.ArtifactArchiver')
XML.SubElement(archiver, 'artifacts').text = 'dist/*.tar.gz'
XML.SubElement(archiver, 'latestOnly').text = 'false'
scp = XML.SubElement(publishers, 'be.certipost.hudson.plugin.SCPRepositoryPublisher')
XML.SubElement(scp, 'siteName').text = '{proj}.{site}.org'.format(proj=project, site=self.data['main']['site'])
XML.SubElement(scp, 'siteName').text = site
entries = XML.SubElement(scp, 'entries')
entry = XML.SubElement(entries, 'be.certipost.hudson.plugin.Entry')
XML.SubElement(entry, 'filePath').text = 'tarballs/{proj}/'.format(proj=project)
XML.SubElement(entry, 'filePath').text = 'tarballs/{proj}/'.format(proj=self.data['main']['project'])
XML.SubElement(entry, 'sourceFile').text = 'dist/*.tar.gz'
XML.SubElement(entry, 'keepHierarchy').text = 'false'
btrigger = XML.SubElement(publishers, 'hudson.plugins.parameterizedtrigger.BuildTrigger')

View File

@ -36,7 +36,7 @@ class scm(object):
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
XML.SubElement(huser, 'name').text = 'origin'
XML.SubElement(huser, 'refspec').text = '+refs/heads/*:refs/remotes/origin/*'
XML.SubElement(huser, 'url').text = 'git://github.com/{site}/{project}.git'.format(site=main['site'], project=main['project'])
XML.SubElement(huser, 'url').text = 'git://github.com/{org}/{project}.git'.format(org=main['github_org'], project=main['project'])
branches = XML.SubElement(scm, 'branches')
bspec = XML.SubElement(branches, 'hudson.plugins.git.BranchSpec')
XML.SubElement(bspec, 'name').text = '**'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'cinder'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-cinderclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-glanceclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-keystoneclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-novaclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-openstackclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-quantumclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'nova.openstack.org'

View File

@ -3,5 +3,7 @@ project:
values:
name: 'python-swiftclient'
site: 'openstack'
disabled: 'false'
github_org: 'openstack'
review_site: 'review.openstack.org'
publisher_site: 'swift.openstack.org'

View File

@ -9,7 +9,8 @@ modules:
main:
name: 'gate-ceilometer-merge'
site: 'stackforge'
github_org: 'stackforge'
review_site: 'review.stackforge.org'
project: 'ceilometer'
authenticatedBuild: 'true'
disabled: 'false'
@ -48,7 +49,8 @@ modules:
main:
name: 'check-ceilometer-merge'
site: 'stackforge'
github_org: 'stackforge'
review_site: 'review.stackforge.org'
project: 'ceilometer'
authenticatedBuild: 'true'
disabled: 'false'

View File

@ -11,7 +11,7 @@ modules:
main:
name: 'heat-coverage'
site: 'heat-api'
github_org: 'heat-api'
project: 'heat'
authenticatedBuild: 'false'
disabled: 'false'
@ -46,8 +46,8 @@ modules:
main:
name: 'gate-heat-pep8'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -88,8 +88,8 @@ modules:
main:
name: 'gate-heat-python26'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -130,8 +130,8 @@ modules:
main:
name: 'gate-heat-python27'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -172,7 +172,8 @@ modules:
main:
name: 'heat-docs'
site: 'heat-api'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'false'
disabled: 'false'
@ -189,6 +190,9 @@ scm:
assignednode:
node: 'oneiric'
publisher:
site: TODO.stackforge.org
---
# merge-gate
modules:
@ -201,8 +205,8 @@ modules:
main:
name: 'gate-heat-merge'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -242,7 +246,7 @@ modules:
main:
name: 'heat-ppa'
site: 'heat-api'
github_org: 'heat-api'
project: 'heat'
authenticatedBuild: 'false'
disabled: 'false'
@ -268,8 +272,8 @@ modules:
main:
name: 'heat-tarball'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBulid: 'false'
disabled: 'false'
@ -298,6 +302,8 @@ scm:
assignednode:
node: 'oneiric'
publisher:
site: TODO.stackforge.org
---
# pep8-check
@ -311,8 +317,8 @@ modules:
main:
name: 'check-heat-pep8'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -354,8 +360,8 @@ modules:
main:
name: 'check-heat-python26'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -397,8 +403,8 @@ modules:
main:
name: 'check-heat-python27'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'
@ -440,8 +446,8 @@ modules:
main:
name: 'check-heat-merge'
site: 'heat-api'
host: 'stackforge'
github_org: 'heat-api'
review_site: 'review.stackforge.org'
project: 'heat'
authenticatedBuild: 'true'
disabled: 'false'

View File

@ -3,6 +3,7 @@ project:
values:
name: 'reddwarf'
site: 'stackforge'
disabled: 'true'
github_org: 'stackforge'
review_site: 'review.stackforge.org'
publisher_site: 'TODO.stackforge.org'

View File

@ -11,10 +11,11 @@ modules:
main:
name: '@NAME@-coverage'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'false'
disabled: '@DISABLED@'
disabled: '@DISABLED@'
trigger:
pollscm: '*/15 * * * *'
@ -46,7 +47,8 @@ modules:
main:
name: 'gate-@NAME@-pep8'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -61,7 +63,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -87,7 +89,8 @@ modules:
main:
name: 'gate-@NAME@-python26'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -102,7 +105,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -128,7 +131,8 @@ modules:
main:
name: 'gate-@NAME@-python27'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -143,7 +147,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -169,7 +173,8 @@ modules:
main:
name: '@NAME@-docs'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'false'
disabled: '@DISABLED@'
@ -186,6 +191,9 @@ scm:
assignednode:
node: 'oneiric'
publisher:
site: '@PUBLISHER_SITE@'
---
# merge-gate
modules:
@ -198,7 +206,8 @@ modules:
main:
name: 'gate-@NAME@-merge'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -213,7 +222,7 @@ trigger:
failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.'
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -238,7 +247,8 @@ modules:
main:
name: '@NAME@-ppa'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'false'
disabled: '@DISABLED@'
@ -264,7 +274,8 @@ modules:
main:
name: '@NAME@-tarball'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBulid: 'false'
disabled: '@DISABLED@'
@ -279,7 +290,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'REG_EXP'
branchPattern: '^(?!refs/).*$'
@ -293,6 +304,9 @@ scm:
assignednode:
node: 'oneiric'
publisher:
site: '@PUBLISHER_SITE@'
---
# pep8-check
@ -306,7 +320,8 @@ modules:
main:
name: 'check-@NAME@-pep8'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -322,7 +337,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -348,7 +363,8 @@ modules:
main:
name: 'check-@NAME@-python26'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -364,7 +380,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -390,7 +406,8 @@ modules:
main:
name: 'check-@NAME@-python27'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -406,7 +423,7 @@ trigger:
failureMessage: ''
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'
@ -432,7 +449,8 @@ modules:
main:
name: 'check-@NAME@-merge'
site: '@SITE@'
review_site: '@REVIEW_SITE@'
github_org: '@GITHUB_ORG@'
project: '@NAME@'
authenticatedBuild: 'true'
disabled: '@DISABLED@'
@ -448,7 +466,7 @@ trigger:
failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.'
projects:
- projectCompareType: 'PLAIN'
projectPattern: '@SITE@/@NAME@'
projectPattern: '@GITHUB_ORG@/@NAME@'
branchCompareType: 'ANT'
branchPattern: '**'

View File

@ -1,9 +1,19 @@
#!/bin/bash -xe
#!/bin/bash -e
# Needed environment variables:
# GERRIT_PROJECT
# GERRIT_BRANCH
# GERRIT_REFSPEC or GERRIT_NEWREV
#
# Optional params:
# DEPENDENT_CHANGES="gtest-org/test:master:refs/changes/20/420/1^gtest-org/test:master:refs/changes/21/421/1"
# DEPENDENT_CHANGES="gtest-org/test:master:refs/changes/21/421/1"
# DEPENDENT_CHANGES=""
SITE=$1
if [ -z "$SITE" ]
then
echo "The site name (eg 'openstack') must be the first argument."
echo "The site name (eg 'review.openstack.org') must be the first argument."
exit 1
fi
@ -13,9 +23,44 @@ then
exit 1
fi
function merge_change {
PROJECT=$1
REFSPEC=$2
git fetch https://$SITE/p/$PROJECT $REFSPEC
git merge FETCH_HEAD
}
function merge_dependent_changes {
set +x
OIFS=$IFS
IFS='^'
for change in $DEPENDENT_CHANGES
do
OIFS2=$IFS
IFS=':'
change_array=($change)
IFS=$OIFS2
CHANGE_PROJECT=${change_array[0]}
CHANGE_BRANCH=${change_array[1]}
CHANGE_REFSPEC=${change_array[2]}
if [ "$CHANGE_PROJECT" = "$GERRIT_PROJECT" ] &&
[ "$CHANGE_BRANCH" = "$GERRIT_BRANCH" ]; then
set -x
merge_change $CHANGE_PROJECT $CHANGE_REFSPEC
set +x
fi
done
IFS=$OIFS
set -x
}
set -x
if [[ ! -e .git ]]
then
git clone https://review.$SITE.org/p/$GERRIT_PROJECT .
git clone https://$SITE/p/$GERRIT_PROJECT .
fi
git remote update || git remote update # attempt to work around bug #925790
git reset --hard
@ -26,8 +71,9 @@ then
git checkout $GERRIT_BRANCH
git reset --hard remotes/origin/$GERRIT_BRANCH
git clean -x -f -d -q
git fetch https://review.$SITE.org/p/$GERRIT_PROJECT $GERRIT_REFSPEC
git merge FETCH_HEAD
merge_dependent_changes
merge_change $GERRIT_PROJECT $GERRIT_REFSPEC
else
git checkout $GERRIT_NEWREV
git reset --hard $GERRIT_NEWREV