From 52db16762bc1da7f6670d0f1449bf1f18933a11a Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Thu, 22 Nov 2012 10:45:10 -0800
Subject: [PATCH] Consume jeepyb.
Instead of keeping many of these files directly in the tree, use them
from the out-of-tree jeepyb project, which makes them easier to consume
for other people who are not us.
Change-Id: Id704f2e17dd80709ef63cbbf2c5475a08a835f91
Reviewed-on: https://review.openstack.org/16777
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
---
doc/source/gerrit.rst | 2 +-
.../files/scripts/expire_old_reviews.py | 78 -----
modules/gerrit/files/scripts/fetch_remotes.py | 76 ----
.../gerrit/files/scripts/get_group_uuid.py | 29 --
.../gerrit/files/scripts/manage_projects.py | 327 ------------------
modules/gerrit/files/scripts/notify_impact.py | 92 -----
.../gerrit/files/scripts/update_blueprint.py | 134 -------
modules/gerrit/files/scripts/update_bug.py | 244 -------------
.../gerrit/files/scripts/update_cla_group.py | 72 ----
modules/gerrit/manifests/cron.pp | 4 +-
modules/gerrit/manifests/init.pp | 9 +-
modules/gerrit/manifests/remotes.pp | 4 +-
.../files/scripts/close_pull_requests.py | 94 -----
modules/github/manifests/init.pp | 13 +-
modules/jeepyb/manifests/init.pp | 54 +++
.../files/gerrit/change-merged | 2 +-
.../files/gerrit/scripts/trivial_rebase.py | 265 --------------
modules/openstack_project/manifests/gerrit.pp | 14 +-
.../templates/gerrit_patchset-created.erb | 10 +-
modules/pypimirror/files/process_cache.py | 90 -----
modules/pypimirror/files/pull-repo.sh | 29 --
modules/pypimirror/files/run_mirror.py | 81 -----
modules/pypimirror/manifests/init.pp | 22 +-
.../pypimirror/templates/run-mirror.sh.erb | 4 +-
24 files changed, 79 insertions(+), 1670 deletions(-)
delete mode 100644 modules/gerrit/files/scripts/expire_old_reviews.py
delete mode 100755 modules/gerrit/files/scripts/fetch_remotes.py
delete mode 100644 modules/gerrit/files/scripts/get_group_uuid.py
delete mode 100755 modules/gerrit/files/scripts/manage_projects.py
delete mode 100755 modules/gerrit/files/scripts/notify_impact.py
delete mode 100755 modules/gerrit/files/scripts/update_blueprint.py
delete mode 100755 modules/gerrit/files/scripts/update_bug.py
delete mode 100755 modules/gerrit/files/scripts/update_cla_group.py
delete mode 100755 modules/github/files/scripts/close_pull_requests.py
create mode 100644 modules/jeepyb/manifests/init.pp
delete mode 100644 modules/openstack_project/files/gerrit/scripts/trivial_rebase.py
delete mode 100644 modules/pypimirror/files/process_cache.py
delete mode 100644 modules/pypimirror/files/pull-repo.sh
delete mode 100755 modules/pypimirror/files/run_mirror.py
diff --git a/doc/source/gerrit.rst b/doc/source/gerrit.rst
index c3104246a3..f0eca0bfcc 100644
--- a/doc/source/gerrit.rst
+++ b/doc/source/gerrit.rst
@@ -452,7 +452,7 @@ to use this build step.
Auto Review Expiry
==================
-Puppet automatically installs a daily cron job called ``expire_old_reviews.py``
+Puppet automatically installs a daily cron job called ``expire-old-reviews``
onto the gerrit servers. This script follows two rules:
#. If the review hasn't been touched in 2 weeks, mark as abandoned.
diff --git a/modules/gerrit/files/scripts/expire_old_reviews.py b/modules/gerrit/files/scripts/expire_old_reviews.py
deleted file mode 100644
index f9e908a67f..0000000000
--- a/modules/gerrit/files/scripts/expire_old_reviews.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This script is designed to expire old code reviews that have not been touched
-# using the following rules:
-# 1. if open and no activity in 2 weeks, expire
-# 2. if negative comment and no activity in 1 week, expire
-
-import paramiko
-import json
-import logging
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument('user', help='The gerrit admin user')
-parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
-options = parser.parse_args()
-
-GERRIT_USER = options.user
-GERRIT_SSH_KEY = options.ssh_key
-
-logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log')
-logger= logging.getLogger('expire_reviews')
-logger.setLevel(logging.INFO)
-
-logger.info('Starting expire reviews')
-logger.info('Connecting to Gerrit')
-
-ssh = paramiko.SSHClient()
-ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418)
-
-def expire_patch_set(patch_id, patch_subject, has_negative):
- if has_negative:
- message= 'code review expired after 1 week of no activity after a negative review, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface'
- else:
- message= 'code review expired after 2 weeks of no activity, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface'
- command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id)
- logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message)
- stdin, stdout, stderr = ssh.exec_command(command)
- if stdout.channel.recv_exit_status() != 0:
- logger.error(stderr.read())
-
-# Query all open with no activity for 2 weeks
-logger.info('Searching no activity for 2 weeks')
-stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w')
-
-for line in stdout:
- row= json.loads(line)
- if not row.has_key('rowCount'):
- expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False)
-
-# Query all reviewed with no activity for 1 week
-logger.info('Searching no activity on negative review for 1 week')
-stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w')
-
-for line in stdout:
- row= json.loads(line)
- if not row.has_key('rowCount'):
- # Search for negative approvals
- for approval in row['currentPatchSet']['approvals']:
- if approval['value'] == '-1':
- expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True)
- break
-
-logger.info('End expire review')
diff --git a/modules/gerrit/files/scripts/fetch_remotes.py b/modules/gerrit/files/scripts/fetch_remotes.py
deleted file mode 100755
index 6f8ab8b4ed..0000000000
--- a/modules/gerrit/files/scripts/fetch_remotes.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Fetch remotes reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# team-id: 153703
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# ---
-# - project: PROJECT_NAME
-# options:
-# - remote: https://gerrit.googlesource.com/gerrit
-
-
-import logging
-import os
-import subprocess
-import shlex
-import yaml
-
-def run_command(cmd, status=False, env={}):
- cmd_list = shlex.split(str(cmd))
- newenv = os.environ
- newenv.update(env)
- p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, env=newenv)
- (out, nothing) = p.communicate()
- if status:
- return (p.returncode, out.strip())
- return out.strip()
-
-
-def run_command_status(cmd, env={}):
- return run_command(cmd, True, env)
-
-
-logging.basicConfig(level=logging.ERROR)
-
-REPO_ROOT = os.environ.get('REPO_ROOT',
- '/home/gerrit2/review_site/git')
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
-
-(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
-
-for section in config:
- project = section['project']
-
- if 'remote' not in section:
- continue
-
- project_git = "%s.git" % project
- os.chdir(os.path.join(REPO_ROOT, project_git))
-
- # Make sure that the specified remote exists
- remote_url = section['remote']
- # We could check if it exists first, but we're ignoring output anyway
- # So just try to make it, and it'll either make a new one or do nothing
- run_command("git remote add -f upstream %s" % remote_url)
- # Fetch new revs from it
- run_command("git remote update upstream")
diff --git a/modules/gerrit/files/scripts/get_group_uuid.py b/modules/gerrit/files/scripts/get_group_uuid.py
deleted file mode 100644
index 335e944089..0000000000
--- a/modules/gerrit/files/scripts/get_group_uuid.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import argparse
-import paramiko
-import json
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--host", dest="host", default="review.openstack.org",
- help="gerrit host to connect to")
-parser.add_argument("--port", dest="port", action='store', type=int,
- default=29418, help="gerrit port to connect to")
-parser.add_argument("groups", nargs=1)
-
-options = parser.parse_args()
-
-
-client = paramiko.SSHClient()
-client.load_system_host_keys()
-client.set_missing_host_key_policy(paramiko.WarningPolicy())
-client.connect(options.host, port=options.port)
-
-group = options.groups[0]
-query = "select group_uuid from account_groups where name = '%s'" % group
-command = 'gerrit gsql --format JSON -c "%s"' % query
-stdin, stdout, stderr = client.exec_command(command)
-
-for line in stdout:
- row = json.loads(line)
- if row['type'] == 'row':
- print row['columns']['group_uuid']
- ret = stdout.channel.recv_exit_status()
diff --git a/modules/gerrit/files/scripts/manage_projects.py b/modules/gerrit/files/scripts/manage_projects.py
deleted file mode 100755
index 9f5c518085..0000000000
--- a/modules/gerrit/files/scripts/manage_projects.py
+++ /dev/null
@@ -1,327 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# manage_projects.py reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# gerrit-host: review.openstack.org
-# local-git-dir: /var/lib/git
-# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# ---
-# - project: PROJECT_NAME
-# options:
-# - has-wiki
-# - has-issues
-# - has-downloads
-# - has-pull-requests
-# homepage: Some homepage that isn't http://openstack.org
-# description: This is a great project
-# remote: https://gerrit.googlesource.com/gerrit
-# upstream: git://github.com/bushy/beards.git
-# acl_config: /path/to/gerrit/project.config
-
-
-import ConfigParser
-import logging
-import os
-import re
-import shlex
-import subprocess
-import tempfile
-import yaml
-
-import github
-import gerritlib.gerrit
-
-
-logging.basicConfig(level=logging.ERROR)
-log = logging.getLogger("manage_projects")
-
-
-def run_command(cmd, status=False, env={}):
- cmd_list = shlex.split(str(cmd))
- newenv = os.environ
- newenv.update(env)
- log.debug("Executing command: %s" % " ".join(cmd_list))
- p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, env=newenv)
- (out, nothing) = p.communicate()
- log.debug("Return code: %s" % p.returncode)
- log.debug("Command said: %s" % out.strip())
- if status:
- return (p.returncode, out.strip())
- return out.strip()
-
-
-def run_command_status(cmd, env={}):
- return run_command(cmd, True, env)
-
-
-def git_command(repo_dir, sub_cmd, env={}):
- git_dir = os.path.join(repo_dir, '.git')
- cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
- status, _ = run_command(cmd, True, env)
- return status
-
-
-def git_command_output(repo_dir, sub_cmd, env={}):
- git_dir = os.path.join(repo_dir, '.git')
- cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
- status, out = run_command(cmd, True, env)
- return (status, out)
-
-
-def fetch_config(project, remote_url, repo_path, env={}):
- status = git_command(repo_path, "fetch %s +refs/meta/config:"
- "refs/remotes/gerrit-meta/config" % remote_url, env)
- if status != 0:
- print "Failed to fetch refs/meta/config for project: %s" % project
- return False
- # Because the following fails if executed more than once you should only
- # run fetch_config once in each repo.
- status = git_command(repo_path, "checkout -b config "
- "remotes/gerrit-meta/config")
- if status != 0:
- print "Failed to checkout config for project: %s" % project
- return False
-
- return True
-
-
-def copy_acl_config(project, repo_path, acl_config):
- if not os.path.exists(acl_config):
- return False
-
- acl_dest = os.path.join(repo_path, "project.config")
- status, _ = run_command("cp %s %s" %
- (acl_config, acl_dest), status=True)
- if status == 0:
- status = git_command(repo_path, "diff --quiet HEAD")
- if status != 0:
- return True
- return False
-
-
-def push_acl_config(project, remote_url, repo_path, env={}):
- cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
- "Creator <openstack-infra@lists.openstack.org>'"
- status = git_command(repo_path, cmd)
- if status != 0:
- print "Failed to commit config for project: %s" % project
- return False
- status, out = git_command_output(repo_path,
- "push %s HEAD:refs/meta/config" %
- remote_url, env)
- if status != 0:
- print "Failed to push config for project: %s" % project
- print out
- return False
- return True
-
-
-def _get_group_uuid(gerrit, group):
- query = "select group_uuid from account_groups where name = '%s'" % group
- data = gerrit.dbQuery(query)
- if data:
- for row in data:
- if row["type"] == "row":
- return row["columns"]["group_uuid"]
- return None
-
-
-def get_group_uuid(gerrit, group):
- uuid = _get_group_uuid(gerrit, group)
- if uuid:
- return uuid
- gerrit.createGroup(group)
- uuid = _get_group_uuid(gerrit, group)
- if uuid:
- return uuid
- return None
-
-
-def create_groups_file(project, gerrit, repo_path):
- acl_config = os.path.join(repo_path, "project.config")
- group_file = os.path.join(repo_path, "groups")
- uuids = {}
- for line in open(acl_config, 'r'):
- r = re.match(r'^\s+.*group\s+(.*)$', line)
- if r:
- group = r.group(1)
- if group in uuids.keys():
- continue
- uuid = get_group_uuid(gerrit, group)
- if uuid:
- uuids[group] = uuid
- else:
- return False
- if uuids:
- with open(group_file, 'w') as fp:
- for group, uuid in uuids.items():
- fp.write("%s\t%s\n" % (uuid, group))
- status = git_command(repo_path, "add groups")
- if status != 0:
- print "Failed to add groups file for project: %s" % project
- return False
- return True
-
-
-def make_ssh_wrapper(gerrit_user, gerrit_key):
- (fd, name) = tempfile.mkstemp(text=True)
- os.write(fd, '#!/bin/bash\n')
- os.write(fd,
- 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
- (gerrit_key, gerrit_user))
- os.close(fd)
- os.chmod(name, 755)
- return dict(GIT_SSH=name)
-
-
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
-configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
-defaults = configs[0][0]
-default_has_issues = defaults.get('has-issues', False)
-default_has_downloads = defaults.get('has-downloads', False)
-default_has_wiki = defaults.get('has-wiki', False)
-
-LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
-GERRIT_HOST = defaults.get('gerrit-host')
-GERRIT_USER = defaults.get('gerrit-user')
-GERRIT_KEY = defaults.get('gerrit-key')
-GITHUB_SECURE_CONFIG = defaults.get('github-config',
- '/etc/github/github-projects.secure.config')
-
-secure_config = ConfigParser.ConfigParser()
-secure_config.read(GITHUB_SECURE_CONFIG)
-
-# Project creation doesn't work via oauth
-ghub = github.Github(secure_config.get("github", "username"),
- secure_config.get("github", "password"))
-orgs = ghub.get_user().get_orgs()
-orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
-
-gerrit = gerritlib.gerrit.Gerrit('localhost',
- GERRIT_USER,
- 29418,
- GERRIT_KEY)
-project_list = gerrit.listProjects()
-ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
-try:
-
- for section in configs[1]:
- project = section['project']
- options = section.get('options', dict())
- description = section.get('description', None)
- homepage = section.get('homepage', defaults.get('homepage', None))
- upstream = section.get('upstream', None)
-
- project_git = "%s.git" % project
- project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
-
- # Find the project's repo
- project_split = project.split('/', 1)
- if len(project_split) > 1:
- repo_name = project_split[1]
- else:
- repo_name = project
- has_issues = 'has-issues' in options or default_has_issues
- has_downloads = 'has-downloads' in options or default_has_downloads
- has_wiki = 'has-wiki' in options or default_has_wiki
- try:
- org = orgs_dict[project_split[0].lower()]
- except KeyError:
- # We do not have control of this github org ignore the project.
- continue
- try:
- repo = org.get_repo(repo_name)
- except github.GithubException:
- repo = org.create_repo(repo_name,
- homepage=homepage,
- has_issues=has_issues,
- has_downloads=has_downloads,
- has_wiki=has_wiki)
- if description:
- repo.edit(repo_name, description=description)
- if homepage:
- repo.edit(repo_name, homepage=homepage)
-
- repo.edit(repo_name, has_issues=has_issues,
- has_downloads=has_downloads,
- has_wiki=has_wiki)
-
- if 'gerrit' not in [team.name for team in repo.get_teams()]:
- teams = org.get_teams()
- teams_dict = dict(zip([t.name.lower() for t in teams], teams))
- teams_dict['gerrit'].add_to_repos(repo)
-
- remote_url = "ssh://localhost:29418/%s" % project
- if project not in project_list:
- tmpdir = tempfile.mkdtemp()
- try:
- repo_path = os.path.join(tmpdir, 'repo')
- if upstream:
- run_command("git clone %(upstream)s %(repo_path)s" %
- dict(upstream=upstream, repo_path=repo_path))
- else:
- run_command("git init %s" % repo_path)
- with open(os.path.join(repo_path,
- ".gitreview"), 'w') as gitreview:
- gitreview.write("""
-[gerrit]
-host=%s
-port=29418
-project=%s
-""" % (GERRIT_HOST, project_git))
- git_command(repo_path, "add .gitreview")
- cmd = "commit -a -m'Added .gitreview' --author=" \
- "'Openstack Project Creator " \
- "<openstack-infra@lists.openstack.org>'"
- git_command(repo_path, cmd)
- gerrit.createProject(project)
-
- if not os.path.exists(project_dir):
- run_command("git --bare init %s" % project_dir)
- run_command("chown -R gerrit2:gerrit2 %s" % project_dir)
-
- git_command(repo_path,
- "push --all %s" % remote_url,
- env=ssh_env)
- git_command(repo_path,
- "push --tags %s" % remote_url, env=ssh_env)
- finally:
- run_command("rm -fr %s" % tmpdir)
-
- if 'acl_config' in section:
- tmpdir = tempfile.mkdtemp()
- try:
- repo_path = os.path.join(tmpdir, 'repo')
- ret, _ = run_command_status("git init %s" % repo_path)
- if ret != 0:
- continue
- if (fetch_config(project, remote_url, repo_path, ssh_env) and
- copy_acl_config(project, repo_path,
- section['acl_config']) and
- create_groups_file(project, gerrit, repo_path)):
- push_acl_config(project, remote_url, repo_path, ssh_env)
- finally:
- run_command("rm -fr %s" % tmpdir)
-finally:
- os.unlink(ssh_env['GIT_SSH'])
diff --git a/modules/gerrit/files/scripts/notify_impact.py b/modules/gerrit/files/scripts/notify_impact.py
deleted file mode 100755
index 5c8ba8c76e..0000000000
--- a/modules/gerrit/files/scripts/notify_impact.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This is designed to be called by a gerrit hook. It searched new
-# patchsets for strings like "bug FOO" and updates corresponding Launchpad
-# bugs status.
-
-import argparse
-import re
-import subprocess
-import smtplib
-
-from email.mime.text import MIMEText
-
-BASE_DIR = '/home/gerrit2/review_site'
-EMAIL_TEMPLATE = """
-Hi, I'd like you to take a look at this patch for potential
-%s.
-%s
-
-Log:
-%s
-"""
-
-def process_impact(git_log, args):
- """Notify mail list of impact"""
- email_content = EMAIL_TEMPLATE % (args.impact, args.change_url, git_log)
- msg = MIMEText(email_content)
- msg['Subject'] = '[%s] %s review request change %s' % \
- (args.project, args.impact, args.change)
- msg['From'] = 'gerrit2@review.openstack.org'
- msg['To'] = args.dest_address
-
- s = smtplib.SMTP('localhost')
- s.sendmail('gerrit2@review.openstack.org',
- args.dest_address, msg.as_string())
- s.quit()
-
-def impacted(git_log, impact_string):
- """Determine if a changes log indicates there is an impact"""
- return re.search(impact_string, git_log, re.IGNORECASE)
-
-def extract_git_log(args):
- """Extract git log of all merged commits"""
- cmd = ['git',
- '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
- 'log', '--no-merges', args.commit + '^1..' + args.commit]
- return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('hook')
- #common
- parser.add_argument('--change', default=None)
- parser.add_argument('--change-url', default=None)
- parser.add_argument('--project', default=None)
- parser.add_argument('--branch', default=None)
- parser.add_argument('--commit', default=None)
- #change-merged
- parser.add_argument('--submitter', default=None)
- #patchset-created
- parser.add_argument('--uploader', default=None)
- parser.add_argument('--patchset', default=None)
- # Not passed by gerrit:
- parser.add_argument('--impact', default=None)
- parser.add_argument('--dest-address', default=None)
-
- args = parser.parse_args()
-
- # Get git log
- git_log = extract_git_log(args)
-
- # Process impacts found in git log
- if impacted(git_log, args.impact):
- process_impact(git_log, args)
-
-
-if __name__ == '__main__':
- main()
diff --git a/modules/gerrit/files/scripts/update_blueprint.py b/modules/gerrit/files/scripts/update_blueprint.py
deleted file mode 100755
index be70ec2deb..0000000000
--- a/modules/gerrit/files/scripts/update_blueprint.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This is designed to be called by a gerrit hook. It searched new
-# patchsets for strings like "blueprint FOO" or "bp FOO" and updates
-# corresponding Launchpad blueprints with links back to the change.
-
-from launchpadlib.launchpad import Launchpad
-from launchpadlib.uris import LPNET_SERVICE_ROOT
-import os
-import argparse
-import re
-import subprocess
-
-import StringIO
-import ConfigParser
-import MySQLdb
-
-BASE_DIR = '/home/gerrit2/review_site'
-GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
- '~/.launchpadlib/cache'))
-GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
- '~/.launchpadlib/creds'))
-GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
- '/home/gerrit2/review_site/etc/gerrit.config')
-GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
- '/home/gerrit2/review_site/etc/secure.config')
-SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I)
-BODY_RE = re.compile(r'^\s+.*$')
-
-def get_broken_config(filename):
- """ gerrit config ini files are broken and have leading tabs """
- text = ""
- with open(filename,"r") as conf:
- for line in conf.readlines():
- text = "%s%s" % (text, line.lstrip())
-
- fp = StringIO.StringIO(text)
- c=ConfigParser.ConfigParser()
- c.readfp(fp)
- return c
-
-GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG)
-SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG)
-DB_USER = GERRIT_CONFIG.get("database", "username")
-DB_PASS = SECURE_CONFIG.get("database","password")
-DB_DB = GERRIT_CONFIG.get("database","database")
-
-def update_spec(launchpad, project, name, subject, link, topic=None):
- group, project = project.split('/')
- spec = launchpad.projects[project].getSpecification(name=name)
- if not spec: return
-
- if spec.whiteboard:
- wb = spec.whiteboard.strip()
- else:
- wb = ''
- changed = False
- if topic:
- topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)],
- topic)
- if topiclink not in wb:
- wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink)
- changed = True
-
- if link not in wb:
- wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject,
- link=link)
- changed = True
-
- if changed:
- spec.whiteboard = wb
- spec.lp_save()
-
-def find_specs(launchpad, dbconn, args):
- git_log = subprocess.Popen(['git',
- '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
- 'log', '--no-merges',
- args.commit + '^1..' + args.commit],
- stdout=subprocess.PIPE).communicate()[0]
-
- cur = dbconn.cursor()
- cur.execute("select subject, topic from changes where change_key=%s", args.change)
- subject, topic = cur.fetchone()
- specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)])
-
- if topic:
- topicspec = topic.split('/')[-1]
- specs |= set([topicspec])
-
- for spec in specs:
- update_spec(launchpad, args.project, spec, subject,
- args.change_url, topic)
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('hook')
- #common
- parser.add_argument('--change', default=None)
- parser.add_argument('--change-url', default=None)
- parser.add_argument('--project', default=None)
- parser.add_argument('--branch', default=None)
- parser.add_argument('--commit', default=None)
- #change-merged
- parser.add_argument('--submitter', default=None)
- # patchset-created
- parser.add_argument('--uploader', default=None)
- parser.add_argument('--patchset', default=None)
-
- args = parser.parse_args()
-
- launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
- GERRIT_CACHE_DIR,
- credentials_file = GERRIT_CREDENTIALS,
- version='devel')
-
- conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB)
-
- find_specs(launchpad, conn, args)
-
-if __name__ == '__main__':
- main()
diff --git a/modules/gerrit/files/scripts/update_bug.py b/modules/gerrit/files/scripts/update_bug.py
deleted file mode 100755
index 9479a81225..0000000000
--- a/modules/gerrit/files/scripts/update_bug.py
+++ /dev/null
@@ -1,244 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This is designed to be called by a gerrit hook. It searched new
-# patchsets for strings like "bug FOO" and updates corresponding Launchpad
-# bugs status.
-
-from launchpadlib.launchpad import Launchpad
-from launchpadlib.uris import LPNET_SERVICE_ROOT
-import os
-import argparse
-import re
-import subprocess
-
-
-BASE_DIR = '/home/gerrit2/review_site'
-GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
- '~/.launchpadlib/cache'))
-GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
- '~/.launchpadlib/creds'))
-
-
-def add_change_proposed_message(bugtask, change_url, project, branch):
- subject = 'Fix proposed to %s (%s)' % (short_project(project), branch)
- body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url)
- bugtask.bug.newMessage(subject=subject, content=body)
-
-
-def add_change_merged_message(bugtask, change_url, project, commit,
- submitter, branch, git_log):
- subject = 'Fix merged to %s (%s)' % (short_project(project), branch)
- git_url = 'http://github.com/%s/commit/%s' % (project, commit)
- body = '''Reviewed: %s
-Committed: %s
-Submitter: %s
-Branch: %s\n''' % (change_url, git_url, submitter, branch)
- body = body + '\n' + git_log
- bugtask.bug.newMessage(subject=subject, content=body)
-
-
-def set_in_progress(bugtask, launchpad, uploader, change_url):
- """Set bug In progress with assignee being the uploader"""
-
- # Retrieve uploader from Launchpad. Use email as search key if
- # provided, and only set if there is a clear match.
- try:
- searchkey = uploader[uploader.rindex("(") + 1:-1]
- except ValueError:
- searchkey = uploader
- persons = launchpad.people.findPerson(text=searchkey)
- if len(persons) == 1:
- bugtask.assignee = persons[0]
-
- bugtask.status = "In Progress"
- bugtask.lp_save()
-
-
-def set_fix_committed(bugtask):
- """Set bug fix committed"""
-
- bugtask.status = "Fix Committed"
- bugtask.lp_save()
-
-
-def set_fix_released(bugtask):
- """Set bug fix released"""
-
- bugtask.status = "Fix Released"
- bugtask.lp_save()
-
-
-def release_fixcommitted(bugtask):
- """Set bug FixReleased if it was FixCommitted"""
-
- if bugtask.status == u'Fix Committed':
- set_fix_released(bugtask)
-
-
-def tag_in_branchname(bugtask, branch):
- """Tag bug with in-branch-name tag (if name is appropriate)"""
-
- lp_bug = bugtask.bug
- branch_name = branch.replace('/', '-')
- if branch_name.replace('-', '').isalnum():
- lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name]
- lp_bug.tags.append("in-%s" % branch_name)
- lp_bug.lp_save()
-
-
-def short_project(full_project_name):
- """Return the project part of the git repository name"""
- return full_project_name.split('/')[-1]
-
-
-def git2lp(full_project_name):
- """Convert Git repo name to Launchpad project"""
- project_map = {
- 'openstack/openstack-ci-puppet': 'openstack-ci',
- 'openstack-ci/devstack-gate': 'openstack-ci',
- 'openstack-ci/gerrit': 'openstack-ci',
- 'openstack-ci/lodgeit': 'openstack-ci',
- 'openstack-ci/meetbot': 'openstack-ci',
- }
- return project_map.get(full_project_name, short_project(full_project_name))
-
-
-def is_direct_release(full_project_name):
- """Test against a list of projects who directly release changes."""
- return full_project_name in [
- 'openstack-ci/devstack-gate',
- 'openstack-ci/lodgeit',
- 'openstack-ci/meetbot',
- 'openstack-dev/devstack',
- 'openstack/openstack-ci',
- 'openstack/openstack-ci-puppet',
- 'openstack/openstack-manuals',
- 'openstack/tempest',
- ]
-
-
-def process_bugtask(launchpad, bugtask, git_log, args):
- """Apply changes to bugtask, based on hook / branch..."""
-
- if args.hook == "change-merged":
- if args.branch == 'master':
- if is_direct_release(args.project):
- set_fix_released(bugtask)
- else:
- if bugtask.status != u'Fix Released':
- set_fix_committed(bugtask)
- elif args.branch == 'milestone-proposed':
- release_fixcommitted(bugtask)
- elif args.branch.startswith('stable/'):
- series = args.branch[7:]
- # Look for a related task matching the series
- for reltask in bugtask.related_tasks:
- if (reltask.bug_target_name.endswith("/" + series) and
- reltask.status != u'Fix Released'):
- # Use fixcommitted if there is any
- set_fix_committed(reltask)
- break
- else:
- # Use tagging if there isn't any
- tag_in_branchname(bugtask, args.branch)
-
- add_change_merged_message(bugtask, args.change_url, args.project,
- args.commit, args.submitter, args.branch,
- git_log)
-
- if args.hook == "patchset-created":
- if args.branch == 'master':
- if bugtask.status not in [u'Fix Committed', u'Fix Released']:
- set_in_progress(bugtask, launchpad, args.uploader,
- args.change_url)
- elif args.branch.startswith('stable/'):
- series = args.branch[7:]
- for reltask in bugtask.related_tasks:
- if (reltask.bug_target_name.endswith("/" + series) and
- reltask.status not in [u'Fix Committed', u'Fix Released']):
- set_in_progress(reltask, launchpad,
- args.uploader, args.change_url)
- break
-
- if args.patchset == '1':
- add_change_proposed_message(bugtask, args.change_url,
- args.project, args.branch)
-
-
-def find_bugs(launchpad, git_log, args):
- """Find bugs referenced in the git log and return related bugtasks"""
-
- bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)'
- tokens = re.split(bug_regexp, git_log)
-
- # Extract unique bug tasks
- bugtasks = {}
- for token in tokens:
- if re.match('^\d+$', token) and (token not in bugtasks):
- try:
- lp_bug = launchpad.bugs[token]
- for lp_task in lp_bug.bug_tasks:
- if lp_task.bug_target_name == git2lp(args.project):
- bugtasks[token] = lp_task
- break
- except KeyError:
- # Unknown bug
- pass
-
- return bugtasks.values()
-
-
-def extract_git_log(args):
- """Extract git log of all merged commits"""
- cmd = ['git',
- '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
- 'log', '--no-merges', args.commit + '^1..' + args.commit]
- return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('hook')
- #common
- parser.add_argument('--change', default=None)
- parser.add_argument('--change-url', default=None)
- parser.add_argument('--project', default=None)
- parser.add_argument('--branch', default=None)
- parser.add_argument('--commit', default=None)
- #change-merged
- parser.add_argument('--submitter', default=None)
- #patchset-created
- parser.add_argument('--uploader', default=None)
- parser.add_argument('--patchset', default=None)
-
- args = parser.parse_args()
-
- # Connect to Launchpad
- launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
- GERRIT_CACHE_DIR,
- credentials_file=GERRIT_CREDENTIALS,
- version='devel')
-
- # Get git log
- git_log = extract_git_log(args)
-
- # Process bugtasks found in git log
- for bugtask in find_bugs(launchpad, git_log, args):
- process_bugtask(launchpad, bugtask, git_log, args)
-
-
-if __name__ == '__main__':
- main()
diff --git a/modules/gerrit/files/scripts/update_cla_group.py b/modules/gerrit/files/scripts/update_cla_group.py
deleted file mode 100755
index 615b25688c..0000000000
--- a/modules/gerrit/files/scripts/update_cla_group.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Add launchpad ids listed in the wiki CLA page to the CLA group in LP.
-
-import os
-import urllib
-import re
-
-from launchpadlib.launchpad import Launchpad
-from launchpadlib.uris import LPNET_SERVICE_ROOT
-
-DEBUG = False
-
-LP_CACHE_DIR = '~/.launchpadlib/cache'
-LP_CREDENTIALS = '~/.launchpadlib/creds'
-CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P<name>.*?)\s*\|\|\s*(?P<login>.*?)\s*\|\|\s*(?P<trans>.*?)\s*\|\|.*?')
-LINK_RE = re.compile(r'\[\[.*\|\s*(?P<name>.*)\s*\]\]')
-
-for check_path in (os.path.dirname(LP_CACHE_DIR),
- os.path.dirname(LP_CREDENTIALS)):
- if not os.path.exists(check_path):
- os.makedirs(check_path)
-
-wiki_members = []
-for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'):
- m = CONTRIBUTOR_RE.match(line)
- if m and m.group('login') and m.group('trans'):
- login = m.group('login')
- if login=="<#c0c0c0>'''Launchpad ID'''": continue
- l = LINK_RE.match(login)
- if l:
- login = l.group('name')
- wiki_members.append(login)
-
-launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT,
- LP_CACHE_DIR,
- credentials_file = LP_CREDENTIALS,
- version='devel')
-
-lp_members = []
-
-team = launchpad.people['openstack-cla']
-for detail in team.members_details:
- user = None
- # detail.self_link ==
- # 'https://api.launchpad.net/1.0/~team/+member/${username}'
- login = detail.self_link.split('/')[-1]
- status = detail.status
- lp_members.append(login)
-
-for wm in wiki_members:
- if wm not in lp_members:
- print "Need to add %s to LP" % (wm)
- try:
- person = launchpad.people[wm]
- except:
- print 'Unable to find %s on LP'%wm
- continue
- status = team.addMember(person=person, status="Approved")
diff --git a/modules/gerrit/manifests/cron.pp b/modules/gerrit/manifests/cron.pp
index 451c8521b4..d659110cfe 100644
--- a/modules/gerrit/manifests/cron.pp
+++ b/modules/gerrit/manifests/cron.pp
@@ -9,8 +9,8 @@ class gerrit::cron(
user => 'gerrit2',
hour => '6',
minute => '3',
- command => "python /usr/local/gerrit/scripts/expire_old_reviews.py ${script_user} ${script_key_file}",
- require => File['/usr/local/gerrit/scripts'],
+ command => "python /usr/local/bin/expire-old-reviews ${script_user} ${script_key_file}",
+ require => Class['jeepyb'],
}
cron { 'gerrit_repack':
diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp
index ded3577748..9305e93aca 100644
--- a/modules/gerrit/manifests/init.pp
+++ b/modules/gerrit/manifests/init.pp
@@ -111,6 +111,7 @@ class gerrit(
$testmode = false
) {
include apache
+ include jeepyb
include pip
$java_home = $::lsbdistcodename ? {
@@ -490,13 +491,7 @@ class gerrit(
}
file { '/usr/local/gerrit/scripts':
- ensure => directory,
- owner => 'root',
- group => 'root',
- mode => '0755',
- recurse => true,
- require => File['/usr/local/gerrit'],
- source => 'puppet:///modules/gerrit/scripts',
+ ensure => absent,
}
# Install Bouncy Castle's OpenPGP plugin and populate the contact store
diff --git a/modules/gerrit/manifests/remotes.pp b/modules/gerrit/manifests/remotes.pp
index ec3ac26b8c..9637f0b88b 100644
--- a/modules/gerrit/manifests/remotes.pp
+++ b/modules/gerrit/manifests/remotes.pp
@@ -5,8 +5,8 @@ class gerrit::remotes($ensure=present) {
ensure => $ensure,
user => 'gerrit2',
minute => '*/30',
- command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/fetch_remotes.py',
- require => File['/usr/local/gerrit/scripts'],
+ command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/fetch-remotes',
+ require => Class['jeepyb'],
}
file { '/home/gerrit2/remotes.config':
diff --git a/modules/github/files/scripts/close_pull_requests.py b/modules/github/files/scripts/close_pull_requests.py
deleted file mode 100755
index cecbd324cc..0000000000
--- a/modules/github/files/scripts/close_pull_requests.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Github pull requests closer reads a project config file called projects.yaml
-# It should look like:
-
-# - homepage: http://openstack.org
-# team-id: 153703
-# has-wiki: False
-# has-issues: False
-# has-downloads: False
-# ---
-# - project: PROJECT_NAME
-# options:
-# - has-pull-requests
-
-# Github authentication information is read from github.secure.config,
-# which should look like:
-
-# [github]
-# username = GITHUB_USERNAME
-# password = GITHUB_PASSWORD
-#
-# or
-#
-# [github]
-# oauth_token = GITHUB_OAUTH_TOKEN
-
-import ConfigParser
-import github
-import os
-import yaml
-import logging
-
-logging.basicConfig(level=logging.ERROR)
-
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/home/gerrit2/projects.yaml')
-GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
- '/etc/github/github.secure.config')
-
-MESSAGE = """Thank you for contributing to OpenStack!
-
-%(project)s uses Gerrit for code review.
-
-Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit.
-"""
-
-secure_config = ConfigParser.ConfigParser()
-secure_config.read(GITHUB_SECURE_CONFIG)
-(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
-
-if secure_config.has_option("github", "oauth_token"):
- ghub = github.Github(secure_config.get("github", "oauth_token"))
-else:
- ghub = github.Github(secure_config.get("github", "username"),
- secure_config.get("github", "password"))
-
-orgs = ghub.get_user().get_orgs()
-orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
-for section in config:
- project = section['project']
-
- # Make sure we're supposed to close pull requests for this project:
- if 'options' in section and 'has-pull-requests' in section['options']:
- continue
-
- # Find the project's repo
- project_split = project.split('/', 1)
- if len(project_split) > 1:
- repo = orgs_dict[project_split[0].lower()].get_repo(project_split[1])
- else:
- repo = ghub.get_user().get_repo(project)
-
- # Close each pull request
- pull_requests = repo.get_pulls("open")
- for req in pull_requests:
- vars = dict(project=project)
- issue_data = {"url": repo.url + "/issues/" + str(req.number)}
- issue = github.Issue.Issue(req._requester, issue_data, completed = True)
- issue.create_comment(MESSAGE % vars)
- req.edit(state = "closed")
diff --git a/modules/github/manifests/init.pp b/modules/github/manifests/init.pp
index dabceedab5..845a9c5aac 100644
--- a/modules/github/manifests/init.pp
+++ b/modules/github/manifests/init.pp
@@ -7,6 +7,7 @@ class github(
$project_password = '',
$projects = []
) {
+ include jeepyb
include pip
package { 'PyGithub':
@@ -80,20 +81,14 @@ class github(
}
file { '/usr/local/github/scripts':
- ensure => directory,
- group => 'root',
- mode => '0755',
- owner => 'root',
- recurse => true,
- require => File['/usr/local/github'],
- source => 'puppet:///modules/github/scripts',
+ ensure => absent,
}
cron { 'githubclosepull':
- command => 'sleep $((RANDOM\%60+90)) && python /usr/local/github/scripts/close_pull_requests.py',
+ command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/close-pull-requests',
minute => '*/5',
require => [
- File['/usr/local/github/scripts'],
+ Class['jeepyb'],
Package['python-yaml'],
Package['PyGithub'],
],
diff --git a/modules/jeepyb/manifests/init.pp b/modules/jeepyb/manifests/init.pp
new file mode 100644
index 0000000000..81801b86f8
--- /dev/null
+++ b/modules/jeepyb/manifests/init.pp
@@ -0,0 +1,54 @@
+# == Class: jeepyb
+#
+class jeepyb (
+ $git_source_repo = 'https://github.com/openstack-ci/jeepyb.git',
+) {
+ $packages = [
+ 'python-mysqldb',
+ 'python-paramiko',
+ ]
+
+ package { $packages:
+ ensure => present,
+ }
+
+ if ! defined(Package['Pygithub']) {
+ package { 'PyGithub':
+ ensure => latest,
+ provider => pip,
+ require => Class['pip'],
+ }
+ }
+
+ if ! defined(Package['gerritlib']) {
+ package { 'gerritlib':
+ ensure => latest,
+ provider => pip,
+ require => Class['pip'],
+ }
+ }
+
+ # A lot of things need yaml, be conservative requiring this package to avoid
+ # conflicts with other modules.
+ if ! defined(Package['python-yaml']) {
+ package { 'python-yaml':
+ ensure => present,
+ }
+ }
+
+ vcsrepo { '/opt/jeepyb':
+ ensure => latest,
+ provider => git,
+ revision => 'master',
+ source => $git_source_repo,
+ }
+
+ exec { 'install_jeepyb' :
+ command => 'python setup.py install',
+ cwd => '/opt/jeepyb',
+ path => '/bin:/usr/bin',
+ refreshonly => true,
+ subscribe => Vcsrepo['/opt/jeepyb'],
+ }
+
+}
diff --git a/modules/openstack_project/files/gerrit/change-merged b/modules/openstack_project/files/gerrit/change-merged
index ef0cd3a296..a35424f302 100755
--- a/modules/openstack_project/files/gerrit/change-merged
+++ b/modules/openstack_project/files/gerrit/change-merged
@@ -1,4 +1,4 @@
#!/bin/sh
# Use timeout to kill any process running longer than 10 minutes.
-timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py change-merged "$@"
+timeout -k 2m 10m /usr/local/bin/update-bug change-merged "$@"
diff --git a/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py b/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py
deleted file mode 100644
index 7575935232..0000000000
--- a/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py
+++ /dev/null
@@ -1,265 +0,0 @@
-#!/usr/bin/env python2.6
-
-# Copyright (c) 2010, Code Aurora Forum. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-# # Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# # Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following
-# disclaimer in the documentation and/or other materials provided
-# with the distribution.
-# # Neither the name of Code Aurora Forum, Inc. nor the names of its
-# contributors may be used to endorse or promote products derived
-# from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
-# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# This script is designed to detect when a patchset uploaded to Gerrit is
-# 'identical' (determined via git-patch-id) and reapply reviews onto the new
-# patchset from the previous patchset.
-
-# Get usage and help info by running: ./trivial_rebase.py --help
-# Documentation is available here: https://www.codeaurora.org/xwiki/bin/QAEP/Gerrit
-
-import json
-import subprocess
-from sys import exit
-
-from optparse import OptionParser as _realOptionParser, AmbiguousOptionError, \
- BadOptionError
-class OptionParser(_realOptionParser):
- """Make OptionParser silently swallow unrecognized options."""
- def _process_args(self, largs, rargs, values):
- while rargs:
- try:
- _realOptionParser._process_args(self, largs, rargs, values)
- except (AmbiguousOptionError, BadOptionError), e:
- largs.append(e.opt_str)
-
-class CheckCallError(OSError):
- """CheckCall() returned non-0."""
- def __init__(self, command, cwd, retcode, stdout, stderr=None):
- OSError.__init__(self, command, cwd, retcode, stdout, stderr)
- self.command = command
- self.cwd = cwd
- self.retcode = retcode
- self.stdout = stdout
- self.stderr = stderr
-
-def CheckCall(command, cwd=None):
- """Like subprocess.check_call() but returns stdout.
-
- Works on python 2.4
- """
- try:
- process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE)
- std_out, std_err = process.communicate()
- except OSError, e:
- raise CheckCallError(command, cwd, e.errno, None)
- if process.returncode:
- raise CheckCallError(command, cwd, process.returncode, std_out, std_err)
- return std_out, std_err
-
-def Gssh(options, api_command):
- """Makes a Gerrit API call via SSH and returns the stdout results."""
- ssh_cmd = ['ssh',
- '-l', 'Gerrit Code Review',
- '-p', options.port,
- '-i', options.private_key_path,
- options.server,
- api_command]
- try:
- return CheckCall(ssh_cmd)[0]
- except CheckCallError, e:
- import sys
- err_template = "call: %s\nreturn code: %s\nstdout: %s\nstderr: %s\n"
- sys.stderr.write(err_template%(ssh_cmd, e.retcode, e.stdout, e.stderr))
- raise
-
-def GsqlQuery(sql_query, options):
- """Runs a gerrit gsql query and returns the result"""
- gsql_cmd = "gerrit gsql --format JSON -c %s"%sql_query
- gsql_out = Gssh(options, gsql_cmd)
- new_out = gsql_out.replace('}}\n', '}}\nsplit here\n')
- return new_out.split('split here\n')
-
-def FindPrevRev(options):
- """Finds the revision of the previous patch set on the change"""
- sql_query = ("\"SELECT revision FROM patch_sets,changes WHERE "
- "patch_sets.change_id = changes.change_id AND "
- "patch_sets.patch_set_id = %s AND "
- "changes.change_key = \'%s\'\"" % ((options.patchset - 1),
- options.changeId))
- revisions = GsqlQuery(sql_query, options)
-
- json_dict = json.loads(revisions[0], strict=False)
- return json_dict["columns"]["revision"]
-
-def GetApprovals(options):
- """Get all the approvals on a specific patch set
-
- Returns a list of approval dicts"""
- sql_query = ("\"SELECT value,account_id,category_id FROM patch_set_approvals "
- "WHERE patch_set_id = %s AND change_id = (SELECT change_id FROM "
- "changes WHERE change_key = \'%s\') AND value <> 0\""
- % ((options.patchset - 1), options.changeId))
- gsql_out = GsqlQuery(sql_query, options)
- approvals = []
- for json_str in gsql_out:
- dict = json.loads(json_str, strict=False)
- if dict["type"] == "row":
- approvals.append(dict["columns"])
- return approvals
-
-def GetPatchId(revision, consider_whitespace=False):
- git_show_cmd = ['git', 'show', revision]
- patch_id_cmd = ['git', 'patch-id']
- patch_id_process = subprocess.Popen(patch_id_cmd, stdout=subprocess.PIPE,
- stdin=subprocess.PIPE)
- git_show_process = subprocess.Popen(git_show_cmd, stdout=subprocess.PIPE)
- if consider_whitespace:
- # This matches on change lines in the patch (those starting with "+"
- # or "-" but not followed by another of the same), then replaces any
- # space or tab characters with "%" before calculating a patch-id.
- replace_ws_cmd = ['sed', r'/^\(+[^+]\|-[^-]\)/y/ \t/%%/']
- replace_ws_process = subprocess.Popen(replace_ws_cmd,
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE)
- return patch_id_process.communicate(
- replace_ws_process.communicate(git_show_process.communicate()[0])[0]
- )[0]
- else:
- return patch_id_process.communicate(git_show_process.communicate()[0])[0]
-
-def SuExec(options, as_user, cmd):
- suexec_cmd = "suexec --as %s -- %s"%(as_user, cmd)
- Gssh(options, suexec_cmd)
-
-def DiffCommitMessages(commit1, commit2):
- log_cmd1 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
- commit1 + '^!']
- commit1_log = CheckCall(log_cmd1)
- log_cmd2 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
- commit2 + '^!']
- commit2_log = CheckCall(log_cmd2)
- if commit1_log != commit2_log:
- return True
- return False
-
-def Main():
- usage = "usage: %prog <required options> [optional options]"
- parser = OptionParser(usage=usage)
- parser.add_option("--change", dest="changeId", help="Change identifier")
- parser.add_option("--project", help="Project path in Gerrit")
- parser.add_option("--commit", help="Git commit-ish for this patchset")
- parser.add_option("--patchset", type="int", help="The patchset number")
- parser.add_option("--role-user", dest="role_user",
- help="E-mail/ID of user commenting on commit messages")
- parser.add_option("--private-key-path", dest="private_key_path",
- help="Full path to Gerrit SSH daemon's private host key")
- parser.add_option("--server-port", dest="port", default='29418',
- help="Port to connect to Gerrit's SSH daemon "
- "[default: %default]")
- parser.add_option("--server", dest="server", default="localhost",
- help="Server name/address for Gerrit's SSH daemon "
- "[default: %default]")
- parser.add_option("--whitespace", action="store_true",
- help="Treat whitespace as significant")
-
- (options, args) = parser.parse_args()
-
- if not options.changeId:
- parser.print_help()
- exit(0)
-
- if options.patchset == 1:
- # Nothing to detect on first patchset
- exit(0)
- prev_revision = None
- prev_revision = FindPrevRev(options)
- if not prev_revision:
- # Couldn't find a previous revision
- exit(0)
- prev_patch_id = GetPatchId(prev_revision)
- cur_patch_id = GetPatchId(options.commit)
- if cur_patch_id.split()[0] != prev_patch_id.split()[0]:
- # patch-ids don't match
- exit(0)
- # Patch ids match. This is a trivial rebase.
- # In addition to patch-id we should check if whitespace content changed. Some
- # languages are more sensitive to whitespace than others, and some changes
- # may either introduce or be intended to fix style problems specifically
- # involving whitespace as well.
- if options.whitespace:
- prev_patch_ws = GetPatchId(prev_revision, consider_whitespace=True)
- cur_patch_ws = GetPatchId(options.commit, consider_whitespace=True)
- if cur_patch_ws.split()[0] != prev_patch_ws.split()[0]:
- # Insert a comment into the change letting the approvers know only the
- # whitespace changed
- comment_msg = "\"New patchset patch-id matches previous patchset, " \
- "but whitespace content has changed.\""
- comment_cmd = ['gerrit', 'approve', '--project', options.project,
- '--message', comment_msg, options.commit]
- SuExec(options, options.role_user, ' '.join(comment_cmd))
- exit(0)
-
- # We should also check if the commit message changed. Most approvers would
- # want to re-review changes when the commit message changes.
- changed = DiffCommitMessages(prev_revision, options.commit)
- if changed:
- # Insert a comment into the change letting the approvers know only the
- # commit message changed
- comment_msg = "\"New patchset patch-id matches previous patchset, " \
- "but commit message has changed.\""
- comment_cmd = ['gerrit', 'approve', '--project', options.project,
- '--message', comment_msg, options.commit]
- SuExec(options, options.role_user, ' '.join(comment_cmd))
- exit(0)
-
- # Need to get all approvals on prior patch set, then suexec them onto
- # this patchset.
- approvals = GetApprovals(options)
- gerrit_approve_msg = ("\'Automatically re-added by Gerrit trivial rebase "
- "detection script.\'")
- for approval in approvals:
- # Note: Sites with different 'copy_min_score' values in the
- # approval_categories DB table might want different behavior here.
- # Additional categories should also be added if desired.
- if approval["category_id"] == "CRVW":
- approve_category = '--code-review'
- elif approval["category_id"] == "VRIF":
- # Don't re-add verifies
- #approve_category = '--verified'
- continue
- elif approval["category_id"] == "SUBM":
- # We don't care about previous submit attempts
- continue
- elif approval["category_id"] == "APRV":
- # Similarly squash old approvals
- continue
- else:
- print "Unsupported category: %s" % approval
- exit(0)
-
- score = approval["value"]
- gerrit_approve_cmd = ['gerrit', 'approve', '--project', options.project,
- '--message', gerrit_approve_msg, approve_category,
- score, options.commit]
- SuExec(options, approval["account_id"], ' '.join(gerrit_approve_cmd))
- exit(0)
-
-if __name__ == "__main__":
- Main()
diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp
index 76e07a8c09..bed9997927 100644
--- a/modules/openstack_project/manifests/gerrit.pp
+++ b/modules/openstack_project/manifests/gerrit.pp
@@ -214,17 +214,6 @@ class openstack_project::gerrit (
require => Class['::gerrit'],
}
- file { '/usr/local/gerrit/scripts/trivial_rebase.py':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0444',
- source =>
- 'puppet:///modules/openstack_project/gerrit/scripts/trivial_rebase.py',
- replace => true,
- require => Class['::gerrit'],
- }
-
if ($projects_file != 'UNDEF') {
if ($replicate_local) {
file { $local_git_dir:
@@ -264,7 +253,7 @@ class openstack_project::gerrit (
}
exec { 'manage_projects':
- command => '/usr/local/gerrit/scripts/manage_projects.py',
+ command => '/usr/local/bin/manage-projects',
subscribe => [
File['/home/gerrit2/projects.yaml'],
File['/home/gerrit2/acls'],
@@ -273,6 +262,7 @@ class openstack_project::gerrit (
require => [
File['/home/gerrit2/projects.yaml'],
File['/home/gerrit2/acls'],
+ Class['jeepyb'],
],
}
}
diff --git a/modules/openstack_project/templates/gerrit_patchset-created.erb b/modules/openstack_project/templates/gerrit_patchset-created.erb
index fabfceebf6..13f5fbbe38 100755
--- a/modules/openstack_project/templates/gerrit_patchset-created.erb
+++ b/modules/openstack_project/templates/gerrit_patchset-created.erb
@@ -1,11 +1,11 @@
#!/bin/sh
# Use timeout to kill any process running longer than 10 minutes.
-timeout -k 2m 10m python /usr/local/gerrit/scripts/update_blueprint.py patchset-created "$@"
-timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py patchset-created "$@"
-timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org'
-timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net'
-timeout -k 2m 10m python /usr/local/gerrit/scripts/trivial_rebase.py \
+timeout -k 2m 10m /usr/local/bin/update-blueprint patchset-created "$@"
+timeout -k 2m 10m /usr/local/bin/update-bug patchset-created "$@"
+timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org'
+timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net'
+timeout -k 2m 10m /usr/local/bin/trivial-rebase \
patchset-created \
--whitespace \
--private-key-path=<%= ssh_host_key %> \
diff --git a/modules/pypimirror/files/process_cache.py b/modules/pypimirror/files/process_cache.py
deleted file mode 100644
index 2b4c054fc1..0000000000
--- a/modules/pypimirror/files/process_cache.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2012 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-import urllib
-import datetime
-import sys
-import re
-import md5
-
-source_cache = sys.argv[1]
-destination_mirror = sys.argv[2]
-
-PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]')
-
-packages = {}
-package_count = 0
-
-for filename in os.listdir(source_cache):
- if filename.endswith('content-type'):
- continue
-
- realname = urllib.unquote(filename)
- # The ? accounts for sourceforge downloads
- tarball = os.path.basename(realname).split("?")[0]
- name_match = PACKAGE_VERSION_RE.search(tarball)
-
- if name_match is None:
- continue
- package_name = name_match.group(1)
-
- version_list = packages.get(package_name,{})
- version_list[tarball] = filename
- packages[package_name] = version_list
- package_count = package_count + 1
-
-full_html = open(os.path.join(destination_mirror, "full.html"), 'w')
-simple_html = open(os.path.join(destination_mirror, "index.html"), 'w')
-
-header = "<html><head><title>PyPI Mirror</title></head><body><h1>PyPI Mirror</h1><h2>Last update: %s</h2>\n\n" % datetime.datetime.utcnow().strftime("%c UTC")
-full_html.write(header)
-simple_html.write(header)
-
-for package_name, versions in packages.items():
- destination_dir = os.path.join(destination_mirror, package_name)
- if not os.path.isdir(destination_dir):
- os.makedirs(destination_dir)
- safe_dir = urllib.quote(package_name)
- simple_html.write("<a href='%s'>%s</a><br />\n" % (safe_dir, safe_dir))
- with open(os.path.join(destination_dir, "index.html"), 'w') as index:
- index.write("""<html><head>
- <title>%s – PyPI Mirror</title>
-</head><body>\n""" % package_name)
- for tarball, filename in versions.items():
- source_path = os.path.join(source_cache, filename)
- destination_path = os.path.join(destination_dir, tarball)
- with open(destination_path, 'w') as dest:
- src = open(source_path, 'r').read()
- md5sum = md5.md5(src).hexdigest()
- dest.write(src)
-
- safe_name = urllib.quote(tarball)
-
- full_html.write("<a href='%s/%s'>%s</a><br />\n" % (safe_dir,
- safe_name,
- safe_name))
- index.write("<a href='%s#md5=%s'>%s</a>\n" % (safe_name,
- md5sum,
- safe_name))
- index.write("</body></html>\n")
-footer = """<p class='footer'>Generated by process_cache.py; %d
-packages mirrored. </p>
-</body></html>\n""" % package_count
-full_html.write(footer)
-full_html.close()
-simple_html.write(footer)
-simple_html.close()
diff --git a/modules/pypimirror/files/pull-repo.sh b/modules/pypimirror/files/pull-repo.sh
deleted file mode 100644
index 4c1b12d672..0000000000
--- a/modules/pypimirror/files/pull-repo.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# This file is managed by puppet.
-# https://github.com/openstack/openstack-ci-puppet
-
-export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip}
-export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download}
-
-project=$1
-pip_command='/usr/local/bin/pip install -M -U -I --exists-action=w --no-install'
-
-cd ${PIP_TEMP_DOWNLOAD}
-short_project=`echo ${project} | cut -f2 -d/`
-if [ ! -d ${short_project} ] ; then
- git clone git://github.com/${project}.git ${short_project} >/dev/null 2>&1
-fi
-cd ${short_project}
-$pip_command pip
-git fetch origin
-for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do
- git reset --hard $branch
- git clean -x -f -d -q
- echo "*********************"
- echo "Fetching pip requires for $project:$branch"
- for requires_file in tools/pip-requires tools/test-requires ; do
- if [ -f ${requires_file} ] ; then
- $pip_command -r $requires_file
- fi
- done
-done
diff --git a/modules/pypimirror/files/run_mirror.py b/modules/pypimirror/files/run_mirror.py
deleted file mode 100755
index b6b07e84c8..0000000000
--- a/modules/pypimirror/files/run_mirror.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#! /usr/bin/env python
-# Copyright (C) 2011 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# run_mirrors reads a project config file called projects.yaml
-# It should look like:
-
-# - project: PROJECT_NAME
-
-import logging
-import os
-import subprocess
-import shlex
-import yaml
-
-def run_command(cmd, status=False, env={}):
- cmd_list = shlex.split(str(cmd))
- newenv = os.environ
- newenv.update(env)
- p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT, env=newenv)
- (out, nothing) = p.communicate()
- if status:
- return (p.returncode, out.strip())
- return out.strip()
-
-
-def run_command_status(cmd, env={}):
- return run_command(cmd, True, env)
-
-
-logging.basicConfig(level=logging.ERROR)
-
-PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
- '/etc/openstackci/projects.yaml')
-PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD',
- '/var/lib/pip-download')
-GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com')
-pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \
- '--no-install %s'
-
-run_command(pip_command % "pip")
-
-(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
-
-for section in config:
- project = section['project']
-
- os.chdir(PIP_TEMP_DOWNLOAD)
- short_project = project.split('/')[1]
- if not os.path.isdir(short_project):
- run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project,
- short_project))
- os.chdir(short_project)
- run_command("git fetch origin")
-
- for branch in run_command("git branch -a").split("\n"):
- branch = branch.strip()
- if (not branch.startswith("remotes/origin")
- or "origin/HEAD" in branch):
- continue
- run_command("git reset --hard %s" % branch)
- run_command("git clean -x -f -d -q")
- print("*********************")
- print("Fetching pip requires for %s:%s" % (project, branch))
- for requires_file in ("tools/pip-requires", "tools/test-requires"):
- if os.path.exists(requires_file):
- stanza = "-r %s" % requires_file
- run_command(pip_command % stanza)
-
diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp
index e4acd2234c..f4856f067c 100644
--- a/modules/pypimirror/manifests/init.pp
+++ b/modules/pypimirror/manifests/init.pp
@@ -15,6 +15,7 @@ class pypimirror(
include apache
include pip
include remove_nginx
+ include jeepyb
package { 'python-yaml':
ensure => present,
@@ -68,30 +69,15 @@ class pypimirror(
}
file { '/usr/local/mirror_scripts/run_mirror.py':
- ensure => present,
- mode => '0755',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/pypimirror/run_mirror.py',
- require => File['/usr/local/mirror_scripts'],
+ ensure => absent,
}
file { '/usr/local/mirror_scripts/pull-repo.sh':
- ensure => present,
- mode => '0755',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/pypimirror/pull-repo.sh',
- require => File['/usr/local/mirror_scripts'],
+ ensure => absent,
}
file { '/usr/local/mirror_scripts/process_cache.py':
- ensure => present,
- mode => '0755',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/pypimirror/process_cache.py',
- require => File['/usr/local/mirror_scripts'],
+ ensure => absent,
}
# Add cron job to update the mirror
diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb
index 0c2b25bbcb..c6609d33f6 100644
--- a/modules/pypimirror/templates/run-mirror.sh.erb
+++ b/modules/pypimirror/templates/run-mirror.sh.erb
@@ -6,5 +6,5 @@ export PIP_DOWNLOAD_CACHE=<%= pip_cache %>
export PIP_TEMP_DOWNLOAD=<%= pip_download %>
export MIRROR_FILE_PATH=<%= mirror_file_path %>
export LOG_FILENAME=<%= log_filename %>
-python /usr/local/mirror_scripts/run_mirror.py <%= git_source %> >>$LOG_FILENAME
-python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}
+/usr/local/bin/run-mirror <%= git_source %> >>$LOG_FILENAME
+/usr/local/bin/process-cache ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}