x/stackalytics
Code Issues Proposed changes

Merge "Get project list from gerrit"

Browse Source
This commit is contained in:
Jenkins 2014-06-17 11:13:11 +00:00 committed by Gerrit Code Review
commit a77502ea10
7 changed files with 52 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/stackalytics.conf
View File

@ -26,6 +26,9 @@
# URI of review system
# review_uri = gerrit://review.openstack.org
# git base location
# git_base_uri = git://git.openstack.org
# SSH key for gerrit review system access
# ssh_key_filename = /home/user/.ssh/id_rsa

1
requirements.txt
View File

@ -4,7 +4,6 @@ oslo.config>=1.2.0
paramiko>=1.8.0
pbr>=0.6,<1.0
psutil>=1.1.1
PyGithub
python-memcached>=1.48
PyYAML>=3.1.0
sh

2
stackalytics/processor/config.py
View File

@ -37,6 +37,8 @@ OPTS = [
help='The address of file with corrections data'),
cfg.StrOpt('review-uri', default='gerrit://review.openstack.org',
help='URI of review system'),
cfg.StrOpt('git-base-uri', default='git://git.openstack.org',
help='git base location'),
cfg.StrOpt('ssh-key-filename', default='/home/user/.ssh/id_rsa',
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',

43
stackalytics/processor/default_data_processor.py
View File

@ -17,7 +17,6 @@ import collections
import hashlib
import json
from github import MainClass
import six
from stackalytics.openstack.common import log as logging
@ -42,30 +41,34 @@ def _check_default_data_change(runtime_storage_inst, default_data):
return True
def _retrieve_project_list_from_github(project_sources):
LOG.info('Retrieving project list from GitHub')
github = MainClass.Github(timeout=60)
repos = []
for project_source in project_sources:
organization = project_source['organization']
LOG.debug('Get list of projects for organization %s', organization)
def _retrieve_project_list_from_gerrit(project_sources, git_base_uri, gerrit):
LOG.info('Retrieving project list from Gerrit')
try:
github_repos = github.get_organization(organization).get_repos()
project_list = gerrit.get_project_list()
except Exception as e:
LOG.exception(e)
LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
return False
repos = []
for project_source in project_sources:
organization = project_source['organization']
LOG.debug('Get list of projects for organization %s', organization)
git_repos = [
f for f in project_list if f.startswith(organization + "/")]
exclude = set(project_source.get('exclude', []))
for repo in github_repos:
if repo.name not in exclude:
for repo in git_repos:
(org, name) = repo.split('/')
if name not in exclude:
url = '%(git_base_uri)s/%(repo)s.git' % dict(
git_base_uri=git_base_uri, repo=repo)
r = {
'branches': ['master'],
'module': repo.name,
'organization': organization,
'uri': repo.git_url,
'module': name,
'organization': org,
'uri': url,
'releases': []
}
repos.append(r)
@ -92,11 +95,12 @@ def _create_module_groups_for_project_sources(project_sources, repos):
return module_groups
def _update_project_list(default_data):
def _update_project_list(default_data, git_base_uri, gerrit):
configured_repos = set([r['uri'] for r in default_data['repos']])
repos = _retrieve_project_list_from_github(default_data['project_sources'])
repos = _retrieve_project_list_from_gerrit(
default_data['project_sources'], git_base_uri, gerrit)
if repos:
default_data['repos'] += [r for r in repos
if r['uri'] not in configured_repos]
@ -160,10 +164,11 @@ def _store_default_data(runtime_storage_inst, default_data):
runtime_storage_inst.set_by_key(key, value)
def process(runtime_storage_inst, default_data):
def process(runtime_storage_inst, default_data,
git_base_uri, gerrit):
LOG.debug('Process default data')
if 'project_sources' in default_data:
_update_project_list(default_data)
_update_project_list(default_data, git_base_uri, gerrit)
_store_default_data(runtime_storage_inst, default_data)

9
stackalytics/processor/main.py
View File

@ -302,8 +302,15 @@ def main():
if not default_data:
LOG.critical('Unable to load default data')
return not 0
gerrit = rcs.get_rcs(None, cfg.CONF.review_uri)
gerrit.setup(key_filename=cfg.CONF.ssh_key_filename,
username=cfg.CONF.ssh_username)
default_data_processor.process(runtime_storage_inst,
default_data)
default_data,
cfg.CONF.git_base_uri,
gerrit)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)

13
stackalytics/processor/rcs.py
View File

@ -135,6 +135,19 @@ class Gerrit(Rcs):
if not proceed:
break
def get_project_list(self):
if not self._connect():
return
exec_result = self._exec_command('gerrit ls-projects')
if not exec_result:
raise Exception("Unable to retrieve list of projects from gerrit.")
stdin, stdout, stderr = exec_result
result = [line.strip() for line in stdout]
self.client.close()
return result
def log(self, branch, last_id):
if not self._connect():
return

4
tests/unit/test_default_data_processor.py
View File

@ -54,7 +54,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
def test_update_project_list(self):
with mock.patch('stackalytics.processor.default_data_processor.'
'_retrieve_project_list_from_github') as retriever:
'_retrieve_project_list_from_gerrit') as retriever:
retriever.return_value = [
{'module': 'nova',
'uri': 'git://git.openstack.org/openstack/nova',
@ -76,7 +76,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
'module_groups': [],
}
default_data_processor._update_project_list(dd)
default_data_processor._update_project_list(dd, None, None)
self.assertEqual(3, len(dd['repos']))
self.assertIn('qa', set([r['module'] for r in dd['repos']]))