From 3edba4b907cf05b2bffef442addd2ebd38671a77 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 11 Jun 2014 10:10:40 -0700 Subject: [PATCH] Get project list from gerrit There is no need to interact with github. The full project list can be retreived from a single query to gerrit to get the full project list. Change-Id: I49a7c3a7ff7d070b8a166c11e66475870cdcd6ca --- etc/stackalytics.conf | 3 ++ requirements.txt | 1 - stackalytics/processor/config.py | 2 + .../processor/default_data_processor.py | 43 +++++++++++-------- stackalytics/processor/main.py | 9 +++- stackalytics/processor/rcs.py | 13 ++++++ tests/unit/test_default_data_processor.py | 4 +- 7 files changed, 52 insertions(+), 23 deletions(-) diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index d9a997637..9ab56c097 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -26,6 +26,9 @@ # URI of review system # review_uri = gerrit://review.openstack.org +# git base location +# git_base_uri = git://git.openstack.org + # SSH key for gerrit review system access # ssh_key_filename = /home/user/.ssh/id_rsa diff --git a/requirements.txt b/requirements.txt index 73d7460a5..3d77ede85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ oslo.config>=1.2.0 paramiko>=1.8.0 pbr>=0.6,<1.0 psutil>=1.1.1 -PyGithub python-memcached>=1.48 PyYAML>=3.1.0 sh diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index 78947a716..63fce1703 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -37,6 +37,8 @@ OPTS = [ help='The address of file with corrections data'), cfg.StrOpt('review-uri', default='gerrit://review.openstack.org', help='URI of review system'), + cfg.StrOpt('git-base-uri', default='git://git.openstack.org', + help='git base location'), cfg.StrOpt('ssh-key-filename', default='/home/user/.ssh/id_rsa', help='SSH key for gerrit review system access'), cfg.StrOpt('ssh-username', default='user', diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py index 5d629abd8..5bb681312 100644 --- a/stackalytics/processor/default_data_processor.py +++ b/stackalytics/processor/default_data_processor.py @@ -17,7 +17,6 @@ import collections import hashlib import json -from github import MainClass import six from stackalytics.openstack.common import log as logging @@ -42,30 +41,34 @@ def _check_default_data_change(runtime_storage_inst, default_data): return True -def _retrieve_project_list_from_github(project_sources): - LOG.info('Retrieving project list from GitHub') - github = MainClass.Github(timeout=60) +def _retrieve_project_list_from_gerrit(project_sources, git_base_uri, gerrit): + LOG.info('Retrieving project list from Gerrit') + try: + project_list = gerrit.get_project_list() + except Exception as e: + LOG.exception(e) + LOG.warn('Fail to retrieve list of projects. Keep it unmodified') + return False repos = [] for project_source in project_sources: organization = project_source['organization'] LOG.debug('Get list of projects for organization %s', organization) - try: - github_repos = github.get_organization(organization).get_repos() - except Exception as e: - LOG.exception(e) - LOG.warn('Fail to retrieve list of projects. Keep it unmodified') - return False + git_repos = [ + f for f in project_list if f.startswith(organization + "/")] exclude = set(project_source.get('exclude', [])) - for repo in github_repos: - if repo.name not in exclude: + for repo in git_repos: + (org, name) = repo.split('/') + if name not in exclude: + url = '%(git_base_uri)s/%(repo)s.git' % dict( + git_base_uri=git_base_uri, repo=repo) r = { 'branches': ['master'], - 'module': repo.name, - 'organization': organization, - 'uri': repo.git_url, + 'module': name, + 'organization': org, + 'uri': url, 'releases': [] } repos.append(r) @@ -92,11 +95,12 @@ def _create_module_groups_for_project_sources(project_sources, repos): return module_groups -def _update_project_list(default_data): +def _update_project_list(default_data, git_base_uri, gerrit): configured_repos = set([r['uri'] for r in default_data['repos']]) - repos = _retrieve_project_list_from_github(default_data['project_sources']) + repos = _retrieve_project_list_from_gerrit( + default_data['project_sources'], git_base_uri, gerrit) if repos: default_data['repos'] += [r for r in repos if r['uri'] not in configured_repos] @@ -160,10 +164,11 @@ def _store_default_data(runtime_storage_inst, default_data): runtime_storage_inst.set_by_key(key, value) -def process(runtime_storage_inst, default_data): +def process(runtime_storage_inst, default_data, + git_base_uri, gerrit): LOG.debug('Process default data') if 'project_sources' in default_data: - _update_project_list(default_data) + _update_project_list(default_data, git_base_uri, gerrit) _store_default_data(runtime_storage_inst, default_data) diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 936431afb..35331b313 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -302,8 +302,15 @@ def main(): if not default_data: LOG.critical('Unable to load default data') return not 0 + + gerrit = rcs.get_rcs(None, cfg.CONF.review_uri) + gerrit.setup(key_filename=cfg.CONF.ssh_key_filename, + username=cfg.CONF.ssh_username) + default_data_processor.process(runtime_storage_inst, - default_data) + default_data, + cfg.CONF.git_base_uri, + gerrit) process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index 541bcbfab..f57d22f40 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -135,6 +135,19 @@ class Gerrit(Rcs): if not proceed: break + def get_project_list(self): + if not self._connect(): + return + + exec_result = self._exec_command('gerrit ls-projects') + if not exec_result: + raise Exception("Unable to retrieve list of projects from gerrit.") + stdin, stdout, stderr = exec_result + result = [line.strip() for line in stdout] + self.client.close() + + return result + def log(self, branch, last_id): if not self._connect(): return diff --git a/tests/unit/test_default_data_processor.py b/tests/unit/test_default_data_processor.py index 2067e2701..0af1453b1 100644 --- a/tests/unit/test_default_data_processor.py +++ b/tests/unit/test_default_data_processor.py @@ -54,7 +54,7 @@ class TestDefaultDataProcessor(testtools.TestCase): def test_update_project_list(self): with mock.patch('stackalytics.processor.default_data_processor.' - '_retrieve_project_list_from_github') as retriever: + '_retrieve_project_list_from_gerrit') as retriever: retriever.return_value = [ {'module': 'nova', 'uri': 'git://git.openstack.org/openstack/nova', @@ -76,7 +76,7 @@ class TestDefaultDataProcessor(testtools.TestCase): 'module_groups': [], } - default_data_processor._update_project_list(dd) + default_data_processor._update_project_list(dd, None, None) self.assertEqual(3, len(dd['repos'])) self.assertIn('qa', set([r['module'] for r in dd['repos']]))