diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf
index d9a997637..9ab56c097 100644
--- a/etc/stackalytics.conf
+++ b/etc/stackalytics.conf
@@ -26,6 +26,9 @@
# URI of review system
# review_uri = gerrit://review.openstack.org
+# git base location
+# git_base_uri = git://git.openstack.org
+
# SSH key for gerrit review system access
# ssh_key_filename = /home/user/.ssh/id_rsa
diff --git a/requirements.txt b/requirements.txt
index 73d7460a5..3d77ede85 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,6 @@ oslo.config>=1.2.0
paramiko>=1.8.0
pbr>=0.6,<1.0
psutil>=1.1.1
-PyGithub
python-memcached>=1.48
PyYAML>=3.1.0
sh
diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py
index 78947a716..63fce1703 100644
--- a/stackalytics/processor/config.py
+++ b/stackalytics/processor/config.py
@@ -37,6 +37,8 @@ OPTS = [
help='The address of file with corrections data'),
cfg.StrOpt('review-uri', default='gerrit://review.openstack.org',
help='URI of review system'),
+ cfg.StrOpt('git-base-uri', default='git://git.openstack.org',
+ help='git base location'),
cfg.StrOpt('ssh-key-filename', default='/home/user/.ssh/id_rsa',
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',
diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py
index 5d629abd8..5bb681312 100644
--- a/stackalytics/processor/default_data_processor.py
+++ b/stackalytics/processor/default_data_processor.py
@@ -17,7 +17,6 @@ import collections
import hashlib
import json
-from github import MainClass
import six
from stackalytics.openstack.common import log as logging
@@ -42,30 +41,34 @@ def _check_default_data_change(runtime_storage_inst, default_data):
return True
-def _retrieve_project_list_from_github(project_sources):
- LOG.info('Retrieving project list from GitHub')
- github = MainClass.Github(timeout=60)
+def _retrieve_project_list_from_gerrit(project_sources, git_base_uri, gerrit):
+ LOG.info('Retrieving project list from Gerrit')
+ try:
+ project_list = gerrit.get_project_list()
+ except Exception as e:
+ LOG.exception(e)
+ LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
+ return False
repos = []
for project_source in project_sources:
organization = project_source['organization']
LOG.debug('Get list of projects for organization %s', organization)
- try:
- github_repos = github.get_organization(organization).get_repos()
- except Exception as e:
- LOG.exception(e)
- LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
- return False
+ git_repos = [
+ f for f in project_list if f.startswith(organization + "/")]
exclude = set(project_source.get('exclude', []))
- for repo in github_repos:
- if repo.name not in exclude:
+ for repo in git_repos:
+ (org, name) = repo.split('/')
+ if name not in exclude:
+ url = '%(git_base_uri)s/%(repo)s.git' % dict(
+ git_base_uri=git_base_uri, repo=repo)
r = {
'branches': ['master'],
- 'module': repo.name,
- 'organization': organization,
- 'uri': repo.git_url,
+ 'module': name,
+ 'organization': org,
+ 'uri': url,
'releases': []
}
repos.append(r)
@@ -92,11 +95,12 @@ def _create_module_groups_for_project_sources(project_sources, repos):
return module_groups
-def _update_project_list(default_data):
+def _update_project_list(default_data, git_base_uri, gerrit):
configured_repos = set([r['uri'] for r in default_data['repos']])
- repos = _retrieve_project_list_from_github(default_data['project_sources'])
+ repos = _retrieve_project_list_from_gerrit(
+ default_data['project_sources'], git_base_uri, gerrit)
if repos:
default_data['repos'] += [r for r in repos
if r['uri'] not in configured_repos]
@@ -160,10 +164,11 @@ def _store_default_data(runtime_storage_inst, default_data):
runtime_storage_inst.set_by_key(key, value)
-def process(runtime_storage_inst, default_data):
+def process(runtime_storage_inst, default_data,
+ git_base_uri, gerrit):
LOG.debug('Process default data')
if 'project_sources' in default_data:
- _update_project_list(default_data)
+ _update_project_list(default_data, git_base_uri, gerrit)
_store_default_data(runtime_storage_inst, default_data)
diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py
index 936431afb..35331b313 100644
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -302,8 +302,15 @@ def main():
if not default_data:
LOG.critical('Unable to load default data')
return not 0
+
+ gerrit = rcs.get_rcs(None, cfg.CONF.review_uri)
+ gerrit.setup(key_filename=cfg.CONF.ssh_key_filename,
+ username=cfg.CONF.ssh_username)
+
default_data_processor.process(runtime_storage_inst,
- default_data)
+ default_data,
+ cfg.CONF.git_base_uri,
+ gerrit)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py
index 541bcbfab..f57d22f40 100644
--- a/stackalytics/processor/rcs.py
+++ b/stackalytics/processor/rcs.py
@@ -135,6 +135,19 @@ class Gerrit(Rcs):
if not proceed:
break
+ def get_project_list(self):
+ if not self._connect():
+ return
+
+ exec_result = self._exec_command('gerrit ls-projects')
+ if not exec_result:
+ raise Exception("Unable to retrieve list of projects from gerrit.")
+ stdin, stdout, stderr = exec_result
+ result = [line.strip() for line in stdout]
+ self.client.close()
+
+ return result
+
def log(self, branch, last_id):
if not self._connect():
return
diff --git a/tests/unit/test_default_data_processor.py b/tests/unit/test_default_data_processor.py
index 2067e2701..0af1453b1 100644
--- a/tests/unit/test_default_data_processor.py
+++ b/tests/unit/test_default_data_processor.py
@@ -54,7 +54,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
def test_update_project_list(self):
with mock.patch('stackalytics.processor.default_data_processor.'
- '_retrieve_project_list_from_github') as retriever:
+ '_retrieve_project_list_from_gerrit') as retriever:
retriever.return_value = [
{'module': 'nova',
'uri': 'git://git.openstack.org/openstack/nova',
@@ -76,7 +76,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
'module_groups': [],
}
- default_data_processor._update_project_list(dd)
+ default_data_processor._update_project_list(dd, None, None)
self.assertEqual(3, len(dd['repos']))
self.assertIn('qa', set([r['module'] for r in dd['repos']]))