Merge "Get project list from gerrit"

etc/stackalytics.conf

@@ -26,6 +26,9 @@
 # URI of review system
 # review_uri = gerrit://review.openstack.org
 
+# git base location
+# git_base_uri = git://git.openstack.org
+
 # SSH key for gerrit review system access
 # ssh_key_filename = /home/user/.ssh/id_rsa
 

requirements.txt

@@ -4,7 +4,6 @@ oslo.config>=1.2.0
 paramiko>=1.8.0
 pbr>=0.6,<1.0
 psutil>=1.1.1
-PyGithub
 python-memcached>=1.48
 PyYAML>=3.1.0
 sh

stackalytics/processor/config.py

@@ -37,6 +37,8 @@ OPTS = [
                help='The address of file with corrections data'),
     cfg.StrOpt('review-uri', default='gerrit://review.openstack.org',
                help='URI of review system'),
+    cfg.StrOpt('git-base-uri', default='git://git.openstack.org',
+               help='git base location'),
     cfg.StrOpt('ssh-key-filename', default='/home/user/.ssh/id_rsa',
                help='SSH key for gerrit review system access'),
     cfg.StrOpt('ssh-username', default='user',

stackalytics/processor/default_data_processor.py

@@ -17,7 +17,6 @@ import collections
 import hashlib
 import json
 
-from github import MainClass
 import six
 
 from stackalytics.openstack.common import log as logging
@@ -42,30 +41,34 @@ def _check_default_data_change(runtime_storage_inst, default_data):
     return True
 
 
-def _retrieve_project_list_from_github(project_sources):
+def _retrieve_project_list_from_gerrit(project_sources, git_base_uri, gerrit):
-    LOG.info('Retrieving project list from GitHub')
+    LOG.info('Retrieving project list from Gerrit')
-    github = MainClass.Github(timeout=60)
-
-    repos = []
-    for project_source in project_sources:
-        organization = project_source['organization']
-        LOG.debug('Get list of projects for organization %s', organization)
     try:
-            github_repos = github.get_organization(organization).get_repos()
+        project_list = gerrit.get_project_list()
     except Exception as e:
         LOG.exception(e)
         LOG.warn('Fail to retrieve list of projects. Keep it unmodified')
         return False
 
+    repos = []
+    for project_source in project_sources:
+        organization = project_source['organization']
+        LOG.debug('Get list of projects for organization %s', organization)
+        git_repos = [
+            f for f in project_list if f.startswith(organization + "/")]
+
         exclude = set(project_source.get('exclude', []))
 
-        for repo in github_repos:
+        for repo in git_repos:
-            if repo.name not in exclude:
+            (org, name) = repo.split('/')
+            if name not in exclude:
+                url = '%(git_base_uri)s/%(repo)s.git' % dict(
+                    git_base_uri=git_base_uri, repo=repo)
                 r = {
                     'branches': ['master'],
-                    'module': repo.name,
+                    'module': name,
-                    'organization': organization,
+                    'organization': org,
-                    'uri': repo.git_url,
+                    'uri': url,
                     'releases': []
                 }
                 repos.append(r)
@@ -92,11 +95,12 @@ def _create_module_groups_for_project_sources(project_sources, repos):
     return module_groups
 
 
-def _update_project_list(default_data):
+def _update_project_list(default_data, git_base_uri, gerrit):
 
     configured_repos = set([r['uri'] for r in default_data['repos']])
 
-    repos = _retrieve_project_list_from_github(default_data['project_sources'])
+    repos = _retrieve_project_list_from_gerrit(
+        default_data['project_sources'], git_base_uri, gerrit)
     if repos:
         default_data['repos'] += [r for r in repos
                                   if r['uri'] not in configured_repos]
@@ -160,10 +164,11 @@ def _store_default_data(runtime_storage_inst, default_data):
             runtime_storage_inst.set_by_key(key, value)
 
 
-def process(runtime_storage_inst, default_data):
+def process(runtime_storage_inst, default_data,
+            git_base_uri, gerrit):
     LOG.debug('Process default data')
 
     if 'project_sources' in default_data:
-        _update_project_list(default_data)
+        _update_project_list(default_data, git_base_uri, gerrit)
 
     _store_default_data(runtime_storage_inst, default_data)

stackalytics/processor/main.py

@@ -302,8 +302,15 @@ def main():
     if not default_data:
         LOG.critical('Unable to load default data')
         return not 0
+
+    gerrit = rcs.get_rcs(None, cfg.CONF.review_uri)
+    gerrit.setup(key_filename=cfg.CONF.ssh_key_filename,
+                 username=cfg.CONF.ssh_username)
+
     default_data_processor.process(runtime_storage_inst,
-                                   default_data)
+                                   default_data,
+                                   cfg.CONF.git_base_uri,
+                                   gerrit)
 
     process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
 

stackalytics/processor/rcs.py

@@ -135,6 +135,19 @@ class Gerrit(Rcs):
             if not proceed:
                 break
 
+    def get_project_list(self):
+        if not self._connect():
+            return
+
+        exec_result = self._exec_command('gerrit ls-projects')
+        if not exec_result:
+            raise Exception("Unable to retrieve list of projects from gerrit.")
+        stdin, stdout, stderr = exec_result
+        result = [line.strip() for line in stdout]
+        self.client.close()
+
+        return result
+
     def log(self, branch, last_id):
         if not self._connect():
             return

tests/unit/test_default_data_processor.py

@@ -54,7 +54,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
 
     def test_update_project_list(self):
         with mock.patch('stackalytics.processor.default_data_processor.'
-                        '_retrieve_project_list_from_github') as retriever:
+                        '_retrieve_project_list_from_gerrit') as retriever:
             retriever.return_value = [
                 {'module': 'nova',
                  'uri': 'git://git.openstack.org/openstack/nova',
@@ -76,7 +76,7 @@ class TestDefaultDataProcessor(testtools.TestCase):
                 'module_groups': [],
             }
 
-            default_data_processor._update_project_list(dd)
+            default_data_processor._update_project_list(dd, None, None)
 
             self.assertEqual(3, len(dd['repos']))
             self.assertIn('qa', set([r['module'] for r in dd['repos']]))