diff --git a/dashboard/vault.py b/dashboard/vault.py index 0858609a2..9ee7db035 100644 --- a/dashboard/vault.py +++ b/dashboard/vault.py @@ -94,7 +94,7 @@ def init_module_groups(vault): for module_group in module_groups: module_group_name = module_group['module_group_name'] - module_group_id = utils.safe_encode(module_group_name.lower()) + module_group_id = module_group_name.lower() module_id_index[module_group_id] = { 'group': True, @@ -112,7 +112,7 @@ def init_module_groups(vault): for module in memory_storage_inst.get_modules(): module_id_index[module] = { - 'id': utils.safe_encode(module.lower()), + 'id': module.lower(), 'text': module, 'modules': [module.lower()], } diff --git a/etc/default_data.json b/etc/default_data.json index ddff3b448..01dda4ec1 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -6053,72 +6053,14 @@ } ], "module_groups": [ - { - "module_group_name": "openstack-core", - "modules": [ - "nova", "python-novaclient", - "neutron", "python-neutronclient", - "keystone", "python-keystoneclient", - "cinder", "python-cinderclient", - "glance", "python-glanceclient", - "swift", "python-swiftclient", - "horizon" - ] - }, - { - "module_group_name": "openstack-docs", - "modules": ["openstack-manuals", "api-site", "operations-guide", "openstack-doc-tools", - "compute-api", "netconn-api", "identity-api", "volume-api", "image-api"] - }, - { - "module_group_name": "nova-group", - "modules": ["nova", "python-novaclient"] - }, - { - "module_group_name": "neutron-group", - "modules": ["neutron", "python-neutronclient"] - }, - { - "module_group_name": "keystone-group", - "modules": ["keystone", "python-keystoneclient"] - }, - { - "module_group_name": "cinder-group", - "modules": ["cinder", "python-cinderclient"] - }, - { - "module_group_name": "glance-group", - "modules": ["glance", "python-glanceclient"] - }, - { - "module_group_name": "swift-group", - "modules": ["swift", "python-swiftclient"] - }, { "module_group_name": "libra-group", "modules": ["libra", "python-libraclient"] }, - { - "module_group_name": "ironic-group", - "modules": ["ironic", "python-ironicclient", "pyghmi"] - }, - { - "module_group_name": "tripleo-group", - "modules": ["tripleo-incubator", "tripleo-image-elements", "tripleo-heat-templates", "diskimage-builder", - "os-apply-config", "os-collect-config", "os-refresh-config", "tuskar", "tuskar-ui", "python-tuskarclient"] - }, - { - "module_group_name": "oslo-group", - "modules": ["oslo-incubator", "oslo.config", "oslo.messaging", "oslo.sphinx", "hacking", "pbr"] - }, { "module_group_name": "fuel-group", "modules": ["fuel-main", "fuel-web", "fuel-astute", "fuel-ostf", "fuel-library", "fuel-docs", "fuel-devops"] }, - { - "module_group_name": "savanna-group", - "modules": ["savanna", "python-savannaclient", "savanna-dashboard", "savanna-extra", "savanna-image-elements", "puppet-savanna"] - }, { "module_group_name": "murano-group", "modules": ["murano-agent", "python-muranoclient", "murano-api", "murano-common", "murano-conductor", "murano-dashboard", "murano-deployment", "murano-docs", "murano-tests", "murano-metadataclient", "murano-repository"] diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index b539b1502..73a628c3f 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -31,3 +31,6 @@ # Forcibly read default data and update records # force_update = False + +# The address of file with list of programs +# program_list_uri = https://raw.github.com/openstack/governance/master/reference/programs.yaml diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 84709a218..7d7095dfc 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -110,7 +110,7 @@ "module_groups": [ { - "module_group_name": "glance-group", + "module_group_name": "manual-glance-group", "modules": ["glance", "python-glanceclient"] } ], diff --git a/etc/test_programs.yaml b/etc/test_programs.yaml new file mode 100644 index 000000000..9e6c828a3 --- /dev/null +++ b/etc/test_programs.yaml @@ -0,0 +1,9 @@ +Image Service: + codename: Glance + ptl: Mark Washenberger (markwash) + url: https://wiki.openstack.org/wiki/Glance + projects: + integrated: + - openstack/glance + other: + - openstack/python-glanceclient diff --git a/requirements.txt b/requirements.txt index 0901f7233..47378a6bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,6 @@ paramiko>=1.8.0 psutil<=1.0.1 PyGithub python-memcached>=1.48 +PyYAML>=3.1.0 sh six>=1.4.1 diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index e8dcd296b..6a8273a0d 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -40,4 +40,8 @@ OPTS = [ help='SSH username for gerrit review system access'), cfg.BoolOpt('force-update', default=False, help='Forcibly read default data and update records'), + cfg.StrOpt('program-list-uri', + default=('https://raw.github.com/openstack/governance/' + 'master/reference/programs.yaml'), + help='The address of file with list of programs'), ] diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 3e256fb9a..7101cd0d5 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections + from oslo.config import cfg import psutil +import six +import yaml from stackalytics.openstack.common import log as logging from stackalytics.openstack.common.py3kcompat import urlutils @@ -168,6 +172,53 @@ def apply_corrections(uri, runtime_storage_inst): runtime_storage_inst.apply_corrections(valid_corrections) +def _make_module_group(name, modules): + module_group = {'module_group_name': name, 'modules': modules} + LOG.debug('New module group: %s', module_group) + return module_group + + +def _read_module_groups(program_list_uri): + LOG.debug('Process list of programs from uri: %s', program_list_uri) + content = yaml.safe_load(utils.read_uri(program_list_uri)) + module_groups = [] + modules_by_types = collections.defaultdict(list) + for name, info in six.iteritems(content): + if 'codename' in info: + name += ' (%s)' % info['codename'] + + all_modules = [] + for project_type, project_list in six.iteritems(info['projects']): + module_list = [s.split('/')[1] for s in project_list] + modules_by_types[project_type] += module_list + all_modules += module_list + + module_groups.append(_make_module_group(name, all_modules)) + + all_modules = [] + for project_type, modules_list in six.iteritems(modules_by_types): + all_modules += modules_list + module_groups.append( + _make_module_group('OpenStack ' + project_type.capitalize(), + modules_list)) + module_groups.append(_make_module_group('OpenStack All Official', + all_modules)) + return module_groups + + +def process_program_list(runtime_storage_inst, program_list_uri): + stored_module_groups = runtime_storage_inst.get_by_key('module_groups') + mg_dict = dict([(mg['module_group_name'], mg['modules']) + for mg in stored_module_groups]) + for mg in _read_module_groups(program_list_uri): + mg_dict[mg['module_group_name']] = mg['modules'] + + stored_module_groups = [{'module_group_name': name, 'modules': modules} + for name, modules in six.iteritems(mg_dict)] + + runtime_storage_inst.set_by_key('module_groups', stored_module_groups) + + def main(): # init conf and logging conf = cfg.CONF @@ -190,6 +241,8 @@ def main(): cfg.CONF.sources_root, cfg.CONF.force_update) + process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) + update_pids(runtime_storage_inst) update_records(runtime_storage_inst)