From aeb6c67498417f5f8f0a08cf64a90313c921cbd3 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 5 Jun 2014 17:21:30 +0400 Subject: [PATCH] Optimized performance of record processing in dashboard Change-Id: I2a20f34b9a92feb8460bcabc59b60842b930a772 --- dashboard/decorators.py | 58 +++++++++++++++++++++++-------------- dashboard/memory_storage.py | 15 ++++++++++ dashboard/parameters.py | 2 +- dashboard/vault.py | 51 +++++++++++++++++++------------- dashboard/web.py | 12 ++++++-- tests/api/test_companies.py | 4 +++ tests/api/test_modules.py | 4 +++ tests/api/test_stats.py | 12 ++++++++ tests/api/test_users.py | 4 +++ 9 files changed, 117 insertions(+), 45 deletions(-) diff --git a/dashboard/decorators.py b/dashboard/decorators.py index b531d38b7..1233b8812 100644 --- a/dashboard/decorators.py +++ b/dashboard/decorators.py @@ -101,60 +101,74 @@ def record_filter(ignore=None): return memory_storage_inst.get_record_ids_by_days( six.moves.range(start_day, end_day + 1)) - def _filter_records_by_modules(memory_storage_inst, modules, releases): + def _filter_records_by_modules(memory_storage_inst, mr): selected = set([]) - for m, r in vault.resolve_modules(modules, releases): - y = memory_storage_inst.get_record_ids_by_modules([m]) - if r: - x = memory_storage_inst.get_record_ids_by_releases([r]) - selected |= x & y + for m, r in mr: + if r is None: + selected |= memory_storage_inst.get_record_ids_by_modules( + [m]) else: - selected |= y + selected |= ( + memory_storage_inst.get_record_ids_by_module_release( + m, r)) return selected + def _intersect(first, second): + if first is not None: + return first & second + return second + @functools.wraps(f) def record_filter_decorated_function(*args, **kwargs): memory_storage_inst = vault.get_memory_storage() - record_ids = set(memory_storage_inst.get_record_ids()) # a copy + record_ids = None params = _prepare_params(kwargs, ignore) release = params['release'] if release: if 'all' not in release: - record_ids &= ( + record_ids = ( memory_storage_inst.get_record_ids_by_releases( c.lower() for c in release)) project_type = params['project_type'] + mr = None if project_type: - record_ids &= _filter_records_by_modules( - memory_storage_inst, - vault.resolve_project_types(project_type), release) + mr = set(vault.resolve_modules(vault.resolve_project_types( + project_type), release)) module = params['module'] if module: - record_ids &= _filter_records_by_modules( - memory_storage_inst, module, release) + mr = _intersect(mr, set(vault.resolve_modules( + module, release))) + + if mr is not None: + record_ids = _intersect( + record_ids, _filter_records_by_modules( + memory_storage_inst, mr)) user_id = params['user_id'] user_id = [u for u in user_id if vault.get_user_from_runtime_storage(u)] if user_id: - record_ids &= ( + record_ids = _intersect( + record_ids, memory_storage_inst.get_record_ids_by_user_ids(user_id)) company = params['company'] if company: - record_ids &= ( + record_ids = _intersect( + record_ids, memory_storage_inst.get_record_ids_by_companies(company)) metric = params['metric'] if 'all' not in metric: for metric in metric: if metric in parameters.METRIC_TO_RECORD_TYPE: - record_ids &= ( + record_ids = _intersect( + record_ids, memory_storage_inst.get_record_ids_by_type( parameters.METRIC_TO_RECORD_TYPE[metric])) @@ -172,7 +186,8 @@ def record_filter(ignore=None): blueprint_id = params['blueprint_id'] if blueprint_id: - record_ids &= ( + record_ids = _intersect( + record_ids, memory_storage_inst.get_record_ids_by_blueprint_ids( blueprint_id)) @@ -180,8 +195,9 @@ def record_filter(ignore=None): end_date = params['end_date'] if start_date or end_date: - record_ids &= _filter_records_by_days(start_date, end_date, - memory_storage_inst) + record_ids = _intersect( + record_ids, _filter_records_by_days(start_date, end_date, + memory_storage_inst)) kwargs['record_ids'] = record_ids kwargs['records'] = memory_storage_inst.get_records(record_ids) @@ -287,7 +303,7 @@ def aggregate_filter(): metric = metric_param.lower() metric_to_filters_map = { - 'commits': (incremental_filter, None), + 'commits': (None, None), 'loc': (loc_filter, None), 'marks': (mark_filter, mark_finalize), 'tm_marks': (mark_filter, mark_finalize), diff --git a/dashboard/memory_storage.py b/dashboard/memory_storage.py index 5911244dd..7eab33f01 100644 --- a/dashboard/memory_storage.py +++ b/dashboard/memory_storage.py @@ -41,6 +41,7 @@ class CachedMemoryStorage(MemoryStorage): self.blueprint_id_index = {} self.company_name_mapping = {} self.day_index = {} + self.module_release_index = {} self.indexes = { 'primary_key': self.primary_key_index, @@ -69,6 +70,12 @@ class CachedMemoryStorage(MemoryStorage): else: self.day_index[record_day] = set([record['record_id']]) + mr = (record['module'], record['release']) + if mr in self.module_release_index: + self.module_release_index[mr].add(record['record_id']) + else: + self.module_release_index[mr] = set([record['record_id']]) + def update(self, records): have_updates = False @@ -92,6 +99,8 @@ class CachedMemoryStorage(MemoryStorage): record_day = utils.timestamp_to_day(record['date']) self.day_index[record_day].remove(record['record_id']) + self.module_release_index[ + (record['module'], record['release'])].remove(record['record_id']) def _add_to_index(self, record_index, record, key): record_key = record[key] @@ -129,6 +138,12 @@ class CachedMemoryStorage(MemoryStorage): def get_record_ids_by_days(self, days): return self._get_record_ids_from_index(days, self.day_index) + def get_record_ids_by_module_release(self, module, release): + mr = (module, release) + if mr in self.module_release_index: + return self.module_release_index[mr] + return set() + def get_index_keys_by_record_ids(self, index_name, record_ids): return set([key for key, value in six.iteritems(self.indexes[index_name]) diff --git a/dashboard/parameters.py b/dashboard/parameters.py index 0c6255636..706547e28 100644 --- a/dashboard/parameters.py +++ b/dashboard/parameters.py @@ -35,7 +35,7 @@ METRIC_LABELS = { 'emails': 'Emails', 'bpd': 'Drafted Blueprints', 'bpc': 'Completed Blueprints', - 'person-day': "Person-day effort" + # 'person-day': "Person-day effort" } METRIC_TO_RECORD_TYPE = { diff --git a/dashboard/vault.py b/dashboard/vault.py index 3c9adea72..3c68f0ee5 100644 --- a/dashboard/vault.py +++ b/dashboard/vault.py @@ -68,26 +68,27 @@ def get_vault(): LOG.exception(e) flask.abort(500) - time_now = utils.date_to_timestamp('now') - may_update_by_time = (time_now > vault.get('vault_update_time', 0) + - cfg.CONF.dashboard_update_interval) - if (may_update_by_time and - not getattr(flask.request, 'stackalytics_updated', None)): - flask.request.stackalytics_updated = True - vault['vault_update_time'] = time_now - memory_storage_inst = vault['memory_storage'] - have_updates = memory_storage_inst.update( - compact_records(vault['runtime_storage'].get_update(os.getpid()))) - vault['runtime_storage_update_time'] = ( - vault['runtime_storage'].get_by_key('runtime_storage_update_time')) + if not getattr(flask.request, 'stackalytics_updated', None): + time_now = utils.date_to_timestamp('now') + may_update_by_time = (time_now > vault.get('vault_update_time', 0) + + cfg.CONF.dashboard_update_interval) + if may_update_by_time: + flask.request.stackalytics_updated = True + vault['vault_update_time'] = time_now + memory_storage_inst = vault['memory_storage'] + have_updates = memory_storage_inst.update(compact_records( + vault['runtime_storage'].get_update(os.getpid()))) + vault['runtime_storage_update_time'] = ( + vault['runtime_storage'].get_by_key( + 'runtime_storage_update_time')) - if have_updates: - vault['cache'] = {} - vault['cache_size'] = 0 - _init_releases(vault) - _init_module_groups(vault) - _init_project_types(vault) - _init_user_index(vault) + if have_updates: + vault['cache'] = {} + vault['cache_size'] = 0 + _init_releases(vault) + _init_module_groups(vault) + _init_project_types(vault) + _init_user_index(vault) return vault @@ -171,7 +172,7 @@ def get_user_from_runtime_storage(user_id): return user_index[user_id] -def resolve_modules(module_ids, releases): +def resolve_modules_for_releases(module_ids, releases): module_id_index = get_vault().get('module_id_index') or {} for module_id in module_ids: @@ -197,6 +198,16 @@ def resolve_modules(module_ids, releases): yield module, release +def resolve_modules(module_ids, releases): + all_releases = get_vault()['releases'].keys() + for module, release in resolve_modules_for_releases(module_ids, releases): + if release is None: + for r in all_releases: + yield (module, r) + else: + yield (module, release) + + def resolve_project_types(project_types): modules = set() project_types_index = get_vault()['project_types_index'] diff --git a/dashboard/web.py b/dashboard/web.py index 21b26e19c..208d5b96a 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -82,9 +82,15 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id, param_title = param_title or param_id result = dict((c, {'metric': 0, 'id': c}) for c in keys) context = {} - for record in records: - metric_filter(result, record, param_id, context) - result[record[param_id]]['name'] = record[param_title] + if metric_filter: + for record in records: + metric_filter(result, record, param_id, context) + result[record[param_id]]['name'] = record[param_title] + else: + for record in records: + record_param_id = record[param_id] + result[record_param_id]['metric'] += 1 + result[record_param_id]['name'] = record[param_title] response = [r for r in result.values() if r['metric']] response = [item for item in map(finalize_handler, response) if item] diff --git a/tests/api/test_companies.py b/tests/api/test_companies.py index b359434e9..0b4d63b6e 100644 --- a/tests/api/test_companies.py +++ b/tests/api/test_companies.py @@ -32,6 +32,10 @@ class TestAPICompanies(test_api.TestAPI): {'id': 'openstack', 'title': 'OpenStack', 'modules': ['nova', 'glance']} ], + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'module_groups': { 'openstack': {'module_group_name': 'openstack', 'modules': ['nova', 'glance']}, diff --git a/tests/api/test_modules.py b/tests/api/test_modules.py index 906ae6a05..3ec325bb1 100644 --- a/tests/api/test_modules.py +++ b/tests/api/test_modules.py @@ -35,6 +35,10 @@ class TestAPIModules(test_api.TestAPI): 'nova-cli': test_api.make_module('nova-cli'), 'glance': test_api.make_module('glance'), }, + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'project_types': [ {'id': 'all', 'title': 'All', 'modules': ['nova', 'glance', 'nova-cli']}, diff --git a/tests/api/test_stats.py b/tests/api/test_stats.py index e02c0a157..bda0e7e93 100644 --- a/tests/api/test_stats.py +++ b/tests/api/test_stats.py @@ -26,6 +26,10 @@ class TestAPIStats(test_api.TestAPI): 'uri': 'git://github.com/openstack/nova.git'}, {'module': 'glance', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'module_groups': { 'openstack': {'id': 'openstack', 'module_group_name': 'openstack', @@ -60,6 +64,10 @@ class TestAPIStats(test_api.TestAPI): {'module': 'glance', 'project_type': 'openstack', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'module_groups': { 'openstack': {'id': 'openstack', 'module_group_name': 'openstack', @@ -108,6 +116,10 @@ class TestAPIStats(test_api.TestAPI): {'module': 'glance', 'project_type': 'openstack', 'organization': 'openstack', 'uri': 'git://github.com/openstack/glance.git'}], + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'module_groups': { 'openstack': {'id': 'openstack', 'module_group_name': 'openstack', diff --git a/tests/api/test_users.py b/tests/api/test_users.py index 3adc9b51f..6b4d06623 100644 --- a/tests/api/test_users.py +++ b/tests/api/test_users.py @@ -27,6 +27,10 @@ class TestAPIUsers(test_api.TestAPI): 'project_types': [ {'id': 'openstack', 'title': 'openstack', 'modules': ['nova', 'glance']}], + 'releases': [{'release_name': 'prehistory', + 'end_date': 1234567890}, + {'release_name': 'icehouse', + 'end_date': 1234567890}], 'module_groups': { 'nova': test_api.make_module('nova'), 'glance': test_api.make_module('glance')},