x/stackalytics
Code Issues Proposed changes

Optimized performance of record processing in dashboard

Browse Source 
Change-Id: I2a20f34b9a92feb8460bcabc59b60842b930a772
This commit is contained in:
Ilya Shakhat 2014-06-05 17:21:30 +04:00
parent 6b8b0b1c5a
commit aeb6c67498
9 changed files with 117 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
dashboard/decorators.py
View File

@ -101,60 +101,74 @@ def record_filter(ignore=None):
return memory_storage_inst.get_record_ids_by_days(
six.moves.range(start_day, end_day + 1))
def _filter_records_by_modules(memory_storage_inst, modules, releases):
def _filter_records_by_modules(memory_storage_inst, mr):
selected = set([])
for m, r in vault.resolve_modules(modules, releases):
y = memory_storage_inst.get_record_ids_by_modules([m])
if r:
x = memory_storage_inst.get_record_ids_by_releases([r])
selected |= x & y
for m, r in mr:
if r is None:
selected |= memory_storage_inst.get_record_ids_by_modules(
[m])
else:
selected |= y
selected |= (
memory_storage_inst.get_record_ids_by_module_release(
m, r))
return selected
def _intersect(first, second):
if first is not None:
return first & second
return second
@functools.wraps(f)
def record_filter_decorated_function(*args, **kwargs):
memory_storage_inst = vault.get_memory_storage()
record_ids = set(memory_storage_inst.get_record_ids()) # a copy
record_ids = None
params = _prepare_params(kwargs, ignore)
release = params['release']
if release:
if 'all' not in release:
record_ids &= (
record_ids = (
memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in release))
project_type = params['project_type']
mr = None
if project_type:
record_ids &= _filter_records_by_modules(
memory_storage_inst,
vault.resolve_project_types(project_type), release)
mr = set(vault.resolve_modules(vault.resolve_project_types(
project_type), release))
module = params['module']
if module:
record_ids &= _filter_records_by_modules(
memory_storage_inst, module, release)
mr = _intersect(mr, set(vault.resolve_modules(
module, release)))
if mr is not None:
record_ids = _intersect(
record_ids, _filter_records_by_modules(
memory_storage_inst, mr))
user_id = params['user_id']
user_id = [u for u in user_id
if vault.get_user_from_runtime_storage(u)]
if user_id:
record_ids &= (
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_user_ids(user_id))
company = params['company']
if company:
record_ids &= (
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_companies(company))
metric = params['metric']
if 'all' not in metric:
for metric in metric:
if metric in parameters.METRIC_TO_RECORD_TYPE:
record_ids &= (
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_type(
parameters.METRIC_TO_RECORD_TYPE[metric]))
@ -172,7 +186,8 @@ def record_filter(ignore=None):
blueprint_id = params['blueprint_id']
if blueprint_id:
record_ids &= (
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_blueprint_ids(
blueprint_id))
@ -180,8 +195,9 @@ def record_filter(ignore=None):
end_date = params['end_date']
if start_date or end_date:
record_ids &= _filter_records_by_days(start_date, end_date,
memory_storage_inst)
record_ids = _intersect(
record_ids, _filter_records_by_days(start_date, end_date,
memory_storage_inst))
kwargs['record_ids'] = record_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids)
@ -287,7 +303,7 @@ def aggregate_filter():
metric = metric_param.lower()
metric_to_filters_map = {
'commits': (incremental_filter, None),
'commits': (None, None),
'loc': (loc_filter, None),
'marks': (mark_filter, mark_finalize),
'tm_marks': (mark_filter, mark_finalize),

15
dashboard/memory_storage.py
View File

@ -41,6 +41,7 @@ class CachedMemoryStorage(MemoryStorage):
self.blueprint_id_index = {}
self.company_name_mapping = {}
self.day_index = {}
self.module_release_index = {}
self.indexes = {
'primary_key': self.primary_key_index,
@ -69,6 +70,12 @@ class CachedMemoryStorage(MemoryStorage):
else:
self.day_index[record_day] = set([record['record_id']])
mr = (record['module'], record['release'])
if mr in self.module_release_index:
self.module_release_index[mr].add(record['record_id'])
else:
self.module_release_index[mr] = set([record['record_id']])
def update(self, records):
have_updates = False
@ -92,6 +99,8 @@ class CachedMemoryStorage(MemoryStorage):
record_day = utils.timestamp_to_day(record['date'])
self.day_index[record_day].remove(record['record_id'])
self.module_release_index[
(record['module'], record['release'])].remove(record['record_id'])
def _add_to_index(self, record_index, record, key):
record_key = record[key]
@ -129,6 +138,12 @@ class CachedMemoryStorage(MemoryStorage):
def get_record_ids_by_days(self, days):
return self._get_record_ids_from_index(days, self.day_index)
def get_record_ids_by_module_release(self, module, release):
mr = (module, release)
if mr in self.module_release_index:
return self.module_release_index[mr]
return set()
def get_index_keys_by_record_ids(self, index_name, record_ids):
return set([key
for key, value in six.iteritems(self.indexes[index_name])

2
dashboard/parameters.py
View File

@ -35,7 +35,7 @@ METRIC_LABELS = {
'emails': 'Emails',
'bpd': 'Drafted Blueprints',
'bpc': 'Completed Blueprints',
'person-day': "Person-day effort"
# 'person-day': "Person-day effort"
}
METRIC_TO_RECORD_TYPE = {

23
dashboard/vault.py
View File

@ -68,18 +68,19 @@ def get_vault():
LOG.exception(e)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
time_now = utils.date_to_timestamp('now')
may_update_by_time = (time_now > vault.get('vault_update_time', 0) +
cfg.CONF.dashboard_update_interval)
if (may_update_by_time and
not getattr(flask.request, 'stackalytics_updated', None)):
if may_update_by_time:
flask.request.stackalytics_updated = True
vault['vault_update_time'] = time_now
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
compact_records(vault['runtime_storage'].get_update(os.getpid())))
have_updates = memory_storage_inst.update(compact_records(
vault['runtime_storage'].get_update(os.getpid())))
vault['runtime_storage_update_time'] = (
vault['runtime_storage'].get_by_key('runtime_storage_update_time'))
vault['runtime_storage'].get_by_key(
'runtime_storage_update_time'))
if have_updates:
vault['cache'] = {}
@ -171,7 +172,7 @@ def get_user_from_runtime_storage(user_id):
return user_index[user_id]
def resolve_modules(module_ids, releases):
def resolve_modules_for_releases(module_ids, releases):
module_id_index = get_vault().get('module_id_index') or {}
for module_id in module_ids:
@ -197,6 +198,16 @@ def resolve_modules(module_ids, releases):
yield module, release
def resolve_modules(module_ids, releases):
all_releases = get_vault()['releases'].keys()
for module, release in resolve_modules_for_releases(module_ids, releases):
if release is None:
for r in all_releases:
yield (module, r)
else:
yield (module, release)
def resolve_project_types(project_types):
modules = set()
project_types_index = get_vault()['project_types_index']

6
dashboard/web.py
View File

@ -82,9 +82,15 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
param_title = param_title or param_id
result = dict((c, {'metric': 0, 'id': c}) for c in keys)
context = {}
if metric_filter:
for record in records:
metric_filter(result, record, param_id, context)
result[record[param_id]]['name'] = record[param_title]
else:
for record in records:
record_param_id = record[param_id]
result[record_param_id]['metric'] += 1
result[record_param_id]['name'] = record[param_title]
response = [r for r in result.values() if r['metric']]
response = [item for item in map(finalize_handler, response) if item]

4
tests/api/test_companies.py
View File

@ -32,6 +32,10 @@ class TestAPICompanies(test_api.TestAPI):
{'id': 'openstack', 'title': 'OpenStack',
'modules': ['nova', 'glance']}
],
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'module_groups': {
'openstack': {'module_group_name': 'openstack',
'modules': ['nova', 'glance']},

4
tests/api/test_modules.py
View File

@ -35,6 +35,10 @@ class TestAPIModules(test_api.TestAPI):
'nova-cli': test_api.make_module('nova-cli'),
'glance': test_api.make_module('glance'),
},
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'project_types': [
{'id': 'all', 'title': 'All',
'modules': ['nova', 'glance', 'nova-cli']},

12
tests/api/test_stats.py
View File

@ -26,6 +26,10 @@ class TestAPIStats(test_api.TestAPI):
'uri': 'git://github.com/openstack/nova.git'},
{'module': 'glance', 'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'module_groups': {
'openstack': {'id': 'openstack',
'module_group_name': 'openstack',
@ -60,6 +64,10 @@ class TestAPIStats(test_api.TestAPI):
{'module': 'glance', 'project_type': 'openstack',
'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'module_groups': {
'openstack': {'id': 'openstack',
'module_group_name': 'openstack',
@ -108,6 +116,10 @@ class TestAPIStats(test_api.TestAPI):
{'module': 'glance', 'project_type': 'openstack',
'organization': 'openstack',
'uri': 'git://github.com/openstack/glance.git'}],
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'module_groups': {
'openstack': {'id': 'openstack',
'module_group_name': 'openstack',

4
tests/api/test_users.py
View File

@ -27,6 +27,10 @@ class TestAPIUsers(test_api.TestAPI):
'project_types': [
{'id': 'openstack', 'title': 'openstack',
'modules': ['nova', 'glance']}],
'releases': [{'release_name': 'prehistory',
'end_date': 1234567890},
{'release_name': 'icehouse',
'end_date': 1234567890}],
'module_groups': {
'nova': test_api.make_module('nova'),
'glance': test_api.make_module('glance')},