diff --git a/dashboard/decorators.py b/dashboard/decorators.py new file mode 100644 index 000000000..b45bb92e1 --- /dev/null +++ b/dashboard/decorators.py @@ -0,0 +1,277 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import json + +import flask +from werkzeug import exceptions + +from dashboard import helpers +from dashboard import parameters +from dashboard import vault +from stackalytics.openstack.common import log as logging +from stackalytics import version as stackalytics_version + + +LOG = logging.getLogger(__name__) + + +def record_filter(ignore=None, use_default=True): + if not ignore: + ignore = [] + + def decorator(f): + @functools.wraps(f) + def record_filter_decorated_function(*args, **kwargs): + + vault_inst = vault.get_vault() + memory_storage_inst = vault.get_memory_storage() + record_ids = set(memory_storage_inst.get_record_ids()) # a copy + + if 'module' not in ignore: + param = parameters.get_parameter(kwargs, 'module', 'modules', + use_default) + if param: + record_ids &= ( + memory_storage_inst.get_record_ids_by_modules( + vault.resolve_modules(param))) + + if 'project_type' not in ignore: + param = parameters.get_parameter(kwargs, 'project_type', + 'project_types', use_default) + if param: + ptgi = vault_inst['project_type_group_index'] + modules = set() + for project_type in param: + project_type = project_type.lower() + if project_type in ptgi: + modules |= ptgi[project_type] + record_ids &= ( + memory_storage_inst.get_record_ids_by_modules(modules)) + + if 'user_id' not in ignore: + param = parameters.get_parameter(kwargs, 'user_id', 'user_ids') + param = [u for u in param + if vault.get_user_from_runtime_storage(u)] + if param: + record_ids &= ( + memory_storage_inst.get_record_ids_by_user_ids(param)) + + if 'company' not in ignore: + param = parameters.get_parameter(kwargs, 'company', + 'companies') + if param: + record_ids &= ( + memory_storage_inst.get_record_ids_by_companies(param)) + + if 'release' not in ignore: + param = parameters.get_parameter(kwargs, 'release', 'releases', + use_default) + if param: + if 'all' not in param: + record_ids &= ( + memory_storage_inst.get_record_ids_by_releases( + c.lower() for c in param)) + + if 'metric' not in ignore: + metrics = parameters.get_parameter(kwargs, 'metric') + for metric in metrics: + record_ids &= memory_storage_inst.get_record_ids_by_type( + parameters.METRIC_TO_RECORD_TYPE[metric]) + + if 'tm_marks' in metrics: + filtered_ids = [] + review_nth = int(parameters.get_parameter( + kwargs, 'review_nth')[0]) + for record in memory_storage_inst.get_records(record_ids): + parent = memory_storage_inst.get_record_by_primary_key( + record['review_id']) + if (parent and ('review_number' in parent) and + (parent['review_number'] <= review_nth)): + filtered_ids.append(record['record_id']) + record_ids = filtered_ids + + kwargs['records'] = memory_storage_inst.get_records(record_ids) + return f(*args, **kwargs) + + return record_filter_decorated_function + + return decorator + + +def aggregate_filter(): + def decorator(f): + @functools.wraps(f) + def aggregate_filter_decorated_function(*args, **kwargs): + + def incremental_filter(result, record, param_id): + result[record[param_id]]['metric'] += 1 + + def loc_filter(result, record, param_id): + result[record[param_id]]['metric'] += record['loc'] + + def mark_filter(result, record, param_id): + value = record['value'] + result_by_param = result[record[param_id]] + result_by_param['metric'] += 1 + + if value in result_by_param: + result_by_param[value] += 1 + else: + result_by_param[value] = 1 + + def mark_finalize(record): + new_record = {} + for key in ['id', 'metric', 'name']: + new_record[key] = record[key] + + positive = 0 + mark_distribution = [] + for key in ['-2', '-1', '1', '2']: + if key in record: + if key in ['1', '2']: + positive += record[key] + mark_distribution.append(str(record[key])) + else: + mark_distribution.append('0') + + new_record['mark_ratio'] = ( + '|'.join(mark_distribution) + + ' (%.1f%%)' % ((positive * 100.0) / record['metric'])) + return new_record + + metric_param = (flask.request.args.get('metric') or + parameters.get_default('metric')) + metric = metric_param.lower() + + metric_to_filters_map = { + 'commits': (incremental_filter, None), + 'loc': (loc_filter, None), + 'marks': (mark_filter, mark_finalize), + 'tm_marks': (mark_filter, mark_finalize), + 'emails': (incremental_filter, None), + 'bpd': (incremental_filter, None), + 'bpc': (incremental_filter, None), + } + if metric not in metric_to_filters_map: + raise Exception('Invalid metric %s' % metric) + + kwargs['metric_filter'] = metric_to_filters_map[metric][0] + kwargs['finalize_handler'] = metric_to_filters_map[metric][1] + return f(*args, **kwargs) + + return aggregate_filter_decorated_function + + return decorator + + +def exception_handler(): + def decorator(f): + @functools.wraps(f) + def exception_handler_decorated_function(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as e: + if isinstance(e, exceptions.HTTPException): + raise # ignore Flask exceptions + LOG.exception(e) + flask.abort(404) + + return exception_handler_decorated_function + + return decorator + + +def templated(template=None, return_code=200): + def decorator(f): + @functools.wraps(f) + def templated_decorated_function(*args, **kwargs): + + vault_inst = vault.get_vault() + template_name = template + if template_name is None: + template_name = (flask.request.endpoint.replace('.', '/') + + '.html') + ctx = f(*args, **kwargs) + if ctx is None: + ctx = {} + + # put parameters into template + metric = flask.request.args.get('metric') + if metric not in parameters.METRIC_LABELS: + metric = None + ctx['metric'] = metric or parameters.get_default('metric') + ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']] + + project_type = flask.request.args.get('project_type') + if not vault.is_project_type_valid(project_type): + project_type = parameters.get_default('project_type') + ctx['project_type'] = project_type + + release = flask.request.args.get('release') + releases = vault_inst['releases'] + if release: + release = release.lower() + if release != 'all': + if release not in releases: + release = None + else: + release = releases[release]['release_name'] + ctx['release'] = (release or + parameters.get_default('release')).lower() + ctx['review_nth'] = (flask.request.args.get('review_nth') or + parameters.get_default('review_nth')) + + ctx['project_type_options'] = vault.get_project_type_options() + ctx['release_options'] = vault.get_release_options() + ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(), + key=lambda x: x[0]) + + ctx['company'] = parameters.get_single_parameter(kwargs, 'company') + ctx['module'] = parameters.get_single_parameter(kwargs, 'module') + ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id') + ctx['page_title'] = helpers.make_page_title( + ctx['company'], ctx['user_id'], ctx['module'], ctx['release']) + ctx['stackalytics_version'] = ( + stackalytics_version.version_info.version_string()) + ctx['stackalytics_release'] = ( + stackalytics_version.version_info.release_string()) + + return flask.render_template(template_name, **ctx), return_code + + return templated_decorated_function + + return decorator + + +def jsonify(root='data'): + def decorator(func): + @functools.wraps(func) + def jsonify_decorated_function(*args, **kwargs): + callback = flask.app.request.args.get('callback', False) + data = json.dumps({root: func(*args, **kwargs)}) + + if callback: + data = str(callback) + '(' + data + ')' + mimetype = 'application/javascript' + else: + mimetype = 'application/json' + + return flask.current_app.response_class(data, mimetype=mimetype) + + return jsonify_decorated_function + + return decorator diff --git a/dashboard/helpers.py b/dashboard/helpers.py new file mode 100644 index 000000000..eb861f01f --- /dev/null +++ b/dashboard/helpers.py @@ -0,0 +1,166 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import re +import urllib + +from flask.ext import gravatar as gravatar_ext + +from dashboard import parameters +from dashboard import vault +from stackalytics.processor import utils + + +gravatar = gravatar_ext.Gravatar(None, size=64, rating='g', default='wavatar') + + +def _extend_record_common_fields(record): + record['date_str'] = format_datetime(record['date']) + record['author_link'] = make_link( + record['author_name'], '/', + {'user_id': record['user_id'], 'company': ''}) + record['company_link'] = make_link( + record['company_name'], '/', + {'company': record['company_name'], 'user_id': ''}) + record['module_link'] = make_link( + record['module'], '/', + {'module': record['module'], 'company': '', 'user_id': ''}) + record['gravatar'] = gravatar(record.get('author_email', 'stackalytics')) + record['blueprint_id_count'] = len(record.get('blueprint_id', [])) + record['bug_id_count'] = len(record.get('bug_id', [])) + + +def extend_record(record): + if record['record_type'] == 'commit': + commit = record.copy() + commit['branches'] = ','.join(commit['branches']) + if 'correction_comment' not in commit: + commit['correction_comment'] = '' + commit['message'] = make_commit_message(record) + _extend_record_common_fields(commit) + return commit + elif record['record_type'] == 'mark': + review = record.copy() + parent = vault.get_memory_storage().get_record_by_primary_key( + review['review_id']) + if parent: + review['review_number'] = parent.get('review_number') + review['subject'] = parent['subject'] + review['url'] = parent['url'] + review['parent_author_link'] = make_link( + parent['author_name'], '/', + {'user_id': parent['user_id'], + 'company': ''}) + _extend_record_common_fields(review) + return review + elif record['record_type'] == 'email': + email = record.copy() + _extend_record_common_fields(email) + email['email_link'] = email.get('email_link') or '' + return email + elif ((record['record_type'] == 'bpd') or + (record['record_type'] == 'bpc')): + blueprint = record.copy() + _extend_record_common_fields(blueprint) + blueprint['summary'] = utils.format_text(record['summary']) + if record.get('mention_count'): + blueprint['mention_date_str'] = format_datetime( + record['mention_date']) + blueprint['blueprint_link'] = make_blueprint_link( + blueprint['name'], blueprint['module']) + return blueprint + + +def format_datetime(timestamp): + return datetime.datetime.utcfromtimestamp( + timestamp).strftime('%d %b %Y %H:%M:%S') + + +def format_date(timestamp): + return datetime.datetime.utcfromtimestamp(timestamp).strftime('%d-%b-%y') + + +def format_launchpad_module_link(module): + return '%s' % (module, module) + + +def safe_encode(s): + return urllib.quote_plus(s.encode('utf-8')) + + +def make_link(title, uri=None, options=None): + param_names = ('release', 'project_type', 'module', 'company', 'user_id', + 'metric') + param_values = {} + for param_name in param_names: + v = parameters.get_parameter({}, param_name, param_name) + if v: + param_values[param_name] = ','.join(v) + if options: + param_values.update(options) + if param_values: + uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v)) + for n, v in param_values.iteritems()]) + return '%(title)s' % {'uri': uri, 'title': title} + + +def make_blueprint_link(name, module): + uri = '/report/blueprint/' + module + '/' + name + return '%(title)s' % {'uri': uri, 'title': name} + + +def make_commit_message(record): + s = record['message'] + module = record['module'] + + s = utils.format_text(s) + + # insert links + s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE), + r'\1\2', s) + s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE), + r'\1\2', s) + s = re.sub(r'\s+(I[0-9a-f]{40})', + r' \1', s) + + s = utils.unwrap_text(s) + return s + + +def make_page_title(company, user_id, module, release): + if company: + memory_storage = vault.get_memory_storage() + company = memory_storage.get_original_company_name(company) + if company or user_id: + if user_id: + s = vault.get_user_from_runtime_storage(user_id)['user_name'] + if company: + s += ' (%s)' % company + else: + s = company + else: + s = 'OpenStack community' + s += ' contribution' + if module: + s += ' to %s' % module + if release != 'all': + s += ' in %s release' % release.capitalize() + else: + s += ' in all releases' + return s diff --git a/dashboard/parameters.py b/dashboard/parameters.py new file mode 100644 index 000000000..90029a611 --- /dev/null +++ b/dashboard/parameters.py @@ -0,0 +1,81 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask + +from stackalytics.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + + +DEFAULTS = { + 'metric': 'commits', + 'release': 'icehouse', + 'project_type': 'openstack', + 'review_nth': 5, +} + +METRIC_LABELS = { + 'loc': 'Lines of code', + 'commits': 'Commits', + 'marks': 'Reviews', + 'tm_marks': 'Top Mentors', + 'emails': 'Emails', + 'bpd': 'Drafted Blueprints', + 'bpc': 'Completed Blueprints', +} + +METRIC_TO_RECORD_TYPE = { + 'loc': 'commit', + 'commits': 'commit', + 'marks': 'mark', + 'tm_marks': 'mark', + 'emails': 'email', + 'bpd': 'bpd', + 'bpc': 'bpc', +} + +DEFAULT_RECORDS_LIMIT = 10 + + +def get_default(param_name): + if param_name in DEFAULTS: + return DEFAULTS[param_name] + else: + return None + + +def get_parameter(kwargs, singular_name, plural_name=None, use_default=True): + if singular_name in kwargs: + p = kwargs[singular_name] + else: + p = flask.request.args.get(singular_name) + if (not p) and plural_name: + flask.request.args.get(plural_name) + if p: + return p.split(',') + elif use_default: + default = get_default(singular_name) + return [default] if default else [] + else: + return [] + + +def get_single_parameter(kwargs, singular_name, use_default=True): + param = get_parameter(kwargs, singular_name, use_default) + if param: + return param[0] + else: + return '' diff --git a/dashboard/reports.py b/dashboard/reports.py new file mode 100644 index 000000000..0fc57f0b9 --- /dev/null +++ b/dashboard/reports.py @@ -0,0 +1,87 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +import time + +import flask + +from dashboard import decorators +from dashboard import helpers +from dashboard import vault +from stackalytics.processor import utils + + +blueprint = flask.Blueprint('reports', __name__, url_prefix='/report') + + +@blueprint.route('/blueprint//') +@decorators.templated() +@decorators.exception_handler() +def blueprint_summary(module, blueprint_name): + blueprint_id = module + ':' + blueprint_name + bpd = vault.get_memory_storage().get_record_by_primary_key( + 'bpd:' + blueprint_id) + if not bpd: + flask.abort(404) + return + + bpd = helpers.extend_record(bpd) + record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids( + [blueprint_id]) + activity = [helpers.extend_record(record) for record in + vault.get_memory_storage().get_records(record_ids)] + activity.sort(key=lambda x: x['date'], reverse=True) + + return {'blueprint': bpd, 'activity': activity} + + +@blueprint.route('/reviews/') +@decorators.templated() +@decorators.exception_handler() +def open_reviews(module): + memory_storage = vault.get_memory_storage() + now = int(time.time()) + review_ids = (memory_storage.get_record_ids_by_modules([module]) & + memory_storage.get_record_ids_by_type('review')) + records = [] + for review in memory_storage.get_records(review_ids): + if review['status'] != 'NEW': + continue + processed_review = review.copy() + helpers.extend_record(processed_review) + processed_review['age'] = utils.make_age_string( + now - processed_review['date']) + records.append(processed_review) + + return { + 'module': module, + 'oldest': sorted(records, key=operator.itemgetter('date'))[:5] + } + + +@blueprint.route('/large_commits') +@decorators.jsonify('commits') +@decorators.exception_handler() +@decorators.record_filter() +def get_commit_report(records): + loc_threshold = int(flask.request.args.get('loc_threshold') or 0) + response = [] + for record in records: + if ('loc' in record) and (record['loc'] > loc_threshold): + nr = dict([(k, record[k]) for k in ['loc', 'subject', 'module', + 'primary_key', 'change_id']]) + response.append(nr) + return response diff --git a/dashboard/templates/overview.html b/dashboard/templates/overview.html index 6687530b7..6b8d5d366 100644 --- a/dashboard/templates/overview.html +++ b/dashboard/templates/overview.html @@ -11,6 +11,7 @@ {% set show_module_contribution = (module) and (not user_id) %} {% set show_contribution = (show_user_contribution) or (show_module_contribution) %} {% set show_user_profile = (user_id) %} +{% set show_module_profile = (module) %} {% set show_top_mentors_options = (metric == 'tm_marks') %} {% set show_review_ratio = (metric in ['marks', 'tm_marks']) %} @@ -290,10 +291,6 @@ {% endif %} - {% if show_module_contribution %} -
- {% endif %} - {% endblock %} {% block right_frame %} @@ -319,6 +316,13 @@ {% endif %} + {% if show_module_profile %} +

Module {{ module }}

+
Open reviews report
+ +
+ {% endif %} + {% if show_bp_breakdown %}

Blueprint popularity

diff --git a/dashboard/templates/blueprint_report.html b/dashboard/templates/reports/blueprint_summary.html similarity index 100% rename from dashboard/templates/blueprint_report.html rename to dashboard/templates/reports/blueprint_summary.html diff --git a/dashboard/templates/reports/open_reviews.html b/dashboard/templates/reports/open_reviews.html new file mode 100644 index 000000000..47ec36a50 --- /dev/null +++ b/dashboard/templates/reports/open_reviews.html @@ -0,0 +1,26 @@ + + + + Open reviews report for {{ module }} + + + + + +

Open Reviews Report

+ +

Longest waiting reviews (since first revision, total age):

+
    +{% for item in oldest %} +
  1. {{ item.age }} {{ item.url }} {{ item.subject }}
    2. +{% endfor %} +
\ No newline at end of file diff --git a/dashboard/vault.py b/dashboard/vault.py new file mode 100644 index 000000000..d19053402 --- /dev/null +++ b/dashboard/vault.py @@ -0,0 +1,194 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import flask +from oslo.config import cfg + +from dashboard import memory_storage +from stackalytics.openstack.common import log as logging +from stackalytics.processor import runtime_storage +from stackalytics.processor import utils + +LOG = logging.getLogger(__name__) + + +def get_vault(): + vault = getattr(flask.current_app, 'stackalytics_vault', None) + if not vault: + try: + vault = {} + runtime_storage_inst = runtime_storage.get_runtime_storage( + cfg.CONF.runtime_storage_uri) + vault['runtime_storage'] = runtime_storage_inst + vault['memory_storage'] = memory_storage.get_memory_storage( + memory_storage.MEMORY_STORAGE_CACHED) + + init_project_types(vault) + init_releases(vault) + + flask.current_app.stackalytics_vault = vault + except Exception as e: + LOG.critical('Failed to initialize application: %s', e) + LOG.exception(e) + flask.abort(500) + + if not getattr(flask.request, 'stackalytics_updated', None): + flask.request.stackalytics_updated = True + memory_storage_inst = vault['memory_storage'] + have_updates = memory_storage_inst.update( + vault['runtime_storage'].get_update(os.getpid())) + + if have_updates: + init_project_types(vault) + init_releases(vault) + init_module_groups(vault) + + return vault + + +def get_memory_storage(): + return get_vault()['memory_storage'] + + +def init_releases(vault): + runtime_storage_inst = vault['runtime_storage'] + releases = runtime_storage_inst.get_by_key('releases') + if not releases: + raise Exception('Releases are missing in runtime storage') + vault['start_date'] = releases[0]['end_date'] + vault['end_date'] = releases[-1]['end_date'] + start_date = releases[0]['end_date'] + for r in releases[1:]: + r['start_date'] = start_date + start_date = r['end_date'] + vault['releases'] = dict((r['release_name'].lower(), r) + for r in releases[1:]) + + +def init_project_types(vault): + runtime_storage_inst = vault['runtime_storage'] + project_type_options = {} + project_type_group_index = {'all': set(['unknown'])} + + for repo in utils.load_repos(runtime_storage_inst): + project_type = repo['project_type'].lower() + project_group = None + if ('project_group' in repo) and (repo['project_group']): + project_group = repo['project_group'].lower() + + if project_type in project_type_options: + if project_group: + project_type_options[project_type].add(project_group) + else: + if project_group: + project_type_options[project_type] = set([project_group]) + else: + project_type_options[project_type] = set() + + module = repo['module'] + if project_type in project_type_group_index: + project_type_group_index[project_type].add(module) + else: + project_type_group_index[project_type] = set([module]) + + if project_group: + if project_group in project_type_group_index: + project_type_group_index[project_group].add(module) + else: + project_type_group_index[project_group] = set([module]) + + project_type_group_index['all'].add(module) + + vault['project_type_options'] = project_type_options + vault['project_type_group_index'] = project_type_group_index + + +def init_module_groups(vault): + runtime_storage_inst = vault['runtime_storage'] + module_index = {} + module_id_index = {} + module_groups = runtime_storage_inst.get_by_key('module_groups') or [] + + for module_group in module_groups: + module_group_name = module_group['module_group_name'] + module_group_id = module_group_name.lower() + + module_id_index[module_group_name] = { + 'group': True, + 'id': module_group_id, + 'text': module_group_name, + 'modules': [m.lower() for m in module_group['modules']], + } + + modules = module_group['modules'] + for module in modules: + if module in module_index: + module_index[module].add(module_group_id) + else: + module_index[module] = set([module_group_id]) + + memory_storage_inst = vault['memory_storage'] + for module in memory_storage_inst.get_modules(): + module_id_index[module] = { + 'id': module.lower(), + 'text': module, + 'modules': [module.lower()], + } + + vault['module_group_index'] = module_index + vault['module_id_index'] = module_id_index + vault['module_groups'] = module_groups + + +def get_project_type_options(): + return get_vault()['project_type_options'] + + +def get_release_options(): + runtime_storage_inst = get_vault()['runtime_storage'] + releases = runtime_storage_inst.get_by_key('releases')[1:] + releases.reverse() + return releases + + +def is_project_type_valid(project_type): + if not project_type: + return False + project_type = project_type.lower() + if project_type == 'all': + return True + project_types = get_project_type_options() + if project_type in project_types: + return True + for p, g in project_types.iteritems(): + if project_type in g: + return True + return False + + +def get_user_from_runtime_storage(user_id): + runtime_storage_inst = get_vault()['runtime_storage'] + return utils.load_user(runtime_storage_inst, user_id) + + +def resolve_modules(module_ids): + module_id_index = get_vault()['module_id_index'] + modules = set() + for module_id in module_ids: + if module_id in module_id_index: + modules |= set(module_id_index[module_id]['modules']) + return modules diff --git a/dashboard/web.py b/dashboard/web.py index 6118727db..3c5e7acdb 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -13,58 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime -import functools -import json import operator import os -import re -import urllib +import time import flask from flask.ext import gravatar as gravatar_ext from oslo.config import cfg -import time -from werkzeug import exceptions -from dashboard import memory_storage +from dashboard import decorators +from dashboard import helpers +from dashboard import parameters +from dashboard import reports +from dashboard import vault from stackalytics.openstack.common import log as logging from stackalytics.processor import config -from stackalytics.processor import runtime_storage from stackalytics.processor import utils -from stackalytics import version as stackalytics_version - - -# Constants and Parameters --------- - -DEFAULTS = { - 'metric': 'commits', - 'release': 'icehouse', - 'project_type': 'openstack', - 'review_nth': 5, -} - -METRIC_LABELS = { - 'loc': 'Lines of code', - 'commits': 'Commits', - 'marks': 'Reviews', - 'tm_marks': 'Top Mentors', - 'emails': 'Emails', - 'bpd': 'Drafted Blueprints', - 'bpc': 'Completed Blueprints', -} - -METRIC_TO_RECORD_TYPE = { - 'loc': 'commit', - 'commits': 'commit', - 'marks': 'mark', - 'tm_marks': 'mark', - 'emails': 'email', - 'bpd': 'bpd', - 'bpc': 'bpc', -} - -DEFAULT_RECORDS_LIMIT = 10 # Application objects --------- @@ -72,6 +36,7 @@ DEFAULT_RECORDS_LIMIT = 10 app = flask.Flask(__name__) app.config.from_object(__name__) app.config.from_envvar('DASHBOARD_CONF', silent=True) +app.register_blueprint(reports.blueprint) LOG = logging.getLogger(__name__) @@ -88,484 +53,16 @@ else: LOG.warn('Conf file is empty or not exist') -def get_vault(): - vault = getattr(app, 'stackalytics_vault', None) - if not vault: - try: - vault = {} - runtime_storage_inst = runtime_storage.get_runtime_storage( - cfg.CONF.runtime_storage_uri) - vault['runtime_storage'] = runtime_storage_inst - vault['memory_storage'] = memory_storage.get_memory_storage( - memory_storage.MEMORY_STORAGE_CACHED) - - init_project_types(vault) - init_releases(vault) - - app.stackalytics_vault = vault - except Exception as e: - LOG.critical('Failed to initialize application: %s', e) - LOG.exception(e) - flask.abort(500) - - if not getattr(flask.request, 'stackalytics_updated', None): - flask.request.stackalytics_updated = True - memory_storage_inst = vault['memory_storage'] - have_updates = memory_storage_inst.update( - vault['runtime_storage'].get_update(os.getpid())) - - if have_updates: - init_project_types(vault) - init_releases(vault) - init_module_groups(vault) - - return vault - - -def get_memory_storage(): - return get_vault()['memory_storage'] - - -def init_releases(vault): - runtime_storage_inst = vault['runtime_storage'] - releases = runtime_storage_inst.get_by_key('releases') - if not releases: - raise Exception('Releases are missing in runtime storage') - vault['start_date'] = releases[0]['end_date'] - vault['end_date'] = releases[-1]['end_date'] - start_date = releases[0]['end_date'] - for r in releases[1:]: - r['start_date'] = start_date - start_date = r['end_date'] - vault['releases'] = dict((r['release_name'].lower(), r) - for r in releases[1:]) - - -def init_project_types(vault): - runtime_storage_inst = vault['runtime_storage'] - project_type_options = {} - project_type_group_index = {'all': set(['unknown'])} - - for repo in utils.load_repos(runtime_storage_inst): - project_type = repo['project_type'].lower() - project_group = None - if ('project_group' in repo) and (repo['project_group']): - project_group = repo['project_group'].lower() - - if project_type in project_type_options: - if project_group: - project_type_options[project_type].add(project_group) - else: - if project_group: - project_type_options[project_type] = set([project_group]) - else: - project_type_options[project_type] = set() - - module = repo['module'] - if project_type in project_type_group_index: - project_type_group_index[project_type].add(module) - else: - project_type_group_index[project_type] = set([module]) - - if project_group: - if project_group in project_type_group_index: - project_type_group_index[project_group].add(module) - else: - project_type_group_index[project_group] = set([module]) - - project_type_group_index['all'].add(module) - - vault['project_type_options'] = project_type_options - vault['project_type_group_index'] = project_type_group_index - - -def init_module_groups(vault): - runtime_storage_inst = vault['runtime_storage'] - module_index = {} - module_id_index = {} - module_groups = runtime_storage_inst.get_by_key('module_groups') or [] - - for module_group in module_groups: - module_group_name = module_group['module_group_name'] - module_group_id = module_group_name.lower() - - module_id_index[module_group_name] = { - 'group': True, - 'id': module_group_id, - 'text': module_group_name, - 'modules': [m.lower() for m in module_group['modules']], - } - - modules = module_group['modules'] - for module in modules: - if module in module_index: - module_index[module].add(module_group_id) - else: - module_index[module] = set([module_group_id]) - - memory_storage_inst = vault['memory_storage'] - for module in memory_storage_inst.get_modules(): - module_id_index[module] = { - 'id': module.lower(), - 'text': module, - 'modules': [module.lower()], - } - - vault['module_group_index'] = module_index - vault['module_id_index'] = module_id_index - vault['module_groups'] = module_groups - - -def get_project_type_options(): - return get_vault()['project_type_options'] - - -def get_release_options(): - runtime_storage_inst = get_vault()['runtime_storage'] - releases = runtime_storage_inst.get_by_key('releases')[1:] - releases.reverse() - return releases - - -def is_project_type_valid(project_type): - if not project_type: - return False - project_type = project_type.lower() - if project_type == 'all': - return True - project_types = get_project_type_options() - if project_type in project_types: - return True - for p, g in project_types.iteritems(): - if project_type in g: - return True - return False - - -def get_user_from_runtime_storage(user_id): - runtime_storage_inst = get_vault()['runtime_storage'] - return utils.load_user(runtime_storage_inst, user_id) - - -# Utils --------- - -def get_default(param_name): - if param_name in DEFAULTS: - return DEFAULTS[param_name] - else: - return None - - -def get_parameter(kwargs, singular_name, plural_name=None, use_default=True): - if singular_name in kwargs: - p = kwargs[singular_name] - else: - p = flask.request.args.get(singular_name) - if (not p) and plural_name: - flask.request.args.get(plural_name) - if p: - return p.split(',') - elif use_default: - default = get_default(singular_name) - return [default] if default else [] - else: - return [] - - -def get_single_parameter(kwargs, singular_name, use_default=True): - param = get_parameter(kwargs, singular_name, use_default) - if param: - return param[0] - else: - return '' - - -def resolve_modules(module_ids): - module_id_index = get_vault()['module_id_index'] - modules = set() - for module_id in module_ids: - if module_id in module_id_index: - modules |= set(module_id_index[module_id]['modules']) - return modules - - -# Decorators --------- - -def record_filter(ignore=None, use_default=True): - if not ignore: - ignore = [] - - def decorator(f): - @functools.wraps(f) - def record_filter_decorated_function(*args, **kwargs): - - vault = get_vault() - memory_storage = vault['memory_storage'] - record_ids = set(memory_storage.get_record_ids()) # make a copy - - if 'module' not in ignore: - param = get_parameter(kwargs, 'module', 'modules', use_default) - if param: - record_ids &= (memory_storage.get_record_ids_by_modules( - resolve_modules(param))) - - if 'project_type' not in ignore: - param = get_parameter(kwargs, 'project_type', 'project_types', - use_default) - if param: - ptgi = vault['project_type_group_index'] - modules = set() - for project_type in param: - project_type = project_type.lower() - if project_type in ptgi: - modules |= ptgi[project_type] - record_ids &= ( - memory_storage.get_record_ids_by_modules(modules)) - - if 'user_id' not in ignore: - param = get_parameter(kwargs, 'user_id', 'user_ids') - param = [u for u in param if get_user_from_runtime_storage(u)] - if param: - record_ids &= ( - memory_storage.get_record_ids_by_user_ids(param)) - - if 'company' not in ignore: - param = get_parameter(kwargs, 'company', 'companies') - if param: - record_ids &= ( - memory_storage.get_record_ids_by_companies(param)) - - if 'release' not in ignore: - param = get_parameter(kwargs, 'release', 'releases', - use_default) - if param: - if 'all' not in param: - record_ids &= ( - memory_storage.get_record_ids_by_releases( - c.lower() for c in param)) - - if 'metric' not in ignore: - metrics = get_parameter(kwargs, 'metric') - for metric in metrics: - record_ids &= memory_storage.get_record_ids_by_type( - METRIC_TO_RECORD_TYPE[metric]) - - if 'tm_marks' in metrics: - filtered_ids = [] - review_nth = int(get_parameter(kwargs, 'review_nth')[0]) - for record in memory_storage.get_records(record_ids): - parent = memory_storage.get_record_by_primary_key( - record['review_id']) - if (parent and ('review_number' in parent) and - (parent['review_number'] <= review_nth)): - filtered_ids.append(record['record_id']) - record_ids = filtered_ids - - kwargs['records'] = memory_storage.get_records(record_ids) - return f(*args, **kwargs) - - return record_filter_decorated_function - - return decorator - - -def aggregate_filter(): - def decorator(f): - @functools.wraps(f) - def aggregate_filter_decorated_function(*args, **kwargs): - - def incremental_filter(result, record, param_id): - result[record[param_id]]['metric'] += 1 - - def loc_filter(result, record, param_id): - result[record[param_id]]['metric'] += record['loc'] - - def mark_filter(result, record, param_id): - value = record['value'] - result_by_param = result[record[param_id]] - result_by_param['metric'] += 1 - - if value in result_by_param: - result_by_param[value] += 1 - else: - result_by_param[value] = 1 - - def mark_finalize(record): - new_record = {} - for key in ['id', 'metric', 'name']: - new_record[key] = record[key] - - positive = 0 - mark_distribution = [] - for key in ['-2', '-1', '1', '2']: - if key in record: - if key in ['1', '2']: - positive += record[key] - mark_distribution.append(str(record[key])) - else: - mark_distribution.append('0') - - new_record['mark_ratio'] = ( - '|'.join(mark_distribution) + - ' (%.1f%%)' % ((positive * 100.0) / record['metric'])) - return new_record - - metric_param = (flask.request.args.get('metric') or - get_default('metric')) - metric = metric_param.lower() - - metric_to_filters_map = { - 'commits': (incremental_filter, None), - 'loc': (loc_filter, None), - 'marks': (mark_filter, mark_finalize), - 'tm_marks': (mark_filter, mark_finalize), - 'emails': (incremental_filter, None), - 'bpd': (incremental_filter, None), - 'bpc': (incremental_filter, None), - } - if metric not in metric_to_filters_map: - raise Exception('Invalid metric %s' % metric) - - kwargs['metric_filter'] = metric_to_filters_map[metric][0] - kwargs['finalize_handler'] = metric_to_filters_map[metric][1] - return f(*args, **kwargs) - - return aggregate_filter_decorated_function - - return decorator - - -def exception_handler(): - def decorator(f): - @functools.wraps(f) - def exception_handler_decorated_function(*args, **kwargs): - try: - return f(*args, **kwargs) - except Exception as e: - if isinstance(e, exceptions.HTTPException): - raise # ignore Flask exceptions - LOG.exception(e) - flask.abort(404) - - return exception_handler_decorated_function - - return decorator - - -def make_page_title(company, user_id, module, release): - if company: - memory_storage = get_vault()['memory_storage'] - company = memory_storage.get_original_company_name(company) - if company or user_id: - if user_id: - s = get_user_from_runtime_storage(user_id)['user_name'] - if company: - s += ' (%s)' % company - else: - s = company - else: - s = 'OpenStack community' - s += ' contribution' - if module: - s += ' to %s' % module - if release != 'all': - s += ' in %s release' % release.capitalize() - else: - s += ' in all releases' - return s - - -def templated(template=None, return_code=200): - def decorator(f): - @functools.wraps(f) - def templated_decorated_function(*args, **kwargs): - - vault = get_vault() - template_name = template - if template_name is None: - template_name = (flask.request.endpoint.replace('.', '/') + - '.html') - ctx = f(*args, **kwargs) - if ctx is None: - ctx = {} - - # put parameters into template - metric = flask.request.args.get('metric') - if metric not in METRIC_LABELS: - metric = None - ctx['metric'] = metric or get_default('metric') - ctx['metric_label'] = METRIC_LABELS[ctx['metric']] - - project_type = flask.request.args.get('project_type') - if not is_project_type_valid(project_type): - project_type = get_default('project_type') - ctx['project_type'] = project_type - - release = flask.request.args.get('release') - releases = vault['releases'] - if release: - release = release.lower() - if release != 'all': - if release not in releases: - release = None - else: - release = releases[release]['release_name'] - ctx['release'] = (release or get_default('release')).lower() - ctx['review_nth'] = (flask.request.args.get('review_nth') or - get_default('review_nth')) - - ctx['project_type_options'] = get_project_type_options() - ctx['release_options'] = get_release_options() - ctx['metric_options'] = sorted(METRIC_LABELS.items(), - key=lambda x: x[0]) - - ctx['company'] = get_single_parameter(kwargs, 'company') - ctx['module'] = get_single_parameter(kwargs, 'module') - ctx['user_id'] = get_single_parameter(kwargs, 'user_id') - ctx['page_title'] = make_page_title(ctx['company'], ctx['user_id'], - ctx['module'], ctx['release']) - ctx['stackalytics_version'] = ( - stackalytics_version.version_info.version_string()) - ctx['stackalytics_release'] = ( - stackalytics_version.version_info.release_string()) - - return flask.render_template(template_name, **ctx), return_code - - return templated_decorated_function - - return decorator - - -def jsonify(root='data'): - def decorator(func): - @functools.wraps(func) - def jsonify_decorated_function(*args, **kwargs): - callback = flask.app.request.args.get('callback', False) - data = json.dumps({root: func(*args, **kwargs)}) - - if callback: - data = str(callback) + '(' + data + ')' - content_type = 'application/javascript' - else: - content_type = 'application/json' - - return app.response_class(data, mimetype=content_type) - - return jsonify_decorated_function - - return decorator - - # Handlers --------- @app.route('/') -@templated() +@decorators.templated() def overview(): pass @app.errorhandler(404) -@templated('404.html', 404) +@decorators.templated('404.html', 404) def page_not_found(e): pass @@ -590,43 +87,43 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id, @app.route('/api/1.0/stats/companies') -@jsonify('stats') -@exception_handler() -@record_filter() -@aggregate_filter() +@decorators.jsonify('stats') +@decorators.exception_handler() +@decorators.record_filter() +@decorators.aggregate_filter() def get_companies(records, metric_filter, finalize_handler): return _get_aggregated_stats(records, metric_filter, - get_memory_storage().get_companies(), + vault.get_memory_storage().get_companies(), 'company_name') @app.route('/api/1.0/stats/modules') -@jsonify('stats') -@exception_handler() -@record_filter() -@aggregate_filter() +@decorators.jsonify('stats') +@decorators.exception_handler() +@decorators.record_filter() +@decorators.aggregate_filter() def get_modules(records, metric_filter, finalize_handler): return _get_aggregated_stats(records, metric_filter, - get_memory_storage().get_modules(), + vault.get_memory_storage().get_modules(), 'module') @app.route('/api/1.0/stats/engineers') -@jsonify('stats') -@exception_handler() -@record_filter() -@aggregate_filter() +@decorators.jsonify('stats') +@decorators.exception_handler() +@decorators.record_filter() +@decorators.aggregate_filter() def get_engineers(records, metric_filter, finalize_handler): return _get_aggregated_stats(records, metric_filter, - get_memory_storage().get_user_ids(), + vault.get_memory_storage().get_user_ids(), 'user_id', 'author_name', finalize_handler=finalize_handler) @app.route('/api/1.0/stats/distinct_engineers') -@jsonify('stats') -@exception_handler() -@record_filter() +@decorators.jsonify('stats') +@decorators.exception_handler() +@decorators.record_filter() def get_distinct_engineers(records): result = {} for record in records: @@ -637,74 +134,17 @@ def get_distinct_engineers(records): return result -def _extend_record_common_fields(record): - record['date_str'] = format_datetime(record['date']) - record['author_link'] = make_link( - record['author_name'], '/', - {'user_id': record['user_id'], 'company': ''}) - record['company_link'] = make_link( - record['company_name'], '/', - {'company': record['company_name'], 'user_id': ''}) - record['module_link'] = make_link( - record['module'], '/', - {'module': record['module'], 'company': '', 'user_id': ''}) - record['gravatar'] = gravatar(record.get('author_email', 'stackalytics')) - record['blueprint_id_count'] = len(record.get('blueprint_id', [])) - record['bug_id_count'] = len(record.get('bug_id', [])) - - -def _extend_record(record): - if record['record_type'] == 'commit': - commit = record.copy() - commit['branches'] = ','.join(commit['branches']) - if 'correction_comment' not in commit: - commit['correction_comment'] = '' - commit['message'] = make_commit_message(record) - _extend_record_common_fields(commit) - return commit - elif record['record_type'] == 'mark': - review = record.copy() - parent = get_memory_storage().get_record_by_primary_key( - review['review_id']) - if parent: - review['review_number'] = parent.get('review_number') - review['subject'] = parent['subject'] - review['url'] = parent['url'] - review['parent_author_link'] = make_link( - parent['author_name'], '/', - {'user_id': parent['user_id'], - 'company': ''}) - _extend_record_common_fields(review) - return review - elif record['record_type'] == 'email': - email = record.copy() - _extend_record_common_fields(email) - email['email_link'] = email.get('email_link') or '' - return email - elif ((record['record_type'] == 'bpd') or - (record['record_type'] == 'bpc')): - blueprint = record.copy() - _extend_record_common_fields(blueprint) - blueprint['summary'] = utils.format_text(record['summary']) - if record.get('mention_count'): - blueprint['mention_date_str'] = format_datetime( - record['mention_date']) - blueprint['blueprint_link'] = make_blueprint_link( - blueprint['name'], blueprint['module']) - return blueprint - - @app.route('/api/1.0/activity') -@jsonify('activity') -@exception_handler() -@record_filter() +@decorators.jsonify('activity') +@decorators.exception_handler() +@decorators.record_filter() def get_activity_json(records): start_record = int(flask.request.args.get('start_record') or 0) page_size = int(flask.request.args.get('page_size') or - DEFAULT_RECORDS_LIMIT) + parameters.DEFAULT_RECORDS_LIMIT) result = [] for record in records: - processed_record = _extend_record(record) + processed_record = helpers.extend_record(record) if processed_record: result.append(processed_record) @@ -713,9 +153,9 @@ def get_activity_json(records): @app.route('/api/1.0/contribution') -@jsonify('contribution') -@exception_handler() -@record_filter(ignore='metric') +@decorators.jsonify('contribution') +@decorators.exception_handler() +@decorators.record_filter(ignore='metric') def get_contribution_json(records): marks = dict((m, 0) for m in [-2, -1, 0, 1, 2]) commit_count = 0 @@ -750,9 +190,9 @@ def get_contribution_json(records): @app.route('/api/1.0/companies') -@jsonify('companies') -@exception_handler() -@record_filter(ignore='company') +@decorators.jsonify('companies') +@decorators.exception_handler() +@decorators.record_filter(ignore='company') def get_companies_json(records): query = flask.request.args.get('company_name') or '' options = set() @@ -762,18 +202,18 @@ def get_companies_json(records): continue if name.lower().find(query.lower()) >= 0: options.add(name) - result = [{'id': safe_encode(c.lower()), 'text': c} + result = [{'id': helpers.safe_encode(c.lower()), 'text': c} for c in sorted(options)] return result @app.route('/api/1.0/modules') -@jsonify('modules') -@exception_handler() -@record_filter(ignore='module') +@decorators.jsonify('modules') +@decorators.exception_handler() +@decorators.record_filter(ignore='module') def get_modules_json(records): - module_group_index = get_vault()['module_group_index'] - module_id_index = get_vault()['module_id_index'] + module_group_index = vault.get_vault()['module_group_index'] + module_id_index = vault.get_vault()['module_id_index'] modules_set = set() for record in records: @@ -799,22 +239,23 @@ def get_modules_json(records): @app.route('/api/1.0/companies/') -@jsonify('company') +@decorators.jsonify('company') def get_company(company_name): - memory_storage = get_vault()['memory_storage'] - for company in memory_storage.get_companies(): + memory_storage_inst = vault.get_memory_storage() + for company in memory_storage_inst.get_companies(): if company.lower() == company_name.lower(): return { 'id': company_name, - 'text': memory_storage.get_original_company_name(company_name) + 'text': memory_storage_inst.get_original_company_name( + company_name) } flask.abort(404) @app.route('/api/1.0/modules/') -@jsonify('module') +@decorators.jsonify('module') def get_module(module): - module_id_index = get_vault()['module_id_index'] + module_id_index = vault.get_vault()['module_id_index'] module = module.lower() if module in module_id_index: return module_id_index[module] @@ -822,16 +263,16 @@ def get_module(module): @app.route('/api/1.0/stats/bp') -@jsonify('stats') -@exception_handler() -@record_filter() +@decorators.jsonify('stats') +@decorators.exception_handler() +@decorators.record_filter() def get_bpd(records): result = [] for record in records: if record['record_type'] in ['bpd', 'bpc']: mention_date = record.get('mention_date') if mention_date: - date = format_date(mention_date) + date = helpers.format_date(mention_date) else: date = 'never' result.append({ @@ -840,7 +281,8 @@ def get_bpd(records): 'metric': record.get('mention_count') or 0, 'id': record['name'], 'name': record['name'], - 'link': make_blueprint_link(record['name'], record['module']) + 'link': helpers.make_blueprint_link( + record['name'], record['module']) }) result.sort(key=lambda x: x['metric'], reverse=True) @@ -849,9 +291,9 @@ def get_bpd(records): @app.route('/api/1.0/users') -@jsonify('users') -@exception_handler() -@record_filter(ignore='user_id') +@decorators.jsonify('users') +@decorators.exception_handler() +@decorators.record_filter(ignore='user_id') def get_users_json(records): user_name_query = flask.request.args.get('user_name') or '' user_ids = set() @@ -869,42 +311,42 @@ def get_users_json(records): @app.route('/api/1.0/users/') -@jsonify('user') +@decorators.jsonify('user') def get_user(user_id): - user = get_user_from_runtime_storage(user_id) + user = vault.get_user_from_runtime_storage(user_id) if not user: flask.abort(404) user['id'] = user['user_id'] user['text'] = user['user_name'] if user['companies']: company_name = user['companies'][-1]['company_name'] - user['company_link'] = make_link( + user['company_link'] = helpers.make_link( company_name, '/', {'company': company_name, 'user_id': ''}) else: user['company_link'] = '' if user['emails']: - user['gravatar'] = gravatar(user['emails'][0]) + user['gravatar'] = helpers.gravatar(user['emails'][0]) else: - user['gravatar'] = gravatar('stackalytics') + user['gravatar'] = helpers.gravatar('stackalytics') return user @app.route('/api/1.0/stats/timeline') -@jsonify('timeline') -@exception_handler() -@record_filter(ignore='release') +@decorators.jsonify('timeline') +@decorators.exception_handler() +@decorators.record_filter(ignore='release') def timeline(records, **kwargs): # find start and end dates - release_names = get_parameter(kwargs, 'release', 'releases') - releases = get_vault()['releases'] + release_names = parameters.get_parameter(kwargs, 'release', 'releases') + releases = vault.get_vault()['releases'] if not release_names: flask.abort(404) if 'all' in release_names: start_date = release_start_date = utils.timestamp_to_week( - get_vault()['start_date']) + vault.get_vault()['start_date']) end_date = release_end_date = utils.timestamp_to_week( - get_vault()['end_date']) + vault.get_vault()['end_date']) else: release = releases[release_names[0]] start_date = release_start_date = utils.timestamp_to_week( @@ -929,7 +371,7 @@ def timeline(records, **kwargs): week_stat_commits = dict((c, 0) for c in weeks) week_stat_commits_hl = dict((c, 0) for c in weeks) - param = get_parameter(kwargs, 'metric') + param = parameters.get_parameter(kwargs, 'metric') if ('commits' in param) or ('loc' in param): handler = lambda record: record['loc'] else: @@ -958,106 +400,6 @@ def timeline(records, **kwargs): return [array_commits, array_commits_hl, array_loc] -@app.route('/api/1.0/report/commits') -@jsonify('commits') -@exception_handler() -@record_filter() -def get_commit_report(records): - loc_threshold = int(flask.request.args.get('loc_threshold') or 0) - response = [] - for record in records: - if ('loc' in record) and (record['loc'] > loc_threshold): - nr = dict([(k, record[k]) for k in ['loc', 'subject', 'module', - 'primary_key', 'change_id']]) - response.append(nr) - return response - - -@app.route('/report/blueprint//') -@templated() -@exception_handler() -def blueprint_report(module, blueprint_name): - blueprint_id = module + ':' + blueprint_name - bpd = get_memory_storage().get_record_by_primary_key('bpd:' + blueprint_id) - if not bpd: - flask.abort(404) - return - - bpd = _extend_record(bpd) - record_ids = get_memory_storage().get_record_ids_by_blueprint_ids( - [blueprint_id]) - activity = [_extend_record(record) for record in - get_memory_storage().get_records(record_ids)] - activity.sort(key=lambda x: x['date'], reverse=True) - - return {'blueprint': bpd, 'activity': activity} - - -# Jinja Filters --------- - -@app.template_filter('datetimeformat') -def format_datetime(timestamp): - return datetime.datetime.utcfromtimestamp( - timestamp).strftime('%d %b %Y %H:%M:%S') - - -def format_date(timestamp): - return datetime.datetime.utcfromtimestamp(timestamp).strftime('%d-%b-%y') - - -@app.template_filter('launchpadmodule') -def format_launchpad_module_link(module): - return '%s' % (module, module) - - -@app.template_filter('encode') -def safe_encode(s): - return urllib.quote_plus(s.encode('utf-8')) - - -@app.template_filter('link') -def make_link(title, uri=None, options=None): - param_names = ('release', 'project_type', 'module', 'company', 'user_id', - 'metric') - param_values = {} - for param_name in param_names: - v = get_parameter({}, param_name, param_name) - if v: - param_values[param_name] = ','.join(v) - if options: - param_values.update(options) - if param_values: - uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v)) - for n, v in param_values.iteritems()]) - return '%(title)s' % {'uri': uri, 'title': title} - - -def make_blueprint_link(name, module): - uri = '/report/blueprint/' + module + '/' + name - return '%(title)s' % {'uri': uri, 'title': name} - - -def make_commit_message(record): - s = record['message'] - module = record['module'] - - s = utils.format_text(s) - - # insert links - s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE), - r'\1\2', s) - s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE), - r'\1\2', s) - s = re.sub(r'\s+(I[0-9a-f]{40})', - r' \1', s) - - s = utils.unwrap_text(s) - return s - - gravatar = gravatar_ext.Gravatar(app, size=64, rating='g', default='wavatar') diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py index b0468a5b9..548397002 100644 --- a/stackalytics/processor/utils.py +++ b/stackalytics/processor/utils.py @@ -119,6 +119,13 @@ def format_text(s): return s +def make_age_string(seconds): + days = seconds / (3600 * 24) + hours = (seconds / 3600) - (days * 24) + minutes = (seconds / 60) - (days * 24 * 60) - (hours * 60) + return '%d days, %d hours, %d minutes' % (days, hours, minutes) + + def merge_records(original, new): need_update = False for key, value in new.iteritems(): diff --git a/tests/unit/test_web_utils.py b/tests/unit/test_web_utils.py index f4bda8567..619decd0d 100644 --- a/tests/unit/test_web_utils.py +++ b/tests/unit/test_web_utils.py @@ -16,7 +16,7 @@ import mock import testtools -from dashboard import web +from dashboard import helpers class TestWebUtils(testtools.TestCase): @@ -50,7 +50,7 @@ Fixes bug \ 'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') - observed = web.make_commit_message(record) + observed = helpers.make_commit_message(record) self.assertEqual(expected, observed, 'Commit message should be processed correctly') @@ -77,13 +77,13 @@ Implements Blueprint ''' + ( 'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">' 'Ie49ccd2138905e178843b375a9b16c3fe572d1db') - observed = web.make_commit_message(record) + observed = helpers.make_commit_message(record) self.assertEqual(expected, observed, 'Commit message should be processed correctly') - @mock.patch('dashboard.web.get_vault') - @mock.patch('dashboard.web.get_user_from_runtime_storage') + @mock.patch('dashboard.vault.get_vault') + @mock.patch('dashboard.vault.get_user_from_runtime_storage') def test_make_page_title(self, user_patch, vault_patch): memory_storage_mock = mock.Mock() memory_storage_mock.get_original_company_name = mock.Mock( @@ -93,13 +93,14 @@ Implements Blueprint ''' + ( user_patch.return_value = {'user_name': 'John Doe'} self.assertEqual('OpenStack community contribution in all releases', - web.make_page_title('', '', '', 'all')) + helpers.make_page_title('', '', '', 'all')) self.assertEqual('OpenStack community contribution in Havana release', - web.make_page_title('', '', '', 'Havana')) + helpers.make_page_title('', '', '', 'Havana')) self.assertEqual('Mirantis contribution in Havana release', - web.make_page_title('Mirantis', '', '', 'Havana')) + helpers.make_page_title('Mirantis', '', '', 'Havana')) self.assertEqual('John Doe contribution in Havana release', - web.make_page_title('', 'john_doe', '', 'Havana')) + helpers.make_page_title('', 'john_doe', '', 'Havana')) self.assertEqual( 'John Doe (Mirantis) contribution to neutron in Havana release', - web.make_page_title('Mirantis', 'John Doe', 'neutron', 'Havana')) + helpers.make_page_title( + 'Mirantis', 'John Doe', 'neutron', 'Havana'))