x/stackalytics
Code Issues Proposed changes

Split huge wep.py into several modules

Browse Source 
* Extract all vault functions
* Extract decorators and parameter processing functions
* Extract helper functions
* Extract reports and turn them into Flask Blueprints
* Move report templates under report/ subfolder

Change-Id: I19914731bb6506eb8d37c56edb52c5a796d01f6f
This commit is contained in:
Ilya Shakhat 2013-10-24 13:28:19 +04:00
parent fcca170df9
commit 656bce0238
11 changed files with 930 additions and 745 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
dashboard/decorators.py Normal file
View File

@ -0,0 +1,277 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import json
import flask
from werkzeug import exceptions
from dashboard import helpers
from dashboard import parameters
from dashboard import vault
from stackalytics.openstack.common import log as logging
from stackalytics import version as stackalytics_version
LOG = logging.getLogger(__name__)
def record_filter(ignore=None, use_default=True):
if not ignore:
ignore = []
def decorator(f):
@functools.wraps(f)
def record_filter_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
memory_storage_inst = vault.get_memory_storage()
record_ids = set(memory_storage_inst.get_record_ids()) # a copy
if 'module' not in ignore:
param = parameters.get_parameter(kwargs, 'module', 'modules',
use_default)
if param:
record_ids &= (
memory_storage_inst.get_record_ids_by_modules(
vault.resolve_modules(param)))
if 'project_type' not in ignore:
param = parameters.get_parameter(kwargs, 'project_type',
'project_types', use_default)
if param:
ptgi = vault_inst['project_type_group_index']
modules = set()
for project_type in param:
project_type = project_type.lower()
if project_type in ptgi:
modules |= ptgi[project_type]
record_ids &= (
memory_storage_inst.get_record_ids_by_modules(modules))
if 'user_id' not in ignore:
param = parameters.get_parameter(kwargs, 'user_id', 'user_ids')
param = [u for u in param
if vault.get_user_from_runtime_storage(u)]
if param:
record_ids &= (
memory_storage_inst.get_record_ids_by_user_ids(param))
if 'company' not in ignore:
param = parameters.get_parameter(kwargs, 'company',
'companies')
if param:
record_ids &= (
memory_storage_inst.get_record_ids_by_companies(param))
if 'release' not in ignore:
param = parameters.get_parameter(kwargs, 'release', 'releases',
use_default)
if param:
if 'all' not in param:
record_ids &= (
memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in param))
if 'metric' not in ignore:
metrics = parameters.get_parameter(kwargs, 'metric')
for metric in metrics:
record_ids &= memory_storage_inst.get_record_ids_by_type(
parameters.METRIC_TO_RECORD_TYPE[metric])
if 'tm_marks' in metrics:
filtered_ids = []
review_nth = int(parameters.get_parameter(
kwargs, 'review_nth')[0])
for record in memory_storage_inst.get_records(record_ids):
parent = memory_storage_inst.get_record_by_primary_key(
record['review_id'])
if (parent and ('review_number' in parent) and
(parent['review_number'] <= review_nth)):
filtered_ids.append(record['record_id'])
record_ids = filtered_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids)
return f(*args, **kwargs)
return record_filter_decorated_function
return decorator
def aggregate_filter():
def decorator(f):
@functools.wraps(f)
def aggregate_filter_decorated_function(*args, **kwargs):
def incremental_filter(result, record, param_id):
result[record[param_id]]['metric'] += 1
def loc_filter(result, record, param_id):
result[record[param_id]]['metric'] += record['loc']
def mark_filter(result, record, param_id):
value = record['value']
result_by_param = result[record[param_id]]
result_by_param['metric'] += 1
if value in result_by_param:
result_by_param[value] += 1
else:
result_by_param[value] = 1
def mark_finalize(record):
new_record = {}
for key in ['id', 'metric', 'name']:
new_record[key] = record[key]
positive = 0
mark_distribution = []
for key in ['-2', '-1', '1', '2']:
if key in record:
if key in ['1', '2']:
positive += record[key]
mark_distribution.append(str(record[key]))
else:
mark_distribution.append('0')
new_record['mark_ratio'] = (
'|'.join(mark_distribution) +
' (%.1f%%)' % ((positive * 100.0) / record['metric']))
return new_record
metric_param = (flask.request.args.get('metric') or
parameters.get_default('metric'))
metric = metric_param.lower()
metric_to_filters_map = {
'commits': (incremental_filter, None),
'loc': (loc_filter, None),
'marks': (mark_filter, mark_finalize),
'tm_marks': (mark_filter, mark_finalize),
'emails': (incremental_filter, None),
'bpd': (incremental_filter, None),
'bpc': (incremental_filter, None),
}
if metric not in metric_to_filters_map:
raise Exception('Invalid metric %s' % metric)
kwargs['metric_filter'] = metric_to_filters_map[metric][0]
kwargs['finalize_handler'] = metric_to_filters_map[metric][1]
return f(*args, **kwargs)
return aggregate_filter_decorated_function
return decorator
def exception_handler():
def decorator(f):
@functools.wraps(f)
def exception_handler_decorated_function(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
if isinstance(e, exceptions.HTTPException):
raise # ignore Flask exceptions
LOG.exception(e)
flask.abort(404)
return exception_handler_decorated_function
return decorator
def templated(template=None, return_code=200):
def decorator(f):
@functools.wraps(f)
def templated_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
# put parameters into template
metric = flask.request.args.get('metric')
if metric not in parameters.METRIC_LABELS:
metric = None
ctx['metric'] = metric or parameters.get_default('metric')
ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
project_type = flask.request.args.get('project_type')
if not vault.is_project_type_valid(project_type):
project_type = parameters.get_default('project_type')
ctx['project_type'] = project_type
release = flask.request.args.get('release')
releases = vault_inst['releases']
if release:
release = release.lower()
if release != 'all':
if release not in releases:
release = None
else:
release = releases[release]['release_name']
ctx['release'] = (release or
parameters.get_default('release')).lower()
ctx['review_nth'] = (flask.request.args.get('review_nth') or
parameters.get_default('review_nth'))
ctx['project_type_options'] = vault.get_project_type_options()
ctx['release_options'] = vault.get_release_options()
ctx['metric_options'] = sorted(parameters.METRIC_LABELS.items(),
key=lambda x: x[0])
ctx['company'] = parameters.get_single_parameter(kwargs, 'company')
ctx['module'] = parameters.get_single_parameter(kwargs, 'module')
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
ctx['page_title'] = helpers.make_page_title(
ctx['company'], ctx['user_id'], ctx['module'], ctx['release'])
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
return flask.render_template(template_name, **ctx), return_code
return templated_decorated_function
return decorator
def jsonify(root='data'):
def decorator(func):
@functools.wraps(func)
def jsonify_decorated_function(*args, **kwargs):
callback = flask.app.request.args.get('callback', False)
data = json.dumps({root: func(*args, **kwargs)})
if callback:
data = str(callback) + '(' + data + ')'
mimetype = 'application/javascript'
else:
mimetype = 'application/json'
return flask.current_app.response_class(data, mimetype=mimetype)
return jsonify_decorated_function
return decorator

166
dashboard/helpers.py Normal file
View File

@ -0,0 +1,166 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import re
import urllib
from flask.ext import gravatar as gravatar_ext
from dashboard import parameters
from dashboard import vault
from stackalytics.processor import utils
gravatar = gravatar_ext.Gravatar(None, size=64, rating='g', default='wavatar')
def _extend_record_common_fields(record):
record['date_str'] = format_datetime(record['date'])
record['author_link'] = make_link(
record['author_name'], '/',
{'user_id': record['user_id'], 'company': ''})
record['company_link'] = make_link(
record['company_name'], '/',
{'company': record['company_name'], 'user_id': ''})
record['module_link'] = make_link(
record['module'], '/',
{'module': record['module'], 'company': '', 'user_id': ''})
record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
record['blueprint_id_count'] = len(record.get('blueprint_id', []))
record['bug_id_count'] = len(record.get('bug_id', []))
def extend_record(record):
if record['record_type'] == 'commit':
commit = record.copy()
commit['branches'] = ','.join(commit['branches'])
if 'correction_comment' not in commit:
commit['correction_comment'] = ''
commit['message'] = make_commit_message(record)
_extend_record_common_fields(commit)
return commit
elif record['record_type'] == 'mark':
review = record.copy()
parent = vault.get_memory_storage().get_record_by_primary_key(
review['review_id'])
if parent:
review['review_number'] = parent.get('review_number')
review['subject'] = parent['subject']
review['url'] = parent['url']
review['parent_author_link'] = make_link(
parent['author_name'], '/',
{'user_id': parent['user_id'],
'company': ''})
_extend_record_common_fields(review)
return review
elif record['record_type'] == 'email':
email = record.copy()
_extend_record_common_fields(email)
email['email_link'] = email.get('email_link') or ''
return email
elif ((record['record_type'] == 'bpd') or
(record['record_type'] == 'bpc')):
blueprint = record.copy()
_extend_record_common_fields(blueprint)
blueprint['summary'] = utils.format_text(record['summary'])
if record.get('mention_count'):
blueprint['mention_date_str'] = format_datetime(
record['mention_date'])
blueprint['blueprint_link'] = make_blueprint_link(
blueprint['name'], blueprint['module'])
return blueprint
def format_datetime(timestamp):
return datetime.datetime.utcfromtimestamp(
timestamp).strftime('%d %b %Y %H:%M:%S')
def format_date(timestamp):
return datetime.datetime.utcfromtimestamp(timestamp).strftime('%d-%b-%y')
def format_launchpad_module_link(module):
return '<a href="https://launchpad.net/%s">%s</a>' % (module, module)
def safe_encode(s):
return urllib.quote_plus(s.encode('utf-8'))
def make_link(title, uri=None, options=None):
param_names = ('release', 'project_type', 'module', 'company', 'user_id',
'metric')
param_values = {}
for param_name in param_names:
v = parameters.get_parameter({}, param_name, param_name)
if v:
param_values[param_name] = ','.join(v)
if options:
param_values.update(options)
if param_values:
uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v))
for n, v in param_values.iteritems()])
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
def make_blueprint_link(name, module):
uri = '/report/blueprint/' + module + '/' + name
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': name}
def make_commit_message(record):
s = record['message']
module = record['module']
s = utils.format_text(s)
# insert links
s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE),
r'\1<a href="https://blueprints.launchpad.net/' +
module + r'/+spec/\2" class="ext_link">\2</a>', s)
s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE),
r'\1<a href="https://bugs.launchpad.net/bugs/\2" '
r'class="ext_link">\2</a>', s)
s = re.sub(r'\s+(I[0-9a-f]{40})',
r' <a href="https://review.openstack.org/#q,\1,n,z" '
r'class="ext_link">\1</a>', s)
s = utils.unwrap_text(s)
return s
def make_page_title(company, user_id, module, release):
if company:
memory_storage = vault.get_memory_storage()
company = memory_storage.get_original_company_name(company)
if company or user_id:
if user_id:
s = vault.get_user_from_runtime_storage(user_id)['user_name']
if company:
s += ' (%s)' % company
else:
s = company
else:
s = 'OpenStack community'
s += ' contribution'
if module:
s += ' to %s' % module
if release != 'all':
s += ' in %s release' % release.capitalize()
else:
s += ' in all releases'
return s

81
dashboard/parameters.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import flask
from stackalytics.openstack.common import log as logging
LOG = logging.getLogger(__name__)
DEFAULTS = {
'metric': 'commits',
'release': 'icehouse',
'project_type': 'openstack',
'review_nth': 5,
}
METRIC_LABELS = {
'loc': 'Lines of code',
'commits': 'Commits',
'marks': 'Reviews',
'tm_marks': 'Top Mentors',
'emails': 'Emails',
'bpd': 'Drafted Blueprints',
'bpc': 'Completed Blueprints',
}
METRIC_TO_RECORD_TYPE = {
'loc': 'commit',
'commits': 'commit',
'marks': 'mark',
'tm_marks': 'mark',
'emails': 'email',
'bpd': 'bpd',
'bpc': 'bpc',
}
DEFAULT_RECORDS_LIMIT = 10
def get_default(param_name):
if param_name in DEFAULTS:
return DEFAULTS[param_name]
else:
return None
def get_parameter(kwargs, singular_name, plural_name=None, use_default=True):
if singular_name in kwargs:
p = kwargs[singular_name]
else:
p = flask.request.args.get(singular_name)
if (not p) and plural_name:
flask.request.args.get(plural_name)
if p:
return p.split(',')
elif use_default:
default = get_default(singular_name)
return [default] if default else []
else:
return []
def get_single_parameter(kwargs, singular_name, use_default=True):
param = get_parameter(kwargs, singular_name, use_default)
if param:
return param[0]
else:
return ''

87
dashboard/reports.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import operator
import time
import flask
from dashboard import decorators
from dashboard import helpers
from dashboard import vault
from stackalytics.processor import utils
blueprint = flask.Blueprint('reports', __name__, url_prefix='/report')
@blueprint.route('/blueprint/<module>/<blueprint_name>')
@decorators.templated()
@decorators.exception_handler()
def blueprint_summary(module, blueprint_name):
blueprint_id = module + ':' + blueprint_name
bpd = vault.get_memory_storage().get_record_by_primary_key(
'bpd:' + blueprint_id)
if not bpd:
flask.abort(404)
return
bpd = helpers.extend_record(bpd)
record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
[blueprint_id])
activity = [helpers.extend_record(record) for record in
vault.get_memory_storage().get_records(record_ids)]
activity.sort(key=lambda x: x['date'], reverse=True)
return {'blueprint': bpd, 'activity': activity}
@blueprint.route('/reviews/<module>')
@decorators.templated()
@decorators.exception_handler()
def open_reviews(module):
memory_storage = vault.get_memory_storage()
now = int(time.time())
review_ids = (memory_storage.get_record_ids_by_modules([module]) &
memory_storage.get_record_ids_by_type('review'))
records = []
for review in memory_storage.get_records(review_ids):
if review['status'] != 'NEW':
continue
processed_review = review.copy()
helpers.extend_record(processed_review)
processed_review['age'] = utils.make_age_string(
now - processed_review['date'])
records.append(processed_review)
return {
'module': module,
'oldest': sorted(records, key=operator.itemgetter('date'))[:5]
}
@blueprint.route('/large_commits')
@decorators.jsonify('commits')
@decorators.exception_handler()
@decorators.record_filter()
def get_commit_report(records):
loc_threshold = int(flask.request.args.get('loc_threshold') or 0)
response = []
for record in records:
if ('loc' in record) and (record['loc'] > loc_threshold):
nr = dict([(k, record[k]) for k in ['loc', 'subject', 'module',
'primary_key', 'change_id']])
response.append(nr)
return response

12
dashboard/templates/overview.html
View File

@ -11,6 +11,7 @@
{% set show_module_contribution = (module) and (not user_id) %}
{% set show_contribution = (show_user_contribution) or (show_module_contribution) %}
{% set show_user_profile = (user_id) %}
{% set show_module_profile = (module) %}
{% set show_top_mentors_options = (metric == 'tm_marks') %}
{% set show_review_ratio = (metric in ['marks', 'tm_marks']) %}
@ -290,10 +291,6 @@
</div>
{% endif %}
{% if show_module_contribution %}
<div id="contribution_container"></div>
{% endif %}
{% endblock %}
{% block right_frame %}
@ -319,6 +316,13 @@
</div>
{% endif %}
{% if show_module_profile %}
<h2>Module {{ module }}</h2>
<div><a href="/report/reviews/{{ module }}">Open reviews report</a></div>
<div id="contribution_container"></div>
{% endif %}
{% if show_bp_breakdown %}
<div id="bp_container">
<h2>Blueprint popularity</h2>

0
dashboard/templates/blueprint_report.html → dashboard/templates/reports/blueprint_summary.html
View File

26
dashboard/templates/reports/open_reviews.html Normal file
View File

@ -0,0 +1,26 @@
<!DOCTYPE html>
<html>
<head>
<title>Open reviews report for {{ module }}</title>
<link rel=stylesheet type=text/css href="{{ url_for('static', filename='css/style.css') }}">
<style>
.label {
font-weight: bold;
line-height: 135%;
}
.message {
margin-top: 1em;
white-space: pre-wrap;
}
</style>
</head>
<body style="margin: 2em;">
<h1>Open Reviews Report</h1>
<h3>Longest waiting reviews (since first revision, total age): </h3>
<ol>
{% for item in oldest %}
<li>{{ item.age }} <a href="{{ item.url }}">{{ item.url }}</a> {{ item.subject }}</li>
{% endfor %}
</ol>

194
dashboard/vault.py Normal file
View File

@ -0,0 +1,194 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import flask
from oslo.config import cfg
from dashboard import memory_storage
from stackalytics.openstack.common import log as logging
from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
LOG = logging.getLogger(__name__)
def get_vault():
vault = getattr(flask.current_app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
init_project_types(vault)
init_releases(vault)
flask.current_app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s', e)
LOG.exception(e)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
if have_updates:
init_project_types(vault)
init_releases(vault)
init_module_groups(vault)
return vault
def get_memory_storage():
return get_vault()['memory_storage']
def init_releases(vault):
runtime_storage_inst = vault['runtime_storage']
releases = runtime_storage_inst.get_by_key('releases')
if not releases:
raise Exception('Releases are missing in runtime storage')
vault['start_date'] = releases[0]['end_date']
vault['end_date'] = releases[-1]['end_date']
start_date = releases[0]['end_date']
for r in releases[1:]:
r['start_date'] = start_date
start_date = r['end_date']
vault['releases'] = dict((r['release_name'].lower(), r)
for r in releases[1:])
def init_project_types(vault):
runtime_storage_inst = vault['runtime_storage']
project_type_options = {}
project_type_group_index = {'all': set(['unknown'])}
for repo in utils.load_repos(runtime_storage_inst):
project_type = repo['project_type'].lower()
project_group = None
if ('project_group' in repo) and (repo['project_group']):
project_group = repo['project_group'].lower()
if project_type in project_type_options:
if project_group:
project_type_options[project_type].add(project_group)
else:
if project_group:
project_type_options[project_type] = set([project_group])
else:
project_type_options[project_type] = set()
module = repo['module']
if project_type in project_type_group_index:
project_type_group_index[project_type].add(module)
else:
project_type_group_index[project_type] = set([module])
if project_group:
if project_group in project_type_group_index:
project_type_group_index[project_group].add(module)
else:
project_type_group_index[project_group] = set([module])
project_type_group_index['all'].add(module)
vault['project_type_options'] = project_type_options
vault['project_type_group_index'] = project_type_group_index
def init_module_groups(vault):
runtime_storage_inst = vault['runtime_storage']
module_index = {}
module_id_index = {}
module_groups = runtime_storage_inst.get_by_key('module_groups') or []
for module_group in module_groups:
module_group_name = module_group['module_group_name']
module_group_id = module_group_name.lower()
module_id_index[module_group_name] = {
'group': True,
'id': module_group_id,
'text': module_group_name,
'modules': [m.lower() for m in module_group['modules']],
}
modules = module_group['modules']
for module in modules:
if module in module_index:
module_index[module].add(module_group_id)
else:
module_index[module] = set([module_group_id])
memory_storage_inst = vault['memory_storage']
for module in memory_storage_inst.get_modules():
module_id_index[module] = {
'id': module.lower(),
'text': module,
'modules': [module.lower()],
}
vault['module_group_index'] = module_index
vault['module_id_index'] = module_id_index
vault['module_groups'] = module_groups
def get_project_type_options():
return get_vault()['project_type_options']
def get_release_options():
runtime_storage_inst = get_vault()['runtime_storage']
releases = runtime_storage_inst.get_by_key('releases')[1:]
releases.reverse()
return releases
def is_project_type_valid(project_type):
if not project_type:
return False
project_type = project_type.lower()
if project_type == 'all':
return True
project_types = get_project_type_options()
if project_type in project_types:
return True
for p, g in project_types.iteritems():
if project_type in g:
return True
return False
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
return utils.load_user(runtime_storage_inst, user_id)
def resolve_modules(module_ids):
module_id_index = get_vault()['module_id_index']
modules = set()
for module_id in module_ids:
if module_id in module_id_index:
modules |= set(module_id_index[module_id]['modules'])
return modules

804
dashboard/web.py
View File

@ -13,58 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import functools
import json
import operator
import os
import re
import urllib
import time
import flask
from flask.ext import gravatar as gravatar_ext
from oslo.config import cfg
import time
from werkzeug import exceptions
from dashboard import memory_storage
from dashboard import decorators
from dashboard import helpers
from dashboard import parameters
from dashboard import reports
from dashboard import vault
from stackalytics.openstack.common import log as logging
from stackalytics.processor import config
from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
from stackalytics import version as stackalytics_version
# Constants and Parameters ---------
DEFAULTS = {
'metric': 'commits',
'release': 'icehouse',
'project_type': 'openstack',
'review_nth': 5,
}
METRIC_LABELS = {
'loc': 'Lines of code',
'commits': 'Commits',
'marks': 'Reviews',
'tm_marks': 'Top Mentors',
'emails': 'Emails',
'bpd': 'Drafted Blueprints',
'bpc': 'Completed Blueprints',
}
METRIC_TO_RECORD_TYPE = {
'loc': 'commit',
'commits': 'commit',
'marks': 'mark',
'tm_marks': 'mark',
'emails': 'email',
'bpd': 'bpd',
'bpc': 'bpc',
}
DEFAULT_RECORDS_LIMIT = 10
# Application objects ---------
@ -72,6 +36,7 @@ DEFAULT_RECORDS_LIMIT = 10
app = flask.Flask(__name__)
app.config.from_object(__name__)
app.config.from_envvar('DASHBOARD_CONF', silent=True)
app.register_blueprint(reports.blueprint)
LOG = logging.getLogger(__name__)
@ -88,484 +53,16 @@ else:
LOG.warn('Conf file is empty or not exist')
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
init_project_types(vault)
init_releases(vault)
app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s', e)
LOG.exception(e)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
if have_updates:
init_project_types(vault)
init_releases(vault)
init_module_groups(vault)
return vault
def get_memory_storage():
return get_vault()['memory_storage']
def init_releases(vault):
runtime_storage_inst = vault['runtime_storage']
releases = runtime_storage_inst.get_by_key('releases')
if not releases:
raise Exception('Releases are missing in runtime storage')
vault['start_date'] = releases[0]['end_date']
vault['end_date'] = releases[-1]['end_date']
start_date = releases[0]['end_date']
for r in releases[1:]:
r['start_date'] = start_date
start_date = r['end_date']
vault['releases'] = dict((r['release_name'].lower(), r)
for r in releases[1:])
def init_project_types(vault):
runtime_storage_inst = vault['runtime_storage']
project_type_options = {}
project_type_group_index = {'all': set(['unknown'])}
for repo in utils.load_repos(runtime_storage_inst):
project_type = repo['project_type'].lower()
project_group = None
if ('project_group' in repo) and (repo['project_group']):
project_group = repo['project_group'].lower()
if project_type in project_type_options:
if project_group:
project_type_options[project_type].add(project_group)
else:
if project_group:
project_type_options[project_type] = set([project_group])
else:
project_type_options[project_type] = set()
module = repo['module']
if project_type in project_type_group_index:
project_type_group_index[project_type].add(module)
else:
project_type_group_index[project_type] = set([module])
if project_group:
if project_group in project_type_group_index:
project_type_group_index[project_group].add(module)
else:
project_type_group_index[project_group] = set([module])
project_type_group_index['all'].add(module)
vault['project_type_options'] = project_type_options
vault['project_type_group_index'] = project_type_group_index
def init_module_groups(vault):
runtime_storage_inst = vault['runtime_storage']
module_index = {}
module_id_index = {}
module_groups = runtime_storage_inst.get_by_key('module_groups') or []
for module_group in module_groups:
module_group_name = module_group['module_group_name']
module_group_id = module_group_name.lower()
module_id_index[module_group_name] = {
'group': True,
'id': module_group_id,
'text': module_group_name,
'modules': [m.lower() for m in module_group['modules']],
}
modules = module_group['modules']
for module in modules:
if module in module_index:
module_index[module].add(module_group_id)
else:
module_index[module] = set([module_group_id])
memory_storage_inst = vault['memory_storage']
for module in memory_storage_inst.get_modules():
module_id_index[module] = {
'id': module.lower(),
'text': module,
'modules': [module.lower()],
}
vault['module_group_index'] = module_index
vault['module_id_index'] = module_id_index
vault['module_groups'] = module_groups
def get_project_type_options():
return get_vault()['project_type_options']
def get_release_options():
runtime_storage_inst = get_vault()['runtime_storage']
releases = runtime_storage_inst.get_by_key('releases')[1:]
releases.reverse()
return releases
def is_project_type_valid(project_type):
if not project_type:
return False
project_type = project_type.lower()
if project_type == 'all':
return True
project_types = get_project_type_options()
if project_type in project_types:
return True
for p, g in project_types.iteritems():
if project_type in g:
return True
return False
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
return utils.load_user(runtime_storage_inst, user_id)
# Utils ---------
def get_default(param_name):
if param_name in DEFAULTS:
return DEFAULTS[param_name]
else:
return None
def get_parameter(kwargs, singular_name, plural_name=None, use_default=True):
if singular_name in kwargs:
p = kwargs[singular_name]
else:
p = flask.request.args.get(singular_name)
if (not p) and plural_name:
flask.request.args.get(plural_name)
if p:
return p.split(',')
elif use_default:
default = get_default(singular_name)
return [default] if default else []
else:
return []
def get_single_parameter(kwargs, singular_name, use_default=True):
param = get_parameter(kwargs, singular_name, use_default)
if param:
return param[0]
else:
return ''
def resolve_modules(module_ids):
module_id_index = get_vault()['module_id_index']
modules = set()
for module_id in module_ids:
if module_id in module_id_index:
modules |= set(module_id_index[module_id]['modules'])
return modules
# Decorators ---------
def record_filter(ignore=None, use_default=True):
if not ignore:
ignore = []
def decorator(f):
@functools.wraps(f)
def record_filter_decorated_function(*args, **kwargs):
vault = get_vault()
memory_storage = vault['memory_storage']
record_ids = set(memory_storage.get_record_ids()) # make a copy
if 'module' not in ignore:
param = get_parameter(kwargs, 'module', 'modules', use_default)
if param:
record_ids &= (memory_storage.get_record_ids_by_modules(
resolve_modules(param)))
if 'project_type' not in ignore:
param = get_parameter(kwargs, 'project_type', 'project_types',
use_default)
if param:
ptgi = vault['project_type_group_index']
modules = set()
for project_type in param:
project_type = project_type.lower()
if project_type in ptgi:
modules |= ptgi[project_type]
record_ids &= (
memory_storage.get_record_ids_by_modules(modules))
if 'user_id' not in ignore:
param = get_parameter(kwargs, 'user_id', 'user_ids')
param = [u for u in param if get_user_from_runtime_storage(u)]
if param:
record_ids &= (
memory_storage.get_record_ids_by_user_ids(param))
if 'company' not in ignore:
param = get_parameter(kwargs, 'company', 'companies')
if param:
record_ids &= (
memory_storage.get_record_ids_by_companies(param))
if 'release' not in ignore:
param = get_parameter(kwargs, 'release', 'releases',
use_default)
if param:
if 'all' not in param:
record_ids &= (
memory_storage.get_record_ids_by_releases(
c.lower() for c in param))
if 'metric' not in ignore:
metrics = get_parameter(kwargs, 'metric')
for metric in metrics:
record_ids &= memory_storage.get_record_ids_by_type(
METRIC_TO_RECORD_TYPE[metric])
if 'tm_marks' in metrics:
filtered_ids = []
review_nth = int(get_parameter(kwargs, 'review_nth')[0])
for record in memory_storage.get_records(record_ids):
parent = memory_storage.get_record_by_primary_key(
record['review_id'])
if (parent and ('review_number' in parent) and
(parent['review_number'] <= review_nth)):
filtered_ids.append(record['record_id'])
record_ids = filtered_ids
kwargs['records'] = memory_storage.get_records(record_ids)
return f(*args, **kwargs)
return record_filter_decorated_function
return decorator
def aggregate_filter():
def decorator(f):
@functools.wraps(f)
def aggregate_filter_decorated_function(*args, **kwargs):
def incremental_filter(result, record, param_id):
result[record[param_id]]['metric'] += 1
def loc_filter(result, record, param_id):
result[record[param_id]]['metric'] += record['loc']
def mark_filter(result, record, param_id):
value = record['value']
result_by_param = result[record[param_id]]
result_by_param['metric'] += 1
if value in result_by_param:
result_by_param[value] += 1
else:
result_by_param[value] = 1
def mark_finalize(record):
new_record = {}
for key in ['id', 'metric', 'name']:
new_record[key] = record[key]
positive = 0
mark_distribution = []
for key in ['-2', '-1', '1', '2']:
if key in record:
if key in ['1', '2']:
positive += record[key]
mark_distribution.append(str(record[key]))
else:
mark_distribution.append('0')
new_record['mark_ratio'] = (
'|'.join(mark_distribution) +
' (%.1f%%)' % ((positive * 100.0) / record['metric']))
return new_record
metric_param = (flask.request.args.get('metric') or
get_default('metric'))
metric = metric_param.lower()
metric_to_filters_map = {
'commits': (incremental_filter, None),
'loc': (loc_filter, None),
'marks': (mark_filter, mark_finalize),
'tm_marks': (mark_filter, mark_finalize),
'emails': (incremental_filter, None),
'bpd': (incremental_filter, None),
'bpc': (incremental_filter, None),
}
if metric not in metric_to_filters_map:
raise Exception('Invalid metric %s' % metric)
kwargs['metric_filter'] = metric_to_filters_map[metric][0]
kwargs['finalize_handler'] = metric_to_filters_map[metric][1]
return f(*args, **kwargs)
return aggregate_filter_decorated_function
return decorator
def exception_handler():
def decorator(f):
@functools.wraps(f)
def exception_handler_decorated_function(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
if isinstance(e, exceptions.HTTPException):
raise # ignore Flask exceptions
LOG.exception(e)
flask.abort(404)
return exception_handler_decorated_function
return decorator
def make_page_title(company, user_id, module, release):
if company:
memory_storage = get_vault()['memory_storage']
company = memory_storage.get_original_company_name(company)
if company or user_id:
if user_id:
s = get_user_from_runtime_storage(user_id)['user_name']
if company:
s += ' (%s)' % company
else:
s = company
else:
s = 'OpenStack community'
s += ' contribution'
if module:
s += ' to %s' % module
if release != 'all':
s += ' in %s release' % release.capitalize()
else:
s += ' in all releases'
return s
def templated(template=None, return_code=200):
def decorator(f):
@functools.wraps(f)
def templated_decorated_function(*args, **kwargs):
vault = get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
# put parameters into template
metric = flask.request.args.get('metric')
if metric not in METRIC_LABELS:
metric = None
ctx['metric'] = metric or get_default('metric')
ctx['metric_label'] = METRIC_LABELS[ctx['metric']]
project_type = flask.request.args.get('project_type')
if not is_project_type_valid(project_type):
project_type = get_default('project_type')
ctx['project_type'] = project_type
release = flask.request.args.get('release')
releases = vault['releases']
if release:
release = release.lower()
if release != 'all':
if release not in releases:
release = None
else:
release = releases[release]['release_name']
ctx['release'] = (release or get_default('release')).lower()
ctx['review_nth'] = (flask.request.args.get('review_nth') or
get_default('review_nth'))
ctx['project_type_options'] = get_project_type_options()
ctx['release_options'] = get_release_options()
ctx['metric_options'] = sorted(METRIC_LABELS.items(),
key=lambda x: x[0])
ctx['company'] = get_single_parameter(kwargs, 'company')
ctx['module'] = get_single_parameter(kwargs, 'module')
ctx['user_id'] = get_single_parameter(kwargs, 'user_id')
ctx['page_title'] = make_page_title(ctx['company'], ctx['user_id'],
ctx['module'], ctx['release'])
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
return flask.render_template(template_name, **ctx), return_code
return templated_decorated_function
return decorator
def jsonify(root='data'):
def decorator(func):
@functools.wraps(func)
def jsonify_decorated_function(*args, **kwargs):
callback = flask.app.request.args.get('callback', False)
data = json.dumps({root: func(*args, **kwargs)})
if callback:
data = str(callback) + '(' + data + ')'
content_type = 'application/javascript'
else:
content_type = 'application/json'
return app.response_class(data, mimetype=content_type)
return jsonify_decorated_function
return decorator
# Handlers ---------
@app.route('/')
@templated()
@decorators.templated()
def overview():
pass
@app.errorhandler(404)
@templated('404.html', 404)
@decorators.templated('404.html', 404)
def page_not_found(e):
pass
@ -590,43 +87,43 @@ def _get_aggregated_stats(records, metric_filter, keys, param_id,
@app.route('/api/1.0/stats/companies')
@jsonify('stats')
@exception_handler()
@record_filter()
@aggregate_filter()
@decorators.jsonify('stats')
@decorators.exception_handler()
@decorators.record_filter()
@decorators.aggregate_filter()
def get_companies(records, metric_filter, finalize_handler):
return _get_aggregated_stats(records, metric_filter,
get_memory_storage().get_companies(),
vault.get_memory_storage().get_companies(),
'company_name')
@app.route('/api/1.0/stats/modules')
@jsonify('stats')
@exception_handler()
@record_filter()
@aggregate_filter()
@decorators.jsonify('stats')
@decorators.exception_handler()
@decorators.record_filter()
@decorators.aggregate_filter()
def get_modules(records, metric_filter, finalize_handler):
return _get_aggregated_stats(records, metric_filter,
get_memory_storage().get_modules(),
vault.get_memory_storage().get_modules(),
'module')
@app.route('/api/1.0/stats/engineers')
@jsonify('stats')
@exception_handler()
@record_filter()
@aggregate_filter()
@decorators.jsonify('stats')
@decorators.exception_handler()
@decorators.record_filter()
@decorators.aggregate_filter()
def get_engineers(records, metric_filter, finalize_handler):
return _get_aggregated_stats(records, metric_filter,
get_memory_storage().get_user_ids(),
vault.get_memory_storage().get_user_ids(),
'user_id', 'author_name',
finalize_handler=finalize_handler)
@app.route('/api/1.0/stats/distinct_engineers')
@jsonify('stats')
@exception_handler()
@record_filter()
@decorators.jsonify('stats')
@decorators.exception_handler()
@decorators.record_filter()
def get_distinct_engineers(records):
result = {}
for record in records:
@ -637,74 +134,17 @@ def get_distinct_engineers(records):
return result
def _extend_record_common_fields(record):
record['date_str'] = format_datetime(record['date'])
record['author_link'] = make_link(
record['author_name'], '/',
{'user_id': record['user_id'], 'company': ''})
record['company_link'] = make_link(
record['company_name'], '/',
{'company': record['company_name'], 'user_id': ''})
record['module_link'] = make_link(
record['module'], '/',
{'module': record['module'], 'company': '', 'user_id': ''})
record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
record['blueprint_id_count'] = len(record.get('blueprint_id', []))
record['bug_id_count'] = len(record.get('bug_id', []))
def _extend_record(record):
if record['record_type'] == 'commit':
commit = record.copy()
commit['branches'] = ','.join(commit['branches'])
if 'correction_comment' not in commit:
commit['correction_comment'] = ''
commit['message'] = make_commit_message(record)
_extend_record_common_fields(commit)
return commit
elif record['record_type'] == 'mark':
review = record.copy()
parent = get_memory_storage().get_record_by_primary_key(
review['review_id'])
if parent:
review['review_number'] = parent.get('review_number')
review['subject'] = parent['subject']
review['url'] = parent['url']
review['parent_author_link'] = make_link(
parent['author_name'], '/',
{'user_id': parent['user_id'],
'company': ''})
_extend_record_common_fields(review)
return review
elif record['record_type'] == 'email':
email = record.copy()
_extend_record_common_fields(email)
email['email_link'] = email.get('email_link') or ''
return email
elif ((record['record_type'] == 'bpd') or
(record['record_type'] == 'bpc')):
blueprint = record.copy()
_extend_record_common_fields(blueprint)
blueprint['summary'] = utils.format_text(record['summary'])
if record.get('mention_count'):
blueprint['mention_date_str'] = format_datetime(
record['mention_date'])
blueprint['blueprint_link'] = make_blueprint_link(
blueprint['name'], blueprint['module'])
return blueprint
@app.route('/api/1.0/activity')
@jsonify('activity')
@exception_handler()
@record_filter()
@decorators.jsonify('activity')
@decorators.exception_handler()
@decorators.record_filter()
def get_activity_json(records):
start_record = int(flask.request.args.get('start_record') or 0)
page_size = int(flask.request.args.get('page_size') or
DEFAULT_RECORDS_LIMIT)
parameters.DEFAULT_RECORDS_LIMIT)
result = []
for record in records:
processed_record = _extend_record(record)
processed_record = helpers.extend_record(record)
if processed_record:
result.append(processed_record)
@ -713,9 +153,9 @@ def get_activity_json(records):
@app.route('/api/1.0/contribution')
@jsonify('contribution')
@exception_handler()
@record_filter(ignore='metric')
@decorators.jsonify('contribution')
@decorators.exception_handler()
@decorators.record_filter(ignore='metric')
def get_contribution_json(records):
marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
commit_count = 0
@ -750,9 +190,9 @@ def get_contribution_json(records):
@app.route('/api/1.0/companies')
@jsonify('companies')
@exception_handler()
@record_filter(ignore='company')
@decorators.jsonify('companies')
@decorators.exception_handler()
@decorators.record_filter(ignore='company')
def get_companies_json(records):
query = flask.request.args.get('company_name') or ''
options = set()
@ -762,18 +202,18 @@ def get_companies_json(records):
continue
if name.lower().find(query.lower()) >= 0:
options.add(name)
result = [{'id': safe_encode(c.lower()), 'text': c}
result = [{'id': helpers.safe_encode(c.lower()), 'text': c}
for c in sorted(options)]
return result
@app.route('/api/1.0/modules')
@jsonify('modules')
@exception_handler()
@record_filter(ignore='module')
@decorators.jsonify('modules')
@decorators.exception_handler()
@decorators.record_filter(ignore='module')
def get_modules_json(records):
module_group_index = get_vault()['module_group_index']
module_id_index = get_vault()['module_id_index']
module_group_index = vault.get_vault()['module_group_index']
module_id_index = vault.get_vault()['module_id_index']
modules_set = set()
for record in records:
@ -799,22 +239,23 @@ def get_modules_json(records):
@app.route('/api/1.0/companies/<company_name>')
@jsonify('company')
@decorators.jsonify('company')
def get_company(company_name):
memory_storage = get_vault()['memory_storage']
for company in memory_storage.get_companies():
memory_storage_inst = vault.get_memory_storage()
for company in memory_storage_inst.get_companies():
if company.lower() == company_name.lower():
return {
'id': company_name,
'text': memory_storage.get_original_company_name(company_name)
'text': memory_storage_inst.get_original_company_name(
company_name)
}
flask.abort(404)
@app.route('/api/1.0/modules/<module>')
@jsonify('module')
@decorators.jsonify('module')
def get_module(module):
module_id_index = get_vault()['module_id_index']
module_id_index = vault.get_vault()['module_id_index']
module = module.lower()
if module in module_id_index:
return module_id_index[module]
@ -822,16 +263,16 @@ def get_module(module):
@app.route('/api/1.0/stats/bp')
@jsonify('stats')
@exception_handler()
@record_filter()
@decorators.jsonify('stats')
@decorators.exception_handler()
@decorators.record_filter()
def get_bpd(records):
result = []
for record in records:
if record['record_type'] in ['bpd', 'bpc']:
mention_date = record.get('mention_date')
if mention_date:
date = format_date(mention_date)
date = helpers.format_date(mention_date)
else:
date = 'never'
result.append({
@ -840,7 +281,8 @@ def get_bpd(records):
'metric': record.get('mention_count') or 0,
'id': record['name'],
'name': record['name'],
'link': make_blueprint_link(record['name'], record['module'])
'link': helpers.make_blueprint_link(
record['name'], record['module'])
})
result.sort(key=lambda x: x['metric'], reverse=True)
@ -849,9 +291,9 @@ def get_bpd(records):
@app.route('/api/1.0/users')
@jsonify('users')
@exception_handler()
@record_filter(ignore='user_id')
@decorators.jsonify('users')
@decorators.exception_handler()
@decorators.record_filter(ignore='user_id')
def get_users_json(records):
user_name_query = flask.request.args.get('user_name') or ''
user_ids = set()
@ -869,42 +311,42 @@ def get_users_json(records):
@app.route('/api/1.0/users/<user_id>')
@jsonify('user')
@decorators.jsonify('user')
def get_user(user_id):
user = get_user_from_runtime_storage(user_id)
user = vault.get_user_from_runtime_storage(user_id)
if not user:
flask.abort(404)
user['id'] = user['user_id']
user['text'] = user['user_name']
if user['companies']:
company_name = user['companies'][-1]['company_name']
user['company_link'] = make_link(
user['company_link'] = helpers.make_link(
company_name, '/', {'company': company_name, 'user_id': ''})
else:
user['company_link'] = ''
if user['emails']:
user['gravatar'] = gravatar(user['emails'][0])
user['gravatar'] = helpers.gravatar(user['emails'][0])
else:
user['gravatar'] = gravatar('stackalytics')
user['gravatar'] = helpers.gravatar('stackalytics')
return user
@app.route('/api/1.0/stats/timeline')
@jsonify('timeline')
@exception_handler()
@record_filter(ignore='release')
@decorators.jsonify('timeline')
@decorators.exception_handler()
@decorators.record_filter(ignore='release')
def timeline(records, **kwargs):
# find start and end dates
release_names = get_parameter(kwargs, 'release', 'releases')
releases = get_vault()['releases']
release_names = parameters.get_parameter(kwargs, 'release', 'releases')
releases = vault.get_vault()['releases']
if not release_names:
flask.abort(404)
if 'all' in release_names:
start_date = release_start_date = utils.timestamp_to_week(
get_vault()['start_date'])
vault.get_vault()['start_date'])
end_date = release_end_date = utils.timestamp_to_week(
get_vault()['end_date'])
vault.get_vault()['end_date'])
else:
release = releases[release_names[0]]
start_date = release_start_date = utils.timestamp_to_week(
@ -929,7 +371,7 @@ def timeline(records, **kwargs):
week_stat_commits = dict((c, 0) for c in weeks)
week_stat_commits_hl = dict((c, 0) for c in weeks)
param = get_parameter(kwargs, 'metric')
param = parameters.get_parameter(kwargs, 'metric')
if ('commits' in param) or ('loc' in param):
handler = lambda record: record['loc']
else:
@ -958,106 +400,6 @@ def timeline(records, **kwargs):
return [array_commits, array_commits_hl, array_loc]
@app.route('/api/1.0/report/commits')
@jsonify('commits')
@exception_handler()
@record_filter()
def get_commit_report(records):
loc_threshold = int(flask.request.args.get('loc_threshold') or 0)
response = []
for record in records:
if ('loc' in record) and (record['loc'] > loc_threshold):
nr = dict([(k, record[k]) for k in ['loc', 'subject', 'module',
'primary_key', 'change_id']])
response.append(nr)
return response
@app.route('/report/blueprint/<module>/<blueprint_name>')
@templated()
@exception_handler()
def blueprint_report(module, blueprint_name):
blueprint_id = module + ':' + blueprint_name
bpd = get_memory_storage().get_record_by_primary_key('bpd:' + blueprint_id)
if not bpd:
flask.abort(404)
return
bpd = _extend_record(bpd)
record_ids = get_memory_storage().get_record_ids_by_blueprint_ids(
[blueprint_id])
activity = [_extend_record(record) for record in
get_memory_storage().get_records(record_ids)]
activity.sort(key=lambda x: x['date'], reverse=True)
return {'blueprint': bpd, 'activity': activity}
# Jinja Filters ---------
@app.template_filter('datetimeformat')
def format_datetime(timestamp):
return datetime.datetime.utcfromtimestamp(
timestamp).strftime('%d %b %Y %H:%M:%S')
def format_date(timestamp):
return datetime.datetime.utcfromtimestamp(timestamp).strftime('%d-%b-%y')
@app.template_filter('launchpadmodule')
def format_launchpad_module_link(module):
return '<a href="https://launchpad.net/%s">%s</a>' % (module, module)
@app.template_filter('encode')
def safe_encode(s):
return urllib.quote_plus(s.encode('utf-8'))
@app.template_filter('link')
def make_link(title, uri=None, options=None):
param_names = ('release', 'project_type', 'module', 'company', 'user_id',
'metric')
param_values = {}
for param_name in param_names:
v = get_parameter({}, param_name, param_name)
if v:
param_values[param_name] = ','.join(v)
if options:
param_values.update(options)
if param_values:
uri += '?' + '&'.join(['%s=%s' % (n, safe_encode(v))
for n, v in param_values.iteritems()])
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
def make_blueprint_link(name, module):
uri = '/report/blueprint/' + module + '/' + name
return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': name}
def make_commit_message(record):
s = record['message']
module = record['module']
s = utils.format_text(s)
# insert links
s = re.sub(re.compile('(blueprint\s+)([\w-]+)', flags=re.IGNORECASE),
r'\1<a href="https://blueprints.launchpad.net/' +
module + r'/+spec/\2" class="ext_link">\2</a>', s)
s = re.sub(re.compile('(bug[\s#:]*)([\d]{5,7})', flags=re.IGNORECASE),
r'\1<a href="https://bugs.launchpad.net/bugs/\2" '
r'class="ext_link">\2</a>', s)
s = re.sub(r'\s+(I[0-9a-f]{40})',
r' <a href="https://review.openstack.org/#q,\1,n,z" '
r'class="ext_link">\1</a>', s)
s = utils.unwrap_text(s)
return s
gravatar = gravatar_ext.Gravatar(app, size=64, rating='g', default='wavatar')

7
stackalytics/processor/utils.py
View File

@ -119,6 +119,13 @@ def format_text(s):
return s
def make_age_string(seconds):
days = seconds / (3600 * 24)
hours = (seconds / 3600) - (days * 24)
minutes = (seconds / 60) - (days * 24 * 60) - (hours * 60)
return '%d days, %d hours, %d minutes' % (days, hours, minutes)
def merge_records(original, new):
need_update = False
for key, value in new.iteritems():

21
tests/unit/test_web_utils.py
View File

@ -16,7 +16,7 @@
import mock
import testtools
from dashboard import web
from dashboard import helpers
class TestWebUtils(testtools.TestCase):
@ -50,7 +50,7 @@ Fixes bug <a href="https://bugs.launchpad.net/bugs/1076801" class="ext_link">\
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
observed = helpers.make_commit_message(record)
self.assertEqual(expected, observed,
'Commit message should be processed correctly')
@ -77,13 +77,13 @@ Implements Blueprint ''' + (
'Ie49ccd2138905e178843b375a9b16c3fe572d1db,n,z" class="ext_link">'
'Ie49ccd2138905e178843b375a9b16c3fe572d1db</a>')
observed = web.make_commit_message(record)
observed = helpers.make_commit_message(record)
self.assertEqual(expected, observed,
'Commit message should be processed correctly')
@mock.patch('dashboard.web.get_vault')
@mock.patch('dashboard.web.get_user_from_runtime_storage')
@mock.patch('dashboard.vault.get_vault')
@mock.patch('dashboard.vault.get_user_from_runtime_storage')
def test_make_page_title(self, user_patch, vault_patch):
memory_storage_mock = mock.Mock()
memory_storage_mock.get_original_company_name = mock.Mock(
@ -93,13 +93,14 @@ Implements Blueprint ''' + (
user_patch.return_value = {'user_name': 'John Doe'}
self.assertEqual('OpenStack community contribution in all releases',
web.make_page_title('', '', '', 'all'))
helpers.make_page_title('', '', '', 'all'))
self.assertEqual('OpenStack community contribution in Havana release',
web.make_page_title('', '', '', 'Havana'))
helpers.make_page_title('', '', '', 'Havana'))
self.assertEqual('Mirantis contribution in Havana release',
web.make_page_title('Mirantis', '', '', 'Havana'))
helpers.make_page_title('Mirantis', '', '', 'Havana'))
self.assertEqual('John Doe contribution in Havana release',
web.make_page_title('', 'john_doe', '', 'Havana'))
helpers.make_page_title('', 'john_doe', '', 'Havana'))
self.assertEqual(
'John Doe (Mirantis) contribution to neutron in Havana release',
web.make_page_title('Mirantis', 'John Doe', 'neutron', 'Havana'))
helpers.make_page_title(
'Mirantis', 'John Doe', 'neutron', 'Havana'))