x/stackalytics
Code Issues Proposed changes

Remove CI votes metric

Browse Source 
This patch removes processing of CI votes. The reasons are:
 * CI votes rely on DriverLog database which is deprecated
 * CI votes are collected from comments to patch sets, querying
   comments significantly increases the load on Gerrit
 * CI votes are not actively used by community
 * CI votes provide only partial look into state of third-party
   drivers.

This reverts commits:
   a6ff499de33baaac82a170ec3f1ba7017d1d65f3
   1b6a5fe764ef0faf34f0431f531bba4afd33ecd7
   03be2b4e19648e00e6016a86311798953079ff68
   94f371d54372ff7bcb64c89e6634de379ba9f440
   5ad1cbe79c6cf8ae64319a82969a22e4123c69f2

Change-Id: Ie30b68c54b8d6fac6330481bf83bd1b2c4bfa190
This commit is contained in:
Ilya Shakhat 2017-08-24 10:19:27 +02:00
parent db0a300d02
commit 30976caaae
18 changed files with 6 additions and 682 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
doc/source/tools/stackalytics-processor.txt
View File

@ -2,7 +2,6 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-data-uri DEFAULT_DATA_URI]
[--driverlog-data-uri DRIVERLOG_DATA_URI]
[--fetching-user-source FETCHING_USER_SOURCE]
[--gerrit-retry GERRIT_RETRY]
[--git-base-uri GIT_BASE_URI]
@ -44,8 +43,6 @@ optional arguments:
URI for default data. A local file can be used with
the prefix "file://". For example, default_data_uri =
file:///path/to/default_data.json
--driverlog-data-uri DRIVERLOG_DATA_URI
URI for default data
--fetching-user-source FETCHING_USER_SOURCE
Source for fetching user profiles
--gerrit-retry GERRIT_RETRY

3
etc/stackalytics.conf
View File

@ -170,9 +170,6 @@
# SSH username for gerrit review system access (string value)
#ssh_username = user
# URI for default data (string value)
#driverlog_data_uri = https://git.openstack.org/cgit/openstack/driverlog/plain/etc/default_data.json
# URI of translation team data (string value)
#translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml

23
stackalytics/dashboard/decorators.py
View File

@ -321,28 +321,6 @@ def mark_finalize(record):
return new_record
def ci_filter(result, record, param_id, context):
result_by_param = result[getattr(record, param_id)]
result_by_param['metric'] += 1
key = 'success' if record.value else 'failure'
result_by_param[key] = result_by_param.get(key, 0) + 1
def ci_finalize(record):
new_record = record.copy()
metric = record.get('metric')
if metric:
new_record['success_ratio'] = '%.1f%%' % (
(record.get('success', 0) * 100.0) / metric)
else:
new_record['success_rate'] = helpers.INFINITY_HTML
return new_record
def person_day_filter(result, record, param_id, context):
day = utils.timestamp_to_day(record.date)
# fact that record-days are grouped by days in some order is used
@ -389,7 +367,6 @@ def aggregate_filter():
'resolved-bugs': (incremental_filter, None),
'members': (incremental_filter, None),
'person-day': (person_day_filter, None),
'ci': (ci_filter, ci_finalize),
'patches': (None, None),
'translations': (loc_filter, None),
}

2
stackalytics/dashboard/parameters.py
View File

@ -37,7 +37,6 @@ METRIC_LABELS = {
'filed-bugs': 'Filed Bugs',
'resolved-bugs': 'Resolved Bugs',
'person-day': "Person-day effort",
'ci': 'CI votes',
'patches': 'Patch Sets',
'translations': 'Translations',
}
@ -53,7 +52,6 @@ METRIC_TO_RECORD_TYPE = {
'resolved-bugs': ['bugr'],
'members': ['member'],
'person-day': ['mark', 'patch', 'email', 'bpd', 'bugf'],
'ci': ['ci'],
'patches': ['patch'],
'translations': ['tr'],
}

16
stackalytics/dashboard/reports.py
View File

@ -148,20 +148,6 @@ def contribution(module, days):
}
@blueprint.route('/ci/<module>/<days>')
@decorators.templated()
@decorators.exception_handler()
def external_ci(module, days):
if int(days) > 100:
days = 100
return {
'module': module,
'days': days,
'start_date': int(time.time()) - int(days) * 24 * 60 * 60
}
@blueprint.route('/members')
@decorators.exception_handler()
@decorators.templated()
@ -230,7 +216,7 @@ def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])
['mark', 'patch', 'email', 'bpd', 'bpc'])
record_ids &= record_ids_by_type
punch_card_data = _get_punch_card_data(

9
stackalytics/dashboard/templates/_macros/activity_log.html
View File

@ -109,7 +109,7 @@ show_twitter=False) -%}
<div class="header">{%html author_link %} ({%html company_link %})</div>
{%/if%}
<div class="header">${date_str} in {%html module_link%}
{%if record_type == "mark" || record_type == "review" || record_type == "patch" || record_type == "ci" || record_type == "tr" %}
{%if record_type == "mark" || record_type == "review" || record_type == "patch" || record_type == "tr" %}
{%if branch != "master" %}(${branch}){%/if%}
{%/if%}
{%if record_type == "commit" %}
@ -192,13 +192,6 @@ show_twitter=False) -%}
<div class="header">Bug &ldquo;${title}&rdquo; (<a href="${web_link}" class="ext_link">${number}</a>)</div>
<div>Status: <span class="status${status_class}">${status}</span></div>
<div>Importance: <span class="importance${importance}">${importance}</span></div>
{%elif record_type == "ci" %}
<div class="header">CI vote in merged change request
<a href="https://review.openstack.org/#/c/${review_number}" target="_blank">${review_number}</a>
</div>
<div>{%if value == true %}<span style="color: green">Success</span>{%else%}<span style="color: red">Failure</span>{%/if%}</div>
<div class="message">${message}</div>
<div>Change Id: <a href="https://review.openstack.org/#/c/${review_number}" target="_blank">${review_id}</a></div>
{%elif record_type == "member" %}
<div class="header"><a href="${member_uri}" target="_blank">Registered</a> in OpenStack Foundation</div>
{%elif record_type == "tr" %}

12
stackalytics/dashboard/templates/overview.html
View File

@ -17,16 +17,12 @@
{% set show_user_profile = (user_id) %}
{% set show_module_details = (module) %}
{% set show_review_ratio = (metric in ['marks']) %}
{% set show_ci = (metric in ['ci']) %}
{% macro show_report_links(module=None, company=None, user_id=None) -%}
{% if module %}
<div><b><a href="/report/reviews/{{ module }}/open" target="_blank">Show open reviews for {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/30" target="_blank">Contribution for the last 30 days in {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/90" target="_blank">Contribution for the last 90 days in {{ module_inst.module_group_name }}</a></b></div>
{% if module_inst.has_drivers %}
<div><b><a href="/report/ci/{{ module }}/7" target="_blank">Status of drivers testing in {{ module_inst.module_group_name }} during recent 7 days</a></b></div>
{% endif %}
<div><b><a href="/report/activity?module={{ module }}&project_type={{ project_type }}&release={{ release }}" target="_blank">Show activity report for {{ module }}</a></b></div>
{% endif %}
{% if company %}
@ -49,9 +45,6 @@
{% if show_review_ratio %}
renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id",
["index", "link", "mark_ratio", "metric"]);
{% elif show_ci %}
renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id",
["index", "link", "metric", "success_ratio"]);
{% else %}
renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id");
{% endif %}
@ -104,14 +97,11 @@
<thead>
<tr>
<th>#</th>
<th>{% if show_ci %}Driver{% else %}Contributor{% endif %}</th>
<th>Contributor</th>
{% if show_review_ratio %}
<th>-2|-1|+1|+2|A|x (+ ratio)</th>
{% endif %}
<th>{{ metric_label }}</th>
{% if show_ci %}
<th>Success</th>
{% endif %}
</tr>
</thead>
<tbody>

245
stackalytics/dashboard/templates/reports/external_ci.html
View File

@ -1,245 +0,0 @@
{% extends "reports/base_report.html" %}
{% set page_title = "Status of drivers testing in " + module.title() + " during recent " + days + " days" %}
{% block scripts %}
<script type="text/javascript">
jQuery.extend(jQuery.fn.dataTableExt.oSort, {
"ratio-pre": function (a) {
if (a == "&#x221E;" || a == "-")
return -1.0;
a = a.substr(0, a.length - 1);
return parseFloat(a);
},
"ratio-asc": function (a, b) {
return ((a < b) ? -1 : ((a > b) ? 1 : 0));
},
"ratio-desc": function (a, b) {
return ((a < b) ? 1 : ((a > b) ? -1 : 0));
}
});
$(document).ready(function () {
var table_column_names = ["name", "merge_run_count", "merge_success_rate", "merge_date_str", "merge_result"];
var table_id = "summary_stats_table";
$.ajax({
url: makeURI("/api/1.0/activity?project_type=all&metric=ci&module={{ module }}&release=all&start_date={{ start_date }}&page_size=-1"),
dataType: "json",
success: function (data) {
var activity = data["activity"];
var tableColumns = [];
for (var i = 0; i < table_column_names.length; i++) {
tableColumns.push({"mData": table_column_names[i]});
}
var ci_map = {}; // ci_id -> statistics
var res_map = {}; // ci_id -> map: review_id -> res
var ci_id_to_name = {}; // ci_id -> name
var review_ids = [];
var review_ids_set = {};
for (i = 0; i < activity.length; i++) {
var ai = activity[i];
var ci_id = ai.user_id;
var ci = ci_map[ci_id];
if (!ci) {
ci = {merge_run_count: 0, merge_success: 0,
merge_date: 0, merge_result: "N/A",
merge_date_str: "N/A", "total_date_str": "N/A"};
}
ci.id = ai.user_id;
ci.name = "<a href=\"/?metric=ci&user_id=" + ai.user_id +
"&module={{ module }}\">" + ai.driver_name + "</a>";
ci_id_to_name[ci_id] = ci.name;
var review_id = ai.review_number;
if (!review_ids_set[review_id]) {
review_ids_set[review_id] = review_id;
review_ids.push({review_id: review_id, date_str: ai.date_str});
}
if (!res_map[ci_id]) {
res_map[ci_id] = {};
}
res_map[ci_id][review_id] = ai.value;
ci.merge_run_count ++;
if (ai.value) {
ci.merge_success ++;
}
if (ai.date > ci.merge_date) {
ci.merge_date = ai.date;
ci.merge_date_str = ai.date_str;
ci.merge_result = (ai.value)? "<span style=\"color: green\">&#x2714;</span>":
"<span style=\"color: red\">&#x2716;</span>";
}
ci_map[ci_id] = ci;
}
var tableData = [];
for (ci_id in ci_map) {
var c = ci_map[ci_id];
if (c.merge_run_count > 0) {
c.merge_success_rate = Math.round(100 * c.merge_success / c.merge_run_count) + "%";
} else {
c.merge_success_rate = "-";
}
tableData.push(c);
}
$("#" + table_id).dataTable({
"aaSorting": [
[ 0, "asc"]
],
"bFilter": true,
"bInfo": false,
"bAutoWidth": false,
"bPaginate": false,
"iDisplayLength": -1,
"aaData": tableData,
"aoColumns": tableColumns,
"aoColumnDefs": [
{ "sClass": "center", "aTargets": [1, 2, 3, 4] },
{ "sType": "ratio", "aTargets": [2]}
]
}).show();
$("#" + table_id + "_loading").hide();
// make table with per-change-request status
var table = $("<table id='table_status'></table>");
var table_head = $('<thead></thead>');
table.append(table_head);
var table_head_row = $("<tr></tr>");
table_head.append(table_head_row);
table_head_row.append($("<th rowspan='2'>Driver</th>"));
var table_second_head_row = $("<tr></tr>");
table_head.append(table_second_head_row);
var table_body = $("<tbody></tbody>");
table.append(table_body);
var prev_date = null;
var count = 0;
for (i = review_ids.length - 1; i >=0; i--) {
var date_str = review_ids[i].date_str;
var dp = date_str.split(" ");
var date = dp[0] + " " + dp[1];
if (date != prev_date && count > 0) {
table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
prev_date = date;
count = 0;
}
prev_date = date;
count++;
table_second_head_row.append($("<th title='Change request " + review_ids[i].review_id +
"'><span style='font-size: 60%;'>" + review_ids[i].review_id + "</span></th>"));
}
if (count > 0) {
table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
}
for (ci_id in ci_id_to_name) {
var table_row = $("<tr></tr>");
table_row.append($("<td>" + ci_id_to_name[ci_id] + "</td>"));
for (i = review_ids.length - 1; i >=0; i--) {
var review_id = review_ids[i].review_id;
var color = "#cfcfcf";
var title = "N/A";
if (res_map[ci_id]) {
var res = res_map[ci_id][review_id];
if (res == true) {
color = "#40bf00";
title = "SUCCESS";
} else if (res == false) {
color = "#bf4000";
title = "FAILURE";
}
}
const url = "https://review.openstack.org/#/c/" + review_id;
var cell = $("<td></td>");
cell.css("background-color", color).css("cursor", "pointer").prop("url", url).prop("title", title + " - " + url);
cell.click(function(evt) {
window.open($(this).prop("url"));
});
table_row.append(cell);
}
table_body.append(table_row);
}
$("#table_status_container").append(table);
$("#table_status_container_loading").hide();
table.dataTable({
"bPaginate": false,
"iDisplayLength": -1
});
}
});
});
</script>
<style type="text/css">
table.dataTable tr.even {
background-color: #EEF1F4;
}
table.dataTable tr.even:hover, table.dataTable tr.odd:hover {
background-color: #F8FFEC;
}
table.dataTable tr.even td.sorting_1 {
background-color: #E0E8E8;
}
</style>
{% endblock %}
{% block content %}
<h1>Status of drivers testing in {{ module|title }} during recent {{ days }} days</h1>
<p>List of drivers is taken from <a href="/report/driverlog" target="_blank">DriverLog</a>.
The status is based on job results from the latest patch in merged change requests (thus assuming that the job is executed on code close to the master).
Missing or misconfigured drivers can be updated by commit into DriverLog's
<a href="https://git.openstack.org/cgit/openstack/driverlog/tree/etc/default_data.json" target="_blank">default_data.json</a>.
</p>
<h2>Drivers summary</h2>
<table id="summary_stats_table" style="display: none;">
<thead>
<tr>
<th>Driver</th>
<th>CI votes</th>
<th>Success, %</th>
<th>Latest run</th>
<th>Latest result</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<div id="summary_stats_table_loading" class="select2-loading" style="width: 7em;">Loading...</div>
<h2>Status per change requests</h2>
<div id="table_status_container"></div>
<div id="table_status_container_loading" class="select2-loading" style="width: 7em;">Loading...</div>
{% endblock %}

2
stackalytics/dashboard/web.py
View File

@ -623,8 +623,6 @@ def timeline(records, **kwargs):
if ('commits' in metric) or ('loc' in metric):
loc_handler = lambda record: record.loc
elif 'ci' in metric:
loc_handler = lambda record: 0 if record.value else 1
else:
loc_handler = lambda record: 0

4
stackalytics/processor/config.py
View File

@ -46,10 +46,6 @@ PROCESSOR_OPTS = [
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',
help='SSH username for gerrit review system access'),
cfg.StrOpt('driverlog-data-uri',
default='https://git.openstack.org/cgit/'
'openstack/driverlog/plain/etc/default_data.json',
help='URI for default data'),
cfg.StrOpt('translation-team-uri',
default='https://git.openstack.org/cgit/openstack/i18n/'
'plain/tools/zanata/translation_team.yaml',

34
stackalytics/processor/default_data_processor.py
View File

@ -175,36 +175,6 @@ def _update_project_list(default_data):
default_data['project_sources'], default_data['repos'])
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
if not driverlog_data:
LOG.warning('DriverLog data is not available')
return
module_cis = collections.defaultdict(list)
for driver in driverlog_data['drivers']:
if 'ci' not in driver:
continue
module = (driver.get('repo') or driver['project_id']).split('/')[1]
module_cis[module].append(driver)
default_data['users'].append({
'user_id': user_processor.make_user_id(ci_id=driver['name']),
'user_name': driver['name'],
'static': True,
'companies': [
{'company_name': driver['vendor'], 'end_date': None}],
})
for repo in default_data['repos']:
if repo['module'] in module_cis:
repo['drivers'] = module_cis[repo['module']]
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = user_processor.load_user(runtime_storage_inst,
@ -259,12 +229,10 @@ def _store_default_data(runtime_storage_inst, default_data):
runtime_storage_inst.set_by_key(key, value)
def process(runtime_storage_inst, default_data, driverlog_data_uri):
def process(runtime_storage_inst, default_data):
LOG.debug('Process default data')
if 'project_sources' in default_data:
_update_project_list(default_data)
_update_with_driverlog_data(default_data, driverlog_data_uri)
_store_default_data(runtime_storage_inst, default_data)

100
stackalytics/processor/driverlog.py
View File

@ -1,100 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from stackalytics.processor import user_processor
def _find_ci_result(review, drivers):
"""For a given stream of reviews yields results produced by CIs."""
review_id = review['id']
review_number = review['number']
ci_id_set = set(d['ci']['id'] for d in drivers)
candidate_drivers = [d for d in drivers]
last_patch_set_number = review['patchSets'][-1]['number']
for comment in reversed(review.get('comments') or []):
comment_author = comment['reviewer'].get('username')
if comment_author not in ci_id_set:
continue # not any of registered CIs
message = comment['message']
prefix = 'Patch Set'
if comment['message'].find(prefix) != 0:
continue # look for special messages only
prefix = 'Patch Set %s:' % last_patch_set_number
if comment['message'].find(prefix) != 0:
break # all comments from the latest patch set already parsed
message = message[len(prefix):].strip()
result = None
matched_drivers = set()
for driver in candidate_drivers:
ci = driver['ci']
if ci['id'] != comment_author:
continue
# try to get result by parsing comment message
success_pattern = ci.get('success_pattern')
failure_pattern = ci.get('failure_pattern')
message_lines = (l for l in message.split('\n') if l.strip())
line = ''
for line in message_lines:
if success_pattern and re.search(success_pattern, line):
result = True
break
elif failure_pattern and re.search(failure_pattern, line):
result = False
break
if result is not None:
matched_drivers.add(driver['name'])
record = {
'user_id': user_processor.make_user_id(
ci_id=driver['name']),
'value': result,
'message': line,
'date': comment['timestamp'],
'branch': review['branch'],
'review_id': review_id,
'review_number': review_number,
'driver_name': driver['name'],
'driver_vendor': driver['vendor'],
'module': review['module']
}
if review['branch'].find('/') > 0:
record['release'] = review['branch'].split('/')[1]
yield record
candidate_drivers = [d for d in candidate_drivers
if d['name'] not in matched_drivers]
if not candidate_drivers:
break # found results from all drivers
def log(review_iterator, drivers):
for record in review_iterator:
for driver_info in _find_ci_result(record, drivers):
yield driver_info

1
stackalytics/processor/dump.py
View File

@ -96,7 +96,6 @@ def get_repo_keys(memcached_inst):
for branch in branches:
yield 'vcs:%s:%s' % (quoted_uri, branch)
yield 'rcs:%s:%s' % (quoted_uri, branch)
yield 'ci:%s:%s' % (quoted_uri, branch)
def export_data(memcached_inst, fd):

34
stackalytics/processor/main.py
View File

@ -24,7 +24,6 @@ import six
from stackalytics.processor import bps
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
from stackalytics.processor import driverlog
from stackalytics.processor import governance
from stackalytics.processor import lp
from stackalytics.processor import mls
@ -139,29 +138,6 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst,
runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)
def _process_repo_ci_votes(repo, runtime_storage_inst, record_processor_inst,
rcs_inst):
for branch in _get_repo_branches(repo):
LOG.info('Processing CI votes for repo: %s, branch: %s',
repo['uri'], branch)
quoted_uri = six.moves.urllib.parse.quote_plus(repo['uri'])
rcs_key = 'ci:%s:%s' % (quoted_uri, branch)
last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key)
current_retrieval_time = utils.date_to_timestamp('now')
review_iterator = rcs_inst.log(repo, branch, last_retrieval_time,
status='merged', grab_comments=True)
review_iterator = driverlog.log(review_iterator, repo['drivers'])
review_iterator_typed = _record_typer(review_iterator, 'ci')
processed_review_iterator = record_processor_inst.process(
review_iterator_typed)
runtime_storage_inst.set_records(processed_review_iterator,
utils.merge_records)
runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)
def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst):
vcs_inst = vcs.get_vcs(repo, CONF.sources_root)
vcs_inst.fetch()
@ -199,10 +175,6 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
_process_repo_reviews(repo, runtime_storage_inst,
record_processor_inst, rcs_inst)
if 'drivers' in repo:
_process_repo_ci_votes(repo, runtime_storage_inst,
record_processor_inst, rcs_inst)
def _process_mail_list(uri, runtime_storage_inst, record_processor_inst):
mail_iterator = mls.log(uri, runtime_storage_inst)
@ -316,9 +288,6 @@ def process_project_list(runtime_storage_inst):
module = repo['module'].lower()
module_groups[module] = utils.make_module_group(module, tag='module')
if 'drivers' in repo:
module_groups[module]['has_drivers'] = True
# register module 'unknown' - used for emails not mapped to any module
module_groups['unknown'] = utils.make_module_group('unknown', tag='module')
@ -344,8 +313,7 @@ def main():
return not 0
default_data_processor.process(runtime_storage_inst,
default_data,
CONF.driverlog_data_uri)
default_data)
process_project_list(runtime_storage_inst)

13
stackalytics/processor/record_processor.py
View File

@ -603,18 +603,6 @@ class RecordProcessor(object):
yield record
def _process_ci(self, record):
ci_vote = dict((k, v) for k, v in six.iteritems(record))
ci_vote['primary_key'] = '%s:%s' % (record['review_id'],
record['driver_name'])
ci_vote['author_name'] = record['driver_name']
ci_vote['author_email'] = record['user_id']
self._update_record_and_user(ci_vote)
yield ci_vote
def _process_translation(self, record):
# todo split translation and approval
translation = record.copy()
@ -646,7 +634,6 @@ class RecordProcessor(object):
'bp': self._process_blueprint,
'bug': self._process_bug,
'member': self._process_member,
'ci': self._process_ci,
'i18n': self._process_translation,
}

5
stackalytics/processor/user_processor.py
View File

@ -14,7 +14,6 @@
# limitations under the License.
import copy
import re
from oslo_log import log as logging
@ -22,7 +21,7 @@ LOG = logging.getLogger(__name__)
def make_user_id(emails=None, launchpad_id=None, gerrit_id=None,
member_id=None, github_id=None, ci_id=None, zanata_id=None):
member_id=None, github_id=None, zanata_id=None):
if launchpad_id or emails:
return launchpad_id or emails[0]
if gerrit_id:
@ -33,8 +32,6 @@ def make_user_id(emails=None, launchpad_id=None, gerrit_id=None,
return 'github:%s' % github_id
if zanata_id:
return 'zanata:%s' % zanata_id
if ci_id:
return 'ci:%s' % re.sub(r'[^\w]', '_', ci_id.lower())
return None

58
stackalytics/tests/unit/test_default_data_processor.py
View File

@ -137,61 +137,3 @@ class TestDefaultDataProcessor(testtools.TestCase):
'module_group_name': 'kubernetes',
'modules': ['kubernetes'],
'tag': 'organization'}, dd['module_groups'])
@mock.patch('stackalytics.processor.utils.read_json_from_uri')
def test_update_with_driverlog(self, mock_read_from_json):
default_data = {'repos': [{'module': 'cinder', }], 'users': []}
driverlog_dd = {'drivers': [{
'project_id': 'openstack/cinder',
'vendor': 'VMware',
'name': 'VMware VMDK Driver',
'ci': {
'id': 'vmwareminesweeper',
'success_pattern': 'Build successful',
'failure_pattern': 'Build failed'
}
}]}
mock_read_from_json.return_value = driverlog_dd
default_data_processor._update_with_driverlog_data(default_data, 'uri')
expected_user = {
'user_id': 'ci:vmware_vmdk_driver',
'user_name': 'VMware VMDK Driver',
'static': True,
'companies': [
{'company_name': 'VMware', 'end_date': None}],
}
self.assertIn(expected_user, default_data['users'])
self.assertIn(driverlog_dd['drivers'][0],
default_data['repos'][0]['drivers'])
@mock.patch('stackalytics.processor.utils.read_json_from_uri')
def test_update_with_driverlog_specific_repo(self, mock_read_from_json):
default_data = {'repos': [{'module': 'fuel-plugin-mellanox', }],
'users': []}
driverlog_dd = {'drivers': [{
'project_id': 'openstack/fuel',
'repo': 'stackforge/fuel-plugin-mellanox',
'vendor': 'Mellanox',
'name': 'ConnectX-3 Pro Network Adapter Support plugin',
'ci': {
'id': 'mellanox',
'success_pattern': 'SUCCESS',
'failure_pattern': 'FAILURE'
}
}]}
mock_read_from_json.return_value = driverlog_dd
default_data_processor._update_with_driverlog_data(default_data, 'uri')
expected_user = {
'user_id': 'ci:connectx_3_pro_network_adapter_support_plugin',
'user_name': 'ConnectX-3 Pro Network Adapter Support plugin',
'static': True,
'companies': [
{'company_name': 'Mellanox', 'end_date': None}],
}
self.assertIn(expected_user, default_data['users'])
self.assertIn(driverlog_dd['drivers'][0],
default_data['repos'][0]['drivers'])

124
stackalytics/tests/unit/test_driverlog.py
View File

@ -1,124 +0,0 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import testtools
from stackalytics.processor import driverlog
COMMENT_SUCCESS = {
'message': 'Patch Set 2: build successful',
'reviewer': {'username': 'virt-ci'},
'timestamp': 1234567890
}
COMMENT_FAILURE = {
'message': 'Patch Set 2: build failed',
'reviewer': {'username': 'virt-ci'},
'timestamp': 1234567880
}
REVIEW = {
'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'module': 'nova',
'branch': 'master',
'status': 'MERGED',
'number': '97860',
'patchSets': [{'number': '1'}, {'number': '2'}],
'comments': [
{'message': 'Patch Set 2: build successful',
'reviewer': {'username': 'other-ci'}, },
{'message': 'Patch Set 2: job started',
'reviewer': {'username': 'virt-ci'}, }]
}
DRIVER = {
'name': 'Virt Nova Driver',
'vendor': 'Virt Inc',
'ci': {
'id': 'virt-ci',
'success_pattern': 'successful',
'failure_pattern': 'failed',
}
}
DRIVER_NON_EXISTENT = {
'name': 'No Virt Nova Driver',
'vendor': 'No Virt Inc',
'ci': {
'id': 'no-virt-ci',
'success_pattern': 'successful',
'failure_pattern': 'failed',
}
}
class TestDriverlog(testtools.TestCase):
def test_find_ci_result_success(self):
drivers = [DRIVER]
review = copy.deepcopy(REVIEW)
review['comments'].append(COMMENT_SUCCESS)
res = list(driverlog.log([review], drivers))
expected_result = {
'user_id': 'ci:virt_nova_driver',
'value': True,
'message': 'build successful',
'date': 1234567890,
'branch': 'master',
'review_id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'review_number': '97860',
'driver_name': 'Virt Nova Driver',
'driver_vendor': 'Virt Inc',
'module': 'nova',
}
self.assertEqual(1, len(res), 'One CI result is expected')
self.assertEqual(expected_result, res[0])
def test_find_ci_result_failure(self):
drivers = [DRIVER]
review = copy.deepcopy(REVIEW)
review['comments'].append(COMMENT_FAILURE)
res = list(driverlog.log([review], drivers))
self.assertEqual(1, len(res), 'One CI result is expected')
self.assertFalse(res[0]['value'])
def test_find_ci_result_non_existent(self):
drivers = [DRIVER_NON_EXISTENT]
review = copy.deepcopy(REVIEW)
review['comments'].append(COMMENT_SUCCESS)
res = list(driverlog.log([REVIEW], drivers))
self.assertEqual(0, len(res), 'No CI results expected')
def test_find_ci_result_last_vote_only(self):
# there may be multiple comments from the same CI,
# only the last one is important
drivers = [DRIVER]
review = copy.deepcopy(REVIEW)
review['comments'].append(COMMENT_FAILURE)
review['comments'].append(COMMENT_SUCCESS)
res = list(driverlog.log([review], drivers))
self.assertEqual(1, len(res), 'One CI result is expected')
self.assertTrue(res[0]['value'])