diff --git a/doc/source/tools/stackalytics-processor.txt b/doc/source/tools/stackalytics-processor.txt
index cf9973805..ece5f313e 100644
--- a/doc/source/tools/stackalytics-processor.txt
+++ b/doc/source/tools/stackalytics-processor.txt
@@ -2,7 +2,6 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
[--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
[--debug] [--default-data-uri DEFAULT_DATA_URI]
- [--driverlog-data-uri DRIVERLOG_DATA_URI]
[--fetching-user-source FETCHING_USER_SOURCE]
[--gerrit-retry GERRIT_RETRY]
[--git-base-uri GIT_BASE_URI]
@@ -44,8 +43,6 @@ optional arguments:
URI for default data. A local file can be used with
the prefix "file://". For example, default_data_uri =
file:///path/to/default_data.json
- --driverlog-data-uri DRIVERLOG_DATA_URI
- URI for default data
--fetching-user-source FETCHING_USER_SOURCE
Source for fetching user profiles
--gerrit-retry GERRIT_RETRY
diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf
index 23ac59f5a..7fde7798e 100644
--- a/etc/stackalytics.conf
+++ b/etc/stackalytics.conf
@@ -170,9 +170,6 @@
# SSH username for gerrit review system access (string value)
#ssh_username = user
-# URI for default data (string value)
-#driverlog_data_uri = https://git.openstack.org/cgit/openstack/driverlog/plain/etc/default_data.json
-
# URI of translation team data (string value)
#translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml
diff --git a/stackalytics/dashboard/decorators.py b/stackalytics/dashboard/decorators.py
index 18094a5b0..27b027a30 100644
--- a/stackalytics/dashboard/decorators.py
+++ b/stackalytics/dashboard/decorators.py
@@ -321,28 +321,6 @@ def mark_finalize(record):
return new_record
-def ci_filter(result, record, param_id, context):
- result_by_param = result[getattr(record, param_id)]
-
- result_by_param['metric'] += 1
-
- key = 'success' if record.value else 'failure'
- result_by_param[key] = result_by_param.get(key, 0) + 1
-
-
-def ci_finalize(record):
- new_record = record.copy()
-
- metric = record.get('metric')
- if metric:
- new_record['success_ratio'] = '%.1f%%' % (
- (record.get('success', 0) * 100.0) / metric)
- else:
- new_record['success_rate'] = helpers.INFINITY_HTML
-
- return new_record
-
-
def person_day_filter(result, record, param_id, context):
day = utils.timestamp_to_day(record.date)
# fact that record-days are grouped by days in some order is used
@@ -389,7 +367,6 @@ def aggregate_filter():
'resolved-bugs': (incremental_filter, None),
'members': (incremental_filter, None),
'person-day': (person_day_filter, None),
- 'ci': (ci_filter, ci_finalize),
'patches': (None, None),
'translations': (loc_filter, None),
}
diff --git a/stackalytics/dashboard/parameters.py b/stackalytics/dashboard/parameters.py
index 433dd86e5..cae6cd7f6 100644
--- a/stackalytics/dashboard/parameters.py
+++ b/stackalytics/dashboard/parameters.py
@@ -37,7 +37,6 @@ METRIC_LABELS = {
'filed-bugs': 'Filed Bugs',
'resolved-bugs': 'Resolved Bugs',
'person-day': "Person-day effort",
- 'ci': 'CI votes',
'patches': 'Patch Sets',
'translations': 'Translations',
}
@@ -53,7 +52,6 @@ METRIC_TO_RECORD_TYPE = {
'resolved-bugs': ['bugr'],
'members': ['member'],
'person-day': ['mark', 'patch', 'email', 'bpd', 'bugf'],
- 'ci': ['ci'],
'patches': ['patch'],
'translations': ['tr'],
}
diff --git a/stackalytics/dashboard/reports.py b/stackalytics/dashboard/reports.py
index 0a2d6f028..df7480c6d 100644
--- a/stackalytics/dashboard/reports.py
+++ b/stackalytics/dashboard/reports.py
@@ -148,20 +148,6 @@ def contribution(module, days):
}
-@blueprint.route('/ci/<module>/<days>')
-@decorators.templated()
-@decorators.exception_handler()
-def external_ci(module, days):
- if int(days) > 100:
- days = 100
-
- return {
- 'module': module,
- 'days': days,
- 'start_date': int(time.time()) - int(days) * 24 * 60 * 60
- }
-
-
@blueprint.route('/members')
@decorators.exception_handler()
@decorators.templated()
@@ -230,7 +216,7 @@ def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
- ['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])
+ ['mark', 'patch', 'email', 'bpd', 'bpc'])
record_ids &= record_ids_by_type
punch_card_data = _get_punch_card_data(
diff --git a/stackalytics/dashboard/templates/_macros/activity_log.html b/stackalytics/dashboard/templates/_macros/activity_log.html
index a38dc5890..44e11eb0f 100644
--- a/stackalytics/dashboard/templates/_macros/activity_log.html
+++ b/stackalytics/dashboard/templates/_macros/activity_log.html
@@ -109,7 +109,7 @@ show_twitter=False) -%}
<div class="header">{%html author_link %} ({%html company_link %})</div>
{%/if%}
<div class="header">${date_str} in {%html module_link%}
- {%if record_type == "mark" || record_type == "review" || record_type == "patch" || record_type == "ci" || record_type == "tr" %}
+ {%if record_type == "mark" || record_type == "review" || record_type == "patch" || record_type == "tr" %}
{%if branch != "master" %}(${branch}){%/if%}
{%/if%}
{%if record_type == "commit" %}
@@ -192,13 +192,6 @@ show_twitter=False) -%}
<div class="header">Bug “${title}” (<a href="${web_link}" class="ext_link">${number}</a>)</div>
<div>Status: <span class="status${status_class}">${status}</span></div>
<div>Importance: <span class="importance${importance}">${importance}</span></div>
- {%elif record_type == "ci" %}
- <div class="header">CI vote in merged change request
- <a href="https://review.openstack.org/#/c/${review_number}" target="_blank">${review_number}</a>
- </div>
- <div>{%if value == true %}<span style="color: green">Success</span>{%else%}<span style="color: red">Failure</span>{%/if%}</div>
- <div class="message">${message}</div>
- <div>Change Id: <a href="https://review.openstack.org/#/c/${review_number}" target="_blank">${review_id}</a></div>
{%elif record_type == "member" %}
<div class="header"><a href="${member_uri}" target="_blank">Registered</a> in OpenStack Foundation</div>
{%elif record_type == "tr" %}
diff --git a/stackalytics/dashboard/templates/overview.html b/stackalytics/dashboard/templates/overview.html
index 381546ad0..ff60b37ac 100644
--- a/stackalytics/dashboard/templates/overview.html
+++ b/stackalytics/dashboard/templates/overview.html
@@ -17,16 +17,12 @@
{% set show_user_profile = (user_id) %}
{% set show_module_details = (module) %}
{% set show_review_ratio = (metric in ['marks']) %}
-{% set show_ci = (metric in ['ci']) %}
{% macro show_report_links(module=None, company=None, user_id=None) -%}
{% if module %}
<div><b><a href="/report/reviews/{{ module }}/open" target="_blank">Show open reviews for {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/30" target="_blank">Contribution for the last 30 days in {{ module_inst.module_group_name }}</a></b></div>
<div><b><a href="/report/contribution/{{ module }}/90" target="_blank">Contribution for the last 90 days in {{ module_inst.module_group_name }}</a></b></div>
- {% if module_inst.has_drivers %}
- <div><b><a href="/report/ci/{{ module }}/7" target="_blank">Status of drivers testing in {{ module_inst.module_group_name }} during recent 7 days</a></b></div>
- {% endif %}
<div><b><a href="/report/activity?module={{ module }}&project_type={{ project_type }}&release={{ release }}" target="_blank">Show activity report for {{ module }}</a></b></div>
{% endif %}
{% if company %}
@@ -49,9 +45,6 @@
{% if show_review_ratio %}
renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id",
["index", "link", "mark_ratio", "metric"]);
- {% elif show_ci %}
- renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id",
- ["index", "link", "metric", "success_ratio"]);
{% else %}
renderTableAndChart("/api/1.0/stats/engineers", "engineer_container", "engineer_table", "engineer_chart", "user_id");
{% endif %}
@@ -104,14 +97,11 @@
<thead>
<tr>
<th>#</th>
- <th>{% if show_ci %}Driver{% else %}Contributor{% endif %}</th>
+ <th>Contributor</th>
{% if show_review_ratio %}
<th>-2|-1|+1|+2|A|x (+ ratio)</th>
{% endif %}
<th>{{ metric_label }}</th>
- {% if show_ci %}
- <th>Success</th>
- {% endif %}
</tr>
</thead>
<tbody>
diff --git a/stackalytics/dashboard/templates/reports/external_ci.html b/stackalytics/dashboard/templates/reports/external_ci.html
deleted file mode 100644
index 3048b0958..000000000
--- a/stackalytics/dashboard/templates/reports/external_ci.html
+++ /dev/null
@@ -1,245 +0,0 @@
-{% extends "reports/base_report.html" %}
-
-{% set page_title = "Status of drivers testing in " + module.title() + " during recent " + days + " days" %}
-
-{% block scripts %}
- <script type="text/javascript">
-
- jQuery.extend(jQuery.fn.dataTableExt.oSort, {
- "ratio-pre": function (a) {
- if (a == "∞" || a == "-")
- return -1.0;
- a = a.substr(0, a.length - 1);
- return parseFloat(a);
- },
-
- "ratio-asc": function (a, b) {
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
-
- "ratio-desc": function (a, b) {
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
- });
-
- $(document).ready(function () {
- var table_column_names = ["name", "merge_run_count", "merge_success_rate", "merge_date_str", "merge_result"];
- var table_id = "summary_stats_table";
-
- $.ajax({
- url: makeURI("/api/1.0/activity?project_type=all&metric=ci&module={{ module }}&release=all&start_date={{ start_date }}&page_size=-1"),
- dataType: "json",
- success: function (data) {
- var activity = data["activity"];
-
- var tableColumns = [];
- for (var i = 0; i < table_column_names.length; i++) {
- tableColumns.push({"mData": table_column_names[i]});
- }
-
- var ci_map = {}; // ci_id -> statistics
- var res_map = {}; // ci_id -> map: review_id -> res
- var ci_id_to_name = {}; // ci_id -> name
- var review_ids = [];
- var review_ids_set = {};
-
- for (i = 0; i < activity.length; i++) {
- var ai = activity[i];
-
- var ci_id = ai.user_id;
-
- var ci = ci_map[ci_id];
- if (!ci) {
- ci = {merge_run_count: 0, merge_success: 0,
- merge_date: 0, merge_result: "N/A",
- merge_date_str: "N/A", "total_date_str": "N/A"};
- }
- ci.id = ai.user_id;
- ci.name = "<a href=\"/?metric=ci&user_id=" + ai.user_id +
- "&module={{ module }}\">" + ai.driver_name + "</a>";
- ci_id_to_name[ci_id] = ci.name;
-
- var review_id = ai.review_number;
- if (!review_ids_set[review_id]) {
- review_ids_set[review_id] = review_id;
- review_ids.push({review_id: review_id, date_str: ai.date_str});
- }
-
- if (!res_map[ci_id]) {
- res_map[ci_id] = {};
- }
- res_map[ci_id][review_id] = ai.value;
-
- ci.merge_run_count ++;
- if (ai.value) {
- ci.merge_success ++;
- }
- if (ai.date > ci.merge_date) {
- ci.merge_date = ai.date;
- ci.merge_date_str = ai.date_str;
- ci.merge_result = (ai.value)? "<span style=\"color: green\">✔</span>":
- "<span style=\"color: red\">✖</span>";
- }
-
- ci_map[ci_id] = ci;
- }
-
- var tableData = [];
- for (ci_id in ci_map) {
- var c = ci_map[ci_id];
-
- if (c.merge_run_count > 0) {
- c.merge_success_rate = Math.round(100 * c.merge_success / c.merge_run_count) + "%";
- } else {
- c.merge_success_rate = "-";
- }
-
- tableData.push(c);
- }
-
- $("#" + table_id).dataTable({
- "aaSorting": [
- [ 0, "asc"]
- ],
- "bFilter": true,
- "bInfo": false,
- "bAutoWidth": false,
- "bPaginate": false,
- "iDisplayLength": -1,
- "aaData": tableData,
- "aoColumns": tableColumns,
- "aoColumnDefs": [
- { "sClass": "center", "aTargets": [1, 2, 3, 4] },
- { "sType": "ratio", "aTargets": [2]}
- ]
- }).show();
- $("#" + table_id + "_loading").hide();
-
- // make table with per-change-request status
-
- var table = $("<table id='table_status'></table>");
- var table_head = $('<thead></thead>');
- table.append(table_head);
- var table_head_row = $("<tr></tr>");
- table_head.append(table_head_row);
- table_head_row.append($("<th rowspan='2'>Driver</th>"));
- var table_second_head_row = $("<tr></tr>");
- table_head.append(table_second_head_row);
- var table_body = $("<tbody></tbody>");
- table.append(table_body);
-
- var prev_date = null;
- var count = 0;
-
- for (i = review_ids.length - 1; i >=0; i--) {
- var date_str = review_ids[i].date_str;
- var dp = date_str.split(" ");
- var date = dp[0] + " " + dp[1];
-
- if (date != prev_date && count > 0) {
- table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
- prev_date = date;
- count = 0;
- }
- prev_date = date;
- count++;
- table_second_head_row.append($("<th title='Change request " + review_ids[i].review_id +
- "'><span style='font-size: 60%;'>" + review_ids[i].review_id + "</span></th>"));
- }
-
- if (count > 0) {
- table_head_row.append($("<th colspan='" + count + "'>" + prev_date + "</th>"));
- }
-
- for (ci_id in ci_id_to_name) {
- var table_row = $("<tr></tr>");
- table_row.append($("<td>" + ci_id_to_name[ci_id] + "</td>"));
-
- for (i = review_ids.length - 1; i >=0; i--) {
- var review_id = review_ids[i].review_id;
-
- var color = "#cfcfcf";
- var title = "N/A";
-
- if (res_map[ci_id]) {
- var res = res_map[ci_id][review_id];
- if (res == true) {
- color = "#40bf00";
- title = "SUCCESS";
- } else if (res == false) {
- color = "#bf4000";
- title = "FAILURE";
- }
- }
- const url = "https://review.openstack.org/#/c/" + review_id;
- var cell = $("<td></td>");
- cell.css("background-color", color).css("cursor", "pointer").prop("url", url).prop("title", title + " - " + url);
- cell.click(function(evt) {
- window.open($(this).prop("url"));
- });
- table_row.append(cell);
- }
- table_body.append(table_row);
- }
-
- $("#table_status_container").append(table);
- $("#table_status_container_loading").hide();
-
- table.dataTable({
- "bPaginate": false,
- "iDisplayLength": -1
- });
- }
- });
- });
- </script>
-
- <style type="text/css">
- table.dataTable tr.even {
- background-color: #EEF1F4;
- }
-
- table.dataTable tr.even:hover, table.dataTable tr.odd:hover {
- background-color: #F8FFEC;
- }
-
- table.dataTable tr.even td.sorting_1 {
- background-color: #E0E8E8;
- }
- </style>
-
-{% endblock %}
-
-{% block content %}
-<h1>Status of drivers testing in {{ module|title }} during recent {{ days }} days</h1>
-
-<p>List of drivers is taken from <a href="/report/driverlog" target="_blank">DriverLog</a>.
- The status is based on job results from the latest patch in merged change requests (thus assuming that the job is executed on code close to the master).
- Missing or misconfigured drivers can be updated by commit into DriverLog's
- <a href="https://git.openstack.org/cgit/openstack/driverlog/tree/etc/default_data.json" target="_blank">default_data.json</a>.
-</p>
-
-<h2>Drivers summary</h2>
-
- <table id="summary_stats_table" style="display: none;">
- <thead>
- <tr>
- <th>Driver</th>
- <th>CI votes</th>
- <th>Success, %</th>
- <th>Latest run</th>
- <th>Latest result</th>
- </tr>
- </thead>
- <tbody>
- </tbody>
- </table>
-
- <div id="summary_stats_table_loading" class="select2-loading" style="width: 7em;">Loading...</div>
-
-<h2>Status per change requests</h2>
-
- <div id="table_status_container"></div>
- <div id="table_status_container_loading" class="select2-loading" style="width: 7em;">Loading...</div>
-
-{% endblock %}
diff --git a/stackalytics/dashboard/web.py b/stackalytics/dashboard/web.py
index 1f3e84a28..e72308c03 100644
--- a/stackalytics/dashboard/web.py
+++ b/stackalytics/dashboard/web.py
@@ -623,8 +623,6 @@ def timeline(records, **kwargs):
if ('commits' in metric) or ('loc' in metric):
loc_handler = lambda record: record.loc
- elif 'ci' in metric:
- loc_handler = lambda record: 0 if record.value else 1
else:
loc_handler = lambda record: 0
diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py
index c7b630bd9..8f238c03d 100644
--- a/stackalytics/processor/config.py
+++ b/stackalytics/processor/config.py
@@ -46,10 +46,6 @@ PROCESSOR_OPTS = [
help='SSH key for gerrit review system access'),
cfg.StrOpt('ssh-username', default='user',
help='SSH username for gerrit review system access'),
- cfg.StrOpt('driverlog-data-uri',
- default='https://git.openstack.org/cgit/'
- 'openstack/driverlog/plain/etc/default_data.json',
- help='URI for default data'),
cfg.StrOpt('translation-team-uri',
default='https://git.openstack.org/cgit/openstack/i18n/'
'plain/tools/zanata/translation_team.yaml',
diff --git a/stackalytics/processor/default_data_processor.py b/stackalytics/processor/default_data_processor.py
index 2af711701..3c22d1722 100644
--- a/stackalytics/processor/default_data_processor.py
+++ b/stackalytics/processor/default_data_processor.py
@@ -175,36 +175,6 @@ def _update_project_list(default_data):
default_data['project_sources'], default_data['repos'])
-def _update_with_driverlog_data(default_data, driverlog_data_uri):
- LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
- driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
-
- if not driverlog_data:
- LOG.warning('DriverLog data is not available')
- return
-
- module_cis = collections.defaultdict(list)
- for driver in driverlog_data['drivers']:
- if 'ci' not in driver:
- continue
-
- module = (driver.get('repo') or driver['project_id']).split('/')[1]
-
- module_cis[module].append(driver)
-
- default_data['users'].append({
- 'user_id': user_processor.make_user_id(ci_id=driver['name']),
- 'user_name': driver['name'],
- 'static': True,
- 'companies': [
- {'company_name': driver['vendor'], 'end_date': None}],
- })
-
- for repo in default_data['repos']:
- if repo['module'] in module_cis:
- repo['drivers'] = module_cis[repo['module']]
-
-
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = user_processor.load_user(runtime_storage_inst,
@@ -259,12 +229,10 @@ def _store_default_data(runtime_storage_inst, default_data):
runtime_storage_inst.set_by_key(key, value)
-def process(runtime_storage_inst, default_data, driverlog_data_uri):
+def process(runtime_storage_inst, default_data):
LOG.debug('Process default data')
if 'project_sources' in default_data:
_update_project_list(default_data)
- _update_with_driverlog_data(default_data, driverlog_data_uri)
-
_store_default_data(runtime_storage_inst, default_data)
diff --git a/stackalytics/processor/driverlog.py b/stackalytics/processor/driverlog.py
deleted file mode 100644
index fb429bc55..000000000
--- a/stackalytics/processor/driverlog.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# Copyright (c) 2014 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-
-from stackalytics.processor import user_processor
-
-
-def _find_ci_result(review, drivers):
- """For a given stream of reviews yields results produced by CIs."""
-
- review_id = review['id']
- review_number = review['number']
-
- ci_id_set = set(d['ci']['id'] for d in drivers)
- candidate_drivers = [d for d in drivers]
-
- last_patch_set_number = review['patchSets'][-1]['number']
-
- for comment in reversed(review.get('comments') or []):
- comment_author = comment['reviewer'].get('username')
- if comment_author not in ci_id_set:
- continue # not any of registered CIs
-
- message = comment['message']
-
- prefix = 'Patch Set'
- if comment['message'].find(prefix) != 0:
- continue # look for special messages only
-
- prefix = 'Patch Set %s:' % last_patch_set_number
- if comment['message'].find(prefix) != 0:
- break # all comments from the latest patch set already parsed
- message = message[len(prefix):].strip()
-
- result = None
- matched_drivers = set()
-
- for driver in candidate_drivers:
- ci = driver['ci']
- if ci['id'] != comment_author:
- continue
-
- # try to get result by parsing comment message
- success_pattern = ci.get('success_pattern')
- failure_pattern = ci.get('failure_pattern')
-
- message_lines = (l for l in message.split('\n') if l.strip())
-
- line = ''
- for line in message_lines:
- if success_pattern and re.search(success_pattern, line):
- result = True
- break
- elif failure_pattern and re.search(failure_pattern, line):
- result = False
- break
-
- if result is not None:
- matched_drivers.add(driver['name'])
- record = {
- 'user_id': user_processor.make_user_id(
- ci_id=driver['name']),
- 'value': result,
- 'message': line,
- 'date': comment['timestamp'],
- 'branch': review['branch'],
- 'review_id': review_id,
- 'review_number': review_number,
- 'driver_name': driver['name'],
- 'driver_vendor': driver['vendor'],
- 'module': review['module']
- }
- if review['branch'].find('/') > 0:
- record['release'] = review['branch'].split('/')[1]
-
- yield record
-
- candidate_drivers = [d for d in candidate_drivers
- if d['name'] not in matched_drivers]
- if not candidate_drivers:
- break # found results from all drivers
-
-
-def log(review_iterator, drivers):
- for record in review_iterator:
- for driver_info in _find_ci_result(record, drivers):
- yield driver_info
diff --git a/stackalytics/processor/dump.py b/stackalytics/processor/dump.py
index 13e1bed99..3b1a38918 100644
--- a/stackalytics/processor/dump.py
+++ b/stackalytics/processor/dump.py
@@ -96,7 +96,6 @@ def get_repo_keys(memcached_inst):
for branch in branches:
yield 'vcs:%s:%s' % (quoted_uri, branch)
yield 'rcs:%s:%s' % (quoted_uri, branch)
- yield 'ci:%s:%s' % (quoted_uri, branch)
def export_data(memcached_inst, fd):
diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py
index da2a2de28..077e422a0 100644
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -24,7 +24,6 @@ import six
from stackalytics.processor import bps
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
-from stackalytics.processor import driverlog
from stackalytics.processor import governance
from stackalytics.processor import lp
from stackalytics.processor import mls
@@ -139,29 +138,6 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst,
runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)
-def _process_repo_ci_votes(repo, runtime_storage_inst, record_processor_inst,
- rcs_inst):
- for branch in _get_repo_branches(repo):
- LOG.info('Processing CI votes for repo: %s, branch: %s',
- repo['uri'], branch)
-
- quoted_uri = six.moves.urllib.parse.quote_plus(repo['uri'])
- rcs_key = 'ci:%s:%s' % (quoted_uri, branch)
- last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key)
- current_retrieval_time = utils.date_to_timestamp('now')
-
- review_iterator = rcs_inst.log(repo, branch, last_retrieval_time,
- status='merged', grab_comments=True)
- review_iterator = driverlog.log(review_iterator, repo['drivers'])
- review_iterator_typed = _record_typer(review_iterator, 'ci')
- processed_review_iterator = record_processor_inst.process(
- review_iterator_typed)
-
- runtime_storage_inst.set_records(processed_review_iterator,
- utils.merge_records)
- runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)
-
-
def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst):
vcs_inst = vcs.get_vcs(repo, CONF.sources_root)
vcs_inst.fetch()
@@ -199,10 +175,6 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
_process_repo_reviews(repo, runtime_storage_inst,
record_processor_inst, rcs_inst)
- if 'drivers' in repo:
- _process_repo_ci_votes(repo, runtime_storage_inst,
- record_processor_inst, rcs_inst)
-
def _process_mail_list(uri, runtime_storage_inst, record_processor_inst):
mail_iterator = mls.log(uri, runtime_storage_inst)
@@ -316,9 +288,6 @@ def process_project_list(runtime_storage_inst):
module = repo['module'].lower()
module_groups[module] = utils.make_module_group(module, tag='module')
- if 'drivers' in repo:
- module_groups[module]['has_drivers'] = True
-
# register module 'unknown' - used for emails not mapped to any module
module_groups['unknown'] = utils.make_module_group('unknown', tag='module')
@@ -344,8 +313,7 @@ def main():
return not 0
default_data_processor.process(runtime_storage_inst,
- default_data,
- CONF.driverlog_data_uri)
+ default_data)
process_project_list(runtime_storage_inst)
diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py
index 8246ff5eb..75d0e7db2 100644
--- a/stackalytics/processor/record_processor.py
+++ b/stackalytics/processor/record_processor.py
@@ -603,18 +603,6 @@ class RecordProcessor(object):
yield record
- def _process_ci(self, record):
- ci_vote = dict((k, v) for k, v in six.iteritems(record))
-
- ci_vote['primary_key'] = '%s:%s' % (record['review_id'],
- record['driver_name'])
- ci_vote['author_name'] = record['driver_name']
- ci_vote['author_email'] = record['user_id']
-
- self._update_record_and_user(ci_vote)
-
- yield ci_vote
-
def _process_translation(self, record):
# todo split translation and approval
translation = record.copy()
@@ -646,7 +634,6 @@ class RecordProcessor(object):
'bp': self._process_blueprint,
'bug': self._process_bug,
'member': self._process_member,
- 'ci': self._process_ci,
'i18n': self._process_translation,
}
diff --git a/stackalytics/processor/user_processor.py b/stackalytics/processor/user_processor.py
index 65e1c4165..634beef76 100644
--- a/stackalytics/processor/user_processor.py
+++ b/stackalytics/processor/user_processor.py
@@ -14,7 +14,6 @@
# limitations under the License.
import copy
-import re
from oslo_log import log as logging
@@ -22,7 +21,7 @@ LOG = logging.getLogger(__name__)
def make_user_id(emails=None, launchpad_id=None, gerrit_id=None,
- member_id=None, github_id=None, ci_id=None, zanata_id=None):
+ member_id=None, github_id=None, zanata_id=None):
if launchpad_id or emails:
return launchpad_id or emails[0]
if gerrit_id:
@@ -33,8 +32,6 @@ def make_user_id(emails=None, launchpad_id=None, gerrit_id=None,
return 'github:%s' % github_id
if zanata_id:
return 'zanata:%s' % zanata_id
- if ci_id:
- return 'ci:%s' % re.sub(r'[^\w]', '_', ci_id.lower())
return None
diff --git a/stackalytics/tests/unit/test_default_data_processor.py b/stackalytics/tests/unit/test_default_data_processor.py
index 7b8d3baba..498efe9a0 100644
--- a/stackalytics/tests/unit/test_default_data_processor.py
+++ b/stackalytics/tests/unit/test_default_data_processor.py
@@ -137,61 +137,3 @@ class TestDefaultDataProcessor(testtools.TestCase):
'module_group_name': 'kubernetes',
'modules': ['kubernetes'],
'tag': 'organization'}, dd['module_groups'])
-
- @mock.patch('stackalytics.processor.utils.read_json_from_uri')
- def test_update_with_driverlog(self, mock_read_from_json):
- default_data = {'repos': [{'module': 'cinder', }], 'users': []}
- driverlog_dd = {'drivers': [{
- 'project_id': 'openstack/cinder',
- 'vendor': 'VMware',
- 'name': 'VMware VMDK Driver',
- 'ci': {
- 'id': 'vmwareminesweeper',
- 'success_pattern': 'Build successful',
- 'failure_pattern': 'Build failed'
- }
- }]}
- mock_read_from_json.return_value = driverlog_dd
-
- default_data_processor._update_with_driverlog_data(default_data, 'uri')
-
- expected_user = {
- 'user_id': 'ci:vmware_vmdk_driver',
- 'user_name': 'VMware VMDK Driver',
- 'static': True,
- 'companies': [
- {'company_name': 'VMware', 'end_date': None}],
- }
- self.assertIn(expected_user, default_data['users'])
- self.assertIn(driverlog_dd['drivers'][0],
- default_data['repos'][0]['drivers'])
-
- @mock.patch('stackalytics.processor.utils.read_json_from_uri')
- def test_update_with_driverlog_specific_repo(self, mock_read_from_json):
- default_data = {'repos': [{'module': 'fuel-plugin-mellanox', }],
- 'users': []}
- driverlog_dd = {'drivers': [{
- 'project_id': 'openstack/fuel',
- 'repo': 'stackforge/fuel-plugin-mellanox',
- 'vendor': 'Mellanox',
- 'name': 'ConnectX-3 Pro Network Adapter Support plugin',
- 'ci': {
- 'id': 'mellanox',
- 'success_pattern': 'SUCCESS',
- 'failure_pattern': 'FAILURE'
- }
- }]}
- mock_read_from_json.return_value = driverlog_dd
-
- default_data_processor._update_with_driverlog_data(default_data, 'uri')
-
- expected_user = {
- 'user_id': 'ci:connectx_3_pro_network_adapter_support_plugin',
- 'user_name': 'ConnectX-3 Pro Network Adapter Support plugin',
- 'static': True,
- 'companies': [
- {'company_name': 'Mellanox', 'end_date': None}],
- }
- self.assertIn(expected_user, default_data['users'])
- self.assertIn(driverlog_dd['drivers'][0],
- default_data['repos'][0]['drivers'])
diff --git a/stackalytics/tests/unit/test_driverlog.py b/stackalytics/tests/unit/test_driverlog.py
deleted file mode 100644
index e626b8004..000000000
--- a/stackalytics/tests/unit/test_driverlog.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# Copyright (c) 2013 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import copy
-
-import testtools
-
-from stackalytics.processor import driverlog
-
-COMMENT_SUCCESS = {
- 'message': 'Patch Set 2: build successful',
- 'reviewer': {'username': 'virt-ci'},
- 'timestamp': 1234567890
-}
-
-COMMENT_FAILURE = {
- 'message': 'Patch Set 2: build failed',
- 'reviewer': {'username': 'virt-ci'},
- 'timestamp': 1234567880
-}
-
-REVIEW = {
- 'record_type': 'review',
- 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
- 'module': 'nova',
- 'branch': 'master',
- 'status': 'MERGED',
- 'number': '97860',
- 'patchSets': [{'number': '1'}, {'number': '2'}],
- 'comments': [
- {'message': 'Patch Set 2: build successful',
- 'reviewer': {'username': 'other-ci'}, },
- {'message': 'Patch Set 2: job started',
- 'reviewer': {'username': 'virt-ci'}, }]
-}
-
-DRIVER = {
- 'name': 'Virt Nova Driver',
- 'vendor': 'Virt Inc',
- 'ci': {
- 'id': 'virt-ci',
- 'success_pattern': 'successful',
- 'failure_pattern': 'failed',
- }
-}
-
-DRIVER_NON_EXISTENT = {
- 'name': 'No Virt Nova Driver',
- 'vendor': 'No Virt Inc',
- 'ci': {
- 'id': 'no-virt-ci',
- 'success_pattern': 'successful',
- 'failure_pattern': 'failed',
- }
-}
-
-
-class TestDriverlog(testtools.TestCase):
-
- def test_find_ci_result_success(self):
- drivers = [DRIVER]
- review = copy.deepcopy(REVIEW)
- review['comments'].append(COMMENT_SUCCESS)
-
- res = list(driverlog.log([review], drivers))
-
- expected_result = {
- 'user_id': 'ci:virt_nova_driver',
- 'value': True,
- 'message': 'build successful',
- 'date': 1234567890,
- 'branch': 'master',
- 'review_id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
- 'review_number': '97860',
- 'driver_name': 'Virt Nova Driver',
- 'driver_vendor': 'Virt Inc',
- 'module': 'nova',
- }
- self.assertEqual(1, len(res), 'One CI result is expected')
- self.assertEqual(expected_result, res[0])
-
- def test_find_ci_result_failure(self):
- drivers = [DRIVER]
- review = copy.deepcopy(REVIEW)
- review['comments'].append(COMMENT_FAILURE)
-
- res = list(driverlog.log([review], drivers))
-
- self.assertEqual(1, len(res), 'One CI result is expected')
- self.assertFalse(res[0]['value'])
-
- def test_find_ci_result_non_existent(self):
- drivers = [DRIVER_NON_EXISTENT]
- review = copy.deepcopy(REVIEW)
- review['comments'].append(COMMENT_SUCCESS)
-
- res = list(driverlog.log([REVIEW], drivers))
-
- self.assertEqual(0, len(res), 'No CI results expected')
-
- def test_find_ci_result_last_vote_only(self):
- # there may be multiple comments from the same CI,
- # only the last one is important
- drivers = [DRIVER]
-
- review = copy.deepcopy(REVIEW)
- review['comments'].append(COMMENT_FAILURE)
- review['comments'].append(COMMENT_SUCCESS)
-
- res = list(driverlog.log([review], drivers))
-
- self.assertEqual(1, len(res), 'One CI result is expected')
- self.assertTrue(res[0]['value'])