opendev/elastic-recheck
Code Issues Proposed changes

Merge "Made elastic-recheck py3 compatible"

Browse Source
This commit is contained in:
Zuul 2018-11-30 00:13:23 +00:00 committed by Gerrit Code Review
commit f8c8cc1cda
17 changed files with 78 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.zuul.yaml
View File

@ -1,6 +1,8 @@
--- ---
- project: - project:
templates: templates:
- openstack-python36-jobs
- openstack-python37-jobs
- publish-tox-docs-infra - publish-tox-docs-infra
check: check:
jobs: jobs:

4
elastic_recheck/bot.py
View File

@ -250,13 +250,13 @@ class ChannelConfig(object):
data['#' + key] = data.pop(key) data['#' + key] = data.pop(key)
self.channels = data.keys() self.channels = data.keys()
self.events = {} self.events = {}
for channel, val in self.data.iteritems(): for channel, val in self.data.items():
for event in val['events']: for event in val['events']:
event_set = self.events.get(event, set()) event_set = self.events.get(event, set())
event_set.add(channel) event_set.add(channel)
self.events[event] = event_set self.events[event] = event_set
self.projects = {} self.projects = {}
for channel, val in self.data.iteritems(): for channel, val in self.data.items():
for project in val['projects']: for project in val['projects']:
project_set = self.projects.get(project, set()) project_set = self.projects.get(project, set())
project_set.add(channel) project_set.add(channel)

26
elastic_recheck/cmd/check_success.py
View File

@ -98,12 +98,12 @@ def classifying_rate(fails, data):
print("Classification percentage: %2.2f%%" % print("Classification percentage: %2.2f%%" %
((float(count) / float(total)) * 100.0)) ((float(count) / float(total)) * 100.0))
sort = sorted( sort = sorted(
bad_jobs.iteritems(), bad_jobs.items(),
key=operator.itemgetter(1), key=operator.itemgetter(1),
reverse=True) reverse=True)
print("Job fails with most unclassified errors") print("Job fails with most unclassified errors")
for s in sort: for s in sort:
print " %3s : %s" % (s[1], s[0]) print(" %3s : %s" % (s[1], s[0]))
def _status_count(results): def _status_count(results):
@ -183,10 +183,10 @@ def collect_metrics(classifier, fails):
def print_metrics(data, with_lp=False): def print_metrics(data, with_lp=False):
print "Elastic recheck known issues" print("Elastic recheck known issues")
print print()
sorted_data = sorted(data.iteritems(), sorted_data = sorted(data.items(),
key=lambda x: -x[1]['fails']) key=lambda x: -x[1]['fails'])
for d in sorted_data: for d in sorted_data:
bug = d[0] bug = d[0]
@ -195,13 +195,13 @@ def print_metrics(data, with_lp=False):
% (bug, data['query'].rstrip())) % (bug, data['query'].rstrip()))
if with_lp: if with_lp:
get_launchpad_bug(d[0]) get_launchpad_bug(d[0])
print "Hits" print("Hits")
for s in data['hits']: for s in data['hits']:
print " %s: %s" % (s, data['hits'][s]) print(" %s: %s" % (s, data['hits'][s]))
print "Percentage of Gate Queue Job failures triggered by this bug" print("Percentage of Gate Queue Job failures triggered by this bug")
for s in data['percentages']: for s in data['percentages']:
print " %s: %2.2f%%" % (s, data['percentages'][s]) print(" %s: %2.2f%%" % (s, data['percentages'][s]))
print print()
def get_launchpad_bug(bug): def get_launchpad_bug(bug):
@ -209,11 +209,11 @@ def get_launchpad_bug(bug):
'production', 'production',
LPCACHEDIR) LPCACHEDIR)
lp_bug = lp.bugs[bug] lp_bug = lp.bugs[bug]
print "Title: %s" % lp_bug.title print("Title: %s" % lp_bug.title)
targets = map(lambda x: (x.bug_target_name, x.status), lp_bug.bug_tasks) targets = map(lambda x: (x.bug_target_name, x.status), lp_bug.bug_tasks)
print "Project: Status" print("Project: Status")
for target, status in targets: for target, status in targets:
print " %s: %s" % (target, status) print(" %s: %s" % (target, status))
def main(): def main():

10
elastic_recheck/cmd/query.py
View File

@ -42,7 +42,7 @@ IGNORED_ATTRIBUTES = [
def analyze_attributes(attributes): def analyze_attributes(attributes):
analysis = {} analysis = {}
for attribute, values in attributes.iteritems(): for attribute, values in attributes.items():
if attribute[0] == '@' or attribute == 'message': if attribute[0] == '@' or attribute == 'message':
# skip meta attributes and raw messages # skip meta attributes and raw messages
continue continue
@ -50,7 +50,7 @@ def analyze_attributes(attributes):
analysis[attribute] = [] analysis[attribute] = []
total_hits = sum(values.values()) total_hits = sum(values.values())
for value_hash, hits in values.iteritems(): for value_hash, hits in values.items():
value = json.loads(value_hash) value = json.loads(value_hash)
analysis[attribute].append((100.0 * hits / total_hits, value)) analysis[attribute].append((100.0 * hits / total_hits, value))
@ -78,13 +78,13 @@ def query(query_file_name, config=None, days=DEFAULT_NUMBER_OF_DAYS,
attributes = {} attributes = {}
for hit in r.hits['hits']: for hit in r.hits['hits']:
for key, value in hit['_source'].iteritems(): for key, value in hit['_source'].items():
value_hash = json.dumps(value) value_hash = json.dumps(value)
attributes.setdefault(key, {}).setdefault(value_hash, 0) attributes.setdefault(key, {}).setdefault(value_hash, 0)
attributes[key][value_hash] += 1 attributes[key][value_hash] += 1
analysis = analyze_attributes(attributes) analysis = analyze_attributes(attributes)
for attribute, results in sorted(analysis.iteritems()): for attribute, results in sorted(analysis.items()):
if not verbose and attribute in IGNORED_ATTRIBUTES: if not verbose and attribute in IGNORED_ATTRIBUTES:
# skip less-than-useful attributes to reduce noise in the report # skip less-than-useful attributes to reduce noise in the report
continue continue
@ -92,7 +92,7 @@ def query(query_file_name, config=None, days=DEFAULT_NUMBER_OF_DAYS,
print(attribute) print(attribute)
for percentage, value in itertools.islice(results, quantity): for percentage, value in itertools.islice(results, quantity):
if isinstance(value, list): if isinstance(value, list):
value = ' '.join(unicode(x) for x in value) value = ' '.join(str(x) for x in value)
print(' %d%% %s' % (percentage, value)) print(' %d%% %s' % (percentage, value))

4
elastic_recheck/cmd/uncategorized_fails.py
View File

@ -176,7 +176,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url):
classification rate. For every failure in the gate queue did classification rate. For every failure in the gate queue did
we find a match for it. we find a match for it.
""" """
found_fails = {k: False for (k, v) in fails.iteritems()} found_fails = {k: False for (k, v) in fails.items()}
for bugnum in data: for bugnum in data:
bug = data[bugnum] bug = data[bugnum]
@ -237,7 +237,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url):
(float(bad_jobs[job]) / float(total_job_failures[job])) (float(bad_jobs[job]) / float(total_job_failures[job]))
* 100.0) * 100.0)
sort = sorted( sort = sorted(
bad_jobs.iteritems(), bad_jobs.items(),
key=operator.itemgetter(1), key=operator.itemgetter(1),
reverse=True) reverse=True)

18
elastic_recheck/config.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import ConfigParser from six.moves import configparser
import os import os
import re import re
@ -26,6 +26,7 @@ JOBS_RE = 'dsvm'
CI_USERNAME = 'jenkins' CI_USERNAME = 'jenkins'
GERRIT_QUERY_FILE = 'queries' GERRIT_QUERY_FILE = 'queries'
GERRIT_HOST = 'review.openstack.org'
PID_FN = '/var/run/elastic-recheck/elastic-recheck.pid' PID_FN = '/var/run/elastic-recheck/elastic-recheck.pid'
@ -101,7 +102,7 @@ class Config(object):
if config_obj: if config_obj:
config = config_obj config = config_obj
else: else:
config = ConfigParser.ConfigParser( config = configparser.ConfigParser(
{'es_url': ES_URL, {'es_url': ES_URL,
'ls_url': LS_URL, 'ls_url': LS_URL,
'db_uri': DB_URI, 'db_uri': DB_URI,
@ -110,7 +111,7 @@ class Config(object):
'jobs_re': JOBS_RE, 'jobs_re': JOBS_RE,
'pidfile': PID_FN, 'pidfile': PID_FN,
'index_format': DEFAULT_INDEX_FORMAT, 'index_format': DEFAULT_INDEX_FORMAT,
'query_file': GERRIT_QUERY_FILE, 'query_file': GERRIT_QUERY_FILE
} }
) )
config.read(config_file) config.read(config_file)
@ -129,8 +130,15 @@ class Config(object):
if config.has_section('gerrit'): if config.has_section('gerrit'):
self.gerrit_user = config.get('gerrit', 'user') self.gerrit_user = config.get('gerrit', 'user')
self.gerrit_query_file = config.get('gerrit', 'query_file') self.gerrit_query_file = config.get('gerrit', 'query_file')
self.gerrit_host = config.get('gerrit', 'host', # workaround for python api change https://docs.python.org/3/library/configparser.html#fallback-values
'review.openstack.org') try:
self.gerrit_host = config.get('gerrit',
'host',
fallback=GERRIT_HOST)
except TypeError:
self.gerrit_host = config.get('gerrit',
'host',
GERRIT_HOST)
self.gerrit_host_key = config.get('gerrit', 'key') self.gerrit_host_key = config.get('gerrit', 'key')
if config.has_section('ircbot'): if config.has_section('ircbot'):

2
elastic_recheck/elasticRecheck.py
View File

@ -90,7 +90,7 @@ class FailJob(object):
self.url = url self.url = url
# The last set of characters of the URL are the first 7 digits # The last set of characters of the URL are the first 7 digits
# of the build_uuid. # of the build_uuid.
self.build_short_uuid = filter(None, url.split('/'))[-1] self.build_short_uuid = list(filter(None, url.split('/')))[-1]
def __repr__(self): def __repr__(self):
return self.name return self.name

4
elastic_recheck/tests/functional/test_gerrit_comment.py
View File

@ -12,8 +12,8 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import ConfigParser
import gerritlib import gerritlib
from six.moves import configparser
import testtools import testtools
from elastic_recheck import elasticRecheck from elastic_recheck import elasticRecheck
@ -23,7 +23,7 @@ class TestGerritComment(testtools.TestCase):
def setUp(self): def setUp(self):
super(TestGerritComment, self).setUp() super(TestGerritComment, self).setUp()
config = ConfigParser.ConfigParser({'server_password': None}) config = configparser.ConfigParser({'server_password': None})
config.read('elasticRecheck.conf') config.read('elasticRecheck.conf')
self.user = config.get('gerrit', 'user') self.user = config.get('gerrit', 'user')
key = config.get('gerrit', 'key') key = config.get('gerrit', 'key')

2
elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py
View File

@ -58,4 +58,4 @@ class TestUncategorizedFails(testtools.TestCase):
self.assertThat(all_fails['integrated_gate'], self.assertThat(all_fails['integrated_gate'],
testtools.matchers.HasLength(1)) testtools.matchers.HasLength(1))
self.assertIn('gate-tempest-dsvm-full', self.assertIn('gate-tempest-dsvm-full',
all_fails['integrated_gate'].keys()[0]) list(all_fails['integrated_gate'].keys())[0])

10
elastic_recheck/tests/unit/test_bot.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import ConfigParser from six.moves import configparser
import unittest import unittest
import yaml import yaml
@ -49,7 +49,9 @@ def _set_fake_config(fake_config):
class TestBot(unittest.TestCase): class TestBot(unittest.TestCase):
def setUp(self): def setUp(self):
super(TestBot, self).setUp() super(TestBot, self).setUp()
self.fake_config = ConfigParser.ConfigParser({'server_password': None}) self.fake_config = configparser.ConfigParser(
{'server_password': None},
allow_no_value=True)
_set_fake_config(self.fake_config) _set_fake_config(self.fake_config)
config = er_conf.Config(config_obj=self.fake_config) config = er_conf.Config(config_obj=self.fake_config)
self.channel_config = bot.ChannelConfig(yaml.load( self.channel_config = bot.ChannelConfig(yaml.load(
@ -93,7 +95,9 @@ class TestBotWithTestTools(tests.TestCase):
self.useFixture(fixtures.MonkeyPatch( self.useFixture(fixtures.MonkeyPatch(
'gerritlib.gerrit.Gerrit', 'gerritlib.gerrit.Gerrit',
fg.Gerrit)) fg.Gerrit))
self.fake_config = ConfigParser.ConfigParser({'server_password': None}) self.fake_config = configparser.ConfigParser(
{'server_password': None},
allow_no_value=True)
_set_fake_config(self.fake_config) _set_fake_config(self.fake_config)
config = er_conf.Config(config_obj=self.fake_config) config = er_conf.Config(config_obj=self.fake_config)
self.channel_config = bot.ChannelConfig(yaml.load( self.channel_config = bot.ChannelConfig(yaml.load(

4
elastic_recheck/tests/unit/test_query.py
View File

@ -11,7 +11,7 @@
# under the License. # under the License.
import json import json
import StringIO from six import StringIO
import sys import sys
import mock import mock
@ -25,7 +25,7 @@ class TestQueryCmd(unit.UnitTestCase):
def setUp(self): def setUp(self):
super(TestQueryCmd, self).setUp() super(TestQueryCmd, self).setUp()
self._stdout = sys.stdout self._stdout = sys.stdout
sys.stdout = StringIO.StringIO() sys.stdout = StringIO()
def tearDown(self): def tearDown(self):
super(TestQueryCmd, self).tearDown() super(TestQueryCmd, self).tearDown()

14
elastic_recheck/tests/unit/test_results.py
View File

@ -59,7 +59,7 @@ class TestBasicParsing(tests.TestCase):
facets = results.FacetSet() facets = results.FacetSet()
facets.detect_facets(result_set, ["build_status"]) facets.detect_facets(result_set, ["build_status"])
self.assertEqual(facets.keys(), ['FAILURE']) self.assertEqual(list(facets.keys()), ['FAILURE'])
data = load_sample(1226337) data = load_sample(1226337)
result_set = results.ResultSet(data) result_set = results.ResultSet(data)
@ -87,18 +87,15 @@ class TestBasicParsing(tests.TestCase):
facets.detect_facets(result_set, facets.detect_facets(result_set,
["timestamp", "build_status", "build_uuid"]) ["timestamp", "build_status", "build_uuid"])
self.assertEqual(len(facets.keys()), 14) self.assertEqual(len(facets.keys()), 14)
print facets[1382104800000].keys() print(facets[1382104800000].keys())
self.assertEqual(facets[1382104800000].keys(), ["FAILURE"]) self.assertEqual(list(facets[1382104800000].keys()), ["FAILURE"])
self.assertEqual(len(facets[1382104800000]["FAILURE"]), 2) self.assertEqual(len(facets[1382104800000]["FAILURE"]), 2)
self.assertEqual(facets[1382101200000].keys(), ["FAILURE"]) self.assertEqual(list(facets[1382101200000].keys()), ["FAILURE"])
# NOTE(mriedem): We can't mock built-ins so we have to override utcnow(). # NOTE(mriedem): We can't mock built-ins so we have to override utcnow().
class MockDatetimeToday(datetime.datetime): class MockDatetimeToday(datetime.datetime):
def __init__(self, *args):
super(MockDatetimeToday, self).__init__(*args)
@classmethod @classmethod
def utcnow(cls): def utcnow(cls):
# One hour and one second into today. # One hour and one second into today.
@ -108,9 +105,6 @@ class MockDatetimeToday(datetime.datetime):
class MockDatetimeYesterday(datetime.datetime): class MockDatetimeYesterday(datetime.datetime):
def __init__(self, *args):
super(MockDatetimeYesterday, self).__init__(*args)
@classmethod @classmethod
def utcnow(cls): def utcnow(cls):
# 59 minutes and 59 seconds into today. # 59 minutes and 59 seconds into today.

5
requirements.txt
View File

@ -9,8 +9,9 @@ irc>=15.0.1
pyyaml pyyaml
lockfile lockfile
Babel>=0.9.6 Babel>=0.9.6
httplib2<0.12.0 # required by launchpadlib, see https://bugs.launchpad.net/lazr.restfulclient/+bug/1803402 lazr.restfulclient>=0.14.2 # LGPL
launchpadlib httplib2>=0.12.0 # MIT License
launchpadlib>=1.10.6 # LGPL
Jinja2 Jinja2
requests requests
subunit2sql>=0.9.0 subunit2sql>=0.9.0

3
setup.cfg
View File

@ -16,7 +16,8 @@ classifier =
Programming Language :: Python :: 2 Programming Language :: Python :: 2
Programming Language :: Python :: 2.7 Programming Language :: Python :: 2.7
Programming Language :: Python :: 3 Programming Language :: Python :: 3
Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
[files] [files]
packages = packages =

22
tools/unaccounted_rechecks.py
View File

@ -101,12 +101,12 @@ def cross_ref_with_er(changes, dirname):
def summarize_changes(changes): def summarize_changes(changes):
no_er = {} no_er = {}
print "Summary" print("Summary")
print "%4.4s - Total Rechecks" % (len(changes)) print("%4.4s - Total Rechecks" % (len(changes)))
print "%4.4s - Total w/Bug" % ( print("%4.4s - Total w/Bug" % (
len([c for c in changes if c['bug'] != 'no bug'])) len([c for c in changes if c['bug'] != 'no bug'])))
print "%4.4s - Total w/Bug and new recheck" % ( print("%4.4s - Total w/Bug and new recheck" % (
len([c for c in changes if (c['bug'] != 'no bug' and not c['er'])])) len([c for c in changes if (c['bug'] != 'no bug' and not c['er'])])))
for c in changes: for c in changes:
bug = c['bug'] bug = c['bug']
@ -115,12 +115,12 @@ def summarize_changes(changes):
no_er[bug] = {'count': 0, 'reviews': []} no_er[bug] = {'count': 0, 'reviews': []}
no_er[bug]['count'] += 1 no_er[bug]['count'] += 1
no_er[bug]['reviews'].append(c['review']) no_er[bug]['reviews'].append(c['review'])
print print()
print "New bugs" print("New bugs")
for k, v in no_er.iteritems(): for k, v in no_er.items():
print "Bug %s found %d times" % (k, v['count']) print("Bug %s found %d times" % (k, v['count']))
for rev in v['reviews']: for rev in v['reviews']:
print " - %s" % rev print(" - %s" % rev)
def main(): def main():

2
tox.ini
View File

@ -1,6 +1,6 @@
[tox] [tox]
minversion = 1.6 minversion = 1.6
envlist = py27,pep8,queries,docs envlist = pep8,py37,py36,py27,queries,docs
skipsdist = True skipsdist = True
[testenv] [testenv]

14
web_server.py
View File

@ -40,7 +40,7 @@ class ERHandler(BaseHTTPServer.BaseHTTPRequestHandler):
# if the file exists locally, we'll serve it up directly # if the file exists locally, we'll serve it up directly
fname = "web/share" + self.path fname = "web/share" + self.path
if os.path.isfile(fname): if os.path.isfile(fname):
print "found local file %s" % (fname) print("found local file %s" % (fname))
self.send_response(200, "Success") self.send_response(200, "Success")
self.end_headers() self.end_headers()
with open(fname) as f: with open(fname) as f:
@ -73,7 +73,7 @@ class ERHandler(BaseHTTPServer.BaseHTTPRequestHandler):
return return
# Fall through for paths we don't understand # Fall through for paths we don't understand
print "Unknown path requested: %s" % self.path print("Unknown path requested: %s" % self.path)
def parse_opts(): def parse_opts():
@ -90,9 +90,9 @@ def main():
server_address = ('', opts.port) server_address = ('', opts.port)
httpd = BaseHTTPServer.HTTPServer(server_address, ERHandler) httpd = BaseHTTPServer.HTTPServer(server_address, ERHandler)
print "Test Server is running at http://localhost:%s" % opts.port print("Test Server is running at http://localhost:%s" % opts.port)
print "Ctrl-C to exit" print("Ctrl-C to exit")
print print()
while True: while True:
httpd.handle_request() httpd.handle_request()
@ -101,5 +101,5 @@ if __name__ == '__main__':
try: try:
main() main()
except KeyboardInterrupt: except KeyboardInterrupt:
print "\n" print("\n")
print "Thanks for testing! Please come again." print("Thanks for testing! Please come again.")