opendev/elastic-recheck
Code Issues Proposed changes

Make Elastic Recheck Watch more reusable

Browse Source 
Refactor to use a config class to hold all the
params needed so that they can be more easily
overridden and reused across all the
elastic-recheck tools.

In addition, use the new class to make the
jobs_regex and ci_username configurable.

Change-Id: Ic6f115a6882494bf4c087ded4d7cafa557765c28
This commit is contained in:
Ramy Asselin 2015-11-30 10:34:18 -08:00
parent 35e31cf56b
commit 49999256f4
10 changed files with 187 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
elasticRecheck.conf.sample
View File

@ -6,6 +6,12 @@ server=irc.freenode.net
port=6667
channel_config=/home/mtreinish/elasticRecheck/recheckwatchbot.yaml
[recheckwatch]
#Any project that has a job that matches this regex will have all their
#jobs included in the recheck algorithm
jobs_regex=(tempest-dsvm-full|gate-tempest-dsvm-virtual-ironic)
ci_username=jenkins
[gerrit]
user=treinish
host=review.openstack.org

66
elastic_recheck/bot.py
View File

@ -42,7 +42,6 @@ openstack-qa:
"""
import argparse
import ConfigParser
import daemon
import os
import textwrap
@ -53,6 +52,7 @@ import yaml
import irc.bot
from launchpadlib import launchpad
import elastic_recheck.config as er_conf
from elastic_recheck import log as logging
LPCACHEDIR = os.path.expanduser('~/.launchpadlib/cache')
@ -73,13 +73,16 @@ class ElasticRecheckException(Exception):
class RecheckWatchBot(irc.bot.SingleServerIRCBot):
def __init__(self, channels, nickname, password, server, port=6667,
server_password=None):
def __init__(self, channels, config):
super(RecheckWatchBot, self).__init__(
[(server, port, server_password)], nickname, nickname)
[(config.ircbot_server,
config.ircbot_port,
config.ircbot_server_password)],
config.ircbot_nick,
config.ircbot_nick)
self.channel_list = channels
self.nickname = nickname
self.password = password
self.nickname = config.ircbot_nick
self.password = config.ircbot_pass
self.log = logging.getLogger('recheckwatchbot')
def on_nicknameinuse(self, c, e):
@ -111,26 +114,24 @@ class RecheckWatchBot(irc.bot.SingleServerIRCBot):
class RecheckWatch(threading.Thread):
def __init__(self, ircbot, channel_config, msgs, username,
queries, host, key, commenting=True, es_url=None,
db_uri=None):
def __init__(self, ircbot, channel_config, msgs, config=None,
commenting=True):
super(RecheckWatch, self).__init__()
self.config = config or er_conf.Config()
self.ircbot = ircbot
self.channel_config = channel_config
self.msgs = msgs
self.log = logging.getLogger('recheckwatchbot')
self.username = username
self.queries = queries
self.host = host
self.username = config.gerrit_user
self.queries = config.gerrit_query_file
self.host = config.gerrit_host
self.connected = False
self.commenting = commenting
self.key = key
self.key = config.gerrit_host_key
self.lp = launchpad.Launchpad.login_anonymously('grabbing bugs',
'production',
LPCACHEDIR,
timeout=60)
self.es_url = es_url
self.db_uri = db_uri
def display(self, channel, event):
display = False
@ -200,10 +201,9 @@ class RecheckWatch(threading.Thread):
def run(self):
# Import here because it needs to happen after daemonization
import elastic_recheck.elasticRecheck as er
classifier = er.Classifier(self.queries, es_url=self.es_url,
db_uri=self.db_uri)
classifier = er.Classifier(self.queries, config=self.config)
stream = er.Stream(self.username, self.host, self.key,
es_url=self.es_url)
config=self.config)
while True:
try:
event = stream.get_failed_tempest()
@ -285,7 +285,7 @@ def get_options():
def _main(args, config):
logging.setup_logging(config)
fp = config.get('ircbot', 'channel_config')
fp = config.ircbot_channel_config
if fp:
fp = os.path.expanduser(fp)
if not os.path.exists(fp):
@ -301,11 +301,7 @@ def _main(args, config):
if not args.noirc:
bot = RecheckWatchBot(
channel_config.channels,
config.get('ircbot', 'nick'),
config.get('ircbot', 'pass'),
config.get('ircbot', 'server'),
config.getint('ircbot', 'port'),
config.get('ircbot', 'server_password'))
config=config)
else:
bot = None
@ -313,16 +309,8 @@ def _main(args, config):
bot,
channel_config,
msgs,
config.get('gerrit', 'user'),
config.get('gerrit', 'query_file'),
config.get('gerrit', 'host', 'review.openstack.org'),
config.get('gerrit', 'key'),
not args.nocomment,
config.get('data_source', 'es_url',
'http://logstash.openstack.org:80/elasticsearch'),
config.get('data_source', 'db_uri',
'mysql+pymysql://query:query@logstash.openstack.org/'
'subunit2sql'),
config=config,
commenting=not args.nocomment,
)
recheck.start()
@ -333,18 +321,12 @@ def _main(args, config):
def main():
args = get_options()
config = ConfigParser.ConfigParser({'server_password': None})
config.read(args.conffile)
if config.has_option('ircbot', 'pidfile'):
pid_fn = os.path.expanduser(config.get('ircbot', 'pidfile'))
else:
pid_fn = '/var/run/elastic-recheck/elastic-recheck.pid'
config = er_conf.Config(config_file=args.conffile)
if args.foreground:
_main(args, config)
else:
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
pid = pid_file_module.TimeoutPIDLockFile(config.pid_fn, 10)
with daemon.DaemonContext(pidfile=pid):
_main(args, config)

24
elastic_recheck/cmd/graph.py
View File

@ -15,7 +15,6 @@
# under the License.
import argparse
import ConfigParser
from datetime import datetime
import json
import os
@ -38,6 +37,8 @@ except ImportError:
from urllib3.exceptions import InsecurePlatformWarning
urllib3.disable_warnings(InsecurePlatformWarning)
import elastic_recheck.config as er_conf
import elastic_recheck.elasticRecheck as er
from elastic_recheck import log as logging
import elastic_recheck.query_builder as qb
@ -111,22 +112,9 @@ def main():
help='print out details as we go')
args = parser.parse_args()
# Start with defaults
es_url = er.ES_URL
ls_url = er.LS_URL
db_uri = er.DB_URI
config = er_conf.Config(config_file=args.conf)
if args.conf:
config = ConfigParser.ConfigParser({'es_url': er.ES_URL,
'ls_url': er.LS_URL,
'db_uri': er.DB_URI})
config.read(args.conf)
if config.has_section('data_source'):
es_url = config.get('data_source', 'es_url')
ls_url = config.get('data_source', 'ls_url')
db_uri = config.get('data_source', 'db_uri')
classifier = er.Classifier(args.queries, es_url=es_url, db_uri=db_uri)
classifier = er.Classifier(args.queries, config=config)
buglist = []
@ -160,7 +148,7 @@ def main():
}
# Get the cluster health for the header
es = pyelasticsearch.ElasticSearch(es_url)
es = pyelasticsearch.ElasticSearch(config.es_url)
jsondata['status'] = es.health()['status']
for query in classifier.queries:
@ -174,7 +162,7 @@ def main():
logstash_query = qb.encode_logstash_query(query['query'],
timeframe=timeframe)
logstash_url = ("%s/#/dashboard/file/logstash.json?%s"
% (ls_url, logstash_query))
% (config.ls_url, logstash_query))
bug_data = get_launchpad_bug(query['bug'])
bug = dict(number=query['bug'],
query=query['query'],

31
elastic_recheck/cmd/query.py
View File

@ -15,18 +15,16 @@
# under the License.
import argparse
import ConfigParser
import itertools
import json
import yaml
import elastic_recheck.elasticRecheck as er
import elastic_recheck.config as er_conf
import elastic_recheck.log as logging
import elastic_recheck.results as er_results
LOG = logging.getLogger('erquery')
DEFAULT_INDEX_FORMAT = 'logstash-%Y.%m.%d'
DEFAULT_NUMBER_OF_DAYS = 10
DEFAULT_MAX_QUANTITY = 5
IGNORED_ATTRIBUTES = [
@ -64,11 +62,11 @@ def analyze_attributes(attributes):
return analysis
def query(query_file_name, days=DEFAULT_NUMBER_OF_DAYS, es_url=er.ES_URL,
quantity=DEFAULT_MAX_QUANTITY, verbose=False,
indexfmt=DEFAULT_INDEX_FORMAT):
es = er_results.SearchEngine(url=es_url, indexfmt=indexfmt)
def query(query_file_name, config=None, days=DEFAULT_NUMBER_OF_DAYS,
quantity=DEFAULT_MAX_QUANTITY, verbose=False,):
_config = config or er_conf.Config()
es = er_results.SearchEngine(url=_config.es_url,
indexfmt=_config.es_index_format)
with open(query_file_name) as f:
query_file = yaml.load(f.read())
@ -119,21 +117,10 @@ def main():
"elastic search url, logstash url, and database uri.")
args = parser.parse_args()
# Start with defaults
es_url = er.ES_URL
es_index_format = DEFAULT_INDEX_FORMAT
config = er_conf.Config(config_file=args.conf)
if args.conf:
config = ConfigParser.ConfigParser({
'es_url': er.ES_URL,
'index_format': DEFAULT_INDEX_FORMAT})
config.read(args.conf)
if config.has_section('data_source'):
es_url = config.get('data_source', 'es_url')
es_index_format = config.get('data_source', 'index_format')
query(args.query_file.name, days=args.days, quantity=args.quantity,
verbose=args.verbose, es_url=es_url, indexfmt=es_index_format)
query(args.query_file.name, config=config, days=args.days,
quantity=args.quantity, verbose=args.verbose)
if __name__ == "__main__":

20
elastic_recheck/cmd/uncategorized_fails.py
View File

@ -16,7 +16,6 @@
import argparse
import collections
import ConfigParser
import datetime
import logging
import operator
@ -27,6 +26,7 @@ import requests
import dateutil.parser as dp
import jinja2
import elastic_recheck.config as er_config
import elastic_recheck.elasticRecheck as er
import elastic_recheck.query_builder as qb
import elastic_recheck.results as er_results
@ -315,22 +315,10 @@ def collect_metrics(classifier, fails):
def main():
opts = get_options()
# Start with defaults
es_url = er.ES_URL
ls_url = er.LS_URL
db_uri = er.DB_URI
if opts.conf:
config = ConfigParser.ConfigParser({'es_url': er.ES_URL,
'ls_url': er.LS_URL,
'db_uri': er.DB_URI})
config.read(opts.conf)
if config.has_section('data_source'):
es_url = config.get('data_source', 'es_url')
ls_url = config.get('data_source', 'ls_url')
db_uri = config.get('data_source', 'db_uri')
config = er_config.Config(config_file=opts.conf)
classifier = er.Classifier(opts.dir, es_url=es_url, db_uri=db_uri)
classifier = er.Classifier(opts.dir, config=config)
all_gate_fails = all_fails(classifier)
for group in all_gate_fails:
fails = all_gate_fails[group]
@ -338,7 +326,7 @@ def main():
continue
data = collect_metrics(classifier, fails)
engine = setup_template_engine(opts.templatedir, group=group)
html = classifying_rate(fails, data, engine, classifier, ls_url)
html = classifying_rate(fails, data, engine, classifier, config.ls_url)
if opts.output:
out_dir = opts.output
else:

100
elastic_recheck/config.py Normal file
View File

@ -0,0 +1,100 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import os
DEFAULT_INDEX_FORMAT = 'logstash-%Y.%m.%d'
ES_URL = 'http://logstash.openstack.org:80/elasticsearch'
LS_URL = 'http://logstash.openstack.org'
DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
JOBS_RE = '(tempest-dsvm-full|gate-tempest-dsvm-virtual-ironic)'
CI_USERNAME = 'jenkins'
PID_FN = '/var/run/elastic-recheck/elastic-recheck.pid'
class Config(object):
def __init__(self,
config_file=None,
config_obj=None,
es_url=None,
ls_url=None,
db_uri=None,
jobs_re=None,
ci_username=None,
pid_fn=None,
es_index_format=None):
self.es_url = es_url or ES_URL
self.ls_url = ls_url or LS_URL
self.db_uri = db_uri or DB_URI
self.jobs_re = jobs_re or JOBS_RE
self.ci_username = ci_username or CI_USERNAME
self.es_index_format = es_index_format or DEFAULT_INDEX_FORMAT
self.pid_fn = pid_fn or PID_FN
self.ircbot_channel_config = None
self.irc_log_config = None
if config_file or config_obj:
if config_obj:
config = config_obj
else:
config = ConfigParser.ConfigParser(
{'es_url': ES_URL,
'ls_url': LS_URL,
'db_uri': DB_URI,
'server_password': None,
'ci_username': CI_USERNAME,
'jobs_regex': JOBS_RE,
'pidfile': PID_FN,
'index_format': DEFAULT_INDEX_FORMAT,
}
)
config.read(config_file)
if config.has_section('data_source'):
self.es_url = config.get('data_source', 'es_url')
self.ls_url = config.get('data_source', 'ls_url')
self.db_uri = config.get('data_source', 'db_uri')
self.es_index_format = config.get('data_source',
'index_format')
if config.has_section('recheckwatch'):
self.ci_username = config.get('recheckwatch', 'ci_username')
self.jobs_regex = config.get('recheckwatch', 'jobs_regex')
if config.has_section('gerrit'):
self.gerrit_user = config.get('gerrit', 'user')
self.gerrit_query_file = config.get('gerrit', 'query_file')
self.gerrit_host = config.get('gerrit', 'host',
'review.openstack.org')
self.gerrit_host_key = config.get('gerrit', 'key')
if config.has_section('ircbot'):
self.pid_fn = os.path.expanduser(config.get('ircbot',
'pidfile'))
self.ircbot_nick = config.get('ircbot', 'nick')
self.ircbot_pass = config.get('ircbot', 'pass')
self.ircbot_server = config.get('ircbot', 'server')
self.ircbot_port = config.getint('ircbot', 'port')
self.ircbot_server_password = config.get('ircbot',
'server_password')
self.ircbot_channel_config = config.get('ircbot',
'channel_config')
if config.has_option('ircbot', 'log_config'):
self.irc_log_config = config.get('ircbot', 'log_config')

40
elastic_recheck/elasticRecheck.py
View File

@ -24,14 +24,11 @@ import logging
import re
import time
import elastic_recheck.config as er_conf
import elastic_recheck.loader as loader
import elastic_recheck.query_builder as qb
from elastic_recheck import results
ES_URL = 'http://logstash.openstack.org:80/elasticsearch'
LS_URL = 'http://logstash.openstack.org'
DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
def required_files(job):
files = ['console.html']
@ -112,7 +109,7 @@ class FailEvent(object):
comment = None
failed_jobs = []
def __init__(self, event, failed_jobs):
def __init__(self, event, failed_jobs, config=None):
self.change = int(event['change']['number'])
self.rev = int(event['patchSet']['number'])
self.project = event['change']['project']
@ -120,10 +117,10 @@ class FailEvent(object):
self.comment = event["comment"]
# TODO(jogo): make FailEvent generate the jobs
self.failed_jobs = failed_jobs
self.config = config or er_conf.Config()
def is_openstack_project(self):
return ("tempest-dsvm-full" in self.comment or
"gate-tempest-dsvm-virtual-ironic" in self.comment)
def is_included_job(self):
return re.search(self.config.jobs_re, self.comment)
def name(self):
return "%d,%d" % (self.change, self.rev)
@ -201,22 +198,22 @@ class Stream(object):
log = logging.getLogger("recheckwatchbot")
def __init__(self, user, host, key, thread=True, es_url=None):
self.es_url = es_url or ES_URL
def __init__(self, user, host, key, config=None, thread=True):
self.config = config or er_conf.Config()
port = 29418
self.gerrit = gerritlib.gerrit.Gerrit(host, user, port, key)
self.es = results.SearchEngine(self.es_url)
self.es = results.SearchEngine(self.config.es_url)
if thread:
self.gerrit.startWatching()
@staticmethod
def parse_jenkins_failure(event):
def parse_jenkins_failure(event, ci_username=er_conf.CI_USERNAME):
"""Is this comment a jenkins failure comment."""
if event.get('type', '') != 'comment-added':
return False
username = event['author'].get('username', '')
if (username not in ['jenkins', 'zuul']):
if (username not in [ci_username, 'zuul']):
return False
if not ("Build failed" in
@ -324,15 +321,17 @@ class Stream(object):
while True:
event = self.gerrit.getEvent()
failed_jobs = Stream.parse_jenkins_failure(event)
failed_jobs = Stream.parse_jenkins_failure(
event, ci_username=self.config.ci_username)
if not failed_jobs:
# nothing to see here, lets try the next event
continue
fevent = FailEvent(event, failed_jobs)
# bail if it's not an openstack project
if not fevent.is_openstack_project():
# bail if the failure is from a project
# that hasn't run any of the included jobs
if not fevent.is_included_job():
continue
self.log.info("Looking for failures in %d,%d on %s" %
@ -379,10 +378,9 @@ class Classifier(object):
queries = None
def __init__(self, queries_dir, es_url=None, db_uri=None):
self.es_url = es_url or ES_URL
self.db_uri = db_uri or DB_URI
self.es = results.SearchEngine(self.es_url)
def __init__(self, queries_dir, config=None):
self.config = config or er_conf.Config()
self.es = results.SearchEngine(self.config.es_url)
self.queries_dir = queries_dir
self.queries = loader.load(self.queries_dir)
@ -409,7 +407,7 @@ class Classifier(object):
# Reload each time
self.queries = loader.load(self.queries_dir)
bug_matches = []
engine = sqlalchemy.create_engine(self.db_uri)
engine = sqlalchemy.create_engine(self.config.db_uri)
Session = orm.sessionmaker(bind=engine)
session = Session()
for x in self.queries:

5
elastic_recheck/log.py
View File

@ -38,9 +38,8 @@ def setup_logging(config=None):
"urllib3.connectionpool": logging.WARN
}
if config is not None and config.has_option('ircbot', 'log_config'):
log_config = config.get('ircbot', 'log_config')
fp = os.path.expanduser(log_config)
if config and config.irc_log_config:
fp = os.path.expanduser(config.irc_log_config)
if not os.path.exists(fp):
raise Exception("Unable to read logging config file at %s" % fp)
logging.config.fileConfig(fp)

26
elastic_recheck/tests/unit/test_bot.py
View File

@ -20,6 +20,7 @@ import fixtures
import mock
from elastic_recheck import bot
import elastic_recheck.config as er_conf
from elastic_recheck import elasticRecheck
from elastic_recheck import tests
import elastic_recheck.tests.unit.fake_gerrit as fg
@ -29,10 +30,11 @@ def _set_fake_config(fake_config):
fake_config.add_section('ircbot')
fake_config.add_section('gerrit')
# Set fake ircbot config
fake_config.set('ircbot', 'pidfile', er_conf.PID_FN)
fake_config.set('ircbot', 'nick', 'Fake_User')
fake_config.set('ircbot', 'pass', '')
fake_config.set('ircbot', 'server', 'irc.fake.net')
fake_config.set('ircbot', 'port', 6667)
fake_config.set('ircbot', 'port', '6667')
fake_config.set('ircbot', 'channel_config',
'fake_recheck_watch_bot.yaml')
# Set fake gerrit config
@ -49,22 +51,21 @@ class TestBot(unittest.TestCase):
super(TestBot, self).setUp()
self.fake_config = ConfigParser.ConfigParser({'server_password': None})
_set_fake_config(self.fake_config)
config = er_conf.Config(config_obj=self.fake_config)
self.channel_config = bot.ChannelConfig(yaml.load(
open('recheckwatchbot.yaml')))
with mock.patch('launchpadlib.launchpad.Launchpad'):
self.recheck_watch = bot.RecheckWatch(
None,
self.channel_config,
self.fake_config.get('gerrit', 'user'),
self.fake_config.get('gerrit', 'query_file'),
self.fake_config.get('gerrit', 'host'),
self.fake_config.get('gerrit', 'key'),
False)
None,
config=config,
commenting=False)
def test_read_channel_config_not_specified(self):
self.fake_config.set('ircbot', 'channel_config', None)
with self.assertRaises(bot.ElasticRecheckException) as exc:
bot._main([], self.fake_config)
bot._main([], er_conf.Config(config_obj=self.fake_config))
raised_exc = exc.exception
self.assertEqual(str(raised_exc), "Channel Config must be specified "
"in config file.")
@ -72,7 +73,7 @@ class TestBot(unittest.TestCase):
def test_read_channel_config_invalid_path(self):
self.fake_config.set('ircbot', 'channel_config', 'fake_path.yaml')
with self.assertRaises(bot.ElasticRecheckException) as exc:
bot._main([], self.fake_config)
bot._main([], er_conf.Config(config_obj=self.fake_config))
raised_exc = exc.exception
error_msg = "Unable to read layout config file at fake_path.yaml"
self.assertEqual(str(raised_exc), error_msg)
@ -94,17 +95,16 @@ class TestBotWithTestTools(tests.TestCase):
fg.Gerrit))
self.fake_config = ConfigParser.ConfigParser({'server_password': None})
_set_fake_config(self.fake_config)
config = er_conf.Config(config_obj=self.fake_config)
self.channel_config = bot.ChannelConfig(yaml.load(
open('recheckwatchbot.yaml')))
with mock.patch('launchpadlib.launchpad.Launchpad'):
self.recheck_watch = bot.RecheckWatch(
None,
self.channel_config,
self.fake_config.get('gerrit', 'user'),
self.fake_config.get('gerrit', 'query_file'),
self.fake_config.get('gerrit', 'host'),
self.fake_config.get('gerrit', 'key'),
False)
None,
config=config,
commenting=False)
def fake_print(self, cls, channel, msg):
reference = ("openstack/keystone change: https://review.openstack.org/"

8
elastic_recheck/tests/unit/test_stream.py
View File

@ -47,7 +47,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.url, "https://review.openstack.org/64750")
self.assertEqual(sorted(event.build_short_uuids()),
["5dd41fe", "d3fd328"])
self.assertTrue(event.is_openstack_project())
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "gate")
self.assertEqual(event.bug_urls(), None)
self.assertEqual(event.bug_urls_map(), None)
@ -65,7 +65,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.url, "https://review.openstack.org/64749")
self.assertEqual(sorted(event.build_short_uuids()),
["5dd41fe", "d3fd328"])
self.assertTrue(event.is_openstack_project())
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "check")
self.assertEqual(event.bug_urls(), None)
self.assertEqual(event.bug_urls_map(), None)
@ -147,7 +147,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.url, "https://review.openstack.org/64750")
self.assertEqual(sorted(event.build_short_uuids()),
["5dd41fe", "d3fd328"])
self.assertTrue(event.is_openstack_project())
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "gate")
self.assertEqual(event.bug_urls(),
['https://bugs.launchpad.net/bugs/123456'])
@ -175,7 +175,7 @@ class TestStream(tests.TestCase):
self.assertEqual(event.url, "https://review.openstack.org/64749")
self.assertEqual(sorted(event.build_short_uuids()),
["5dd41fe", "d3fd328"])
self.assertTrue(event.is_openstack_project())
self.assertTrue(event.is_included_job())
self.assertEqual(event.queue(), "check")
self.assertEqual(event.bug_urls(),
['https://bugs.launchpad.net/bugs/123456'])