x/packstack
Code Issues Proposed changes

Change message in certain cases

Browse Source 
In some cases Puppet error messages are useless.
This patch implements algorithm which makes certain error
messages more userfriendly.

Change-Id: Iac6f1c5382b3c3d605df3477477558858eca69ef
Fixes: rhbz#989334, rhbz#1006476, rhbz#1003959
This commit is contained in:
Martin Magr 2013-09-23 11:43:31 +02:00
parent 6b19e22919
commit 5859915898
10 changed files with 213 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packstack/installer/exceptions.py
View File

@ -24,6 +24,9 @@ class PackStackError(Exception):
self.stderr = kwargs.get('stderr', None)
class PuppetError(Exception):
"""Raised when Puppet will have some problems."""
class MissingRequirements(PackStackError):
"""Raised when minimum install requirements are not met."""

70
packstack/modules/ospluginutils.py
View File

@ -90,73 +90,3 @@ def gethostlist(CONF):
if host and host not in hosts:
hosts.append(host)
return hosts
_error_exceptions = [
# puppet preloads a provider using the mysql command before it is installed
re.compile('Command mysql is missing'),
# puppet preloads a database_grant provider which fails if /root/.my.cnf
# this is ok because it will be retried later if needed
re.compile('Could not prefetch database_grant provider.*?\\.my\\.cnf'),
# swift puppet module tries to install swift-plugin-s3, there is no such
# pakage on RHEL, fixed in the upstream puppet module
re.compile('yum.*?install swift-plugin-s3'),
]
def isErrorException(line):
for ee in _error_exceptions:
if ee.search(line):
return True
return False
_re_color = re.compile('\x1b.*?\d\dm')
_re_errorline = re.compile('err: | Syntax error at|^Duplicate definition:|'
'^No matching value for selector param|'
'^Parameter name failed:|Error: |^Invalid tag |'
'^Invalid parameter |^Duplicate declaration: '
'^Could not find resource |^Could not parse for ')
def validate_puppet_logfile(logfile):
"""
Check a puppet log file for errors and raise an error if we find any
"""
fp = open(logfile)
data = fp.read()
fp.close()
manifestfile = os.path.splitext(logfile)[0]
for line in data.split('\n'):
line = line.strip()
if _re_errorline.search(line) is None:
continue
message = _re_color.sub('', line) # remove colors
if isErrorException(line):
logging.info("Ignoring expected error during puppet run %s : %s" %
(manifestfile, message))
continue
message = "Error during puppet run : " + message
logging.error("Error during remote puppet apply of " + manifestfile)
logging.error(data)
raise PackStackError(message)
def scan_puppet_logfile(logfile):
"""
Returns list of packstack_info/packstack_warn notices parsed from
given puppet log file.
"""
output = []
notice = re.compile(r"notice: .*Notify\[packstack_info\]"
"\/message: defined \'message\' as "
"\'(?P<message>.*)\'")
with open(logfile) as content:
for line in content:
match = notice.search(line)
if match:
output.append(match.group('message'))
return output

101
packstack/modules/puppet.py Normal file
View File

@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
import logging
import os
import re
from packstack.installer.exceptions import PuppetError
# TODO: Fill logger name when logging system will be refactored
logger = logging.getLogger()
re_color = re.compile('\x1b.*?\d\dm')
re_error = re.compile(
'err:|Syntax error at|^Duplicate definition:|^Invalid tag|'
'^No matching value for selector param|^Parameter name failed:|Error:|'
'^Invalid parameter|^Duplicate declaration:|^Could not find resource|'
'^Could not parse for|^\/usr\/bin\/env\: jruby\: No such file or directory'
)
re_ignore = re.compile(
# Puppet preloads a provider using the mysql command before it is installed
'Command mysql is missing|'
# Puppet preloads a database_grant provider which fails if /root/.my.cnf
# is missing, this is ok because it will be retried later if needed
'Could not prefetch database_grant provider.*?\\.my\\.cnf|'
# Swift Puppet module tries to install swift-plugin-s3, there is no such
# package on RHEL, fixed in the upstream puppet module
'yum.*?install swift-plugin-s3'
)
re_notice = re.compile(r"notice: .*Notify\[packstack_info\]"
"\/message: defined \'message\' as "
"\'(?P<message>.*)\'")
surrogates = [
# Value in /etc/sysctl.conf cannot be changed
('Sysctl::Value\[.*\]\/Sysctl\[(?P<arg1>.*)\].*Field \'val\' is required',
'Cannot change value of %(arg1)s in /etc/sysctl.conf'),
# Package is not found in yum repos
('Package\[.*\]\/ensure.*yum.*install (?P<arg1>.*)\'.*Nothing to do',
'Package %(arg1)s has not been found in enabled Yum repos.'),
# Packstack does not cooperate with jruby
('jruby', 'Your Puppet installation uses jruby instead of ruby. Package '
'jruby does not cooperate with Packstack well. You will have to '
'fix this manually.'),
]
def validate_logfile(logpath):
"""
Check given Puppet log file for errors and raise PuppetError if there is
any error
"""
manifestpath = os.path.splitext(logpath)[0]
manifestfile = os.path.basename(manifestpath)
with open(logpath) as logfile:
for line in logfile:
line = line.strip()
if re_error.search(line) is None:
continue
error = re_color.sub('', line) # remove colors
if re_ignore.search(line):
msg = ('Ignoring expected error during Puppet run %s: %s' %
(manifestfile, error))
logger.debug(msg)
continue
for regex, surrogate in surrogates:
match = re.search(regex, error)
if match is None:
continue
args = {}
num = 1
while True:
try:
args['arg%d' % num] = match.group(num)
num += 1
except IndexError:
break
error = surrogate % args
message = ('Error appeared during Puppet run: %s\n%s\n'
'You will find full trace in log %s' %
(manifestfile, error, logpath))
raise PuppetError(message)
def scan_logfile(logpath):
"""
Returns list of packstack_info/packstack_warn notices parsed from
given puppet log file.
"""
output = []
with open(logpath) as logfile:
for line in logfile:
match = re_notice.search(line)
if match:
output.append(match.group('message'))
return output

9
packstack/plugins/puppet_950.py
View File

@ -12,9 +12,8 @@ from packstack.installer import basedefs, output_messages
from packstack.installer.exceptions import ScriptRuntimeError
from packstack.modules.common import filtered_hosts
from packstack.modules.ospluginutils import (manifestfiles,
scan_puppet_logfile,
validate_puppet_logfile)
from packstack.modules.ospluginutils import manifestfiles
from packstack.modules.puppet import scan_logfile, validate_logfile
# Controller object will be initialized from main flow
controller = None
@ -162,10 +161,10 @@ def waitforpuppet(currently_running):
continue
# check log file for relevant notices
controller.MESSAGES.extend(scan_puppet_logfile(log))
controller.MESSAGES.extend(scan_logfile(log))
# check the log file for errors
validate_puppet_logfile(log)
validate_logfile(log)
sys.stdout.write(("\r%s : " % log_file).ljust(space_len))
print ("[ " + utils.color_text(output_messages.INFO_DONE, 'green') + " ]")

8
tests/installer/test_run_setup.py
View File

@ -20,7 +20,7 @@ import shutil
import sys
from unittest import TestCase
from packstack.modules import ospluginutils
from packstack.modules import ospluginutils, puppet
from packstack.installer import run_setup, basedefs
from ..test_base import PackstackTestCaseMixin
@ -55,9 +55,9 @@ class CommandLineTestCase(PackstackTestCaseMixin, TestCase):
# There is no puppet logfile to validate, so replace
# ospluginutils.validate_puppet_logfile with a mock function
orig_validate_logfile = ospluginutils.validate_puppet_logfile
ospluginutils.validate_puppet_logfile = lambda a: None
ospluginutils.scan_puppet_logfile = lambda a: []
orig_validate_logfile = puppet.validate_logfile
puppet.validate_logfile = lambda a: None
puppet.scan_logfile = lambda a: []
# If there is a error in a plugin sys.exit() gets called, this masks
# the actual error that should be reported, so we replace it to

8
tests/installer/test_utils.py
View File

@ -106,3 +106,11 @@ class ParameterTestCase(PackstackTestCaseMixin, TestCase):
mask_list=["'text'"],
replace_list=[("'", "'\\''")])
self.assertEqual(masked, 'test %s' % STR_MASK)
def test_shortcuts(self):
"""Test packstack.installer.utils.shortcuts functions"""
conf = {"A_HOST": "1.1.1.1", "B_HOSTS": "2.2.2.2,1.1.1.1",
"C_HOSTS": "3.3.3.3/vdc"}
hostlist = list(hosts(conf))
hostlist.sort()
self.assertEquals(['1.1.1.1', '2.2.2.2', '3.3.3.3'], hostlist)

0
tests/modules/__init__.py Normal file
View File

31
tests/modules/test_ospluginutils.py Normal file
View File

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import TestCase
from ..test_base import PackstackTestCaseMixin
from packstack.modules.ospluginutils import gethostlist
class OSPluginUtilsTestCase(PackstackTestCaseMixin, TestCase):
def test_gethostlist(self):
conf = {"A_HOST": "1.1.1.1", "B_HOSTS": "2.2.2.2,1.1.1.1",
"C_HOSTS": "3.3.3.3/vdc"}
hosts = gethostlist(conf)
hosts.sort()
self.assertEquals(['1.1.1.1', '2.2.2.2', '3.3.3.3'], hosts)

62
tests/modules/test_puppet.py Normal file
View File

@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import TestCase
from ..test_base import PackstackTestCaseMixin
from packstack.installer.exceptions import PuppetError
from packstack.modules.puppet import validate_logfile, scan_logfile
class PuppetTestCase(PackstackTestCaseMixin, TestCase):
def test_validate_logfile(self):
"""Test packstack.modules.validate_logfile"""
filename = os.path.join(self.tempdir, "puppet.log")
# test valid run
with open(filename, "w") as fp:
fp.write("Everything went ok")
validate_logfile(filename)
# test invalid run
with open(filename, "w") as fp:
fp.write("No matching value for selector param 'Fedora' ...")
self.assertRaises(PuppetError, validate_logfile, filename)
# test run with error exception
with open(filename, "w") as fp:
err = ("err: Could not prefetch database_grant provider 'mysql': "
"Execution of '/usr/bin/mysql --defaults-file=/root/.my.cnf"
" mysql -Be describe user' returned 1: Could not open "
"required defaults file: /root/.my.cnf")
fp.write(err)
validate_logfile(filename)
# test surrogate
with open(filename, "w") as fp:
err = ("err: /Stage[main]/Vswitch::Ovs/Package[openvswitch]/ensure"
": change from absent to present failed: Execution of "
"'/usr/bin/yum -d 0 -e 0 -y install openvswitch' returned "
"1: Error: Nothing to do")
fp.write(err)
self.assertRaises(PuppetError, validate_logfile, filename)
try:
validate_logfile(filename)
except PuppetError as ex:
ex_msg = str(ex)
sr_msg = ("Package openvswitch has not been found in enabled Yum "
"repos")
assert sr_msg in ex_msg

60
tests/test_ospluginutils.py
View File

@ -1,60 +0,0 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from unittest import TestCase
from test_base import PackstackTestCaseMixin
from packstack.modules.ospluginutils import gethostlist, \
validate_puppet_logfile, \
PackStackError
class OSPluginUtilsTestCase(PackstackTestCaseMixin, TestCase):
def test_gethostlist(self):
conf = {"A_HOST": "1.1.1.1", "B_HOSTS": "2.2.2.2,1.1.1.1",
"C_HOSTS": "3.3.3.3/vdc"}
hosts = gethostlist(conf)
hosts.sort()
self.assertEquals(['1.1.1.1', '2.2.2.2', '3.3.3.3'], hosts)
def test_validate_puppet_logfile(self):
filename = os.path.join(self.tempdir, "puppet.log")
fp = open(filename, "w")
fp.write("Everything went ok")
fp.close()
validate_puppet_logfile(filename)
def test_validate_puppet_logfile_error(self):
filename = os.path.join(self.tempdir, "puppet.log")
fp = open(filename, "w")
fp.write("No matching value for selector param 'Fedora' ...")
fp.close()
self.assertRaises(PackStackError, validate_puppet_logfile, filename)
def test_validate_puppet_logfile_okerror(self):
filename = os.path.join(self.tempdir, "puppet.log")
fp = open(filename, "w")
fp.write("err: Could not prefetch database_grant provider 'mysql': "
"Execution of '/usr/bin/mysql --defaults-file=/root/.my.cnf "
"mysql -Be describe user' returned 1: Could not open required"
" defaults file: /root/.my.cnf")
fp.close()
validate_puppet_logfile(filename)