diff --git a/discover_test.py b/discover_test.py index ad86f9f..c9b65e3 100644 --- a/discover_test.py +++ b/discover_test.py @@ -1,5 +1,5 @@ -import logging from itertools import groupby +import logging from ostack_validator.common import MarkedIssue, Inspection from ostack_validator.discovery import OpenstackDiscovery diff --git a/doc/source/rules_engine.rst b/doc/source/rules_engine.rst index 06f06b0..1fef778 100644 --- a/doc/source/rules_engine.rst +++ b/doc/source/rules_engine.rst @@ -34,15 +34,8 @@ Rule-based inspection All rule-based inspections are using pre-defined actions written on python, for now they defined in "steps.py" file in the directory: ostack_validator/inspections/lettuce. As you can see they are based on lettuce -framework - bdd framework for python. - -Store and reuse rules ---------------------- -You can store your rules wherever you want and add it through the UI or simply -putting it in directory ostack_validator/inspections/lettuce with name like -this: *.feature. The main requirement is that all you actions in those files -must be written according to the rules in steps.py. -Also you can expand the rules definition by adding your own steps.py. As example: +framework - bdd framework for python. +You can expand the rules definition by adding your own steps.py. As example: #This decorator is for defining step for using them in the scenario. @step(r'Nova has "(.+)" equal to "(.*)"') @@ -50,16 +43,26 @@ def nova_has_property(step, name, value): name = subst(name) value = subst(value) - for nova in [c for c in world.openstack.components if + for nova in [c for c in world.openstack.components if c.name.startswith('nova')]: if not nova.config[name] == value: stop() New methods can use 2 classes from the inspections framework: -ostack_validator.model and ostack_validator.common. There are you can find many -adapters to the services configuration data and all additional information -collected from OpenStack nodes. After that you can use you brand new rule in -the scenarios as described above. +ostack_validator/model.py and ostack_validator/common.py. There are you can +find many adapters to the services configuration data and all additional +information collected from OpenStack nodes. After that you can use you brand +new rule in the scenarios as described above. In common.py you can find +Inspection, Issue, Mark, Error and Version classes for your comfortability in +rule defining. Model.py contains Openstack model based on configuration +schemas. + +Store and reuse rules +--------------------- +You can store your rules wherever you want and add it through the UI or simply +putting it in directory ostack_validator/inspections/lettuce with name like +this: *.feature. The main requirement is that all you actions in those files +must be written according to the rules in steps.py. Sanity checks vs best practices ------------------------------- diff --git a/ostack_validator/__init__.py b/ostack_validator/__init__.py index 16ffa97..c93ada2 100644 --- a/ostack_validator/__init__.py +++ b/ostack_validator/__init__.py @@ -1,5 +1,5 @@ if __name__ == '__main__': - import sys from ostack_validator.main import main + import sys main(sys.argv[1:]) diff --git a/ostack_validator/celery.py b/ostack_validator/celery.py index f56cbc5..ec988cb 100644 --- a/ostack_validator/celery.py +++ b/ostack_validator/celery.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -import os import logging +import os import traceback from celery import Celery @@ -63,7 +63,8 @@ def ostack_inspect_task(request): openstack.report_issue( Issue( Issue.ERROR, - 'Unexpected error running inspection "%s". See log for details' % + 'Unexpected error running inspection "%s". See log for ' + 'details' % inspection.name)) return InspectionResult(request, openstack) diff --git a/ostack_validator/common.py b/ostack_validator/common.py index 1566d16..aa2a99f 100644 --- a/ostack_validator/common.py +++ b/ostack_validator/common.py @@ -32,7 +32,8 @@ def path_relative_to(path, base_path): class Version: def __init__(self, major, minor=0, maintenance=0): - "Create Version object by either passing 3 integers, one string or an another Version object" + """Create Version object by either passing 3 integers, one string or + an another Version object""" if isinstance(major, str): self.parts = [int(x) for x in major.split('.', 3)] while len(self.parts) < 3: @@ -173,8 +174,9 @@ class MarkedIssue(Issue): def __str__(self): return ( super( - MarkedIssue, self).__str__() + (' (source "%s" line %d column %d)' % - (self.mark.source, self.mark.line + 1, self.mark.column + 1)) + MarkedIssue, self).__str__() + + (' (source "%s" line %d column %d)' % + (self.mark.source, self.mark.line + 1, self.mark.column + 1)) ) diff --git a/ostack_validator/config_model.py b/ostack_validator/config_model.py index 9c92c61..309be7d 100644 --- a/ostack_validator/config_model.py +++ b/ostack_validator/config_model.py @@ -140,7 +140,8 @@ class Configuration(object): if section in self._normal and name in self._normal[section]: return True - if not ignoreDefault and section in self._defaults and name in self._defaults[section]: + if not ignoreDefault and section in self._defaults \ + and name in self._defaults[section]: return True return False @@ -149,8 +150,8 @@ class Configuration(object): section, name = self._normalize_name(name) return ( - not (section in self._normal and name in self._normal[section]) and ( - section in self._defaults and name in self._defaults[section]) + not (section in self._normal and name in self._normal[section]) + and (section in self._defaults and name in self._defaults[section]) ) def set_default(self, name, value): @@ -223,8 +224,9 @@ class Element(object): def __eq__(self, other): return ( - (self.__class__ == other.__class__) and ( - self.start_mark == other.start_mark) and (self.end_mark == other.end_mark) + (self.__class__ == other.__class__) + and (self.start_mark == other.start_mark) + and (self.end_mark == other.end_mark) ) def __ne__(self, other): diff --git a/ostack_validator/discovery.py b/ostack_validator/discovery.py index a61ade2..2eeb11e 100644 --- a/ostack_validator/discovery.py +++ b/ostack_validator/discovery.py @@ -6,7 +6,8 @@ import logging import spur -from ostack_validator.common import Issue, Mark, MarkedIssue, index, path_relative_to +from ostack_validator.common import Issue, Mark, MarkedIssue, index, \ + path_relative_to from ostack_validator.model import * @@ -34,7 +35,10 @@ host_port_re = re.compile('(\d+\.\d+\.\d+\.\d+):(\d+)') class OpenstackDiscovery(object): def discover(self, initial_nodes, username, private_key): - "Takes a list of node addresses and returns discovered openstack installation info" + """ + Takes a list of node addresses and returns discovered openstack + installation info + """ openstack = Openstack() private_key_file = None @@ -114,16 +118,19 @@ class OpenstackDiscovery(object): def _find_python_process(self, client, name): processes = self._get_processes(client) for line in processes: - if len(line) > 0 and (line[0] == name or line[0].endswith('/' + name)): + if len(line) > 0 and (line[0] == name + or line[0].endswith('/' + name)): return line - if len(line) > 1 and python_re.match(line[0]) and (line[1] == name or line[1].endswith('/' + name)): + if len(line) > 1 and python_re.match(line[0]) \ + and (line[1] == name or line[1].endswith('/' + name)): return line return None def _find_python_package_version(self, client, package): result = client.run( - ['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % + ['python', '-c', + 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % package]) s = result.output.strip() @@ -169,8 +176,8 @@ class OpenstackDiscovery(object): return None line = ls.output.split("\n")[0] - perm, links, owner, group, size, date, time, timezone, name = line.split( - ) + perm, links, owner, group, size, date, time, timezone, name = \ + line.split() permissions = self._permissions_string_to_number(perm) with client.open(path) as f: @@ -440,7 +447,8 @@ class OpenstackDiscovery(object): mysql.config_files = [] config_locations_result = client.run( - ['bash', '-c', 'mysqld --help --verbose | grep "Default options are read from the following files in the given order" -A 1']) + ['bash', '-c', + 'mysqld --help --verbose | grep "Default options are read from the following files in the given order" -A 1']) config_locations = config_locations_result.output.strip().split( "\n")[-1].split() for path in config_locations: diff --git a/ostack_validator/inspections/keystone_authtoken.py b/ostack_validator/inspections/keystone_authtoken.py index 717df28..769a9e9 100644 --- a/ostack_validator/inspections/keystone_authtoken.py +++ b/ostack_validator/inspections/keystone_authtoken.py @@ -1,7 +1,8 @@ from ostack_validator.common import Inspection, Issue, find -KEYSTONE_AUTHTOKEN_FILTER_FACTORY = 'keystoneclient.middleware.auth_token:filter_factory' +KEYSTONE_AUTHTOKEN_FILTER_FACTORY = 'keystoneclient.middleware.' \ + 'auth_token:filter_factory' class KeystoneAuthtokenSettingsInspection(Inspection): @@ -30,8 +31,9 @@ class KeystoneAuthtokenSettingsInspection(Inspection): (authtoken_section, _) = find( nova.paste_config.items(), - lambda name_values: name_values[0].startswith('filter:') and name_values[ - 1].get('paste.filter_factory') == KEYSTONE_AUTHTOKEN_FILTER_FACTORY + lambda name_values: name_values[0].startswith('filter:') + and name_values[1].get('paste.filter_factory') == + KEYSTONE_AUTHTOKEN_FILTER_FACTORY ) if not authtoken_section: @@ -63,46 +65,53 @@ class KeystoneAuthtokenSettingsInspection(Inspection): Issue( Issue.ERROR, msg_prefix + - ' miss "auth_host" setting in keystone authtoken config')) + ' miss "auth_host" setting in keystone authtoken' + ' config')) elif not auth_host in keystone_addresses: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' has incorrect "auth_host" setting in keystone authtoken config')) + ' has incorrect "auth_host" setting in keystone' + ' authtoken config')) if not auth_port: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' miss "auth_port" setting in keystone authtoken config')) + ' miss "auth_port" setting in keystone authtoken' + ' config')) elif auth_port != keystone.config['admin_port']: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' has incorrect "auth_port" setting in keystone authtoken config')) + ' has incorrect "auth_port" setting in keystone' + ' authtoken config')) if not auth_protocol: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' miss "auth_protocol" setting in keystone authtoken config')) + ' miss "auth_protocol" setting in keystone authtoken' + ' config')) elif not auth_protocol in ['http', 'https']: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' has incorrect "auth_protocol" setting in keystone authtoken config')) + ' has incorrect "auth_protocol" setting in keystone ' + 'authtoken config')) if not admin_user: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' miss "admin_user" setting in keystone authtoken config')) + ' miss "admin_user" setting in keystone authtoken ' + 'config')) else: user = find( keystone.db['users'], @@ -112,14 +121,16 @@ class KeystoneAuthtokenSettingsInspection(Inspection): Issue( Issue.ERROR, msg_prefix + - ' has "admin_user" that is missing in Keystone catalog')) + ' has "admin_user" that is missing in Keystone ' + 'catalog')) if not admin_tenant_name: nova.report_issue( Issue( Issue.ERROR, msg_prefix + - ' miss "admin_tenant_name" setting in keystone authtoken config')) + ' miss "admin_tenant_name" setting in keystone ' + 'authtoken config')) else: tenant = find( keystone.db['tenants'], @@ -129,7 +140,8 @@ class KeystoneAuthtokenSettingsInspection(Inspection): Issue( Issue.ERROR, msg_prefix + - ' has "admin_tenant_name" that is missing in Keystone catalog')) + ' has "admin_tenant_name" that is missing in ' + 'Keystone catalog')) if admin_token: nova.report_issue( diff --git a/ostack_validator/inspections/keystone_endpoints.py b/ostack_validator/inspections/keystone_endpoints.py index c9ea0bb..4d97586 100644 --- a/ostack_validator/inspections/keystone_endpoints.py +++ b/ostack_validator/inspections/keystone_endpoints.py @@ -5,7 +5,8 @@ from ostack_validator.common import Inspection, Issue, find class KeystoneEndpointsInspection(Inspection): name = 'Keystone endpoints' - description = 'Validate that each keystone endpoint leads to proper service' + description = 'Validate that each keystone endpoint leads to' \ + ' proper service' def inspect(self, openstack): keystone = find(openstack.components, lambda c: c.name == 'keystone') @@ -20,8 +21,9 @@ class KeystoneEndpointsInspection(Inspection): if not endpoint: keystone.report_issue( Issue( - Issue.WARNING, 'Keystone catalog contains service "%s" that has no defined endpoints' % - service['name'])) + Issue.WARNING, + 'Keystone catalog contains service "%s" that has ' + 'no defined endpoints' % service['name'])) continue for url_attr in ['adminurl', 'publicurl', 'internalurl']: @@ -34,7 +36,10 @@ class KeystoneEndpointsInspection(Inspection): if not host: keystone.report_issue( Issue( - Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to unknown host' % + Issue.ERROR, 'Keystone catalog has endpoint ' + 'for service "%s" (id %s) that ' + 'has "%s" set pointing to unknown' + ' host' % (service['name'], service['id'], url_attr))) continue @@ -43,12 +48,18 @@ class KeystoneEndpointsInspection(Inspection): if c.name != 'nova-compute': continue - if c.config['osapi_compute_listen'] in ['0.0.0.0', url.hostname] and c.config['osapi_compute_listen_port'] == url.port: + if c.config['osapi_compute_listen'] in ['0.0.0.0', + url.hostname] \ + and c.config['osapi_compute_listen_port'] == \ + url.port: nova_compute = c break if not nova_compute: keystone.report_issue( Issue( - Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to no service' % + Issue.ERROR, 'Keystone catalog has endpoint ' + 'for service "%s" (id %s) that ' + 'has "%s" set pointing to no ' + 'service' % (service['name'], service['id'], url_attr))) diff --git a/ostack_validator/inspections/lettuce/steps.py b/ostack_validator/inspections/lettuce/steps.py index 27fd2eb..1c64179 100644 --- a/ostack_validator/inspections/lettuce/steps.py +++ b/ostack_validator/inspections/lettuce/steps.py @@ -35,7 +35,8 @@ def stop(): @step(r'I use OpenStack (\w+)') def use_openstack_version(step, version): version = Version(version) - for component in [c for c in world.openstack.components if isinstance(c, OpenstackComponent)]: + for component in [c for c in world.openstack.components + if isinstance(c, OpenstackComponent)]: if not Version(component.version) >= version: stop() @@ -75,7 +76,8 @@ def nova_has_property(step, name, value): name = subst(name) value = subst(value) - for nova in [c for c in world.openstack.components if c.name.startswith('nova')]: + for nova in [c for c in world.openstack.components + if c.name.startswith('nova')]: if not nova.config[name] == value: stop() @@ -88,7 +90,8 @@ def nova_property_assertion(self, name, values): if not values: return - for nova in [c for c in world.openstack.components if c.name.startswith('nova')]: + for nova in [c for c in world.openstack.components + if c.name.startswith('nova')]: nova_value = nova.config[name] if not (nova_value and nova_value in values): @@ -102,7 +105,8 @@ def nova_has_property(step, component_name, parameter_name): component_name = subst(component_name) parameter_name = subst(parameter_name) - for component in [c for c in world.openstack.components if c.component.startswith('%s' % component_name)]: + for component in [c for c in world.openstack.components + if c.component.startswith('%s' % component_name)]: component_value = component.config[parameter_name] if component_value is None: @@ -117,10 +121,12 @@ def nova_has_property(step, component_name, parameter_name, value): parameter_name = subst(parameter_name) value = subst(value) - for component in [c for c in world.openstack.components if c.component.startswith('%s' % component_name)]: + for component in [c for c in world.openstack.components + if c.component.startswith('%s' % component_name)]: component_value = component.config[parameter_name] if not component_value == value: component.report_issue( - Issue(Issue.ERROR, '"%s" should have parameter equals "%s" now its "%s"' % - (component_name, component_value, value))) \ No newline at end of file + Issue(Issue.ERROR, + '"%s" should have parameter equals "%s" now its "%s"' % + (component_name, component_value, value))) diff --git a/ostack_validator/inspections/lettuce_runner.py b/ostack_validator/inspections/lettuce_runner.py index 54dbb04..4ff119e 100644 --- a/ostack_validator/inspections/lettuce_runner.py +++ b/ostack_validator/inspections/lettuce_runner.py @@ -1,5 +1,5 @@ -import os.path import lettuce +import os.path from ostack_validator.common import Inspection, Issue @@ -16,7 +16,8 @@ class LettuceRunnerInspection(Inspection): del lettuce.world.openstack for feature_result in result.feature_results: - for scenario_result in [s for s in feature_result.scenario_results if not s.passed]: + for scenario_result in [s for s in feature_result.scenario_results + if not s.passed]: for step in scenario_result.steps_undefined: openstack.report_issue( Issue( diff --git a/ostack_validator/main.py b/ostack_validator/main.py index f886ee0..48fff10 100644 --- a/ostack_validator/main.py +++ b/ostack_validator/main.py @@ -1,9 +1,10 @@ -import sys -import logging import argparse +import logging +import sys + -from ostack_validator.model_parser import ModelParser from ostack_validator.inspection import MainConfigValidationInspection +from ostack_validator.model_parser import ModelParser def main(args): diff --git a/ostack_validator/model.py b/ostack_validator/model.py index e696de7..8621ccc 100644 --- a/ostack_validator/model.py +++ b/ostack_validator/model.py @@ -127,8 +127,8 @@ class OpenstackComponent(Service): schema = ConfigSchemaRegistry.get_schema(self.component, self.version) if not schema: self.logger.debug( - 'No schema for component "%s" main config version %s. Skipping it' % - (self.component, self.version)) + 'No schema for component "%s" main config version %s. ' + 'Skipping it' % (self.component, self.version)) return None return self._parse_config_resources(self.config_files, schema) @@ -216,15 +216,19 @@ class OpenstackComponent(Service): if not (parameter_schema or unknown_section): report_issue( MarkedIssue( - Issue.WARNING, 'Unknown parameter: section "%s" name "%s"' % - (section_name, parameter.name.text), parameter.start_mark)) + Issue.WARNING, + 'Unknown parameter: section "%s" name "%s"' + % (section_name, parameter.name.text), + parameter.start_mark)) continue if parameter.name.text in seen_parameters: report_issue( MarkedIssue( - Issue.WARNING, 'Parameter "%s" in section "%s" redeclared' % - (parameter.name.text, section_name), parameter.start_mark)) + Issue.WARNING, + 'Parameter "%s" in section "%s" redeclared' % + (parameter.name.text, section_name), + parameter.start_mark)) else: seen_parameters.add(parameter.name.text) @@ -241,8 +245,10 @@ class OpenstackComponent(Service): if isinstance(type_validation_result, Issue): type_validation_result.mark = parameter.value.start_mark.merge( type_validation_result.mark) - type_validation_result.message = 'Property "%s" in section "%s": %s' % ( - parameter.name.text, section_name, type_validation_result.message) + type_validation_result.message = \ + 'Property "%s" in section "%s": %s' % ( + parameter.name.text, section_name, + type_validation_result.message) report_issue(type_validation_result) config.set( @@ -258,8 +264,12 @@ class OpenstackComponent(Service): if parameter_schema.deprecation_message: report_issue( MarkedIssue( - Issue.WARNING, 'Deprecated parameter: section "%s" name "%s". %s' % - (section_name, parameter.name.text, parameter_schema.deprecation_message), parameter.start_mark)) + Issue.WARNING, + 'Deprecated parameter: section "%s" name ' + '"%s". %s' % + (section_name, parameter.name.text, + parameter_schema.deprecation_message), + parameter.start_mark)) else: config.set(parameter_fullname, parameter.value.text)