From 0b9824726ae46e72262d6bfc24ff50ea37ef08c3 Mon Sep 17 00:00:00 2001 From: Yaroslav Lobankov Date: Wed, 31 Aug 2016 18:01:26 +0300 Subject: [PATCH] Adding threads to IPMI driver This patch adds threads to poweroff/poweron/reset methods to increase the velocity of IPMI nodes management. The common code with the libvirt driver was moved to a separate module utils.py. Change-Id: Ic6e65079636cef6f32b13876fe3fb56ee302c6e9 --- examples/power_off_on_vm_node.py | 2 +- os_failures/api/error.py | 2 +- os_failures/drivers/ipmi.py | 48 +++++++++++++----------- os_failures/drivers/libvirt_driver.py | 54 +++++---------------------- os_failures/utils.py | 53 ++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 os_failures/utils.py diff --git a/examples/power_off_on_vm_node.py b/examples/power_off_on_vm_node.py index 7329bc3..f93d524 100644 --- a/examples/power_off_on_vm_node.py +++ b/examples/power_off_on_vm_node.py @@ -45,7 +45,7 @@ def main(): nodes = destructor.get_nodes() logging.info('All cluster nodes: %s', nodes) - logging.info('Pick and power off one of cluster nodes') + logging.info('Pick and power off/on one of cluster nodes') one = nodes.pick() one.poweroff() one.poweron() diff --git a/os_failures/api/error.py b/os_failures/api/error.py index 03ab60c..1d3e47c 100644 --- a/os_failures/api/error.py +++ b/os_failures/api/error.py @@ -20,5 +20,5 @@ class OSFError(OSFException): """Base Error class""" -class PowerManagmentError(OSFError): +class PowerManagementError(OSFError): """Base Error class for Power Management API""" diff --git a/os_failures/drivers/ipmi.py b/os_failures/drivers/ipmi.py index 23cfba5..33c45b8 100644 --- a/os_failures/drivers/ipmi.py +++ b/os_failures/drivers/ipmi.py @@ -18,6 +18,7 @@ from pyghmi.ipmi import command as ipmi_command from os_failures.api import error from os_failures.api import power_management +from os_failures import utils class IPMIDriver(power_management.PowerManagement): @@ -26,7 +27,7 @@ class IPMIDriver(power_management.PowerManagement): def _find_bmc_by_mac_address(self, mac_address): if mac_address not in self.mac_to_bmc: - raise error.PowerManagmentError( + raise error.PowerManagementError( 'BMC for Node(%s) not found!' % mac_address) return self.mac_to_bmc[mac_address] @@ -38,12 +39,11 @@ class IPMIDriver(power_management.PowerManagement): userid=bmc['username'], password=bmc['password']) ret = ipmicmd.set_power(cmd, wait=True) - except pyghmi_exception.IpmiException as e: + except pyghmi_exception.IpmiException: msg = 'IPMI cmd {!r} failed on bmc {!r}, Node({})'.format( cmd, bmc['address'], mac_address) logging.error(msg) - logging.exception(e) - raise error.PowerManagmentError(msg) + raise logging.debug('IPMI response: {}'.format(ret)) if ret.get('powerstate') != expected_state or 'error' in ret: @@ -51,26 +51,32 @@ class IPMIDriver(power_management.PowerManagement): 'Node({})'.format(expected_state, bmc['address'], mac_address)) - raise error.PowerManagmentError(msg) + raise error.PowerManagementError(msg) + + def _poweroff(self, mac_address): + logging.info('Power off Node with MAC address: %s', mac_address) + self._run_set_power_cmd( + mac_address, cmd='off', expected_state='off') + logging.info('Node(%s) was powered off' % mac_address) + + def _poweron(self, mac_address): + logging.info('Power on Node with MAC address: %s', mac_address) + self._run_set_power_cmd( + mac_address, cmd='on', expected_state='on') + logging.info('Node(%s) was powered on' % mac_address) + + def _reset(self, mac_address): + logging.info('Reset Node with MAC address: %s', mac_address) + # boot -- If system is off, then 'on', else 'reset' + self._run_set_power_cmd(mac_address, cmd='boot') + # NOTE(astudenov): This command does not wait for node to boot + logging.info('Node(%s) was reset' % mac_address) def poweroff(self, mac_addresses_list): - for mac_address in mac_addresses_list: - logging.info('Power off Node with MAC address: %s', mac_address) - self._run_set_power_cmd( - mac_address, cmd='off', expected_state='off') - logging.info('Node(%s) was powered off' % mac_address) + utils.run(self._poweroff, mac_addresses_list) def poweron(self, mac_addresses_list): - for mac_address in mac_addresses_list: - logging.info('Power on Node with MAC address: %s', mac_address) - self._run_set_power_cmd( - mac_address, cmd='on', expected_state='on') - logging.info('Node(%s) was powered on' % mac_address) + utils.run(self._poweron, mac_addresses_list) def reset(self, mac_addresses_list): - for mac_address in mac_addresses_list: - logging.info('Reset Node with MAC address: %s', mac_address) - # boot -- If system is off, then 'on', else 'reset' - self._run_set_power_cmd(mac_address, cmd='boot') - # NOTE(astudenov): this command does not wait for node to boot - logging.info('Node(%s) was reset' % mac_address) + utils.run(self._reset, mac_addresses_list) diff --git a/os_failures/drivers/libvirt_driver.py b/os_failures/drivers/libvirt_driver.py index 8359a6c..594f619 100644 --- a/os_failures/drivers/libvirt_driver.py +++ b/os_failures/drivers/libvirt_driver.py @@ -12,36 +12,12 @@ # limitations under the License. import logging -import threading -import traceback import libvirt from os_failures.api import error from os_failures.api import power_management - - -class ThreadsWrapper(object): - def __init__(self, target): - self.target = target - self.threads = [] - self.errors = [] - - def _target(self, **kwargs): - try: - self.target(**kwargs) - except Exception as exc: - logging.error(traceback.format_exc()) - self.errors.append(exc) - - def start_thread(self, **kwargs): - thread = threading.Thread(target=self._target, kwargs=kwargs) - thread.start() - self.threads.append(thread) - - def join_threads(self): - for thread in self.threads: - thread.join() +from os_failures import utils class LibvirtDriver(power_management.PowerManagement): @@ -64,44 +40,32 @@ class LibvirtDriver(power_management.PowerManagement): if mac_address in domain.XMLDesc(): return domain - raise error.PowerManagmentError( - 'Node with MAC address %s not found!' % mac_address) + raise error.PowerManagementError( + 'Domain with MAC address %s not found!' % mac_address) def _poweroff(self, mac_address): logging.info('Power off domain with MAC address: %s', mac_address) domain = self._find_domain_by_mac_address(mac_address) domain.destroy() - logging.info('Domain (%s) was powered off' % mac_address) + logging.info('Domain(%s) was powered off' % mac_address) def _poweron(self, mac_address): logging.info('Power on domain with MAC address: %s', mac_address) domain = self._find_domain_by_mac_address(mac_address) domain.create() - logging.info('Domain (%s) was powered on' % mac_address) + logging.info('Domain(%s) was powered on' % mac_address) def _reset(self, mac_address): logging.info('Reset domain with MAC address: %s', mac_address) domain = self._find_domain_by_mac_address(mac_address) domain.reset() - logging.info('Domain (%s) was reset' % mac_address) - - @staticmethod - def _run(target, mac_addresses_list): - tw = ThreadsWrapper(target) - for mac_address in mac_addresses_list: - tw.start_thread(mac_address=mac_address) - tw.join_threads() - - if tw.errors: - raise error.PowerManagmentError( - 'There are some errors when working the libvirt driver. ' - 'Please, check logs for more details.') + logging.info('Domain(%s) was reset' % mac_address) def poweroff(self, mac_addresses_list): - self._run(self._poweroff, mac_addresses_list) + utils.run(self._poweroff, mac_addresses_list) def poweron(self, mac_addresses_list): - self._run(self._poweron, mac_addresses_list) + utils.run(self._poweron, mac_addresses_list) def reset(self, mac_addresses_list): - self._run(self._reset, mac_addresses_list) + utils.run(self._reset, mac_addresses_list) diff --git a/os_failures/utils.py b/os_failures/utils.py new file mode 100644 index 0000000..577c3cc --- /dev/null +++ b/os_failures/utils.py @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading +import traceback + +from os_failures.api import error + + +def run(target, mac_addresses_list): + tw = ThreadsWrapper(target) + for mac_address in mac_addresses_list: + tw.start_thread(mac_address=mac_address) + tw.join_threads() + + if tw.errors: + raise error.PowerManagementError( + 'There are some errors when working the driver. ' + 'Please, check logs for more details.') + + +class ThreadsWrapper(object): + def __init__(self, target): + self.target = target + self.threads = [] + self.errors = [] + + def _target(self, **kwargs): + try: + self.target(**kwargs) + except Exception as exc: + logging.error(traceback.format_exc()) + self.errors.append(exc) + + def start_thread(self, **kwargs): + thread = threading.Thread(target=self._target, kwargs=kwargs) + thread.start() + self.threads.append(thread) + + def join_threads(self): + for thread in self.threads: + thread.join()