From 0ed49056d70f07897749790e3a60170b5c942a42 Mon Sep 17 00:00:00 2001 From: Anton Studenov Date: Wed, 28 Dec 2016 13:57:39 +0300 Subject: [PATCH] Support for multi-node devstack * Added ServiceInScreen class. * Added 'slaves' parameter to devstack driver. This parameter contains a list of ip addresses that will be included to ansible executor. Default: empty list * Added 'iface' parameter to devstack driver. It specifies interface name that is used for communication between nodes. Change-Id: Iad93ac301da3b2c8e9e9a30afcda091788546c56 --- os_faults/drivers/devstack.py | 91 +++++++++---- os_faults/tests/unit/drivers/test_devstack.py | 120 +++++++++++++++++- 2 files changed, 180 insertions(+), 31 deletions(-) diff --git a/os_faults/drivers/devstack.py b/os_faults/drivers/devstack.py index f0f003f..b81c3fc 100644 --- a/os_faults/drivers/devstack.py +++ b/os_faults/drivers/devstack.py @@ -17,6 +17,7 @@ from os_faults.ansible import executor from os_faults.api import cloud_management from os_faults.api import node_collection from os_faults.common import service +from os_faults import utils LOG = logging.getLogger(__name__) @@ -30,16 +31,45 @@ class DevStackNode(node_collection.NodeCollection): raise NotImplementedError +class ServiceInScreen(service.ServiceAsProcess): + + @utils.require_variables('WINDOW_NAME') + def __init__(self, *args, **kwargs): + super(ServiceInScreen, self).__init__(*args, **kwargs) + + # sends ctr+c, arrow up key, enter key + self.RESTART_CMD = ( + "screen -S stack -p {window_name} -X " + "stuff $'\\003'$'\\033[A'$(printf \\\\r)").format( + window_name=self.WINDOW_NAME) + + # sends ctr+c + self.TERMINATE_CMD = ( + "screen -S stack -p {window_name} -X " + "stuff $'\\003'").format( + window_name=self.WINDOW_NAME) + + # sends arrow up key, enter key + self.START_CMD = ( + "screen -S stack -p {window_name} -X " + "stuff $'\\033[A'$(printf \\\\r)").format( + window_name=self.WINDOW_NAME) + + class KeystoneService(service.ServiceAsProcess): SERVICE_NAME = 'keystone' GREP = '[k]eystone-' RESTART_CMD = 'sudo service apache2 restart' + TERMINATE_CMD = 'sudo service apache2 stop' + START_CMD = 'sudo service apache2 start' class MySQLService(service.ServiceAsProcess): SERVICE_NAME = 'mysql' GREP = '[m]ysqld' RESTART_CMD = 'sudo service mysql restart' + TERMINATE_CMD = 'sudo service mysql stop' + START_CMD = 'sudo service mysql start' PORT = ('tcp', 3307) @@ -47,34 +77,32 @@ class RabbitMQService(service.ServiceAsProcess): SERVICE_NAME = 'rabbitmq' GREP = '[r]abbitmq-server' RESTART_CMD = 'sudo service rabbitmq-server restart' + TERMINATE_CMD = 'sudo service rabbitmq-server stop' + START_CMD = 'sudo service rabbitmq-server start' -class NovaAPIService(service.ServiceAsProcess): +class NovaAPIService(ServiceInScreen): SERVICE_NAME = 'nova-api' GREP = '[n]ova-api' - RESTART_CMD = ("screen -S stack -p n-api -X " - "stuff $'\\003'$'\\033[A'$(printf \\\\r)") + WINDOW_NAME = 'n-api' -class GlanceAPIService(service.ServiceAsProcess): +class GlanceAPIService(ServiceInScreen): SERVICE_NAME = 'glance-api' GREP = '[g]lance-api' - RESTART_CMD = ("screen -S stack -p g-api -X " - "stuff $'\\003'$'\\033[A'$(printf \\\\r)") + WINDOW_NAME = 'g-api' -class NovaComputeService(service.ServiceAsProcess): +class NovaComputeService(ServiceInScreen): SERVICE_NAME = 'nova-compute' GREP = '[n]ova-compute' - RESTART_CMD = ("screen -S stack -p n-cpu -X " - "stuff $'\\003'$'\\033[A'$(printf \\\\r)") + WINDOW_NAME = 'n-cpu' -class NovaSchedulerService(service.ServiceAsProcess): +class NovaSchedulerService(ServiceInScreen): SERVICE_NAME = 'nova-scheduler' GREP = '[n]ova-scheduler' - RESTART_CMD = ("screen -S stack -p n-sch -X " - "stuff $'\\003'$'\\033[A'$(printf \\\\r)") + WINDOW_NAME = 'n-sch' class DevStackManagement(cloud_management.CloudManagement): @@ -99,7 +127,11 @@ class DevStackManagement(cloud_management.CloudManagement): 'address': {'type': 'string'}, 'username': {'type': 'string'}, 'private_key_file': {'type': 'string'}, - + 'slaves': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'iface': {'type': 'string'}, }, 'required': ['address', 'username'], 'additionalProperties': False, @@ -111,18 +143,24 @@ class DevStackManagement(cloud_management.CloudManagement): self.address = cloud_management_params['address'] self.username = cloud_management_params['username'] self.private_key_file = cloud_management_params.get('private_key_file') + self.slaves = cloud_management_params.get('slaves', []) + self.iface = cloud_management_params.get('iface', 'eth0') self.cloud_executor = executor.AnsibleRunner( remote_user=self.username, private_key_file=self.private_key_file, become=False) - self.host = None + + self.hosts = [self.address] + if self.slaves: + self.hosts.extend(self.slaves) + self.nodes = None def verify(self): """Verify connection to the cloud.""" - task = {'shell': 'screen -ls | grep stack'} - hostname = self.execute_on_cloud( - [self.address], task)[0].payload['stdout'] - LOG.debug('DevStack hostname: %s', hostname) + task = {'shell': 'screen -ls | grep -P "\\d+\\.stack"'} + results = self.execute_on_cloud(self.hosts, task) + hostnames = [result.host for result in results] + LOG.debug('DevStack hostnames: %s', hostnames) LOG.info('Connected to cloud successfully') def execute_on_cloud(self, hosts, task, raise_on_error=True): @@ -139,14 +177,17 @@ class DevStackManagement(cloud_management.CloudManagement): return self.cloud_executor.execute(hosts, task, []) def get_nodes(self, fqdns=None): - if self.host is None: - task = {'command': 'cat /sys/class/net/eth0/address'} - mac = self.execute_on_cloud( - [self.address], task)[0].payload['stdout'] + if self.nodes is None: + get_mac_cmd = 'cat /sys/class/net/{}/address'.format(self.iface) + task = {'command': get_mac_cmd} + results = self.execute_on_cloud(self.hosts, task) + # TODO(astudenov): support fqdn - self.host = node_collection.Host(ip=self.address, mac=mac, - fqdn='') + self.nodes = [node_collection.Host(ip=r.host, + mac=r.payload['stdout'], + fqdn='') + for r in results] return self.NODE_CLS(cloud_management=self, power_management=self.power_management, - hosts=[self.host]) + hosts=self.nodes) diff --git a/os_faults/tests/unit/drivers/test_devstack.py b/os_faults/tests/unit/drivers/test_devstack.py index fbd8158..4521461 100644 --- a/os_faults/tests/unit/drivers/test_devstack.py +++ b/os_faults/tests/unit/drivers/test_devstack.py @@ -57,13 +57,33 @@ class DevStackManagementTestCase(test.TestCase): def test_verify(self, mock_ansible_runner): ansible_runner_inst = mock_ansible_runner.return_value ansible_runner_inst.execute.side_effect = [ - [fakes.FakeAnsibleResult(payload={'stdout': 'node1.com'})], + [fakes.FakeAnsibleResult(payload={'stdout': ''}, + host='10.0.0.2')], ] devstack_management = devstack.DevStackManagement(self.conf) devstack_management.verify() ansible_runner_inst.execute.assert_called_once_with( - ['10.0.0.2'], {'shell': 'screen -ls | grep stack'}) + ['10.0.0.2'], {'shell': 'screen -ls | grep -P "\\d+\\.stack"'}) + + @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) + def test_verify_slaves(self, mock_ansible_runner): + self.conf['slaves'] = ['10.0.0.3', '10.0.0.4'] + ansible_runner_inst = mock_ansible_runner.return_value + ansible_runner_inst.execute.side_effect = [ + [fakes.FakeAnsibleResult(payload={'stdout': ''}, + host='10.0.0.2'), + fakes.FakeAnsibleResult(payload={'stdout': ''}, + host='10.0.0.3'), + fakes.FakeAnsibleResult(payload={'stdout': ''}, + host='10.0.0.4')], + ] + devstack_management = devstack.DevStackManagement(self.conf) + devstack_management.verify() + + ansible_runner_inst.execute.assert_called_once_with( + ['10.0.0.2', '10.0.0.3', '10.0.0.4'], + {'shell': 'screen -ls | grep -P "\\d+\\.stack"'}) @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) def test_execute_on_cloud(self, mock_ansible_runner): @@ -84,7 +104,8 @@ class DevStackManagementTestCase(test.TestCase): def test_get_nodes(self, mock_ansible_runner): ansible_runner_inst = mock_ansible_runner.return_value ansible_runner_inst.execute.side_effect = [ - [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'})], + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2')], ] devstack_management = devstack.DevStackManagement(self.conf) @@ -99,6 +120,37 @@ class DevStackManagementTestCase(test.TestCase): fqdn='')], nodes.hosts) + @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) + def test_get_nodes_with_slaves(self, mock_ansible_runner): + self.conf['slaves'] = ['10.0.0.3', '10.0.0.4'] + self.conf['iface'] = 'eth1' + ansible_runner_inst = mock_ansible_runner.return_value + ansible_runner_inst.execute.side_effect = [ + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2'), + fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c2'}, + host='10.0.0.3'), + fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c3'}, + host='10.0.0.4')], + ] + + devstack_management = devstack.DevStackManagement(self.conf) + nodes = devstack_management.get_nodes() + + ansible_runner_inst.execute.assert_called_once_with( + ['10.0.0.2', '10.0.0.3', '10.0.0.4'], + {'command': 'cat /sys/class/net/eth1/address'}) + + self.assertIsInstance(nodes, devstack.DevStackNode) + self.assertEqual( + [node_collection.Host(ip='10.0.0.2', mac='09:7b:74:90:63:c1', + fqdn=''), + node_collection.Host(ip='10.0.0.3', mac='09:7b:74:90:63:c2', + fqdn=''), + node_collection.Host(ip='10.0.0.4', mac='09:7b:74:90:63:c3', + fqdn='')], + nodes.hosts) + @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) @ddt.data(*devstack.DevStackManagement.SERVICE_NAME_TO_CLASS.items()) @ddt.unpack @@ -106,7 +158,8 @@ class DevStackManagementTestCase(test.TestCase): mock_ansible_runner): ansible_runner_inst = mock_ansible_runner.return_value ansible_runner_inst.execute.side_effect = [ - [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'})], + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2')], [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')] ] @@ -142,7 +195,8 @@ class DevStackServiceTestCase(test.TestCase): def test_restart(self, service_name, service_cls, mock_ansible_runner): ansible_runner_inst = mock_ansible_runner.return_value ansible_runner_inst.execute.side_effect = [ - [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'})], + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2')], [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')], [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')] ] @@ -159,5 +213,59 @@ class DevStackServiceTestCase(test.TestCase): mock.call( ['10.0.0.2'], {'command': 'cat /sys/class/net/eth0/address'}), mock.call(['10.0.0.2'], {'command': cmd}, []), - mock.call(['10.0.0.2'], {'shell': service_cls.RESTART_CMD}) + mock.call(['10.0.0.2'], {'shell': service.RESTART_CMD}) + ]) + + @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) + @ddt.data(*devstack.DevStackManagement.SERVICE_NAME_TO_CLASS.items()) + @ddt.unpack + def test_terminate(self, service_name, service_cls, mock_ansible_runner): + ansible_runner_inst = mock_ansible_runner.return_value + ansible_runner_inst.execute.side_effect = [ + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2')], + [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')], + [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')] + ] + + devstack_management = devstack.DevStackManagement(self.conf) + + service = devstack_management.get_service(service_name) + self.assertIsInstance(service, service_cls) + + service.terminate() + + cmd = 'bash -c "ps ax | grep \'{}\'"'.format(service_cls.GREP) + ansible_runner_inst.execute.assert_has_calls([ + mock.call( + ['10.0.0.2'], {'command': 'cat /sys/class/net/eth0/address'}), + mock.call(['10.0.0.2'], {'command': cmd}, []), + mock.call(['10.0.0.2'], {'shell': service.TERMINATE_CMD}) + ]) + + @mock.patch('os_faults.ansible.executor.AnsibleRunner', autospec=True) + @ddt.data(*devstack.DevStackManagement.SERVICE_NAME_TO_CLASS.items()) + @ddt.unpack + def test_start(self, service_name, service_cls, mock_ansible_runner): + ansible_runner_inst = mock_ansible_runner.return_value + ansible_runner_inst.execute.side_effect = [ + [fakes.FakeAnsibleResult(payload={'stdout': '09:7b:74:90:63:c1'}, + host='10.0.0.2')], + [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')], + [fakes.FakeAnsibleResult(payload={'stdout': ''}, host='10.0.0.2')] + ] + + devstack_management = devstack.DevStackManagement(self.conf) + + service = devstack_management.get_service(service_name) + self.assertIsInstance(service, service_cls) + + service.start() + + cmd = 'bash -c "ps ax | grep \'{}\'"'.format(service_cls.GREP) + ansible_runner_inst.execute.assert_has_calls([ + mock.call( + ['10.0.0.2'], {'command': 'cat /sys/class/net/eth0/address'}), + mock.call(['10.0.0.2'], {'command': cmd}, []), + mock.call(['10.0.0.2'], {'shell': service.START_CMD}) ])