diff --git a/stacklight_tests/elasticsearch_kibana/test_system.py b/stacklight_tests/elasticsearch_kibana/test_system.py index 78258be..5061593 100644 --- a/stacklight_tests/elasticsearch_kibana/test_system.py +++ b/stacklight_tests/elasticsearch_kibana/test_system.py @@ -41,7 +41,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): 7. Run OSTF Duration 120m - Snapshot add_remove_controller_elasticsearch_kibana """ self.env.revert_snapshot("deploy_ha_elasticsearch_kibana") @@ -62,8 +61,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): self.helpers.run_ostf(should_fail=1) - self.env.make_snapshot("add_remove_controller_elasticsearch_kibana") - @test(depends_on_groups=['deploy_ha_elasticsearch_kibana'], groups=["check_scaling_elasticsearch_kibana", "scaling", "elasticsearch_kibana", "system", @@ -83,7 +80,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): 7. Run OSTF Duration 120m - Snapshot add_remove_compute_elasticsearch_kibana """ self.env.revert_snapshot("deploy_ha_elasticsearch_kibana") @@ -104,8 +100,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): self.helpers.run_ostf(should_fail=1) - self.env.make_snapshot("add_remove_compute_elasticsearch_kibana") - @test(depends_on_groups=['deploy_ha_elasticsearch_kibana'], groups=["check_scaling_elasticsearch_kibana", "scaling", "elasticsearch_kibana", "system", @@ -126,7 +120,6 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): 7. Run OSTF Duration 120m - Snapshot add_remove_elasticsearch_kibana_node """ self.env.revert_snapshot("deploy_ha_elasticsearch_kibana") @@ -152,7 +145,39 @@ class TestNodesElasticsearchPlugin(api.ElasticsearchPluginApi): self.helpers.run_ostf() - self.env.make_snapshot("add_remove_elasticsearch_kibana_node") + @test(depends_on_groups=["deploy_ha_elasticsearch_kibana"], + groups=["check_failover_elasticsearch_kibana" "failover", + "elasticsearch_kibana", "system", "destructive", + "shutdown_elasticsearch_kibana_node"]) + @log_snapshot_after_test + def shutdown_elasticsearch_kibana_node(self): + """Verify that failover for Elasticsearch cluster works. + + Scenario: + 1. Shutdown node were es_vip_mgmt was started. + 2. Check that es_vip_mgmt was started on another + elasticsearch_kibana node. + 3. Check that plugin is working. + 4. Check that no data lost after shutdown. + 5. Run OSTF. + + Duration 30m + """ + self.env.revert_snapshot("deploy_ha_elasticsearch_kibana") + + vip_name = self.helpers.full_vip_name(self.settings.vip_name) + + target_node = self.helpers.get_node_with_vip( + self.settings.role_name, vip_name) + + self.helpers.power_off_node(target_node) + + self.helpers.wait_for_vip_migration( + target_node, self.settings.role_name, vip_name) + + self.check_plugin_online() + + self.helpers.run_ostf() @test(depends_on_groups=['prepare_slaves_3'], groups=["elasticsearch_kibana_createmirror_deploy_plugin", diff --git a/stacklight_tests/helpers/helpers.py b/stacklight_tests/helpers/helpers.py index 1281cad..826452d 100644 --- a/stacklight_tests/helpers/helpers.py +++ b/stacklight_tests/helpers/helpers.py @@ -216,18 +216,46 @@ class PluginHelper(object): if node['fqdn'] in stdout: return node - def hard_shutdown_node(self, fqdn): + @staticmethod + def full_vip_name(vip_name): + return "".join(["vip__", vip_name]) + + def get_node_with_vip(self, role_name, vip, exclude_node=None): + nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( + self.cluster_id, role_name) + lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( + nailgun_nodes) + lma_node = None + if exclude_node: + for node in lma_nodes: + if node.name != exclude_node.name: + lma_node = node + break + else: + lma_node = lma_nodes[0] + return self.fuel_web.get_pacemaker_resource_location( + lma_node.name, vip)[0] + + def wait_for_vip_migration(self, old_master, role_name, vip, + timeout=5 * 60): + logger.info('Waiting for the migration of VIP {}'.format(vip)) + msg = "VIP {0} has not been migrated away from {1}".format( + vip, old_master) + helpers.wait( + lambda: old_master != self.get_node_with_vip( + role_name, vip, exclude_node=old_master), + timeout=timeout, timeout_msg=msg) + + def power_off_node(self, node): """Power off a node. """ - devops_node = self.fuel_web.get_devops_node_by_nailgun_fqdn( - fqdn) msg = 'Node {0} has not become offline after hard shutdown'.format( - devops_node.name) - logger.info('Destroy node %s', devops_node.name) - devops_node.destroy() - logger.info('Wait a %s node offline status', devops_node.name) + node.name) + logger.info('Power off node %s', node.name) + node.destroy() + logger.info('Wait a %s node offline status', node.name) helpers.wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node( - devops_node)['online'], timeout=60 * 5, timeout_msg=msg) + node)['online'], timeout=60 * 5, timeout_msg=msg) def emulate_whole_network_disaster(self, delay_before_recover=5 * 60, wait_become_online=True): diff --git a/stacklight_tests/influxdb_grafana/api.py b/stacklight_tests/influxdb_grafana/api.py index b110138..62f249d 100644 --- a/stacklight_tests/influxdb_grafana/api.py +++ b/stacklight_tests/influxdb_grafana/api.py @@ -105,19 +105,6 @@ class InfluxdbPluginApi(base_test.PluginApi): count, nodes_count_responsed) asserts.assert_equal(count, nodes_count_responsed, msg) - def get_influxdb_master_node(self, excluded_nodes_fqdns=()): - influx_master_node = self.helpers.get_master_node_by_role( - self.settings.role_name, excluded_nodes_fqdns=excluded_nodes_fqdns) - return influx_master_node - - def wait_for_rotation_influx_master(self, old_master, timeout=5 * 60): - logger.info('Wait a influxDB master node rotation') - msg = "Failed influxDB master rotation from {0}".format(old_master) - devops_helpers.wait( - lambda: old_master != self.get_influxdb_master_node( - excluded_nodes_fqdns=(old_master,))['fqdn'], - timeout=timeout, timeout_msg=msg) - def wait_plugin_online(self, timeout=5 * 60): def check_availability(): try: diff --git a/stacklight_tests/influxdb_grafana/test_system.py b/stacklight_tests/influxdb_grafana/test_system.py index dbaa0e8..836592e 100644 --- a/stacklight_tests/influxdb_grafana/test_system.py +++ b/stacklight_tests/influxdb_grafana/test_system.py @@ -164,23 +164,26 @@ class TestNodesInfluxdbPlugin(api.InfluxdbPluginApi): """Verify that failover for InfluxDB cluster works. Scenario: - 1. Revert snapshot with 9 deployed nodes in HA configuration - 2. Determine influx_db master node were vip_influxdb was started - 3. Shutdown influx_db master node - 4. Check that vip_influxdb was started on another node - 5. Check that plugin is working - 6. Check that no data lost after shutdown - 7. Run OSTF + 1. Shutdown node were vip_influxdb was started. + 2. Check that vip_influxdb was started on another influxdb_grafana + node. + 3. Check that plugin is working. + 4. Check that no data lost after shutdown. + 5. Run OSTF. Duration 30m """ self.env.revert_snapshot("deploy_ha_influxdb_grafana") - master_node_hostname = self.get_influxdb_master_node()['fqdn'] + vip_name = self.helpers.full_vip_name(self.settings.vip_name) - self.helpers.hard_shutdown_node(master_node_hostname) + target_node = self.helpers.get_node_with_vip( + self.settings.role_name, vip_name) - self.wait_for_rotation_influx_master(master_node_hostname) + self.helpers.power_off_node(target_node) + + self.helpers.wait_for_vip_migration( + target_node, self.settings.role_name, vip_name) self.check_plugin_online() diff --git a/stacklight_tests/lma_infrastructure_alerting/api.py b/stacklight_tests/lma_infrastructure_alerting/api.py index e5c33b6..b391ab3 100644 --- a/stacklight_tests/lma_infrastructure_alerting/api.py +++ b/stacklight_tests/lma_infrastructure_alerting/api.py @@ -49,21 +49,6 @@ class InfraAlertingPluginApi(base_test.PluginApi): self.settings.nagios_password, self.get_plugin_vip()) - def get_primary_lma_node(self, exclude=None): - nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( - self.helpers.cluster_id, self.settings.role_name) - lma_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( - nailgun_nodes) - if exclude: - for node in lma_nodes: - if node.name != exclude: - lma_node = node - break - else: - lma_node = lma_nodes[0] - return self.fuel_web.get_pacemaker_resource_location( - lma_node.name, "vip__infrastructure_alerting_mgmt_vip")[0] - def open_nagios_page(self, link_text, anchor): driver = self.ui_tester.get_driver(self.get_nagios_url(), "//frame[2]", "Nagios Core") diff --git a/stacklight_tests/lma_infrastructure_alerting/test_system.py b/stacklight_tests/lma_infrastructure_alerting/test_system.py index 164a191..b219546 100644 --- a/stacklight_tests/lma_infrastructure_alerting/test_system.py +++ b/stacklight_tests/lma_infrastructure_alerting/test_system.py @@ -14,7 +14,6 @@ # under the License. from fuelweb_test.helpers.decorators import log_snapshot_after_test -from proboscis import asserts from proboscis import test from stacklight_tests.lma_infrastructure_alerting import api @@ -143,28 +142,26 @@ class TestLMAInfraAlertingPluginSystem(api.InfraAlertingPluginApi): "lma_infrastructure_alerting", "shutdown"]) @log_snapshot_after_test def shutdown_infrastructure_alerting_node(self): - """Shutdown infrastructure alerting node + """Verify that failover for LMA Infrastructure Alerting cluster works. Scenario: - 1. Connect to any infrastructure_alerting node and run - command 'crm status'. - 2. Shutdown node were vip_infrastructure_alerting_mgmt_vip + 1. Shutdown node were vip_infrastructure_alerting_mgmt_vip was started. - 3. Check that vip_infrastructure_alerting was started + 2. Check that vip_infrastructure_alerting was started on another infrastructure_alerting node. - 4. Check that Nagios UI works correctly. - 5. Check that no data lost after shutdown. - 6. Run OSTF. + 3. Check that plugin is working. + 4. Check that no data lost after shutdown. + 5. Run OSTF. - Duration 60m + Duration 30m """ self.env.revert_snapshot("deploy_ha_lma_infrastructure_alerting") - - target_node = self.get_primary_lma_node() - self.fuel_web.warm_shutdown_nodes([target_node]) - new_node = self.get_primary_lma_node(target_node.name) - asserts.assert_not_equal(target_node, new_node) - + vip_name = self.helpers.full_vip_name(self.settings.vip_name) + target_node = self.helpers.get_node_with_vip( + self.settings.role_name, vip_name) + self.helpers.power_off_node(target_node) + self.helpers.wait_for_vip_migration( + target_node, self.settings.role_name, vip_name) self.check_plugin_online() self.helpers.run_ostf()