Add test with Nova logs querying

Change-Id: I787b436f3f617d63747625180fb46862d0154c23

requirements.txt

@@ -1,7 +1,8 @@
+elasticsearch
 git+git://github.com/openstack/fuel-devops.git@2.9.20
 PyYAML
 requests
 selenium
 six
 tox
-xvfbwrapper
+xvfbwrapper

stacklight_tests/elasticsearch_kibana/api.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import elasticsearch
 from fuelweb_test import logger
 from proboscis import asserts
 
@@ -20,6 +21,11 @@ from stacklight_tests.elasticsearch_kibana import plugin_settings
 
 
 class ElasticsearchPluginApi(base_test.PluginApi):
+    def __init__(self):
+        super(ElasticsearchPluginApi, self).__init__()
+        self.es = elasticsearch.Elasticsearch([{'host': self.get_plugin_vip(),
+                                                'port': 9200}])
+
     def get_plugin_settings(self):
         return plugin_settings
 
@@ -69,3 +75,16 @@ class ElasticsearchPluginApi(base_test.PluginApi):
     def check_uninstall_failure(self):
         return self.helpers.check_plugin_cannot_be_uninstalled(
             self.settings.name, self.settings.version)
+
+    def get_current_indices(self, index_type):
+        indices = self.es.indices.get_aliases().keys()
+        return filter(lambda x: index_type in x, sorted(indices))[-2:]
+
+    def query_nova_logs(self, indices):
+        query = {"query": {"filtered": {
+            "query": {"bool": {"should": [{"query_string": {
+                "query": "programname:nova*"}}]}},
+            "filter": {"bool": {"must": [{"range": {"Timestamp": {
+                "from": "now-1h"}}}]}}}}, "size": 100}
+        output = self.es.search(index=indices, body=query)
+        return output

stacklight_tests/toolchain/api.py

@@ -119,3 +119,22 @@ class ToolchainApi(object):
                 instances_found=len(updated_metrics),
                 tests_started=len(instance_tests))
         )
+
+    def check_nova_logs(self):
+        indices = self.plugins_mapping[
+            'elasticsearch_kibana'].get_current_indices('log')
+        logger.info("Found indexes {}".format(indices))
+        output = self.plugins_mapping[
+            'elasticsearch_kibana'].query_nova_logs(indices)
+        msg = "Indexes {} don't contain Nova logs"
+        asserts.assert_not_equal(output['hits']['total'], 0, msg.format(
+            indices))
+        controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
+            self.helpers.cluster_id, ["controller"])
+        computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
+            self.helpers.cluster_id, ["compute"])
+        target_nodes = controllers + computes
+        expected_hostnames = set([node["hostname"] for node in target_nodes])
+        actual_hostnames = set([hit['_source']['Hostname']
+                                for hit in output['hits']['hits']])
+        asserts.assert_equal(expected_hostnames, actual_hostnames)

stacklight_tests/toolchain/test_functional.py

@@ -83,3 +83,24 @@ class TestFunctionalToolchain(api.ToolchainApi):
         self.check_plugins_online()
 
         self.check_nova_metrics()
+
+    @test(depends_on_groups=["deploy_ha_toolchain"],
+          groups=["check_nova_logs_in_elasticsearch", "toolchain",
+                  "functional"])
+    @log_snapshot_after_test
+    def check_nova_logs_in_elasticsearch(self):
+        """Check that Nova logs are present in Elasticsearch
+
+        Scenario:
+            1. Revert snapshot with 9 deployed nodes in HA configuration
+            2. Query Nova logs are present in current Elasticsearch index
+            3. Check that Nova logs are collected from all controller and
+               compute nodes
+
+        Duration 10m
+        """
+        self.env.revert_snapshot("deploy_ha_toolchain")
+
+        self.check_plugins_online()
+
+        self.check_nova_logs()