Add datamodel time_delta, metrics and status with influxdb

Change-Id: I2914c7f488e3becb5fd5d63a92dfc87798d364e5

doc/source/webapi/v2/status.rst

@@ -20,9 +20,6 @@ Hosts
 .. rest-controller:: surveil.api.controllers.v2.status.hosts:ConfigController
    :webprefix: /v2/status/hosts/(host_name)/config
 
-.. rest-controller:: surveil.api.controllers.v2.status.metrics:MetricsController
-   :webprefix: /v2/status/hosts/(host_name)/metrics
-
 .. rest-controller:: surveil.api.controllers.v2.logs:LogsController
    :webprefix: /v2/status/hosts/(host_name)/events
 
@@ -45,15 +42,6 @@ Services
    :webprefix: /v2/status/services
 
 
-Metrics
-=======
-
-.. rest-controller:: surveil.api.controllers.v2.status.metrics:MetricsController
-   :webprefix: /v2/status/metrics
-
-.. rest-controller:: surveil.api.controllers.v2.status.metrics:MetricController
-   :webprefix: /v2/status/metrics/
-
 types documentation
 ===================
 

surveil/api/controllers/v2/status/__init__.py

@@ -15,7 +15,6 @@
 from pecan import rest
 
 from surveil.api.controllers.v2.status import hosts as v2_hosts
-from surveil.api.controllers.v2.status import metrics
 from surveil.api.controllers.v2.status import services as v2_services
 
 
@@ -23,4 +22,3 @@ class StatusController(rest.RestController):
     # events = EventsController()
     hosts = v2_hosts.HostsController()
     services = v2_services.ServicesController()
-    metrics = metrics.MetricsController()

surveil/api/controllers/v2/status/hosts.py

@@ -17,12 +17,14 @@ from pecan import rest
 import wsmeext.pecan as wsme_pecan
 
 from surveil.api.controllers.v2 import logs
-from surveil.api.controllers.v2.status import metrics
 from surveil.api.datamodel.status import live_host
 from surveil.api.datamodel.status import live_query
 from surveil.api.datamodel.status import live_service
+from surveil.api.datamodel.status.metrics import live_metric
+from surveil.api.datamodel.status.metrics import time_delta
 from surveil.api.handlers.status import live_host_handler
 from surveil.api.handlers.status import live_service_handler
+from surveil.api.handlers.status.metrics import live_metric_handler
 from surveil.common import util
 
 
@@ -65,8 +67,24 @@ class HostServicesController(rest.RestController):
         return HostServiceController(service_name), remainder
 
 
+class HostServiceMetricsController(rest.RestController):
+
+    @pecan.expose()
+    def _lookup(self, metric_name, *remainder):
+        return HostServiceMetricController(metric_name), remainder
+
+
+class HostMetricsController(rest.RestController):
+
+    @pecan.expose()
+    def _lookup(self, metric_name, *remainder):
+        return HostMetricController(metric_name), remainder
+
+
 class HostServiceController(rest.RestController):
 
+    metrics = HostServiceMetricsController()
+
     def __init__(self, service_name):
         pecan.request.context['service_name'] = service_name
         self.service_name = service_name
@@ -83,6 +101,72 @@ class HostServiceController(rest.RestController):
         return service
 
 
+class HostServiceMetricController(rest.RestController):
+
+    def __init__(self, metric_name):
+        pecan.request.context['metric_name'] = metric_name
+        self.metric_name = metric_name
+
+    @util.policy_enforce(['authenticated'])
+    @wsme_pecan.wsexpose(live_metric.LiveMetric)
+    def get(self):
+        """Return the last measure for the metric name
+
+            of the service name on the host name.
+            """
+        handler = live_metric_handler.MetricHandler(pecan.request)
+        metric = handler.get(
+            self.metric_name,
+            pecan.request.context['host_name'],
+            pecan.request.context['service_name']
+        )
+        return metric
+
+    @util.policy_enforce(['authenticated'])
+    @wsme_pecan.wsexpose([live_metric.LiveMetric], body=time_delta.TimeDelta)
+    def post(self, time):
+        """Given a LiveQuery, returns all matching s."""
+        handler = live_metric_handler.MetricHandler(pecan.request)
+        metrics = handler.get_all(time_delta=time,
+                                  metric_name=self.metric_name,
+                                  host_name=pecan.request.context['host_name'],
+                                  service_description=pecan.request.
+                                  context['service_name'])
+        return metrics
+
+
+class HostMetricController(rest.RestController):
+
+    def __init__(self, metric_name):
+        pecan.request.context['metric_name'] = metric_name
+        self.metric_name = metric_name
+
+    @util.policy_enforce(['authenticated'])
+    @wsme_pecan.wsexpose(live_metric.LiveMetric)
+    def get(self):
+        """Return the last measure for the metric name
+
+        of the service name on the host name
+        """
+        handler = live_metric_handler.MetricHandler(pecan.request)
+        metric = handler.get(
+            self.metric_name,
+            pecan.request.context['host_name']
+        )
+        return metric
+
+    @util.policy_enforce(['authenticated'])
+    @wsme_pecan.wsexpose([live_metric.LiveMetric], body=time_delta.TimeDelta)
+    def post(self, time):
+        """Given a LiveQuery, returns all matching s."""
+        handler = live_metric_handler.MetricHandler(pecan.request)
+        metrics = handler.get_all(time_delta=time,
+                                  metric_name=self.metric_name,
+                                  host_name=pecan.request.context['host_name']
+                                  )
+        return metrics
+
+
 class HostController(rest.RestController):
 
     services = HostServicesController()
@@ -90,7 +174,7 @@ class HostController(rest.RestController):
     # externalcommands = ExternalCommandsController()
     # config = config.ConfigController()
     events = logs.LogsController()
-    metrics = metrics.MetricsController()
+    metrics = HostMetricsController()
 
     def __init__(self, host_name):
         pecan.request.context['host_name'] = host_name

surveil/api/controllers/v2/status/metrics/__init__.py

@@ -1,69 +0,0 @@
-# Copyright 2014 - Savoir-Faire Linux inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import pecan
-from pecan import rest
-
-
-from surveil.common import util
-
-
-class MetricsController(rest.RestController):
-
-    @util.policy_enforce(['authenticated'])
-    @pecan.expose()
-    def get_all(self):
-        """Returns all metrics."""
-        host_name = pecan.request.context.get("host_name")
-        if host_name is not None:
-            return '{"host_name": "%s",  "metrics" : "22"}' % host_name
-
-        return '{"host_name": "NOHOSTNAME",  "metrics" : "22"}'
-
-    @util.policy_enforce(['authenticated'])
-    @pecan.expose()
-    def _lookup(self, *args):
-        props = {}
-        leftovers = list(args)
-        # print leftovers
-        for attr in ["host_name", "service_description", "metric"]:
-            value = pecan.request.context.get(attr)
-            if value is not None:
-                props[attr] = value
-            else:
-                if len(leftovers) > 0:
-                    props[attr] = leftovers[0]
-                    leftovers.pop(0)
-                else:
-                    props[attr] = None
-
-        return MetricController(**props), leftovers
-
-
-class MetricController(rest.RestController):
-
-    def __init__(self, host_name, service_description=None, metric=None):
-        self._id = host_name
-        self.sd = service_description
-        self.metric = metric
-
-    @util.policy_enforce(['authenticated'])
-    @pecan.expose()
-    def get(self):
-        """Returns (specific) metrics."""
-
-        output = '{"hn": %s, "sd": %s, "metric":%s}' % (
-                 self._id, self.sd, self.metric)
-
-        return output

surveil/api/datamodel/status/metrics/live_metric.py

@@ -0,0 +1,54 @@
+# Copyright 2014 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import wsme
+import wsme.types as wtypes
+
+from surveil.api.datamodel import types
+
+
+class LiveMetric(types.Base):
+
+    metric_name = wsme.wsattr(wtypes.text, mandatory=True)
+    """Name of the metric"""
+
+    max = wsme.wsattr(wtypes.text, mandatory=False)
+    """Maximum value for the metric"""
+
+    min = wsme.wsattr(wtypes.text, mandatory=False)
+    """Minimal value for the metric"""
+
+    critical = wsme.wsattr(wtypes.text, mandatory=False)
+    """Critical value for the metric"""
+
+    warning = wsme.wsattr(wtypes.text, mandatory=False)
+    """Warning value for the metric"""
+
+    value = wsme.wsattr(wtypes.text, mandatory=False)
+    """Current value of the metric"""
+
+    unit = wsme.wsattr(wtypes.text, mandatory=False)
+    """Unit of the metric"""
+
+    @classmethod
+    def sample(cls):
+        return cls(
+            metric_name='pl',
+            max='100',
+            min='0',
+            critical='100',
+            warning='100',
+            value='0',
+            unit='s'
+        )

surveil/api/datamodel/status/metrics/time_delta.py

@@ -0,0 +1,35 @@
+# Copyright 2014 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import wsme
+import wsme.types as wtypes
+
+from surveil.api.datamodel import types
+
+
+class TimeDelta(types.Base):
+    """Hold a time."""
+
+    begin = wsme.wsattr(wtypes.text, mandatory=True)
+    "The begin time of a measure in RFC3339."
+
+    end = wsme.wsattr(wtypes.text, mandatory=True)
+    "The end time of a measure in RFC3339."
+
+    @classmethod
+    def sample(cls):
+        return cls(
+            begin='2015-01-29T21:50:44Z',
+            end='2015-01-29T22:50:44Z'
+        )

surveil/api/handlers/status/metrics/influxdb_time_query.py

@@ -0,0 +1,43 @@
+# Copyright 2014 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+def build_influxdb_query(metric_name,
+                         time_delta,
+                         host_name=None,
+                         service_description=None
+                         ):
+    group_by = []
+    query = ['SELECT max,min,warning,critical,value,unit FROM metric_%s'
+             % metric_name]
+    begin = time_delta.begin
+    end = time_delta.end
+    query.append("WHERE time >= '%s' AND time <= '%s'" % (begin, end))
+
+    if host_name is None:
+        group_by.append('host_name')
+    else:
+        query.append("AND host_name ='%s'" % host_name)
+
+    if service_description is None:
+        group_by.append('service_description')
+    else:
+        query.append("AND service_description ='%s'" % service_description)
+
+    if len(group_by) != 0:
+        query.append('GROUP BY')
+        query.append(', '.join(group_by))
+
+    query.append('ORDER BY time DESC')
+    return ' '.join(query)

surveil/api/handlers/status/metrics/live_metric_handler.py

@@ -0,0 +1,95 @@
+# Copyright 2014 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from surveil.api.datamodel.status.metrics import live_metric
+from surveil.api.handlers import handler
+from surveil.api.handlers.status.metrics import influxdb_time_query
+
+
+class MetricHandler(handler.Handler):
+    """Fulfills a request on the metrics."""
+
+    def get(self, metric_name, host_name, service_description=None):
+        """Return a metric."""
+        cli = self.request.influxdb_client
+
+        if service_description is None:
+            query = ("SELECT max,min,warning,critical,value,unit "
+                     "FROM metric_%s "
+                     "WHERE host_name= '%s' "
+                     "GROUP BY service_description "
+                     "ORDER BY time DESC "
+                     "LIMIT 1" % (metric_name, host_name))
+        else:
+            query = ("SELECT max,min,warning,critical,value,unit "
+                     "FROM metric_%s "
+                     "WHERE host_name= '%s' "
+                     "AND service_description= '%s'"
+                     "ORDER BY time DESC "
+                     "LIMIT 1" % (metric_name, host_name, service_description))
+
+        response = cli.query(query)
+        metric = live_metric.LiveMetric(
+            **self._metric_dict_from_influx_item(response.items()[0],
+                                                 metric_name)
+        )
+
+        return metric
+
+    def get_all(self, metric_name, time_delta, host_name=None,
+                service_description=None):
+        """Return all metrics."""
+
+        cli = self.request.influxdb_client
+        query = influxdb_time_query.build_influxdb_query(
+            metric_name,
+            time_delta,
+            host_name,
+            service_description
+        )
+        response = cli.query(query)
+
+        metric_dicts = []
+
+        for item in response.items():
+            metric_dict = self._metric_dict_from_influx_item(item, metric_name)
+            metric_dicts.append(metric_dict)
+
+        metrics = []
+        for metric_dict in metric_dicts:
+            metric = live_metric.LiveMetric(**metric_dict)
+            metrics.append(metric)
+
+        return metrics
+
+    def _metric_dict_from_influx_item(self, item, metric_name):
+        points = item[1]
+        first_point = next(points)
+        metric_dict = {"metric_name": str(metric_name)}
+        mappings = [
+            ('min', str),
+            ('max', str),
+            ('critical', str),
+            ('warning', str),
+            ('value', str),
+            ('unit', str),
+        ]
+
+        for field in mappings:
+            value = first_point.get(field[0], None)
+            if value is not None:
+                metric_dict[field[0]] = field[1](value)
+
+        return metric_dict

surveil/tests/api/controllers/v2/status/test_hosts_metric.py

@@ -0,0 +1,164 @@
+# Copyright 2015 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+import httpretty
+
+from surveil.tests.api import functionalTest
+
+
+class TestHostMetric(functionalTest.FunctionalTest):
+    def setUp(self):
+        super(TestHostMetric, self).setUp()
+        self.influxdb_response = json.dumps({
+            "results": [
+                {
+                    "series": [
+                        {"name": "metric_load1",
+                         "tags": {"host_name": "srv-monitoring-01",
+                                  "service_description": "load",
+                                  },
+                         "columns": [
+                             "time",
+                             "critical",
+                             "min",
+                             "value",
+                             "warning"
+                         ],
+                         "values": [
+                             ["2015-04-19T01:09:24Z",
+                              "30",
+                              "0",
+                              "0.60",
+                              "15"]
+                         ]},
+                        {"name": "metric_load1",
+                         "tags": {"host_name": "test_keystone",
+                                  "service_description": "unload"},
+                         "columns": [
+                             "time",
+                             "critical",
+                             "min",
+                             "value",
+                             "warning"
+                         ],
+                         "values": [
+                             ["2015-04-19T01:09:23Z",
+                              "60",
+                              "0",
+                              "1.5",
+                              "20"]
+                         ]},
+                        {"name": "metric_load1",
+                         "tags": {"host_name": "ws-arbiter",
+                                  },
+                         "columns": [
+                             "time",
+                             "critical",
+                             "min",
+                             "value",
+                             "warning"
+                         ],
+                         "values": [
+                             ["2015-04-19T01:09:24Z",
+                              "20",
+                              "0",
+                              "6",
+                              "10"]
+                         ]}
+                    ]
+                }
+            ]
+        })
+
+    @httpretty.activate
+    def test_get_metric_hosts(self):
+        httpretty.register_uri(httpretty.GET,
+                               "http://influxdb:8086/query",
+                               body=self.influxdb_response)
+
+        response = self.get("/v2/status/hosts/srv-monitoring-01/metrics/load1")
+
+        expected = {
+            "metric_name": "load1",
+            "min": "0",
+            "critical": "30",
+            "warning": "15",
+            "value": "0.6"
+        }
+
+        self.assert_count_equal_backport(
+            json.loads(response.body.decode()),
+            expected)
+        self.assertEqual(
+            httpretty.last_request().querystring['q'],
+            ["SELECT max,min,warning,critical,value,unit FROM metric_load1 "
+             "WHERE host_name= 'srv-monitoring-01' "
+             "GROUP BY service_description "
+             "ORDER BY time DESC LIMIT 1"]
+        )
+
+    @httpretty.activate
+    def test_time_hosts(self):
+        self.influxdb_response = json.dumps({
+            "results": [
+                {"series": [
+                    {"name": "metric_load1",
+                     "tags": {"host_name": "srv-monitoring-01",
+                              "service_description": "load"},
+                     "columns": ["time",
+                                 "critical",
+                                 "min",
+                                 "value",
+                                 "warning",
+                                 ],
+                     "values": [["2015-04-19T01:09:24Z",
+                                 "30",
+                                 "0",
+                                 "0.6",
+                                 "15"]]}]}]
+
+        })
+        httpretty.register_uri(httpretty.GET,
+                               "http://influxdb:8086/query",
+                               body=self.influxdb_response)
+
+        time = {'begin': '2015-04-19T00:09:24Z',
+                'end': '2015-04-19T02:09:24Z'}
+
+        response = self.post_json("/v2/status/hosts/srv-monitoring-01/"
+                                  "services/load/metrics/load1",
+                                  params=time)
+
+        expected = [{"metric_name": 'load1',
+                     "min": "0",
+                     "critical": "30",
+                     "warning": "15",
+                     "value": "0.6"
+                     }]
+
+        self.assert_count_equal_backport(
+            json.loads(response.body.decode()),
+            expected)
+        self.assertEqual(
+            httpretty.last_request().querystring['q'],
+            ["SELECT max,min,warning,critical,value,unit FROM metric_load1 "
+             "WHERE time >= '2015-04-19T00:09:24Z' "
+             "AND time <= '2015-04-19T02:09:24Z' "
+             "AND host_name ='srv-monitoring-01' "
+             "AND service_description ='load' "
+             "ORDER BY time DESC"
+             ]
+        )

surveil/tests/api/handlers/live/test_influxdb_time_query.py

@@ -0,0 +1,80 @@
+# Copyright 2015 - Savoir-Faire Linux inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from surveil.api.datamodel.status.metrics import time_delta
+from surveil.api.handlers.status.metrics import influxdb_time_query
+from surveil.tests import base
+
+
+class InfluxdbTimeQueryTest(base.BaseTestCase):
+    def test_build_query_basic(self):
+        query_time = time_delta.TimeDelta(begin='2015-01-29T21:50:44Z',
+                                          end='2015-01-29T22:50:44Z', )
+        query_metric_name = 'pl'
+
+        result = influxdb_time_query.build_influxdb_query(query_metric_name,
+                                                          query_time
+                                                          )
+        expected = ("SELECT max,min,value,warning,critical,unit "
+                    "FROM metric_pl "
+                    "WHERE time >= '2015-01-29T21:50:44Z' "
+                    "AND time <= '2015-01-29T22:50:44Z' "
+                    "GROUP BY host_name, "
+                    "service_description ORDER BY time DESC")
+
+        self.assert_count_equal_backport(result, expected)
+
+    def test_build_query_host_name(self):
+        query_time = time_delta.TimeDelta(begin='2015-01-29T21:50:44Z',
+                                          end='2015-01-29T22:50:44Z', )
+        query_metric_name = 'pl'
+        query_host_name = 'localhost'
+
+        result = influxdb_time_query.build_influxdb_query(query_metric_name,
+                                                          query_time,
+                                                          query_host_name
+                                                          )
+        expected = ("SELECT max,min,value,warning,critical,unit "
+                    "FROM metric_pl "
+                    "WHERE time >= '2015-01-29T21:50:44Z' "
+                    "AND time <= '2015-01-29T22:50:44Z' "
+                    "AND host_name ='localhost' "
+                    "GROUP BY service_description "
+                    "ORDER BY time DESC")
+
+        self.assert_count_equal_backport(result, expected)
+
+    def test_build_query_complete(self):
+        query_time = time_delta.TimeDelta(begin='2015-01-29T21:50:44Z',
+                                          end='2015-01-29T22:50:44Z', )
+        query_metric_name = 'pl'
+        query_host_name = 'localhost'
+        query_service_description = 'mySQL'
+
+        result = influxdb_time_query.build_influxdb_query(
+            query_metric_name,
+            query_time,
+            query_host_name,
+            query_service_description
+        )
+        expected = ("SELECT max,min,value,warning,critical,unit "
+                    "FROM metric_pl "
+                    "WHERE time >= '2015-01-29T21:50:44Z' "
+                    "AND time <= '2015-01-29T22:50:44Z' "
+                    "AND host_name ='localhost' "
+                    "AND service_description ='mySQL' "
+                    "ORDER BY time DESC")
+
+        self.assert_count_equal_backport(result, expected)