From 274c5b71bf1a699d5cda442a1535387e4d9204e8 Mon Sep 17 00:00:00 2001 From: Niv Oppenhaim Date: Sun, 31 Dec 2017 14:11:28 +0000 Subject: [PATCH] add support for webhooks Implements: blueprint configurable-notifications Change-Id: I0c808c5e44f9d6092d113bb277c8ab8cf0d69716 --- devstack/gate_hook.sh | 2 +- devstack/post_test_hook.sh | 3 +- doc/source/contributor/configuration.rst | 1 + .../contributor/notifier-webhook-plugin.rst | 116 ++++++++ doc/source/contributor/vitrage-api.rst | 269 +++++++++++++++++- vitrage/api/controllers/v1/alarm.py | 8 +- vitrage/api/controllers/v1/root.py | 2 + vitrage/api/controllers/v1/webhook.py | 141 +++++++++ vitrage/api_handler/apis/alarm.py | 13 +- vitrage/api_handler/apis/base.py | 2 - vitrage/api_handler/apis/rca.py | 6 +- vitrage/api_handler/apis/resource.py | 9 +- vitrage/api_handler/apis/topology.py | 5 +- vitrage/api_handler/apis/webhook.py | 153 ++++++++++ vitrage/api_handler/service.py | 4 +- vitrage/common/constants.py | 6 + vitrage/common/policies/__init__.py | 4 +- vitrage/common/policies/webhook.py | 82 ++++++ vitrage/notifier/plugins/webhook/__init__.py | 27 ++ vitrage/notifier/plugins/webhook/utils.py | 26 ++ vitrage/notifier/plugins/webhook/webhook.py | 139 +++++++++ vitrage/opts.py | 9 +- vitrage/storage/base.py | 59 +++- vitrage/storage/impl_sqlalchemy.py | 38 +++ vitrage/storage/sqlalchemy/models.py | 38 ++- .../tests/api/webhook/__init__.py | 15 + .../tests/api/webhook/test_webhook.py | 149 ++++++++++ .../tests/e2e/test_e2e_webhook.py | 268 +++++++++++++++++ .../resources/templates/api/e2e_webhooks.yaml | 29 ++ 29 files changed, 1584 insertions(+), 39 deletions(-) create mode 100644 doc/source/contributor/notifier-webhook-plugin.rst create mode 100644 vitrage/api/controllers/v1/webhook.py create mode 100644 vitrage/api_handler/apis/webhook.py create mode 100644 vitrage/common/policies/webhook.py create mode 100644 vitrage/notifier/plugins/webhook/__init__.py create mode 100644 vitrage/notifier/plugins/webhook/utils.py create mode 100644 vitrage/notifier/plugins/webhook/webhook.py create mode 100644 vitrage_tempest_tests/tests/api/webhook/__init__.py create mode 100644 vitrage_tempest_tests/tests/api/webhook/test_webhook.py create mode 100644 vitrage_tempest_tests/tests/e2e/test_e2e_webhook.py create mode 100644 vitrage_tempest_tests/tests/resources/templates/api/e2e_webhooks.yaml diff --git a/devstack/gate_hook.sh b/devstack/gate_hook.sh index 00a65a431..c8f28110e 100644 --- a/devstack/gate_hook.sh +++ b/devstack/gate_hook.sh @@ -66,7 +66,7 @@ topics = notifications, vitrage_notifications [[post-config|\$VITRAGE_CONF]] [DEFAULT] -notifiers = mistral,nova +notifiers = mistral,nova,webhook [static_physical] changes_interval = 5 diff --git a/devstack/post_test_hook.sh b/devstack/post_test_hook.sh index 03b658bd1..403a9ad61 100644 --- a/devstack/post_test_hook.sh +++ b/devstack/post_test_hook.sh @@ -17,7 +17,8 @@ DEVSTACK_PATH="$BASE/new" #Argument is received from Zuul if [ "$1" = "api" ]; then - TESTS="topology|test_rca|test_alarms|test_resources|test_template" + TESTS="topology|test_rca|test_alarms|test_resources|test_template + |test_webhook" elif [ "$1" = "datasources" ]; then TESTS="datasources|test_events|notifiers|e2e|database" else diff --git a/doc/source/contributor/configuration.rst b/doc/source/contributor/configuration.rst index 1ac989088..22212a0ca 100644 --- a/doc/source/contributor/configuration.rst +++ b/doc/source/contributor/configuration.rst @@ -37,6 +37,7 @@ Notifiers nova-notifier notifier-snmp-plugin mistral-config + notifier-webhook-plugin Machine_Learning diff --git a/doc/source/contributor/notifier-webhook-plugin.rst b/doc/source/contributor/notifier-webhook-plugin.rst new file mode 100644 index 000000000..6776942dc --- /dev/null +++ b/doc/source/contributor/notifier-webhook-plugin.rst @@ -0,0 +1,116 @@ +===================== +Webhook Configuration +===================== + +Vitrage can be configured to support webhooks for the sending of +notifications regarding raised or cleared alarms to any registered target. + +Enable Webhook Notifier +----------------------- + +To enable the webhook plugin, add it to the list of notifiers in +**/etc/vitrage/vitrage.conf** file: + + .. code:: + + [DEFAULT] + notifiers = webhook + +Webhook API +=========== + +Webhooks can be added, listed and deleted from the database using the +following commands: + +Add +--- +To add a new webhook to the database, use the command 'vitrage webhook add'. +The fields are: + ++------------------+-----------------------------------------------------------------+--------------+ +| Name | Description | Required | ++==================+=================================================================+==============+ +| url | The webhook URL to which notifications will be sent | Yes | ++------------------+-----------------------------------------------------------------+--------------+ +| regex_filter | A JSON string to filter for specific events | No | ++------------------+-----------------------------------------------------------------+--------------+ +| headers | A JSON string specifying additional headers to the notification | No | ++------------------+-----------------------------------------------------------------+--------------+ + + +Usage example:: + + vitrage webhook add --url https://www.myserver.com --headers + "{'content-type': 'application/json'}" --regex_filter "{'vitrage_type': + '.*'}" + +- If no regex filter is supplied, all notifications will be sent. +- The defaults headers are : '{'content-type': 'application/json'}' + +Data is sent by the webhook notifier in the following format. + +* notification: ``vitrage.alarm.activate`` or ``vitrage.alarm.deactivate`` +* payload: The alarm data + + +:: + + { + "notification": "vitrage.alarm.activate", + "payload": { + "vitrage_id": "2def31e9-6d9f-4c16-b007-893caa806cd4", + "resource": { + "vitrage_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac", + "name": "app-1-server-1-jz6qvznkmnif", + "update_timestamp": "2018-01-22 10:00:34.327142+00:00", + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "active", + "vitrage_type": "nova.instance", + "vitrage_sample_timestamp": "2018-01-22 10:00:34.327142+00:00", + "vitrage_aggregated_state": "ACTIVE", + "host_id": "iafek-devstack-pre-queens", + "project_id": "8f007e5ba0944e84baa6f2a4f2b5d03a", + "id": "9b7d93b9-94ec-41e1-9cec-f28d4f8d702c" + }, + "severity": "warning", + "update_timestamp": "2018-01-22T10:00:34Z", + "resource_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac", + "vitrage_category": "ALARM", + "state": "Active", + "vitrage_type": "vitrage", + "vitrage_sample_timestamp": "2018-01-22 10:00:34.366364+00:00", + "vitrage_operational_severity": "WARNING", + "vitrage_aggregated_severity": "WARNING", + "vitrage_resource_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac", + "vitrage_resource_type": "nova.instance", + "name": "Instance memory performance degraded" + } + } + + +Each of the fields listed can be used to filter the data when specifying a +regex filter for the webhook. + + +List +---- +List all webhooks currently in the DB:: + + vitrage webhook list + +Show +---- +Show a webhook with specified id:: + + vitrage webhook show + +ID of webhooks is decided by Vitrage and can be found using the 'list' command + +Delete +------ +Delete a webhook with specified id:: + + vitrage webhook delete + +ID of webhooks is decided by Vitrage and can be found using the 'list' command \ No newline at end of file diff --git a/doc/source/contributor/vitrage-api.rst b/doc/source/contributor/vitrage-api.rst index dd14aa52b..35a4d184b 100755 --- a/doc/source/contributor/vitrage-api.rst +++ b/doc/source/contributor/vitrage-api.rst @@ -1498,7 +1498,7 @@ Resource show Show the details of specified resource. GET /v1/resources/[vitrage_id] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Headers ======= @@ -1562,3 +1562,270 @@ Response Examples "id": "dc35fa2f-4515-1653-ef6b-03b471bb395b", "vitrage_id": "RESOURCE:nova.instance:dc35fa2f-4515-1653-ef6b-03b471bb395b" } + +Webhook List +^^^^^^^^^^^^ +List all webhooks. + +GET /v1/webhook/ +~~~~~~~~~~~~~~~~ + +Headers +======= + +- X-Auth-Token (string, required) - Keystone auth token +- Accept (string) - application/json +- User-Agent (String) +- Content-Type (String): application/json + +Path Parameters +=============== + +None. + +Query Parameters +================ + +None. + +Request Body +============ + +None. + +Request Examples +================ + +:: + + GET /v1/webhook + Host: 135.248.18.122:8999 + User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 + Content-Type: application/json + Accept: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + + +Response Status code +==================== + +- 200 - OK +- 404 - Bad request + +Response Body +============= + +Returns a list with all webhooks. + +Response Examples +================= + +:: + + [ + { + "url":"https://requestb.in/tq3fkvtq", + "headers":"{'content-type': 'application/json'}", + "regex_filter":"{'name':'e2e.*'}", + "created_at":"2018-01-04T12:27:47.000000", + "id":"c35caf11-f34d-440e-a804-0c1a4fdfb95b" + } + ] + +Webhook Show +^^^^^^^^^^^^ +Show the details of specified webhook. + +GET /v1/webhook/[id] +~~~~~~~~~~~~~~~~~~~~ + +Headers +======= + +- X-Auth-Token (string, required) - Keystone auth token +- Accept (string) - application/json +- User-Agent (String) +- Content-Type (String): application/json + +Path Parameters +=============== + +- id. + +Query Parameters +================ + +None. + +Request Body +============ + +None. + +Request Examples +================ + +:: + + GET /v1/resources/`` + Host: 127.0.0.1:8999 + User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 + Accept: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + +Response Status code +==================== + +- 200 - OK +- 404 - Bad request + +Response Body +============= + +Returns details of the requested webhook. + +Response Examples +================= + +:: + + { + "url":"https://requestb.in/tq3fkvtq", + "created_at":"2018-01-04T12:27:47.000000", + "updated_at":null, + "id":"c35caf11-f34d-440e-a804-0c1a4fdfb95b", + "headers":"{'content-type': 'application/json'}", + "regex_filter":"{'name':'e2e.*'}" + } + +Webhook Add +^^^^^^^^^^^ +Add a webhook to the database, to be used by the notifier. + +POST /v1/webhook/ +~~~~~~~~~~~~~~~~~ + +Headers +======= + +- X-Auth-Token (string, required) - Keystone auth token +- Accept (string) - application/json +- User-Agent (String) +- Content-Type (String): application/json + +Path Parameters +=============== + +None. + +Query Parameters +================ + +None. + +Request Body +============ + +A webhook to be added. Will contain the following fields: + ++------------------+-----------------------------------------------------------------+--------------+ +| Name | Description | Required | ++==================+=================================================================+==============+ +| url | The webhook URL to which notifications will be sent | Yes | ++------------------+-----------------------------------------------------------------+--------------+ +| regex_filter | A JSON string to filter for specific events | No | ++------------------+-----------------------------------------------------------------+--------------+ +| headers | A JSON string specifying additional headers to the notification | No | ++------------------+-----------------------------------------------------------------+--------------+ + +- If no regex filter is supplied, all notifications will be sent. +- The defaults headers are : '{'content-type': 'application/json'}' + +Request Examples +================ + +:: + + POST /v1/webhook/ + Host: 135.248.18.122:8999 + User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 + Content-Type: application/json + Accept: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + + +:: + + { + "webhook":{ + "url":"https://requestb.in/tqfkvtqa", + "headers":null, + "regex_filter":"{'name':'e2e.*'}" + } + } + + +Response Status code +==================== + +- 200 - OK +- 400 - Bad request + +Response Body +============= + +Returns webhook details if request was OK, +otherwise returns a detailed error message (e.g. 'headers in bad format'). + +Webhook Delete +^^^^^^^^^^^^^^ +Delete a specified webhook. + +DELETE /v1/webhook/[id] +~~~~~~~~~~~~~~~~~~~~~~~ + +Headers +======= + +- X-Auth-Token (string, required) - Keystone auth token +- Accept (string) - application/json +- User-Agent (String) +- Content-Type (String): application/json + +Path Parameters +=============== + +- id. + +Query Parameters +================ + +None. + +Request Body +============ + +None. + +Request Examples +================ + +:: + + DELETE /v1/resources/`` + Host: 127.0.0.1:8999 + User-Agent: keystoneauth1/2.3.0 python-requests/2.9.1 CPython/2.7.6 + Accept: application/json + X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 + +Response Status code +==================== + +- 200 - OK +- 404 - Bad request + +Response Body +============= + +Returns a success message if the webhook is deleted, otherwise an error +message is returned. \ No newline at end of file diff --git a/vitrage/api/controllers/v1/alarm.py b/vitrage/api/controllers/v1/alarm.py index 9a8b5df54..ae6c559b5 100755 --- a/vitrage/api/controllers/v1/alarm.py +++ b/vitrage/api/controllers/v1/alarm.py @@ -12,9 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. + import json import pecan + from oslo_log import log from oslo_utils import encodeutils from oslo_utils.strutils import bool_from_string @@ -24,6 +26,8 @@ from pecan.core import abort from vitrage.api.controllers.rest import RootRestController from vitrage.api.controllers.v1 import count from vitrage.api.policy import enforce +from vitrage.common.constants import TenantProps +from vitrage.common.constants import VertexProperties as Vprops LOG = log.getLogger(__name__) @@ -36,8 +40,8 @@ class AlarmsController(RootRestController): @pecan.expose('json') def get_all(self, **kwargs): - vitrage_id = kwargs.get('vitrage_id') - all_tenants = kwargs.get('all_tenants', False) + vitrage_id = kwargs.get(Vprops.VITRAGE_ID) + all_tenants = kwargs.get(TenantProps.ALL_TENANTS, False) all_tenants = bool_from_string(all_tenants) if all_tenants: enforce("list alarms:all_tenants", pecan.request.headers, diff --git a/vitrage/api/controllers/v1/root.py b/vitrage/api/controllers/v1/root.py index 148712030..be94fcc5b 100644 --- a/vitrage/api/controllers/v1/root.py +++ b/vitrage/api/controllers/v1/root.py @@ -16,6 +16,7 @@ from vitrage.api.controllers.v1 import rca from vitrage.api.controllers.v1 import resource from vitrage.api.controllers.v1 import template from vitrage.api.controllers.v1 import topology +from vitrage.api.controllers.v1 import webhook class V1Controller(object): @@ -23,5 +24,6 @@ class V1Controller(object): resources = resource.ResourcesController() alarm = alarm.AlarmsController() rca = rca.RCAController() + webhook = webhook.WebhookController() template = template.TemplateController() event = event.EventController() diff --git a/vitrage/api/controllers/v1/webhook.py b/vitrage/api/controllers/v1/webhook.py new file mode 100644 index 000000000..d5abc9da2 --- /dev/null +++ b/vitrage/api/controllers/v1/webhook.py @@ -0,0 +1,141 @@ +# Copyright 2018 - Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import pecan + +from oslo_log import log +from oslo_utils.strutils import bool_from_string +from osprofiler import profiler +from pecan.core import abort + +from vitrage.api.controllers.rest import RootRestController +from vitrage.api.policy import enforce +from vitrage.common.constants import TenantProps + +LOG = log.getLogger(__name__) + + +@profiler.trace_cls("webhook controller", + info={}, hide_args=False, trace_private=False) +class WebhookController(RootRestController): + + @pecan.expose('json') + def get_all(self, **kwargs): + LOG.info('list all webhooks with args: %s', kwargs) + + all_tenants = kwargs.get(TenantProps.ALL_TENANTS, False) + all_tenants = bool_from_string(all_tenants) + + if all_tenants: + enforce('webhook list:all_tenants', pecan.request.headers, + pecan.request.enforcer, {}) + else: + enforce('webhook list', pecan.request.headers, + pecan.request.enforcer, {}) + + try: + return self._get_all(all_tenants) + except Exception as e: + LOG.exception('failed to list webhooks %s', e) + abort(404, str(e)) + + @staticmethod + def _get_all(all_tenants): + webhooks = \ + pecan.request.client.call(pecan.request.context, + 'get_all_webhooks', + all_tenants=all_tenants) + LOG.info(webhooks) + return webhooks + + @pecan.expose('json') + def get(self, id): + LOG.info('Show webhook with id: %s', id) + + enforce('webhook show', pecan.request.headers, + pecan.request.enforcer, {}) + + try: + return self._get(id) + except Exception as e: + LOG.exception('Failed to get webhooks %s', e) + abort(404, str(e)) + + @staticmethod + def _get(id): + webhook = \ + pecan.request.client.call(pecan.request.context, + 'get_webhook', + id=id) + LOG.info(webhook) + if not webhook: + abort(404, "Failed to find webhook with ID: %s" % id) + return webhook + + @pecan.expose('json') + def post(self, **kwargs): + LOG.info("Add webhook with following props: %s" % str( + kwargs)) + enforce('webhook add', pecan.request.headers, + pecan.request.enforcer, {}) + try: + return self._post(**kwargs) + except Exception as e: + LOG.exception('Failed to add webhooks %s', e) + abort(400, str(e)) + + @staticmethod + def _post(**kwargs): + url = kwargs.get('url') + if not url: + abort(400, 'Missing mandatory field: URL') + regex_filter = kwargs.get('regex_filter', None) + headers = kwargs.get('headers', None) + + webhook = \ + pecan.request.client.call(pecan.request.context, + 'add_webhook', + url=url, + regex_filter=regex_filter, + headers=headers) + LOG.info(webhook) + if webhook.get("ERROR"): + abort(400, "Failed to add webhook: %s" % webhook.get("ERROR")) + return webhook + + @pecan.expose('json') + def delete(self, id): + + LOG.info('delete webhook with id: %s', id) + enforce("webhook delete", + pecan.request.headers, + pecan.request.enforcer, + {}) + + try: + return self._delete_registration(id) + except Exception as e: + LOG.exception('Failed to delete webhook %s: ' + '%s' % (id, str(e))) + abort(404, str(e)) + + @staticmethod + def _delete_registration(id): + resource = pecan.request.client.call( + pecan.request.context, + 'delete_webhook', + id=id) + if not resource: + abort(404, "Failed to find resource with ID: %s" % id) + LOG.info("Request returned with: %s" % resource) + return resource diff --git a/vitrage/api_handler/apis/alarm.py b/vitrage/api_handler/apis/alarm.py index fac4837e6..b7c2a3ba2 100755 --- a/vitrage/api_handler/apis/alarm.py +++ b/vitrage/api_handler/apis/alarm.py @@ -20,6 +20,7 @@ from vitrage.api_handler.apis.base import ALARM_QUERY from vitrage.api_handler.apis.base import ALARMS_ALL_QUERY from vitrage.api_handler.apis.base import EntityGraphApisBase from vitrage.common.constants import EntityCategory as ECategory +from vitrage.common.constants import TenantProps from vitrage.common.constants import VertexProperties as VProps from vitrage.entity_graph.mappings.operational_alarm_severity import \ OperationalAlarmSeverity @@ -40,8 +41,8 @@ class AlarmApis(EntityGraphApisBase): LOG.debug("AlarmApis get_alarms - vitrage_id: %s, all_tenants=%s", str(vitrage_id), all_tenants) - project_id = ctx.get(self.TENANT_PROPERTY, None) - is_admin_project = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) + project_id = ctx.get(TenantProps.TENANT, None) + is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) if not vitrage_id or vitrage_id == 'all': if all_tenants: @@ -68,8 +69,8 @@ class AlarmApis(EntityGraphApisBase): LOG.warning('Alarm show - Not found (%s)', vitrage_id) return None - is_admin = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) - curr_project = ctx.get(self.TENANT_PROPERTY, None) + is_admin = ctx.get(TenantProps.IS_ADMIN, False) + curr_project = ctx.get(TenantProps.TENANT, None) alarm_project = alarm.get(VProps.PROJECT_ID) if not is_admin and curr_project != alarm_project: LOG.warning('Alarm show - Authorization failed (%s)', vitrage_id) @@ -80,8 +81,8 @@ class AlarmApis(EntityGraphApisBase): def get_alarm_counts(self, ctx, all_tenants): LOG.debug("AlarmApis get_alarm_counts - all_tenants=%s", all_tenants) - project_id = ctx.get(self.TENANT_PROPERTY, None) - is_admin_project = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) + project_id = ctx.get(TenantProps.TENANT, None) + is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) if all_tenants: alarms = self.entity_graph.get_vertices( diff --git a/vitrage/api_handler/apis/base.py b/vitrage/api_handler/apis/base.py index 693d80be5..cf98d9e86 100644 --- a/vitrage/api_handler/apis/base.py +++ b/vitrage/api_handler/apis/base.py @@ -89,8 +89,6 @@ RESOURCES_ALL_QUERY = { class EntityGraphApisBase(object): - TENANT_PROPERTY = 'tenant' - IS_ADMIN_PROJECT_PROPERTY = 'is_admin' @staticmethod def _get_query_with_project(vitrage_category, project_id, is_admin): diff --git a/vitrage/api_handler/apis/rca.py b/vitrage/api_handler/apis/rca.py index 9dd3223ec..aefcdd0c1 100644 --- a/vitrage/api_handler/apis/rca.py +++ b/vitrage/api_handler/apis/rca.py @@ -15,10 +15,12 @@ from oslo_log import log from osprofiler import profiler + from vitrage.api_handler.apis.base import ALARMS_ALL_QUERY from vitrage.api_handler.apis.base import EDGE_QUERY from vitrage.api_handler.apis.base import EntityGraphApisBase from vitrage.api_handler.apis.base import RCA_QUERY +from vitrage.common.constants import TenantProps from vitrage.graph import Direction @@ -37,8 +39,8 @@ class RcaApis(EntityGraphApisBase): LOG.debug("RcaApis get_rca - root: %s, all_tenants=%s", str(root), all_tenants) - project_id = ctx.get(self.TENANT_PROPERTY, None) - is_admin_project = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) + project_id = ctx.get(TenantProps.TENANT, None) + is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) ga = self.entity_graph.algo found_graph_out = ga.graph_query_vertices(root, diff --git a/vitrage/api_handler/apis/resource.py b/vitrage/api_handler/apis/resource.py index 5e95ebffc..d7a3d92f6 100644 --- a/vitrage/api_handler/apis/resource.py +++ b/vitrage/api_handler/apis/resource.py @@ -19,6 +19,7 @@ from osprofiler import profiler from vitrage.api_handler.apis.base import EntityGraphApisBase from vitrage.api_handler.apis.base import RESOURCES_ALL_QUERY from vitrage.common.constants import EntityCategory +from vitrage.common.constants import TenantProps from vitrage.common.constants import VertexProperties as VProps @@ -37,8 +38,8 @@ class ResourceApis(EntityGraphApisBase): LOG.debug('ResourceApis get_resources - resource_type: %s,' 'all_tenants: %s', str(resource_type), all_tenants) - project_id = ctx.get(self.TENANT_PROPERTY, None) - is_admin_project = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) + project_id = ctx.get(TenantProps.TENANT, None) + is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) if all_tenants: resource_query = RESOURCES_ALL_QUERY @@ -66,8 +67,8 @@ class ResourceApis(EntityGraphApisBase): LOG.warning('Resource show - Not found (%s)', vitrage_id) return None - is_admin = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) - curr_project = ctx.get(self.TENANT_PROPERTY, None) + is_admin = ctx.get(TenantProps.IS_ADMIN, False) + curr_project = ctx.get(TenantProps.TENANT, None) resource_project = resource.get(VProps.PROJECT_ID) if not is_admin and curr_project != resource_project: LOG.warning('Resource show - Authorization failed (%s)', diff --git a/vitrage/api_handler/apis/topology.py b/vitrage/api_handler/apis/topology.py index e723ee1ce..d47bcfb98 100644 --- a/vitrage/api_handler/apis/topology.py +++ b/vitrage/api_handler/apis/topology.py @@ -21,6 +21,7 @@ from vitrage.api_handler.apis.base import EntityGraphApisBase from vitrage.api_handler.apis.base import TOPOLOGY_AND_ALARMS_QUERY from vitrage.common.constants import EdgeProperties as EProps from vitrage.common.constants import EntityCategory +from vitrage.common.constants import TenantProps from vitrage.common.constants import VertexProperties as VProps from vitrage.common.exception import VitrageError from vitrage.datasources.nova.instance import NOVA_INSTANCE_DATASOURCE @@ -41,8 +42,8 @@ class TopologyApis(EntityGraphApisBase): LOG.debug("TopologyApis get_topology - root: %s, all_tenants=%s", str(root), all_tenants) - project_id = ctx.get(self.TENANT_PROPERTY, None) - is_admin_project = ctx.get(self.IS_ADMIN_PROJECT_PROPERTY, False) + project_id = ctx.get(TenantProps.TENANT, None) + is_admin_project = ctx.get(TenantProps.IS_ADMIN, False) ga = self.entity_graph.algo LOG.debug('project_id = %s, is_admin_project %s', diff --git a/vitrage/api_handler/apis/webhook.py b/vitrage/api_handler/apis/webhook.py new file mode 100644 index 000000000..3bab22cfe --- /dev/null +++ b/vitrage/api_handler/apis/webhook.py @@ -0,0 +1,153 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import ast +from collections import namedtuple +import datetime +from oslo_log import log +from oslo_utils import uuidutils +from osprofiler import profiler +import re +from six.moves.urllib.parse import urlparse +from vitrage.common.constants import TenantProps +from vitrage.common.constants import VertexProperties as Vprops +from vitrage.notifier.plugins.webhook.utils import db_row_to_dict +from vitrage import storage +from vitrage.storage.sqlalchemy.models import Webhooks + +LOG = log.getLogger(__name__) + +Result = namedtuple("Result", ["is_valid", "message"]) + + +@profiler.trace_cls("webhook apis", + info={}, hide_args=False, trace_private=False) +class WebhookApis(object): + DELETED_ROWS_SUCCESS = 1 + + def __init__(self, conf): + self.conf = conf + self.db_conn = storage.get_connection_from_config(conf) + + def delete_webhook(self, ctx, id): + + LOG.info("Delete webhook with id: %s", + str(id)) + + deleted_rows_count = self.db_conn.webhooks.delete(id) + + if deleted_rows_count == self.DELETED_ROWS_SUCCESS: + return {'SUCCESS': 'Webhook %s deleted' % id} + else: + return None + + def get_all_webhooks(self, ctx, all_tenants): + LOG.info("List all webhooks") + if all_tenants and ctx.get(TenantProps.IS_ADMIN, False): + res = self.db_conn.webhooks.query() + else: + res = self.db_conn.webhooks.query(project_id=ctx.get( + TenantProps.TENANT, "")) + LOG.info(res) + webhooks = [db_row_to_dict(webhook) for webhook in res] + + return webhooks + + def add_webhook(self, ctx, url, headers=None, regex_filter=None): + res = self._check_valid_webhook(url, headers, regex_filter) + if not res.is_valid: + LOG.exception("Failed to create webhook: %s" % res.message) + return res.message + try: + db_row = self._webhook_to_db_row(url, headers, regex_filter, ctx) + self.db_conn.webhooks.create(db_row) + return db_row_to_dict(db_row) + except Exception as e: + LOG.exception("Failed to add webhook to DB: %s", str(e)) + return {"ERROR": str(e)} + + def get_webhook(self, ctx, id): + try: + webhooks = self.db_conn.webhooks.query(id=id) + # Check that webhook belongs to current tenant or current tenant + # is admin + if len(webhooks) == 0: + LOG.warning("Webhook not found - %s" % id) + return None + if ctx.get(TenantProps.TENANT, "") == \ + webhooks[0][Vprops.PROJECT_ID] or ctx.get( + TenantProps.IS_ADMIN, False): + return (webhooks[0]) + else: + LOG.warning('Webhook show - Authorization failed (%s)', + id) + return None + except Exception as e: + LOG.exception("Failed to get webhook: %s", str(e)) + return {"ERROR": str(e)} + + def _webhook_to_db_row(self, url, headers, regex_filter, ctx): + if not regex_filter: + regex_filter = "" + if not headers: + headers = "" + uuid = uuidutils.generate_uuid() + project_id = ctx.get(TenantProps.TENANT, "") + is_admin = ctx.get(TenantProps.IS_ADMIN, False) + created_at = str(datetime.datetime.now()) + db_row = Webhooks(id=uuid, + project_id=project_id, + is_admin_webhook=is_admin, + created_at=created_at, + url=url, + headers=headers, + regex_filter=regex_filter) + return db_row + + def _check_valid_webhook(self, url, headers, regex_filter): + if not self._validate_url(url): + return Result(False, {"ERROR": "Invalid URL"}) + elif not self._validate_headers(headers): + return Result(False, {"ERROR": "Headers in invalid format"}) + elif not self._validate_regex(regex_filter): + return Result(False, {"ERROR": "Invalid RegEx"}) + return Result(True, "") + + def _validate_url(self, url): + try: + result = urlparse(url) + if not result.scheme or not result.netloc: + return False + except Exception: + return False + return True + + def _validate_regex(self, regex_filter): + if regex_filter: + try: + filter_dict = ast.literal_eval(regex_filter) + if not isinstance(filter_dict, dict): + return False + for filter in filter_dict.values(): + re.compile(filter) + except Exception: + return False + return True + + def _validate_headers(self, headers): + if headers: + try: + return isinstance(ast.literal_eval(headers), dict) + except Exception: + return False + return True diff --git a/vitrage/api_handler/service.py b/vitrage/api_handler/service.py index 9836412c6..dece8d2f3 100644 --- a/vitrage/api_handler/service.py +++ b/vitrage/api_handler/service.py @@ -25,6 +25,7 @@ from vitrage.api_handler.apis.rca import RcaApis from vitrage.api_handler.apis.resource import ResourceApis from vitrage.api_handler.apis.template import TemplateApis from vitrage.api_handler.apis.topology import TopologyApis +from vitrage.api_handler.apis.webhook import WebhookApis from vitrage import messaging from vitrage import rpc as vitrage_rpc @@ -55,7 +56,8 @@ class VitrageApiHandlerService(os_service.Service): RcaApis(self.entity_graph, self.conf), TemplateApis(self.notifier), EventApis(self.conf), - ResourceApis(self.entity_graph, self.conf)] + ResourceApis(self.entity_graph, self.conf), + WebhookApis(self.conf)] server = vitrage_rpc.get_server(target, endpoints, transport) diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index aca4dbb57..57a54a1fd 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -169,3 +169,9 @@ class TemplateStatus(object): DELETING = 'DELETING' DELETED = 'DELETED' LOADING = 'LOADING' + + +class TenantProps(object): + ALL_TENANTS = 'all_tenants' + TENANT = 'tenant' + IS_ADMIN = 'is_admin' diff --git a/vitrage/common/policies/__init__.py b/vitrage/common/policies/__init__.py index 98b29bab1..e584e36be 100644 --- a/vitrage/common/policies/__init__.py +++ b/vitrage/common/policies/__init__.py @@ -18,6 +18,7 @@ from vitrage.common.policies import rca from vitrage.common.policies import resource from vitrage.common.policies import template from vitrage.common.policies import topology +from vitrage.common.policies import webhook def list_rules(): @@ -27,5 +28,6 @@ def list_rules(): rca.list_rules(), template.list_rules(), topology.list_rules(), - resource.list_rules() + resource.list_rules(), + webhook.list_rules() ) diff --git a/vitrage/common/policies/webhook.py b/vitrage/common/policies/webhook.py new file mode 100644 index 000000000..2260b967d --- /dev/null +++ b/vitrage/common/policies/webhook.py @@ -0,0 +1,82 @@ +# Copyright 2018 - Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +from vitrage.common.policies import base + +webhook = 'webhook %s' + +rules = [ + policy.DocumentedRuleDefault( + name=webhook % 'delete', + check_str=base.UNPROTECTED, + description='Delete a webhook registration', + operations=[ + { + 'path': '/webhook/{webhook_id}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=webhook % 'list', + check_str=base.UNPROTECTED, + description='List all webhook registrations', + operations=[ + { + 'path': '/webhook', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=webhook % 'list:all_tenants', + check_str=base.ROLE_ADMIN, + description='List all webhooks (if the user' + ' has the permissions)', + operations=[ + { + 'path': '/webhook', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=webhook % 'show', + check_str=base.UNPROTECTED, + description='Show a webhook registration with a given id', + operations=[ + { + 'path': '/webhook/{webhook_id}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=webhook % 'add', + check_str=base.UNPROTECTED, + description='Add a webhook registration with given info', + operations=[ + { + 'path': '/webhook', + 'method': 'POST' + } + ] + ) +] + + +def list_rules(): + return rules diff --git a/vitrage/notifier/plugins/webhook/__init__.py b/vitrage/notifier/plugins/webhook/__init__.py new file mode 100644 index 000000000..5df598252 --- /dev/null +++ b/vitrage/notifier/plugins/webhook/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2018 - Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +OPTS = [ + cfg.StrOpt('notifier', + default='vitrage.notifier.plugins.webhook.' + 'webhook.Webhook', + help='notifier webhook class path', + required=True), + cfg.IntOpt('max_retries', + default=2, + help='rest http post max retries', + required=False), +] diff --git a/vitrage/notifier/plugins/webhook/utils.py b/vitrage/notifier/plugins/webhook/utils.py new file mode 100644 index 000000000..73adf3011 --- /dev/null +++ b/vitrage/notifier/plugins/webhook/utils.py @@ -0,0 +1,26 @@ +# Copyright 2018 - Nokia Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def db_row_to_dict(row): + + return { + 'id': row.id, + 'project_id': row.project_id, + 'is_admin_webhook': row.is_admin_webhook, + 'created_at': row.created_at, + 'url': row.url, + 'regex_filter': row.regex_filter, + 'headers': row.headers + } diff --git a/vitrage/notifier/plugins/webhook/webhook.py b/vitrage/notifier/plugins/webhook/webhook.py new file mode 100644 index 000000000..260c5bfb1 --- /dev/null +++ b/vitrage/notifier/plugins/webhook/webhook.py @@ -0,0 +1,139 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import ast +import re + +from oslo_log import log as logging +from oslo_serialization import jsonutils +from oslo_utils import uuidutils + +import requests + +from vitrage.common.constants import NotifierEventTypes +from vitrage.common.constants import VertexProperties as VProps +from vitrage.notifier.plugins.base import NotifierBase +from vitrage.notifier.plugins.webhook import utils as webhook_utils +from vitrage import storage + +LOG = logging.getLogger(__name__) +URL = 'url' +IS_ADMIN_WEBHOOK = 'is_admin_webhook' +FIELDS_TO_REMOVE = (VProps.VITRAGE_IS_PLACEHOLDER, + VProps.VITRAGE_IS_DELETED, + VProps.IS_REAL_VITRAGE_ID) +NOTIFICATION_TYPE = 'notification_type' +PAYLOAD = 'payload' + + +class Webhook(NotifierBase): + + @staticmethod + def get_notifier_name(): + return 'webhook' + + def __init__(self, conf): + super(Webhook, self).__init__(conf) + self.conf = conf + self._db = storage.get_connection_from_config(self.conf) + self.max_retries = self.conf.webhook.max_retries + self.default_headers = {'content-type': 'application/json'} + + def process_event(self, data, event_type): + + if event_type == NotifierEventTypes.ACTIVATE_ALARM_EVENT \ + or event_type == NotifierEventTypes.DEACTIVATE_ALARM_EVENT: + + LOG.info('Webhook API starting to process %s', str(data)) + + webhooks = self._load_webhooks() + if webhooks: + for webhook in webhooks: + webhook_filters = self._get_webhook_filters(webhook) + data = self._filter_fields(data) + if self._check_against_filter(webhook_filters, data)\ + and self._check_correct_tenant(webhook, data): + self._post_data(webhook, event_type, data) + + LOG.info('Webhook API finished processing %s', str(data)) + + def _post_data(self, webhook, event_type, data): + try: + webhook_data = {'notification': event_type, 'payload': data} + webhook_headers = self._get_webhook_headers(webhook) + session = requests.Session() + session.mount(str(webhook[URL]), + requests.adapters.HTTPAdapter( + max_retries=self.max_retries)) + resp = session.post(str(webhook[URL]), + data=jsonutils.dumps(webhook_data), + headers=webhook_headers) + LOG.info('posted %s to %s. Response status %s, reason %s', + str(webhook_data), str(webhook[URL]), + resp.status_code, resp.reason) + except Exception as e: + LOG.exception("Could not post to webhook %s %s" % ( + str(webhook['id']), str(e))) + + def _load_webhooks(self): + db_webhooks = self._db.webhooks.query() + return [webhook_utils.db_row_to_dict(webhook) for webhook in + db_webhooks] + + def _get_webhook_headers(self, webhook): + headers = self.default_headers.copy() + headers['x-openstack-request-id'] = b'req-' + \ + uuidutils.generate_uuid().encode( + 'ascii') + if webhook.get('headers') != '': + headers.update(ast.literal_eval(webhook['headers'])) + return headers + + def _get_webhook_filters(self, webhook): + filters = webhook.get('regex_filter') + if filters != '': + filters = ast.literal_eval(filters) + for k, v in filters.items(): + filters[k] = re.compile(v, re.IGNORECASE) + return filters + return None + + def _check_against_filter(self, webhook_filters, event): + # Check if the event matches the specified filters + if webhook_filters: + for field, filter in webhook_filters.items(): + value = event.get(field) + if value is None: + return False + elif filter.match(value) is None: + return False + return True + + def _filter_fields(self, data): + data = {k: v for k, v in data.items() if k not in FIELDS_TO_REMOVE} + if data.get(VProps.RESOURCE): + data[VProps.RESOURCE] = \ + self._filter_fields(data[VProps.RESOURCE]) + return data + + def _check_correct_tenant(self, webhook, data): + # Check that the resource project ID matches the project ID under + # which the webhook was added. + + if webhook.get(IS_ADMIN_WEBHOOK): + return True + if data.get(VProps.RESOURCE): + if data[VProps.RESOURCE].get(VProps.PROJECT_ID): + return data[VProps.RESOURCE][VProps.PROJECT_ID] == \ + webhook.get(VProps.PROJECT_ID) + return True diff --git a/vitrage/opts.py b/vitrage/opts.py index 0178a47d4..599401407 100644 --- a/vitrage/opts.py +++ b/vitrage/opts.py @@ -27,6 +27,7 @@ import vitrage.machine_learning import vitrage.machine_learning.plugins.jaccard_correlation import vitrage.notifier import vitrage.notifier.plugins.snmp +import vitrage.notifier.plugins.webhook import vitrage.os_clients import vitrage.persistency import vitrage.rpc @@ -56,6 +57,7 @@ def list_opts(): ('jaccard_correlation', vitrage.machine_learning.plugins.jaccard_correlation.OPTS), ('snmp', vitrage.notifier.plugins.snmp.OPTS), + ('webhook', vitrage.notifier.plugins.webhook.OPTS), ('snmp_parsing', vitrage.snmp_parsing.OPTS), ('DEFAULT', itertools.chain( vitrage.os_clients.OPTS, @@ -98,6 +100,8 @@ def _normalize_path_to_datasource_name(path_list, top=os.getcwd()): def register_opts(conf, package_name, paths): """register opts of package package_name, with base path in paths""" for path in paths: + LOG.info("package name: %s" % package_name) + LOG.info("path: % s" % path) try: opt = importutils.import_module( "%s.%s" % (path, package_name)).OPTS @@ -107,6 +111,5 @@ def register_opts(conf, package_name, paths): ) return except ImportError: - pass - - LOG.error("Failed to register config options for %s" % package_name) + LOG.error("Failed to register config options for %s" % + package_name) diff --git a/vitrage/storage/base.py b/vitrage/storage/base.py index 2f05f9e8a..d0e74cd16 100644 --- a/vitrage/storage/base.py +++ b/vitrage/storage/base.py @@ -39,17 +39,21 @@ class Connection(object): def graph_snapshots(self): return None + @property + def webhooks(self): + return None + @abc.abstractmethod def upgrade(self, nocreate=False): - raise NotImplementedError('upgrade not implemented') + raise NotImplementedError('upgrade is not implemented') @abc.abstractmethod def disconnect(self): - raise NotImplementedError('disconnect not implemented') + raise NotImplementedError('disconnect is not implemented') @abc.abstractmethod def clear(self): - raise NotImplementedError('clear not implemented') + raise NotImplementedError('clear is not implemented') @six.add_metaclass(abc.ABCMeta) @@ -60,7 +64,7 @@ class ActiveActionsConnection(object): :type active_action: vitrage.storage.sqlalchemy.models.ActiveAction """ - raise NotImplementedError('create active action not implemented') + raise NotImplementedError('create active action is not implemented') @abc.abstractmethod def update(self, active_action): @@ -68,7 +72,7 @@ class ActiveActionsConnection(object): :type active_action: vitrage.storage.sqlalchemy.models.ActiveAction """ - raise NotImplementedError('update active action not implemented') + raise NotImplementedError('update active action is not implemented') @abc.abstractmethod def query(self, @@ -84,7 +88,7 @@ class ActiveActionsConnection(object): :rtype: list of vitrage.storage.sqlalchemy.models.ActiveAction """ - raise NotImplementedError('query active actions not implemented') + raise NotImplementedError('query active actions is not implemented') @abc.abstractmethod def delete(self, @@ -97,7 +101,40 @@ class ActiveActionsConnection(object): trigger=None, ): """Delete all active actions that match the filters.""" - raise NotImplementedError('delete active actions not implemented') + raise NotImplementedError('delete active actions is not implemented') + + +@six.add_metaclass(abc.ABCMeta) +class WebhooksConnection(object): + + @abc.abstractmethod + def create(self, webhook): + """Create a new webhook. + + :type webhook: + vitrage.storage.sqlalchemy.models.Webhook + """ + raise NotImplementedError('create webhook is not implemented') + + @abc.abstractmethod + def query(self, + id=None, + project_id=None, + is_admin_webhook=None, + url=None, + headers=None, + regex_filter=None, + ): + """Yields a lists of webhooks that match filters. + + :rtype: list of vitrage.storage.sqlalchemy.models.Webhook + """ + raise NotImplementedError('query webhook is not implemented') + + @abc.abstractmethod + def delete(self, id=None): + """Delete all webhooks that match the filters.""" + raise NotImplementedError('delete webhook is not implemented') @six.add_metaclass(abc.ABCMeta) @@ -145,14 +182,14 @@ class EventsConnection(object): :type event: vitrage.storage.sqlalchemy.models.Event """ - raise NotImplementedError('create event not implemented') + raise NotImplementedError('create event is not implemented') def update(self, event): """Update an existing event. :type event: vitrage.storage.sqlalchemy.models.Event """ - raise NotImplementedError('update event not implemented') + raise NotImplementedError('update event is not implemented') def query(self, event_id=None, @@ -164,7 +201,7 @@ class EventsConnection(object): :rtype: list of vitrage.storage.sqlalchemy.models.Event """ - raise NotImplementedError('query events not implemented') + raise NotImplementedError('query events is not implemented') def delete(self, event_id=None, @@ -172,7 +209,7 @@ class EventsConnection(object): gt_collector_timestamp=None, lt_collector_timestamp=None): """Delete all events that match the filters.""" - raise NotImplementedError('delete events not implemented') + raise NotImplementedError('delete events is not implemented') @six.add_metaclass(abc.ABCMeta) diff --git a/vitrage/storage/impl_sqlalchemy.py b/vitrage/storage/impl_sqlalchemy.py index 426b007ba..554f599cf 100644 --- a/vitrage/storage/impl_sqlalchemy.py +++ b/vitrage/storage/impl_sqlalchemy.py @@ -44,6 +44,12 @@ class Connection(base.Connection): self._events = EventsConnection(self._engine_facade) self._templates = TemplatesConnection(self._engine_facade) self._graph_snapshots = GraphSnapshotsConnection(self._engine_facade) + self._webhooks = WebhooksConnection( + self._engine_facade) + + @property + def webhooks(self): + return self._webhooks @property def active_actions(self): @@ -191,6 +197,38 @@ class ActiveActionsConnection(base.ActiveActionsConnection, BaseTableConn): return query.delete() +class WebhooksConnection(base.WebhooksConnection, + BaseTableConn): + def __init__(self, engine_facade): + super(WebhooksConnection, self).__init__(engine_facade) + + def create(self, webhook): + session = self._engine_facade.get_session() + with session.begin(): + session.add(webhook) + + def query(self, + id=None, + project_id=None, + is_admin_webhook=None, + url=None, + headers=None, + regex_filter=None): + query = self.query_filter( + models.Webhooks, + id=id, + project_id=project_id, + is_admin_webhook=is_admin_webhook, + url=url, + headers=headers, + regex_filter=regex_filter) + return query.all() + + def delete(self, id=None): + query = self.query_filter(models.Webhooks, id=id) + return query.delete() + + class EventsConnection(base.EventsConnection, BaseTableConn): def __init__(self, engine_facade): super(EventsConnection, self).__init__(engine_facade) diff --git a/vitrage/storage/sqlalchemy/models.py b/vitrage/storage/sqlalchemy/models.py index 51afb0745..eb385f7e7 100644 --- a/vitrage/storage/sqlalchemy/models.py +++ b/vitrage/storage/sqlalchemy/models.py @@ -16,13 +16,13 @@ import json from oslo_db.sqlalchemy import models from sqlalchemy import Column, DateTime, INTEGER, String, \ - SmallInteger, BigInteger, Index + SmallInteger, BigInteger, Index, Boolean, UniqueConstraint from sqlalchemy.ext.declarative import declarative_base import sqlalchemy.types as types -class VitrageBase(models.ModelBase): +class VitrageBase(models.TimestampMixin, models.ModelBase): """Base class for Vitrage Models.""" __table_args__ = {'mysql_charset': "utf8", 'mysql_engine': "InnoDB"} @@ -161,3 +161,37 @@ class Template(Base, models.TimestampMixin): self.status_details, self.file_content, self.template_type,) + + +class Webhooks(Base): + __tablename__ = 'webhooks' + + id = Column(String(128), primary_key=True) + project_id = Column(String(128), nullable=False) + is_admin_webhook = Column(Boolean, nullable=False) + url = Column(String(256), nullable=False) + headers = Column(String(1024)) + regex_filter = Column(String(512)) + constraint = UniqueConstraint('url', 'regex_filter') + + __table_args__ = (UniqueConstraint('url', 'regex_filter'),) + + def __repr__(self): + return \ + " " %\ + ( + self.id, + self.created_at, + self.project_id, + self.is_admin_webhook, + self.url, + self.headers, + self.regex_filter + ) diff --git a/vitrage_tempest_tests/tests/api/webhook/__init__.py b/vitrage_tempest_tests/tests/api/webhook/__init__.py new file mode 100644 index 000000000..bf9f61d74 --- /dev/null +++ b/vitrage_tempest_tests/tests/api/webhook/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage_tempest_tests/tests/api/webhook/test_webhook.py b/vitrage_tempest_tests/tests/api/webhook/test_webhook.py new file mode 100644 index 000000000..2113f6b83 --- /dev/null +++ b/vitrage_tempest_tests/tests/api/webhook/test_webhook.py @@ -0,0 +1,149 @@ +# Copyright 2017 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from vitrage_tempest_tests.tests.base import BaseVitrageTempest +from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients +from vitrageclient.exceptions import ClientException + +LOG = logging.getLogger(__name__) + +URL = 'url' +REGEX_FILTER = 'regex_filter' +HEADERS = 'headers' +HEADERS_PROPS = '{"content": "application/json"}' +REGEX_PROPS = '{"name": "e2e.*"}' + + +class TestWebhook(BaseVitrageTempest): + """Webhook test class for Vitrage API tests.""" + + @classmethod + def setUpClass(cls): + super(TestWebhook, cls).setUpClass() + cls.pre_test_webhook_count = \ + len(TempestClients.vitrage().webhook.list()) + + def test_add_webhook(self): + + webhooks = TempestClients.vitrage().webhook.list() + self.assertEqual(self.pre_test_webhook_count, + len(webhooks), + 'Amount of webhooks should be the same as ' + 'before the test') + + created_webhook = TempestClients.vitrage().webhook.add( + url="https://www.test.com", + regex_filter=REGEX_PROPS, + headers=HEADERS_PROPS + ) + + self.assertIsNone(created_webhook.get('ERROR'), 'webhook not ' + 'created') + self.assertEqual(created_webhook[HEADERS], + HEADERS_PROPS, + 'headers not created correctly') + self.assertEqual(created_webhook[REGEX_FILTER], + REGEX_PROPS, + 'regex not created correctly') + self.assertEqual(created_webhook[URL], + "https://www.test.com", + 'URL not created correctly') + + webhooks = TempestClients.vitrage().webhook.list() + + self.assertEqual(self.pre_test_webhook_count + 1, len(webhooks)) + TempestClients.vitrage().webhook.delete( + created_webhook['id']) + + def test_delete_webhook(self): + webhooks = TempestClients.vitrage().webhook.list() + self.assertEqual(self.pre_test_webhook_count, + len(webhooks), + 'Amount of webhooks should be the same as ' + 'before the test') + + created_webhook = TempestClients.vitrage().webhook.add( + url="https://www.test.com", + regex_filter=REGEX_PROPS, + headers=HEADERS_PROPS + ) + + created_webhook = TempestClients.vitrage().webhook.delete( + id=created_webhook['id']) + self.assertIsNotNone(created_webhook.get('SUCCESS'), + 'failed to delete') + self.assertEqual(self.pre_test_webhook_count, len(webhooks), + 'No webhooks should exist after deletion') + + def test_delete_non_existing_webhook(self): + self.assertRaises(ClientException, + TempestClients.vitrage().webhook.delete, + ('non existant')) + + def test_list_webhook(self): + + webhooks = TempestClients.vitrage().webhook.list() + self.assertEqual(self.pre_test_webhook_count, + len(webhooks), + 'Amount of webhooks should be the same as ' + 'before the test') + + created_webhook = TempestClients.vitrage().webhook.add( + url="https://www.test.com", + regex_filter=REGEX_PROPS, + headers=HEADERS_PROPS + ) + + webhooks = TempestClients.vitrage().webhook.list() + self.assertEqual(self.pre_test_webhook_count + 1, len(webhooks)) + self.assertEqual(created_webhook[HEADERS], webhooks[0][HEADERS]) + self.assertEqual(created_webhook['id'], webhooks[0]['id']) + self.assertEqual(created_webhook[REGEX_FILTER], + webhooks[0][REGEX_FILTER]) + + TempestClients.vitrage().webhook.delete( + created_webhook['id']) + + def test_show_webhook(self): + webhooks = TempestClients.vitrage().webhook.list() + self.assertEqual(self.pre_test_webhook_count, + len(webhooks), + 'Amount of webhooks should be the same as ' + 'before the test') + + created_webhook = TempestClients.vitrage().webhook.add( + url="https://www.test.com", + regex_filter=REGEX_PROPS, + headers=HEADERS_PROPS + ) + + show_webhook = TempestClients.vitrage().webhook.show( + created_webhook['id'] + ) + + self.assertIsNotNone(show_webhook, 'webhook not listed') + self.assertEqual(created_webhook[HEADERS], + show_webhook[HEADERS], + 'headers mismatch') + self.assertEqual(created_webhook[REGEX_FILTER], + show_webhook[REGEX_FILTER], + 'regex mismatch') + self.assertEqual(created_webhook[URL], + show_webhook[URL], + 'URL mismatch') + + TempestClients.vitrage().webhook.delete( + created_webhook['id']) diff --git a/vitrage_tempest_tests/tests/e2e/test_e2e_webhook.py b/vitrage_tempest_tests/tests/e2e/test_e2e_webhook.py new file mode 100644 index 000000000..c888c17f0 --- /dev/null +++ b/vitrage_tempest_tests/tests/e2e/test_e2e_webhook.py @@ -0,0 +1,268 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +import requests +from six.moves import BaseHTTPServer +import socket +from threading import Thread + +from vitrage.common.constants import VertexProperties as VProps +from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients +from vitrage_tempest_tests.tests.e2e.test_actions_base import TestActionsBase +from vitrage_tempest_tests.tests import utils + +LOG = logging.getLogger(__name__) + +TRIGGER_ALARM_1 = 'e2e.test_webhook.alarm1' +TRIGGER_ALARM_2 = 'e2e.test_webhook.alarm2' +TRIGGER_ALARM_WITH_DEDUCED = 'e2e.test_webhook.alarm_with_deduced' +URL = 'url' +REGEX_FILTER = 'regex_filter' +HEADERS = 'headers' +HEADERS_PROPS = '{"content": "application/json"}' +NAME_FILTER = '{"name": "e2e.*"}' +NAME_FILTER_FOR_DEDUCED = '{"name": "e2e.test_webhook.deduced"}' +TYPE_FILTER = '{"vitrage_type": "doctor"}' +FILTER_NO_MATCH = '{"name": "NO MATCH"}' + + +class TestWebhook(TestActionsBase): + + @classmethod + def setUpClass(cls): + super(TestWebhook, cls).setUpClass() + # Configure mock server. + cls.mock_server_port = _get_free_port() + cls.mock_server = MockHTTPServer(('localhost', cls.mock_server_port), + MockServerRequestHandler) + + # Start running mock server in a separate thread. + cls.mock_server_thread = Thread(target=cls.mock_server.serve_forever) + cls.mock_server_thread.setDaemon(True) + cls.mock_server_thread.start() + cls.URL_PROPS = 'http://localhost:%s/' % cls.mock_server_port + + @utils.tempest_logger + def test_webhook_basic_event(self): + + try: + + # Add webhook with filter matching alarm + TempestClients.vitrage().webhook.add( + url=self.URL_PROPS, + regex_filter=NAME_FILTER, + headers=HEADERS_PROPS + ) + + # Raise alarm + self._trigger_do_action(TRIGGER_ALARM_1) + + # Check event received + self.assertEqual(1, len(self.mock_server.requests), + 'Wrong number of notifications for raise alarm') + + # Undo + self._trigger_undo_action(TRIGGER_ALARM_1) + + # Check event undo received + self.assertEqual(2, len(self.mock_server.requests), + 'Wrong number of notifications for clear alarm') + + finally: + self._delete_webhooks() + self._trigger_undo_action(TRIGGER_ALARM_1) + self.mock_server.reset_requests_list() + + def test_webhook_with_no_filter(self): + """Test to see that a webhook with no filter receives all + + notifications + """ + + try: + + # Add webhook + TempestClients.vitrage().webhook.add( + url=self.URL_PROPS, + regex_filter=NAME_FILTER, + ) + + # Raise alarm + self._trigger_do_action(TRIGGER_ALARM_1) + + # Check event received + self.assertEqual(1, len(self.mock_server.requests), + 'Wrong number of notifications for raise alarm') + + # Raise another alarm + self._trigger_do_action(TRIGGER_ALARM_2) + + # Check second event received + self.assertEqual(2, len(self.mock_server.requests), + 'Wrong number of notifications for clear alarm') + + finally: + self._delete_webhooks() + self._trigger_undo_action(TRIGGER_ALARM_1) + self._trigger_undo_action(TRIGGER_ALARM_2) + self.mock_server.reset_requests_list() + + def test_webhook_with_no_match(self): + """Test to check that filters with no match do not send event """ + + try: + + # Add webhook + TempestClients.vitrage().webhook.add( + url=self.URL_PROPS, + regex_filter=FILTER_NO_MATCH, + ) + + # Raise alarm + self._trigger_do_action(TRIGGER_ALARM_1) + + # Check event not received + self.assertEqual(0, len(self.mock_server.requests), + 'event should not have passed filter') + + # Raise another alarm + self._trigger_do_action(TRIGGER_ALARM_2) + + # Check second event not received + self.assertEqual(0, len(self.mock_server.requests), + 'event should not have passed filter') + + finally: + self._delete_webhooks() + self._trigger_undo_action(TRIGGER_ALARM_1) + self._trigger_undo_action(TRIGGER_ALARM_2) + self.mock_server.reset_requests_list() + + def test_multiple_webhooks(self): + """Test to check filter by type and by ID (with 2 different + + webhooks) + """ + + host_id = self.orig_host[VProps.VITRAGE_ID] + ID_FILTER = '{"%s": "%s"}' % (VProps.VITRAGE_RESOURCE_ID, host_id) + + try: + + # Add webhook + TempestClients.vitrage().webhook.add( + url=self.URL_PROPS, + regex_filter=TYPE_FILTER, + ) + + TempestClients.vitrage().webhook.add( + url=self.URL_PROPS, + regex_filter=ID_FILTER, + ) + + # Raise alarm + self._trigger_do_action(TRIGGER_ALARM_1) + + # Check event received + self.assertEqual(2, len(self.mock_server.requests), + 'event not posted to all webhooks') + + # Raise another alarm + self._trigger_do_action(TRIGGER_ALARM_2) + + # Check second event received + self.assertEqual(4, len(self.mock_server.requests), + 'event not posted to all webhooks') + + finally: + self._delete_webhooks() + self._trigger_undo_action(TRIGGER_ALARM_1) + self._trigger_undo_action(TRIGGER_ALARM_2) + self.mock_server.reset_requests_list() + + # Will be un-commented-out in the next change + # + # @utils.tempest_logger + # def test_webhook_for_deduced_alarm(self): + # + # try: + # + # # Add webhook with filter for the deduced alarm + # TempestClients.vitrage().webhook.add( + # url=self.URL_PROPS, + # regex_filter=NAME_FILTER_FOR_DEDUCED, + # headers=HEADERS_PROPS + # ) + # + # # Raise the trigger alarm + # self._trigger_do_action(TRIGGER_ALARM_WITH_DEDUCED) + # + # # Check event received - expected one for the deduced alarm + # # (the trigger alarm does not pass the filter). This test verifies + # # that the webhook is called only once for the deduced alarm. + # self.assertEqual(1, len(self.mock_server.requests), + # 'Wrong number of notifications for deduced alarm') + # + # # Undo + # self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED) + # + # # Check event undo received + # self.assertEqual(2, len(self.mock_server.requests), + # 'Wrong number of notifications for clear deduced ' + # 'alarm') + # + # finally: + # self._delete_webhooks() + # self._trigger_undo_action(TRIGGER_ALARM_WITH_DEDUCED) + # self.mock_server.reset_requests_list() + + def _delete_webhooks(self): + webhooks = TempestClients.vitrage().webhook.list() + for webhook in webhooks: + TempestClients.vitrage().webhook.delete(webhook['id']) + + +def _get_free_port(): + s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM) + s.bind(('localhost', 0)) + address, port = s.getsockname() + s.close() + return port + + +class MockHTTPServer(BaseHTTPServer.HTTPServer): + + def __init__(self, server, handler): + BaseHTTPServer.HTTPServer.__init__(self, server, handler) + self.requests = [] + + def process_request(self, request, client_address): + self.requests.append(request) + LOG.info('received request: %s', str(request)) + BaseHTTPServer.HTTPServer.process_request( + self, client_address=client_address, request=request) + + def reset_requests_list(self): + self.requests = [] + + +class MockServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + def do_POST(self): + + # Process a HTTP Post request and return status code 200 + self.send_response(requests.codes.ok) + self.end_headers() + return diff --git a/vitrage_tempest_tests/tests/resources/templates/api/e2e_webhooks.yaml b/vitrage_tempest_tests/tests/resources/templates/api/e2e_webhooks.yaml new file mode 100644 index 000000000..4bc105efe --- /dev/null +++ b/vitrage_tempest_tests/tests/resources/templates/api/e2e_webhooks.yaml @@ -0,0 +1,29 @@ +metadata: + name: e2e_webhook +definitions: + entities: + - entity: + category: ALARM + name: e2e.test_webhook.alarm_with_deduced + template_id: host_alarm + - entity: + category: RESOURCE + type: nova.host + template_id: host + relationships: + - relationship: + source: host_alarm + target: host + relationship_type: on + template_id : alarm_on_host +scenarios: + - scenario: + condition: alarm_on_host + actions: + - action: + action_type: raise_alarm + action_target: + target: host + properties: + alarm_name: e2e.test_webhook.deduced + severity: WARNING