diff --git a/doc/ext/versioned_notifications.py b/doc/ext/versioned_notifications.py index f3b2d8582..1a8107961 100644 --- a/doc/ext/versioned_notifications.py +++ b/doc/ext/versioned_notifications.py @@ -23,7 +23,7 @@ It is used via a single directive in the .rst file from sphinx.util.compat import Directive from docutils import nodes -from watcher.objects.notifications import base as notification +from watcher.notifications import base as notification from watcher.objects import base diff --git a/doc/notification_samples/audit-update.json b/doc/notification_samples/audit-update.json new file mode 100644 index 000000000..5cf90ac10 --- /dev/null +++ b/doc/notification_samples/audit-update.json @@ -0,0 +1,78 @@ +{ + "publisher_id": "infra-optim:localhost", + "timestamp": "2016-11-04 16:51:38.722986 ", + "payload": { + "watcher_object.name": "AuditUpdatePayload", + "watcher_object.data": { + "strategy": { + "watcher_object.name": "StrategyPayload", + "watcher_object.data": { + "name": "dummy", + "parameters_spec": { + "properties": { + "para2": { + "default": "hello", + "type": "string", + "description": "string parameter example" + }, + "para1": { + "maximum": 10.2, + "default": 3.2, + "minimum": 1.0, + "description": "number parameter example", + "type": "number" + } + } + }, + "updated_at": null, + "display_name": "Dummy strategy", + "deleted_at": null, + "uuid": "75234dfe-87e3-4f11-a0e0-3c3305d86a39", + "created_at": "2016-11-04T16:25:35Z" + }, + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0" + }, + "scope": [], + "created_at": "2016-11-04T16:51:21Z", + "uuid": "f1e0d912-afd9-4bf2-91ef-c99cd08cc1ef", + "goal": { + "watcher_object.name": "GoalPayload", + "watcher_object.data": { + "efficacy_specification": [], + "updated_at": null, + "name": "dummy", + "display_name": "Dummy goal", + "deleted_at": null, + "uuid": "bc830f84-8ae3-4fc6-8bc6-e3dd15e8b49a", + "created_at": "2016-11-04T16:25:35Z" + }, + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0" + }, + "parameters": { + "para2": "hello", + "para1": 3.2 + }, + "deleted_at": null, + "state_update": { + "watcher_object.name": "AuditStateUpdatePayload", + "watcher_object.data": { + "state": "ONGOING", + "old_state": "PENDING" + }, + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0" + }, + "interval": null, + "updated_at": null, + "state": "ONGOING", + "audit_type": "ONESHOT" + }, + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0" + }, + "priority": "INFO", + "event_type": "audit.update", + "message_id": "697fdf55-7252-4b6c-a2c2-5b9e85f6342c" +} diff --git a/tox.ini b/tox.ini index ea1cd04cb..b0f9c9f6d 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,6 @@ setenv = deps = -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.py[c|o]" -delete - find . -type d -name "__pycache__" -delete ostestr --concurrency=6 {posargs} [testenv:pep8] diff --git a/watcher/api/controllers/v1/audit.py b/watcher/api/controllers/v1/audit.py index e457f84b8..6d6de379e 100644 --- a/watcher/api/controllers/v1/audit.py +++ b/watcher/api/controllers/v1/audit.py @@ -544,14 +544,11 @@ class AuditsController(rest.RestController): raise exception.OperationNotPermitted context = pecan.request.context - audit_to_update = api_utils.get_resource('Audit', - audit_uuid) + audit_to_update = api_utils.get_resource( + 'Audit', audit_uuid, eager=True) policy.enforce(context, 'audit:update', audit_to_update, action='audit:update') - audit_to_update = objects.Audit.get_by_uuid(pecan.request.context, - audit_uuid) - try: audit_dict = audit_to_update.as_dict() audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch)) @@ -580,7 +577,8 @@ class AuditsController(rest.RestController): :param audit_uuid: UUID of a audit. """ context = pecan.request.context - audit_to_delete = api_utils.get_resource('Audit', audit_uuid) + audit_to_delete = api_utils.get_resource( + 'Audit', audit_uuid, eager=True) policy.enforce(context, 'audit:update', audit_to_delete, action='audit:update') diff --git a/watcher/api/controllers/v1/utils.py b/watcher/api/controllers/v1/utils.py index 207f79043..b011dbd34 100644 --- a/watcher/api/controllers/v1/utils.py +++ b/watcher/api/controllers/v1/utils.py @@ -15,6 +15,7 @@ import jsonpatch from oslo_config import cfg +from oslo_utils import reflection from oslo_utils import uuidutils import pecan import wsme @@ -81,7 +82,7 @@ def as_filters_dict(**filters): return filters_dict -def get_resource(resource, resource_id): +def get_resource(resource, resource_id, eager=False): """Get the resource from the uuid, id or logical name. :param resource: the resource type. @@ -91,10 +92,17 @@ def get_resource(resource, resource_id): """ resource = getattr(objects, resource) + _get = None if utils.is_int_like(resource_id): - return resource.get(pecan.request.context, int(resource_id)) + resource_id = int(resource_id) + _get = resource.get + elif uuidutils.is_uuid_like(resource_id): + _get = resource.get_by_uuid + else: + _get = resource.get_by_name - if uuidutils.is_uuid_like(resource_id): - return resource.get_by_uuid(pecan.request.context, resource_id) + method_signature = reflection.get_signature(_get) + if 'eager' in method_signature.parameters: + return _get(pecan.request.context, resource_id, eager=eager) - return resource.get_by_name(pecan.request.context, resource_id) + return _get(pecan.request.context, resource_id) diff --git a/watcher/common/exception.py b/watcher/common/exception.py index db7958361..da7905894 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -119,6 +119,10 @@ class WatcherException(Exception): return six.text_type(self) +class UnsupportedError(WatcherException): + msg_fmt = _("Not supported") + + class NotAuthorized(WatcherException): msg_fmt = _("Not authorized") code = 403 @@ -168,6 +172,14 @@ class InvalidStrategy(Invalid): msg_fmt = _("Strategy %(strategy)s is invalid") +class InvalidAudit(Invalid): + msg_fmt = _("Audit %(audit)s is invalid") + + +class EagerlyLoadedAuditRequired(InvalidAudit): + msg_fmt = _("Audit %(audit)s was not eagerly loaded") + + class InvalidUUID(Invalid): msg_fmt = _("Expected a uuid but received %(uuid)s") diff --git a/watcher/db/sqlalchemy/api.py b/watcher/db/sqlalchemy/api.py index 86c2661ec..4807ae1ab 100644 --- a/watcher/db/sqlalchemy/api.py +++ b/watcher/db/sqlalchemy/api.py @@ -233,6 +233,10 @@ class Connection(api.BaseConnection): return query + @staticmethod + def _get_relationships(model): + return inspect(model).relationships + @staticmethod def _set_eager_options(model, query): relationships = inspect(model).relationships @@ -242,6 +246,14 @@ class Connection(api.BaseConnection): query = query.options(joinedload(relationship.key)) return query + def _create(self, model, values): + obj = model() + cleaned_values = {k: v for k, v in values.items() + if k not in self._get_relationships(model)} + obj.update(cleaned_values) + obj.save() + return obj + def _get(self, context, model, fieldname, value, eager): query = model_query(model) if eager: @@ -429,11 +441,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - goal = models.Goal() - goal.update(values) - try: - goal.save() + goal = self._create(models.Goal, values) except db_exc.DBDuplicateEntry: raise exception.GoalAlreadyExists(uuid=values['uuid']) return goal @@ -498,11 +507,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - strategy = models.Strategy() - strategy.update(values) - try: - strategy.save() + strategy = self._create(models.Strategy, values) except db_exc.DBDuplicateEntry: raise exception.StrategyAlreadyExists(uuid=values['uuid']) return strategy @@ -576,11 +582,8 @@ class Connection(api.BaseConnection): raise exception.AuditTemplateAlreadyExists( audit_template=values['name']) - audit_template = models.AuditTemplate() - audit_template.update(values) - try: - audit_template.save() + audit_template = self._create(models.AuditTemplate, values) except db_exc.DBDuplicateEntry: raise exception.AuditTemplateAlreadyExists( audit_template=values['name']) @@ -657,11 +660,8 @@ class Connection(api.BaseConnection): if values.get('state') is None: values['state'] = objects.audit.State.PENDING - audit = models.Audit() - audit.update(values) - try: - audit.save() + audit = self._create(models.Audit, values) except db_exc.DBDuplicateEntry: raise exception.AuditAlreadyExists(uuid=values['uuid']) return audit @@ -740,10 +740,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - action = models.Action() - action.update(values) try: - action.save() + action = self._create(models.Action, values) except db_exc.DBDuplicateEntry: raise exception.ActionAlreadyExists(uuid=values['uuid']) return action @@ -829,11 +827,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - action_plan = models.ActionPlan() - action_plan.update(values) - try: - action_plan.save() + action_plan = self._create(models.ActionPlan, values) except db_exc.DBDuplicateEntry: raise exception.ActionPlanAlreadyExists(uuid=values['uuid']) return action_plan @@ -932,11 +927,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - efficacy_indicator = models.EfficacyIndicator() - efficacy_indicator.update(values) - try: - efficacy_indicator.save() + efficacy_indicator = self._create(models.EfficacyIndicator, values) except db_exc.DBDuplicateEntry: raise exception.EfficacyIndicatorAlreadyExists(uuid=values['uuid']) return efficacy_indicator @@ -1024,11 +1016,8 @@ class Connection(api.BaseConnection): if not values.get('uuid'): values['uuid'] = utils.generate_uuid() - scoring_engine = models.ScoringEngine() - scoring_engine.update(values) - try: - scoring_engine.save() + scoring_engine = self._create(models.ScoringEngine, values) except db_exc.DBDuplicateEntry: raise exception.ScoringEngineAlreadyExists(uuid=values['uuid']) return scoring_engine @@ -1106,10 +1095,8 @@ class Connection(api.BaseConnection): sort_key, sort_dir, query) def create_service(self, values): - service = models.Service() - service.update(values) try: - service.save() + service = self._create(models.Service, values) except db_exc.DBDuplicateEntry: raise exception.ServiceAlreadyExists(name=values['name'], host=values['host']) diff --git a/watcher/decision_engine/audit/continuous.py b/watcher/decision_engine/audit/continuous.py index f0f8ae2a1..362a4167b 100644 --- a/watcher/decision_engine/audit/continuous.py +++ b/watcher/decision_engine/audit/continuous.py @@ -101,7 +101,8 @@ class ContinuousAuditHandler(base.AuditHandler): objects.audit.State.ONGOING, objects.audit.State.SUCCEEDED) } - audits = objects.Audit.list(audit_context, filters=audit_filters) + audits = objects.Audit.list( + audit_context, filters=audit_filters, eager=True) scheduler_job_args = [job.args for job in self.scheduler.get_jobs() if job.name == 'execute_audit'] for audit in audits: diff --git a/watcher/decision_engine/messaging/audit_endpoint.py b/watcher/decision_engine/messaging/audit_endpoint.py index 0d19600bf..ec0934d3e 100644 --- a/watcher/decision_engine/messaging/audit_endpoint.py +++ b/watcher/decision_engine/messaging/audit_endpoint.py @@ -49,7 +49,7 @@ class AuditEndpoint(object): return self._messaging def do_trigger_audit(self, context, audit_uuid): - audit = objects.Audit.get_by_uuid(context, audit_uuid) + audit = objects.Audit.get_by_uuid(context, audit_uuid, eager=True) self._oneshot_handler.execute(audit, context) def trigger_audit(self, context, audit_uuid): diff --git a/watcher/decision_engine/sync.py b/watcher/decision_engine/sync.py index 989bf01cf..48e189e9b 100644 --- a/watcher/decision_engine/sync.py +++ b/watcher/decision_engine/sync.py @@ -318,7 +318,7 @@ class Syncer(object): for goal_id, synced_goal in self.goal_mapping.items(): filters = {"goal_id": goal_id} stale_audits = objects.Audit.list( - self.ctx, filters=filters) + self.ctx, filters=filters, eager=True) # Update the goal ID for the stale audits (w/o saving) for audit in stale_audits: @@ -331,7 +331,8 @@ class Syncer(object): def _find_stale_audits_due_to_strategy(self): for strategy_id, synced_strategy in self.strategy_mapping.items(): filters = {"strategy_id": strategy_id} - stale_audits = objects.Audit.list(self.ctx, filters=filters) + stale_audits = objects.Audit.list( + self.ctx, filters=filters, eager=True) # Update strategy IDs for all stale audits (w/o saving) for audit in stale_audits: if audit.id not in self.stale_audits_map: @@ -396,7 +397,8 @@ class Syncer(object): _LW("Audit Template '%(audit_template)s' references a " "goal that does not exist"), audit_template=at.uuid) - stale_audits = objects.Audit.list(self.ctx, filters=filters) + stale_audits = objects.Audit.list( + self.ctx, filters=filters, eager=True) for audit in stale_audits: LOG.warning( _LW("Audit '%(audit)s' references a " @@ -431,7 +433,8 @@ class Syncer(object): else: self.stale_audit_templates_map[at.id].strategy_id = None - stale_audits = objects.Audit.list(self.ctx, filters=filters) + stale_audits = objects.Audit.list( + self.ctx, filters=filters, eager=True) for audit in stale_audits: LOG.warning( _LW("Audit '%(audit)s' references a " diff --git a/watcher/notifications/__init__.py b/watcher/notifications/__init__.py new file mode 100644 index 000000000..14151440a --- /dev/null +++ b/watcher/notifications/__init__.py @@ -0,0 +1,26 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Authors: Vincent FRANCOISE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Note(gibi): Importing publicly called functions so the caller code does not +# need to be changed after we moved these function inside the package +# Todo(gibi): remove these imports after legacy notifications using these are +# transformed to versioned notifications +from watcher.notifications import audit # noqa +from watcher.notifications import exception # noqa +from watcher.notifications import goal # noqa +from watcher.notifications import strategy # noqa diff --git a/watcher/notifications/audit.py b/watcher/notifications/audit.py new file mode 100644 index 000000000..01be8e14b --- /dev/null +++ b/watcher/notifications/audit.py @@ -0,0 +1,152 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Authors: Vincent FRANCOISE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from watcher.common import exception +from watcher.notifications import base as notificationbase +from watcher.notifications import goal as goal_notifications +from watcher.notifications import strategy as strategy_notifications +from watcher.objects import base +from watcher.objects import fields as wfields + +CONF = cfg.CONF + + +@base.WatcherObjectRegistry.register_notification +class AuditPayload(notificationbase.NotificationPayloadBase): + SCHEMA = { + 'uuid': ('audit', 'uuid'), + + 'audit_type': ('audit', 'audit_type'), + 'state': ('audit', 'state'), + 'parameters': ('audit', 'parameters'), + 'interval': ('audit', 'interval'), + 'scope': ('audit', 'scope'), + # 'goal_uuid': ('audit', 'goal_uuid'), + # 'strategy_uuid': ('audit', 'strategy_uuid'), + + 'created_at': ('audit', 'created_at'), + 'updated_at': ('audit', 'updated_at'), + 'deleted_at': ('audit', 'deleted_at'), + } + + # Version 1.0: Initial version + VERSION = '1.0' + + fields = { + 'uuid': wfields.UUIDField(), + 'audit_type': wfields.StringField(), + 'state': wfields.StringField(), + 'parameters': wfields.FlexibleDictField(nullable=True), + 'interval': wfields.IntegerField(nullable=True), + 'scope': wfields.FlexibleListOfDictField(nullable=True), + 'goal_uuid': wfields.UUIDField(), + 'strategy_uuid': wfields.UUIDField(nullable=True), + 'goal': wfields.ObjectField('GoalPayload'), + 'strategy': wfields.ObjectField('StrategyPayload', nullable=True), + + 'created_at': wfields.DateTimeField(nullable=True), + 'updated_at': wfields.DateTimeField(nullable=True), + 'deleted_at': wfields.DateTimeField(nullable=True), + } + + def __init__(self, audit, **kwargs): + super(AuditPayload, self).__init__(**kwargs) + self.populate_schema(audit=audit) + + +@base.WatcherObjectRegistry.register_notification +class AuditStateUpdatePayload(notificationbase.NotificationPayloadBase): + # Version 1.0: Initial version + VERSION = '1.0' + + fields = { + 'old_state': wfields.StringField(nullable=True), + 'state': wfields.StringField(nullable=True), + } + + +@base.WatcherObjectRegistry.register_notification +class AuditUpdatePayload(AuditPayload): + # Version 1.0: Initial version + VERSION = '1.0' + fields = { + 'state_update': wfields.ObjectField('AuditStateUpdatePayload'), + } + + def __init__(self, audit, state_update, goal, strategy): + super(AuditUpdatePayload, self).__init__( + audit=audit, + state_update=state_update, + goal=goal, + strategy=strategy) + + +@notificationbase.notification_sample('audit-update.json') +@base.WatcherObjectRegistry.register_notification +class AuditUpdateNotification(notificationbase.NotificationBase): + # Version 1.0: Initial version + VERSION = '1.0' + + fields = { + 'payload': wfields.ObjectField('AuditUpdatePayload') + } + + +def send_update(context, audit, service='infra-optim', + host=None, old_state=None): + """Emit an audit.update notification.""" + goal = None + strategy = None + try: + goal = audit.goal + if audit.strategy_id: + strategy = audit.strategy + except NotImplementedError: + raise exception.EagerlyLoadedAuditRequired(audit=audit.uuid) + + goal_payload = goal_notifications.GoalPayload(goal=goal) + + strategy_payload = None + if strategy: + strategy_payload = strategy_notifications.StrategyPayload( + strategy=strategy) + + state_update = AuditStateUpdatePayload( + old_state=old_state, + state=audit.state if old_state else None) + + versioned_payload = AuditUpdatePayload( + audit=audit, + state_update=state_update, + goal=goal_payload, + strategy=strategy_payload, + ) + + notification = AuditUpdateNotification( + priority=wfields.NotificationPriority.INFO, + event_type=notificationbase.EventType( + object='audit', + action=wfields.NotificationAction.UPDATE), + publisher=notificationbase.NotificationPublisher( + host=host or CONF.host, + binary=service), + payload=versioned_payload) + + notification.emit(context) diff --git a/watcher/objects/notifications/base.py b/watcher/notifications/base.py similarity index 97% rename from watcher/objects/notifications/base.py rename to watcher/notifications/base.py index d2ba3cee4..fa56247d2 100644 --- a/watcher/objects/notifications/base.py +++ b/watcher/notifications/base.py @@ -46,6 +46,12 @@ class NotificationObject(base.WatcherObject): # reset the object after creation. self.obj_reset_changes(recursive=False) + def save(self, context): + raise exception.UnsupportedError() + + def obj_load_attr(self, attrname): + raise exception.UnsupportedError() + @base.WatcherObjectRegistry.register_notification class EventType(NotificationObject): diff --git a/watcher/objects/notifications/exception.py b/watcher/notifications/exception.py similarity index 96% rename from watcher/objects/notifications/exception.py rename to watcher/notifications/exception.py index afe954819..be6d4bb85 100644 --- a/watcher/objects/notifications/exception.py +++ b/watcher/notifications/exception.py @@ -14,9 +14,9 @@ import inspect import six +from watcher.notifications import base as notificationbase from watcher.objects import base as base from watcher.objects import fields as wfields -from watcher.objects.notifications import base as notificationbase @base.WatcherObjectRegistry.register_notification diff --git a/watcher/notifications/goal.py b/watcher/notifications/goal.py new file mode 100644 index 000000000..8c76bad54 --- /dev/null +++ b/watcher/notifications/goal.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Authors: Vincent FRANCOISE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.notifications import base as notificationbase +from watcher.objects import base +from watcher.objects import fields as wfields + + +@base.WatcherObjectRegistry.register_notification +class GoalPayload(notificationbase.NotificationPayloadBase): + SCHEMA = { + 'uuid': ('goal', 'uuid'), + 'name': ('goal', 'name'), + 'display_name': ('goal', 'display_name'), + 'efficacy_specification': ('goal', 'efficacy_specification'), + + 'created_at': ('goal', 'created_at'), + 'updated_at': ('goal', 'updated_at'), + 'deleted_at': ('goal', 'deleted_at'), + } + + # Version 1.0: Initial version + VERSION = '1.0' + + fields = { + 'uuid': wfields.UUIDField(), + 'name': wfields.StringField(), + 'display_name': wfields.StringField(), + 'efficacy_specification': wfields.FlexibleListOfDictField(), + + 'created_at': wfields.DateTimeField(nullable=True), + 'updated_at': wfields.DateTimeField(nullable=True), + 'deleted_at': wfields.DateTimeField(nullable=True), + } + + def __init__(self, goal, **kwargs): + super(GoalPayload, self).__init__(**kwargs) + self.populate_schema(goal=goal) diff --git a/watcher/notifications/strategy.py b/watcher/notifications/strategy.py new file mode 100644 index 000000000..f7da10919 --- /dev/null +++ b/watcher/notifications/strategy.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Authors: Vincent FRANCOISE +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.notifications import base as notificationbase +from watcher.objects import base +from watcher.objects import fields as wfields + + +@base.WatcherObjectRegistry.register_notification +class StrategyPayload(notificationbase.NotificationPayloadBase): + SCHEMA = { + 'uuid': ('strategy', 'uuid'), + 'name': ('strategy', 'name'), + 'display_name': ('strategy', 'display_name'), + 'parameters_spec': ('strategy', 'parameters_spec'), + + 'created_at': ('strategy', 'created_at'), + 'updated_at': ('strategy', 'updated_at'), + 'deleted_at': ('strategy', 'deleted_at'), + } + + # Version 1.0: Initial version + VERSION = '1.0' + + fields = { + 'uuid': wfields.UUIDField(), + 'name': wfields.StringField(), + 'display_name': wfields.StringField(), + 'parameters_spec': wfields.FlexibleDictField(nullable=True), + + 'created_at': wfields.DateTimeField(nullable=True), + 'updated_at': wfields.DateTimeField(nullable=True), + 'deleted_at': wfields.DateTimeField(nullable=True), + } + + def __init__(self, strategy, **kwargs): + super(StrategyPayload, self).__init__(**kwargs) + self.populate_schema(strategy=strategy) diff --git a/watcher/objects/audit.py b/watcher/objects/audit.py index fc044ab21..507ad968d 100644 --- a/watcher/objects/audit.py +++ b/watcher/objects/audit.py @@ -53,6 +53,7 @@ import enum from watcher.common import exception from watcher.common import utils from watcher.db import api as db_api +from watcher import notifications from watcher import objects from watcher.objects import base from watcher.objects import fields as wfields @@ -102,6 +103,39 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject, 'strategy': (objects.Strategy, 'strategy_id'), } + # Proxified field so we can keep the previous value after an update + _state = None + _old_state = None + + # NOTE(v-francoise): The way oslo.versionedobjects works is by using a + # __new__ that will automagically create the attributes referenced in + # fields. These attributes are properties that raise an exception if no + # value has been assigned, which means that they store the actual field + # value in an "_obj_%(field)s" attribute. So because we want to proxify a + # value that is already proxified, we have to do what you see below. + @property + def _obj_state(self): + return self._state + + @property + def _obj_old_state(self): + return self._old_state + + @property + def old_state(self): + return self._old_state + + @_obj_old_state.setter + def _obj_old_state(self, value): + self._old_state = value + + @_obj_state.setter + def _obj_state(self, value): + if self._old_state is None and self._state is None: + self._state = value + else: + self._old_state, self._state = self._state, value + @base.remotable_classmethod def get(cls, context, audit_id, eager=False): """Find a audit based on its id or uuid and return a Audit object. @@ -218,6 +252,12 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject, updates = self.obj_get_changes() self.dbapi.update_audit(self.uuid, updates) + def _notify(): + notifications.audit.send_update( + self._context, self, old_state=self.old_state) + + _notify() + self.obj_reset_changes() @base.remotable diff --git a/watcher/tests/api/v1/test_audits.py b/watcher/tests/api/v1/test_audits.py index 6853a8224..cc0596579 100644 --- a/watcher/tests/api/v1/test_audits.py +++ b/watcher/tests/api/v1/test_audits.py @@ -252,7 +252,7 @@ class TestPatch(api_base.FunctionalTest): obj_utils.create_test_goal(self.context) obj_utils.create_test_strategy(self.context) obj_utils.create_test_audit_template(self.context) - self.audit = obj_utils.create_test_audit(self.context, ) + self.audit = obj_utils.create_test_audit(self.context) p = mock.patch.object(db_api.BaseConnection, 'update_audit') self.mock_audit_update = p.start() self.mock_audit_update.side_effect = self._simulate_rpc_audit_update diff --git a/watcher/tests/db/utils.py b/watcher/tests/db/utils.py index 07a9439b1..dba00d1fc 100644 --- a/watcher/tests/db/utils.py +++ b/watcher/tests/db/utils.py @@ -75,7 +75,6 @@ def get_test_audit(**kwargs): 'strategy_id': kwargs.get('strategy_id', None), 'scope': kwargs.get('scope', []), } - # ObjectField doesn't allow None nor dict, so if we want to simulate a # non-eager object loading, the field should not be referenced at all. if kwargs.get('goal'): diff --git a/watcher/tests/decision_engine/audit/test_audit_handlers.py b/watcher/tests/decision_engine/audit/test_audit_handlers.py index 63d7fb26f..be9956296 100644 --- a/watcher/tests/decision_engine/audit/test_audit_handlers.py +++ b/watcher/tests/decision_engine/audit/test_audit_handlers.py @@ -14,10 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid - from apscheduler.schedulers import background import mock +from oslo_utils import uuidutils from watcher.decision_engine.audit import continuous from watcher.decision_engine.audit import oneshot @@ -32,15 +31,19 @@ class TestOneShotAuditHandler(base.DbTestCase): def setUp(self): super(TestOneShotAuditHandler, self).setUp() - obj_utils.create_test_goal(self.context, id=1, name="dummy") + self.goal = obj_utils.create_test_goal( + self.context, id=1, name="dummy") self.strategy = obj_utils.create_test_strategy( - self.context, name='dummy') + self.context, name='dummy', goal_id=self.goal.id) audit_template = obj_utils.create_test_audit_template( self.context, strategy_id=self.strategy.id) self.audit = obj_utils.create_test_audit( self.context, + uuid=uuidutils.generate_uuid(), + goal_id=self.goal.id, strategy_id=self.strategy.id, - audit_template_id=audit_template.id) + audit_template_id=audit_template.id, + goal=self.goal) @mock.patch.object(manager.CollectorManager, "get_cluster_model_collector") def test_trigger_audit_without_errors(self, mock_collector): @@ -58,18 +61,23 @@ class TestOneShotAuditHandler(base.DbTestCase): class TestContinuousAuditHandler(base.DbTestCase): + def setUp(self): super(TestContinuousAuditHandler, self).setUp() - obj_utils.create_test_goal(self.context, id=1, name="dummy") + self.goal = obj_utils.create_test_goal( + self.context, id=1, name="dummy") audit_template = obj_utils.create_test_audit_template( self.context) self.audits = [ obj_utils.create_test_audit( self.context, - uuid=uuid.uuid4(), + id=id_, + uuid=uuidutils.generate_uuid(), audit_template_id=audit_template.id, - audit_type=audit_objects.AuditType.CONTINUOUS.value) - for i in range(2)] + goal_id=self.goal.id, + audit_type=audit_objects.AuditType.CONTINUOUS.value, + goal=self.goal) + for id_ in range(2, 4)] @mock.patch.object(manager.CollectorManager, "get_cluster_model_collector") @mock.patch.object(background.BackgroundScheduler, 'add_job') @@ -78,9 +86,7 @@ class TestContinuousAuditHandler(base.DbTestCase): def test_launch_audits_periodically(self, mock_list, mock_jobs, mock_add_job, mock_collector): audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock()) - audits = [audit_objects.Audit.get_by_uuid(self.context, - self.audits[0].uuid)] - mock_list.return_value = audits + mock_list.return_value = self.audits mock_jobs.return_value = mock.MagicMock() mock_add_job.return_value = audit_handler.execute_audit( self.audits[0], self.context) @@ -95,10 +101,7 @@ class TestContinuousAuditHandler(base.DbTestCase): def test_launch_multiply_audits_periodically(self, mock_list, mock_jobs, mock_add_job): audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock()) - audits = [audit_objects.Audit.get_by_uuid( - self.context, - audit.uuid) for audit in self.audits] - mock_list.return_value = audits + mock_list.return_value = self.audits mock_jobs.return_value = mock.MagicMock() calls = [mock.call(audit_handler.execute_audit, 'interval', args=[mock.ANY, mock.ANY], @@ -114,12 +117,9 @@ class TestContinuousAuditHandler(base.DbTestCase): def test_period_audit_not_called_when_deleted(self, mock_list, mock_jobs, mock_add_job): audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock()) - audits = [audit_objects.Audit.get_by_uuid( - self.context, - audit.uuid) for audit in self.audits] - mock_list.return_value = audits + mock_list.return_value = self.audits mock_jobs.return_value = mock.MagicMock() - audits[1].state = audit_objects.State.CANCELLED + self.audits[1].state = audit_objects.State.CANCELLED calls = [mock.call(audit_handler.execute_audit, 'interval', args=[mock.ANY, mock.ANY], seconds=3600, @@ -128,7 +128,7 @@ class TestContinuousAuditHandler(base.DbTestCase): audit_handler.launch_audits_periodically() mock_add_job.assert_has_calls(calls) - audit_handler.update_audit_state(audits[1], + audit_handler.update_audit_state(self.audits[1], audit_objects.State.CANCELLED) - is_inactive = audit_handler._is_audit_inactive(audits[1]) + is_inactive = audit_handler._is_audit_inactive(self.audits[1]) self.assertTrue(is_inactive) diff --git a/watcher/objects/notifications/__init__.py b/watcher/tests/notifications/__init__.py similarity index 100% rename from watcher/objects/notifications/__init__.py rename to watcher/tests/notifications/__init__.py diff --git a/watcher/tests/notifications/test_audit_notification.py b/watcher/tests/notifications/test_audit_notification.py new file mode 100644 index 000000000..e7a06d661 --- /dev/null +++ b/watcher/tests/notifications/test_audit_notification.py @@ -0,0 +1,163 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import freezegun +import mock + +from watcher.common import exception +from watcher.notifications import audit as auditnotifs +from watcher.tests import base as testbase +from watcher.tests.objects import utils + + +class TestAuditNotification(testbase.TestCase): + + @mock.patch.object(auditnotifs.AuditUpdateNotification, '_emit') + def test_send_version_invalid_audit(self, mock_emit): + audit = utils.get_test_audit(mock.Mock(), state='DOESNOTMATTER', + goal_id=1) + + self.assertRaises( + exception.InvalidAudit, + auditnotifs.send_update, + mock.MagicMock(), audit, 'host', 'node0') + + @freezegun.freeze_time('2016-10-18T09:52:05.219414') + @mock.patch.object(auditnotifs.AuditUpdateNotification, '_emit') + def test_send_version_audit_update_with_strategy(self, mock_emit): + goal = utils.get_test_goal(mock.Mock(), id=1) + strategy = utils.get_test_strategy(mock.Mock(), id=1) + audit = utils.get_test_audit(mock.Mock(), state='ONGOING', + goal_id=goal.id, strategy_id=strategy.id, + goal=goal, strategy=strategy) + auditnotifs.send_update( + mock.MagicMock(), audit, 'host', 'node0', old_state='PENDING') + + self.assertEqual(1, mock_emit.call_count) + notification = mock_emit.call_args_list[0][1] + payload = notification['payload'] + + self.assertDictEqual( + { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "interval": 3600, + "strategy": { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "updated_at": None, + "uuid": "cb3d0b58-4415-4d90-b75b-1e96878730e3", + "name": "TEST", + "parameters_spec": {}, + "created_at": None, + "display_name": "test strategy", + "deleted_at": None + }, + "watcher_object.name": "StrategyPayload" + }, + "parameters": {}, + "uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d", + "goal": { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "updated_at": None, + "uuid": "f7ad87ae-4298-91cf-93a0-f35a852e3652", + "name": "TEST", + "efficacy_specification": [], + "created_at": None, + "display_name": "test goal", + "deleted_at": None + }, + "watcher_object.name": "GoalPayload" + }, + "deleted_at": None, + "scope": [], + "state": "ONGOING", + "updated_at": None, + "created_at": None, + "state_update": { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "old_state": "PENDING", + "state": "ONGOING" + }, + "watcher_object.name": "AuditStateUpdatePayload" + }, + "audit_type": "ONESHOT" + }, + "watcher_object.name": "AuditUpdatePayload" + }, + payload + ) + + @freezegun.freeze_time('2016-10-18T09:52:05.219414') + @mock.patch.object(auditnotifs.AuditUpdateNotification, '_emit') + def test_send_version_audit_update_without_strategy(self, mock_emit): + goal = utils.get_test_goal(mock.Mock(), id=1) + audit = utils.get_test_audit( + mock.Mock(), state='ONGOING', goal_id=goal.id, goal=goal) + auditnotifs.send_update( + mock.MagicMock(), audit, 'host', 'node0', old_state='PENDING') + + self.assertEqual(1, mock_emit.call_count) + notification = mock_emit.call_args_list[0][1] + payload = notification['payload'] + + self.assertDictEqual( + { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "interval": 3600, + "parameters": {}, + "uuid": "10a47dd1-4874-4298-91cf-eff046dbdb8d", + "goal": { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "updated_at": None, + "uuid": "f7ad87ae-4298-91cf-93a0-f35a852e3652", + "name": "TEST", + "efficacy_specification": [], + "created_at": None, + "display_name": "test goal", + "deleted_at": None + }, + "watcher_object.name": "GoalPayload" + }, + "strategy": None, + "deleted_at": None, + "scope": [], + "state": "ONGOING", + "updated_at": None, + "created_at": None, + "state_update": { + "watcher_object.namespace": "watcher", + "watcher_object.version": "1.0", + "watcher_object.data": { + "old_state": "PENDING", + "state": "ONGOING" + }, + "watcher_object.name": "AuditStateUpdatePayload" + }, + "audit_type": "ONESHOT" + }, + "watcher_object.name": "AuditUpdatePayload" + }, + payload + ) diff --git a/watcher/tests/objects/notifications/test_notification.py b/watcher/tests/notifications/test_notification.py similarity index 95% rename from watcher/tests/objects/notifications/test_notification.py rename to watcher/tests/notifications/test_notification.py index dab514dba..084e9a9bc 100644 --- a/watcher/tests/objects/notifications/test_notification.py +++ b/watcher/tests/notifications/test_notification.py @@ -19,9 +19,9 @@ from oslo_versionedobjects import fixture from watcher.common import exception from watcher.common import rpc +from watcher.notifications import base as notificationbase from watcher.objects import base from watcher.objects import fields as wfields -from watcher.objects.notifications import base as notificationbase from watcher.tests import base as testbase from watcher.tests.objects import test_objects @@ -251,7 +251,15 @@ class TestNotificationBase(testbase.TestCase): expected_notification_fingerprints = { 'EventType': '1.0-92100a9f0908da98dfcfff9c42e0018c', + 'ExceptionNotification': '1.0-9b69de0724fda8310d05e18418178866', + 'ExceptionPayload': '1.0-4516ae282a55fe2fd5c754967ee6248b', 'NotificationPublisher': '1.0-bbbc1402fb0e443a3eb227cc52b61545', + 'AuditPayload': '1.0-30c85c834648c8ca11f54fc5e084d86b', + 'AuditStateUpdatePayload': '1.0-1a1b606bf14a2c468800c2b010801ce5', + 'AuditUpdateNotification': '1.0-9b69de0724fda8310d05e18418178866', + 'AuditUpdatePayload': '1.0-d3aace28d9eb978c1ecf833e108f61f7', + 'GoalPayload': '1.0-fa1fecb8b01dd047eef808ded4d50d1a', + 'StrategyPayload': '1.0-94f01c137b083ac236ae82573c1fcfc1', } diff --git a/watcher/tests/objects/notifications/__init__.py b/watcher/tests/objects/notifications/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/watcher/tests/objects/test_audit.py b/watcher/tests/objects/test_audit.py index 144646422..5ba6b6ca4 100644 --- a/watcher/tests/objects/test_audit.py +++ b/watcher/tests/objects/test_audit.py @@ -16,8 +16,10 @@ import mock from watcher.common import exception +from watcher.common import rpc from watcher.common import utils as w_utils from watcher.db.sqlalchemy import api as db_api +from watcher import notifications from watcher import objects from watcher.tests.db import base from watcher.tests.db import utils @@ -46,6 +48,12 @@ class TestAuditObject(base.DbTestCase): def setUp(self): super(TestAuditObject, self).setUp() + + p_audit_notifications = mock.patch.object( + notifications, 'audit', autospec=True) + self.m_audit_notifications = p_audit_notifications.start() + self.addCleanup(p_audit_notifications.stop) + self.m_send_update = self.m_audit_notifications.send_update self.fake_goal = utils.create_test_goal(**self.goal_data) def eager_load_audit_assert(self, audit, goal): @@ -71,6 +79,7 @@ class TestAuditObject(base.DbTestCase): self.context, audit_id, eager=self.eager) self.assertEqual(self.context, audit._context) self.eager_load_audit_assert(audit, self.fake_goal) + self.assertEqual(0, self.m_send_update.call_count) @mock.patch.object(db_api.Connection, 'get_audit_by_uuid') def test_get_by_uuid(self, mock_get_audit): @@ -81,6 +90,7 @@ class TestAuditObject(base.DbTestCase): self.context, uuid, eager=self.eager) self.assertEqual(self.context, audit._context) self.eager_load_audit_assert(audit, self.fake_goal) + self.assertEqual(0, self.m_send_update.call_count) def test_get_bad_id_and_uuid(self): self.assertRaises(exception.InvalidIdentity, @@ -99,6 +109,7 @@ class TestAuditObject(base.DbTestCase): self.assertEqual(self.context, audits[0]._context) for audit in audits: self.eager_load_audit_assert(audit, self.fake_goal) + self.assertEqual(0, self.m_send_update.call_count) @mock.patch.object(db_api.Connection, 'update_audit') @mock.patch.object(db_api.Connection, 'get_audit_by_uuid') @@ -106,15 +117,17 @@ class TestAuditObject(base.DbTestCase): mock_get_audit.return_value = self.fake_audit uuid = self.fake_audit['uuid'] audit = objects.Audit.get_by_uuid(self.context, uuid, eager=self.eager) - audit.state = 'SUCCEEDED' + audit.state = objects.audit.State.SUCCEEDED audit.save() mock_get_audit.assert_called_once_with( self.context, uuid, eager=self.eager) mock_update_audit.assert_called_once_with( - uuid, {'state': 'SUCCEEDED'}) + uuid, {'state': objects.audit.State.SUCCEEDED}) self.assertEqual(self.context, audit._context) self.eager_load_audit_assert(audit, self.fake_goal) + self.m_send_update.assert_called_once_with( + self.context, audit, old_state=self.fake_audit['state']) @mock.patch.object(db_api.Connection, 'get_audit_by_uuid') def test_refresh(self, mock_get_audit): @@ -138,12 +151,18 @@ class TestCreateDeleteAuditObject(base.DbTestCase): def setUp(self): super(TestCreateDeleteAuditObject, self).setUp() + p_audit_notifications = mock.patch.object( + notifications, 'audit', autospec=True) + self.m_audit_notifications = p_audit_notifications.start() + self.addCleanup(p_audit_notifications.stop) + self.m_send_update = self.m_audit_notifications.send_update + self.goal_id = 1 + self.goal = utils.create_test_goal(id=self.goal_id, name="DUMMY") self.fake_audit = utils.get_test_audit(goal_id=self.goal_id) @mock.patch.object(db_api.Connection, 'create_audit') def test_create(self, mock_create_audit): - utils.create_test_goal(id=self.goal_id) mock_create_audit.return_value = self.fake_audit audit = objects.Audit(self.context, **self.fake_audit) audit.create() @@ -157,9 +176,9 @@ class TestCreateDeleteAuditObject(base.DbTestCase): mock_soft_delete_audit, mock_update_audit): mock_get_audit.return_value = self.fake_audit uuid = self.fake_audit['uuid'] - audit = objects.Audit.get_by_uuid(self.context, uuid) + audit = objects.Audit.get_by_uuid(self.context, uuid, eager=True) audit.soft_delete() - mock_get_audit.assert_called_once_with(self.context, uuid, eager=False) + mock_get_audit.assert_called_once_with(self.context, uuid, eager=True) mock_soft_delete_audit.assert_called_once_with(uuid) mock_update_audit.assert_called_once_with(uuid, {'state': 'DELETED'}) self.assertEqual(self.context, audit._context) @@ -176,3 +195,35 @@ class TestCreateDeleteAuditObject(base.DbTestCase): self.context, uuid, eager=False) mock_destroy_audit.assert_called_once_with(uuid) self.assertEqual(self.context, audit._context) + + +class TestAuditObjectSendNotifications(base.DbTestCase): + + def setUp(self): + super(TestAuditObjectSendNotifications, self).setUp() + goal_id = 1 + self.fake_goal = utils.create_test_goal(id=goal_id, name="DUMMY") + self.fake_strategy = utils.create_test_strategy( + id=goal_id, name="DUMMY") + self.fake_audit = utils.get_test_audit( + goal_id=goal_id, goal=utils.get_test_goal(id=goal_id), + strategy_id=self.fake_strategy.id, strategy=self.fake_strategy) + + p_get_notifier = mock.patch.object(rpc, 'get_notifier') + self.m_get_notifier = p_get_notifier.start() + self.m_get_notifier.return_value = mock.Mock(name='m_notifier') + self.m_notifier = self.m_get_notifier.return_value + self.addCleanup(p_get_notifier.stop) + + @mock.patch.object(db_api.Connection, 'update_audit', mock.Mock()) + @mock.patch.object(db_api.Connection, 'get_audit_by_uuid') + def test_send_update_notification(self, mock_get_audit): + mock_get_audit.return_value = self.fake_audit + uuid = self.fake_audit['uuid'] + audit = objects.Audit.get_by_uuid(self.context, uuid, eager=True) + audit.state = objects.audit.State.ONGOING + audit.save() + + self.assertEqual(1, self.m_notifier.info.call_count) + self.assertEqual('audit.update', + self.m_notifier.info.call_args[1]['event_type'])