From e77beaa30a48b98a50fe1de584531307cc0910b6 Mon Sep 17 00:00:00 2001 From: Manali Latkar Date: Wed, 16 Oct 2013 12:53:16 +0530 Subject: [PATCH] stacktach accepts AH response code as a ping from yagi along with the name of the service and stores it accordingly and is also compatible with the earlier format. --- stacktach/dbapi.py | 66 +++-- ..._auto__add_field_imageexists_message_id.py | 226 ++++++++++++++++++ stacktach/models.py | 2 + stacktach/notification.py | 5 +- tests/unit/test_dbapi.py | 121 +++++++--- tests/unit/test_notification.py | 8 +- tests/unit/utils.py | 1 + 7 files changed, 375 insertions(+), 54 deletions(-) create mode 100644 stacktach/migrations/0009_auto__add_field_imageexists_message_id.py diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index dfd99b2..abed81f 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -79,6 +79,13 @@ def _log_api_exception(cls, ex, request): stacklog.error(msg) +def _exists_model_factory(service): + if service == 'glance': + return models.ImageExists + elif service == 'nova': + return models.InstanceExists + + def api_call(func): @functools.wraps(func) @@ -193,27 +200,52 @@ def exists_send_status(request, message_id): raise BadRequestException(message=msg) -def _exists_send_status_batch(request): +def _find_exists_with_message_id(msg_id, exists_model, service): + if service == 'glance': + return exists_model.objects.select_for_update().filter( + message_id=msg_id) + elif service == 'nova': + return [models.InstanceExists.objects.select_for_update() + .get(message_id=msg_id)] + +def _ping_processing_with_service(pings, service): + exists_model = _exists_model_factory(service) + with transaction.commit_on_success(): + for msg_id, status_code in pings.items(): + try: + exists = _find_exists_with_message_id(msg_id, exists_model, + service) + for exists in exists: + exists.send_status = status_code + exists.save() + except exists_model.DoesNotExist: + msg = "Could not find Exists record with message_id = '%s' for %s" + msg = msg % (msg_id, service) + raise NotFoundException(message=msg) + except exists_model.MultipleObjectsReturned: + msg = "Multiple Exists records with message_id = '%s' for %s" + msg = msg % (msg_id, service) + raise APIException(message=msg) + + +def _exists_send_status_batch(request): body = json.loads(request.body) if body.get('messages') is not None: messages = body['messages'] - with transaction.commit_on_success(): - for msg_id, status in messages.items(): - try: - exist = models.InstanceExists.objects\ - .select_for_update()\ - .get(message_id=msg_id) - exist.send_status = status - exist.save() - except models.InstanceExists.DoesNotExist: - msg = "Could not find Exists record with message_id = '%s'" - msg = msg % msg_id - raise NotFoundException(message=msg) - except models.InstanceExists.MultipleObjectsReturned: - msg = "Multiple Exists records with message_id = '%s'" - msg = msg % msg_id - raise APIException(message=msg) + version = body.get('version', 0) + if version == 0: + service = 'nova' + nova_pings = messages + if nova_pings: + _ping_processing_with_service(nova_pings, service) + if version == 1: + nova_pings = messages['nova'] + glance_pings = messages['glance'] + if nova_pings: + _ping_processing_with_service(nova_pings, 'nova') + if glance_pings: + _ping_processing_with_service(glance_pings, 'glance') else: msg = "'messages' missing from request body" raise BadRequestException(message=msg) diff --git a/stacktach/migrations/0009_auto__add_field_imageexists_message_id.py b/stacktach/migrations/0009_auto__add_field_imageexists_message_id.py new file mode 100644 index 0000000..ac67ad0 --- /dev/null +++ b/stacktach/migrations/0009_auto__add_field_imageexists_message_id.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'ImageExists.message_id' + db.add_column(u'stacktach_imageexists', 'message_id', + self.gf('django.db.models.fields.CharField')(db_index=True, max_length=50, null=True, blank=True), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'ImageExists.message_id' + db.delete_column(u'stacktach_imageexists', 'message_id') + + + models = { + u'stacktach.deployment': { + 'Meta': {'object_name': 'Deployment'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'stacktach.genericrawdata': { + 'Meta': {'object_name': 'GenericRawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.glancerawdata': { + 'Meta': {'object_name': 'GlanceRawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'owner': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'db_index': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.imagedeletes': { + 'Meta': {'object_name': 'ImageDeletes'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.imageexists': { + 'Meta': {'object_name': 'ImageExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'created_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': u"orm['stacktach.GlanceRawData']"}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.ImageUsage']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'db_index': 'True'}) + }, + u'stacktach.imageusage': { + 'Meta': {'object_name': 'ImageUsage'}, + 'created_at': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.GlanceRawData']", 'null': 'True'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'db_index': 'True'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'max_length': '20'}), + 'uuid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'stacktach.instancedeletes': { + 'Meta': {'object_name': 'InstanceDeletes'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}) + }, + u'stacktach.instanceexists': { + 'Meta': {'object_name': 'InstanceExists'}, + 'audit_period_beginning': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'audit_period_ending': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'bandwidth_public_out': ('django.db.models.fields.BigIntegerField', [], {'null': 'True'}), + 'delete': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceDeletes']"}), + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'fail_reason': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '300', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'send_status': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '50', 'db_index': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'usage': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.InstanceUsage']"}) + }, + u'stacktach.instancereconcile': { + 'Meta': {'object_name': 'InstanceReconcile'}, + 'deleted_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'row_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'row_updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '150', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.instanceusage': { + 'Meta': {'object_name': 'InstanceUsage'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'instance_type_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'launched_at': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.jsonreport': { + 'Meta': {'object_name': 'JsonReport'}, + 'created': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'period_end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'period_start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'version': ('django.db.models.fields.IntegerField', [], {'default': '1'}) + }, + u'stacktach.lifecycle': { + 'Meta': {'object_name': 'Lifecycle'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']", 'null': 'True'}), + 'last_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'last_task_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}) + }, + u'stacktach.rawdata': { + 'Meta': {'object_name': 'RawData'}, + 'deployment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Deployment']"}), + 'event': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'host': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image_type': ('django.db.models.fields.IntegerField', [], {'default': '0', 'null': 'True', 'db_index': 'True'}), + 'instance': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'json': ('django.db.models.fields.TextField', [], {}), + 'old_state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'old_task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'request_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'routing_key': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'service': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'state': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '20', 'null': 'True', 'blank': 'True'}), + 'task': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '30', 'null': 'True', 'blank': 'True'}), + 'tenant': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'when': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.rawdataimagemeta': { + 'Meta': {'object_name': 'RawDataImageMeta'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'os_architecture': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_distro': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'os_version': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'raw': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.RawData']"}), + 'rax_options': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'stacktach.requesttracker': { + 'Meta': {'object_name': 'RequestTracker'}, + 'completed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'duration': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_timing': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Timing']", 'null': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'request_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start': ('django.db.models.fields.DecimalField', [], {'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}) + }, + u'stacktach.timing': { + 'Meta': {'object_name': 'Timing'}, + 'diff': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6', 'db_index': 'True'}), + 'end_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'end_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lifecycle': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['stacktach.Lifecycle']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'start_raw': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': u"orm['stacktach.RawData']"}), + 'start_when': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '6'}) + } + } + + complete_apps = ['stacktach'] \ No newline at end of file diff --git a/stacktach/models.py b/stacktach/models.py index b30686e..6cb930a 100644 --- a/stacktach/models.py +++ b/stacktach/models.py @@ -487,6 +487,8 @@ class ImageExists(models.Model): send_status = models.IntegerField(default=0, db_index=True) owner = models.CharField(max_length=255, db_index=True, null=True) size = models.BigIntegerField(max_length=20) + message_id = models.CharField(max_length=50, null=True, + blank=True, db_index=True) def update_status(self, new_status): self.status = new_status diff --git a/stacktach/notification.py b/stacktach/notification.py index 23fbbd3..abcce00 100644 --- a/stacktach/notification.py +++ b/stacktach/notification.py @@ -34,6 +34,7 @@ class Notification(object): self.publisher = self.body['publisher_id'] self.event = self.body['event_type'] + @property def when(self): when = self.body.get('timestamp', None) @@ -157,6 +158,7 @@ class GlanceNotification(Notification): 'audit_period_ending', None) audit_period_ending = audit_period_ending and \ utils.str_time_to_unix(audit_period_ending) + message_id = self.message_id images = self.payload.get('images', []) else: stacklog.warn("Received exists with invalid payload " @@ -179,7 +181,8 @@ class GlanceNotification(Notification): 'audit_period_ending': audit_period_ending, 'owner': self.owner, 'size': image['size'], - 'raw': raw + 'raw': raw, + 'message_id': message_id } usage = db.get_image_usage(uuid=uuid) values['usage'] = usage diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 87e0de9..1265d29 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -33,6 +33,7 @@ import utils from utils import INSTANCE_ID_1 from utils import MESSAGE_ID_1 from utils import MESSAGE_ID_2 +from utils import MESSAGE_ID_3 class DBAPITestCase(StacktachBaseTestCase): @@ -43,8 +44,11 @@ class DBAPITestCase(StacktachBaseTestCase): self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) models.InstanceExists.objects = self.mox.CreateMockAnything() + models.ImageExists.objects = self.mox.CreateMockAnything() models.InstanceExists.DoesNotExist = dne_exception + models.ImageExists.DoesNotExist = dne_exception models.InstanceExists.MultipleObjectsReturned = mor_exception + models.ImageExists.MultipleObjectsReturned = mor_exception def tearDown(self): self.mox.UnsetStubs() @@ -508,39 +512,11 @@ class DBAPITestCase(StacktachBaseTestCase): self.assertEqual(body.get("message"), msg) self.mox.VerifyAll() - def test_send_status_batch(self): - fake_request = self.mox.CreateMockAnything() - fake_request.method = 'PUT' - messages = { - MESSAGE_ID_1: 200, - MESSAGE_ID_2: 400 - } - body_dict = {'messages': messages} - body = json.dumps(body_dict) - fake_request.body = body - self.mox.StubOutWithMock(transaction, 'commit_on_success') - trans_obj = self.mox.CreateMockAnything() - transaction.commit_on_success().AndReturn(trans_obj) - trans_obj.__enter__() - results1 = self.mox.CreateMockAnything() - models.InstanceExists.objects.select_for_update().AndReturn(results1) - exists1 = self.mox.CreateMockAnything() - results1.get(message_id=MESSAGE_ID_2).AndReturn(exists1) - exists1.save() - results2 = self.mox.CreateMockAnything() - models.InstanceExists.objects.select_for_update().AndReturn(results2) - exists2 = self.mox.CreateMockAnything() - results2.get(message_id=MESSAGE_ID_1).AndReturn(exists2) - exists2.save() - trans_obj.__exit__(None, None, None) - self.mox.ReplayAll() - - def test_send_status_batch_accepts_post(self): + def test_send_status_batch_accepts_post_when_version_is_not_given(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'POST' messages = { - MESSAGE_ID_1: 200, - MESSAGE_ID_2: 400 + MESSAGE_ID_1: 201, MESSAGE_ID_2: 400 } body_dict = {'messages': messages} body = json.dumps(body_dict) @@ -567,11 +543,86 @@ class DBAPITestCase(StacktachBaseTestCase): exists1.send_status = 200 self.mox.VerifyAll() + def test_send_status_batch_accepts_post_for_nova_and_glance_when_version_is_1(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'POST' + fake_request.GET = {'service': 'glance'} + messages = { + 'nova': {MESSAGE_ID_3: 201}, + 'glance': {MESSAGE_ID_1: 201, MESSAGE_ID_2: 201} + } + body_dict = {'version': 1, 'messages': messages} + body = json.dumps(body_dict) + fake_request.body = body + self.mox.StubOutWithMock(transaction, 'commit_on_success') + trans_obj = self.mox.CreateMockAnything() + transaction.commit_on_success().AndReturn(trans_obj) + trans_obj.__enter__() + results1 = self.mox.CreateMockAnything() + models.InstanceExists.objects.select_for_update().AndReturn(results1) + exists1 = self.mox.CreateMockAnything() + results1.get(message_id=MESSAGE_ID_3).AndReturn(exists1) + exists1.save() + trans_obj.__exit__(None, None, None) + trans_obj = self.mox.CreateMockAnything() + transaction.commit_on_success().AndReturn(trans_obj) + trans_obj.__enter__() + results1 = self.mox.CreateMockAnything() + models.ImageExists.objects.select_for_update().AndReturn(results1) + exists1A = self.mox.CreateMockAnything() + exists1B = self.mox.CreateMockAnything() + results1.filter(message_id=MESSAGE_ID_2).AndReturn([exists1A, exists1B]) + exists1A.save() + exists1B.save() + results2 = self.mox.CreateMockAnything() + models.ImageExists.objects.select_for_update().AndReturn(results2) + exists2A = self.mox.CreateMockAnything() + exists2B = self.mox.CreateMockAnything() + results2.filter(message_id=MESSAGE_ID_1).AndReturn([exists2A, exists2B]) + exists2A.save() + exists2B.save() + trans_obj.__exit__(None, None, None) + self.mox.ReplayAll() + + resp = dbapi.exists_send_status(fake_request, 'batch') + self.assertEqual(resp.status_code, 200) + self.mox.VerifyAll() + + + + def test_send_status_batch_accepts_post_when_version_is_0(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'POST' + messages = {MESSAGE_ID_1: 201, MESSAGE_ID_2: 201} + body_dict = {'version': 0, 'messages': messages} + body = json.dumps(body_dict) + fake_request.body = body + self.mox.StubOutWithMock(transaction, 'commit_on_success') + trans_obj = self.mox.CreateMockAnything() + transaction.commit_on_success().AndReturn(trans_obj) + trans_obj.__enter__() + results1 = self.mox.CreateMockAnything() + models.InstanceExists.objects.select_for_update().AndReturn(results1) + exists1 = self.mox.CreateMockAnything() + results1.get(message_id=MESSAGE_ID_2).AndReturn(exists1) + exists1.save() + results2 = self.mox.CreateMockAnything() + models.InstanceExists.objects.select_for_update().AndReturn(results2) + exists2 = self.mox.CreateMockAnything() + results2.get(message_id=MESSAGE_ID_1).AndReturn(exists2) + exists2.save() + trans_obj.__exit__(None, None, None) + self.mox.ReplayAll() + + resp = dbapi.exists_send_status(fake_request, 'batch') + self.assertEqual(resp.status_code, 200) + self.mox.VerifyAll() + def test_send_status_batch_not_found(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'PUT' messages = { - MESSAGE_ID_1: 200, + MESSAGE_ID_1: '201', } body_dict = {'messages': messages} body = json.dumps(body_dict) @@ -593,7 +644,7 @@ class DBAPITestCase(StacktachBaseTestCase): self.assertEqual(resp.status_code, 404) body = json.loads(resp.content) self.assertEqual(body.get("status"), 404) - msg = "Could not find Exists record with message_id = '%s'" + msg = "Could not find Exists record with message_id = '%s' for nova" msg = msg % MESSAGE_ID_1 self.assertEqual(body.get("message"), msg) self.mox.VerifyAll() @@ -602,7 +653,7 @@ class DBAPITestCase(StacktachBaseTestCase): fake_request = self.mox.CreateMockAnything() fake_request.method = 'PUT' messages = { - MESSAGE_ID_1: 200, + MESSAGE_ID_1: 201, } body_dict = {'messages': messages} body = json.dumps(body_dict) @@ -624,7 +675,7 @@ class DBAPITestCase(StacktachBaseTestCase): self.assertEqual(resp.status_code, 500) body = json.loads(resp.content) self.assertEqual(body.get("status"), 500) - msg = "Multiple Exists records with message_id = '%s'" + msg = "Multiple Exists records with message_id = '%s' for nova" msg = msg % MESSAGE_ID_1 self.assertEqual(body.get("message"), msg) self.mox.VerifyAll() @@ -643,6 +694,7 @@ class DBAPITestCase(StacktachBaseTestCase): def test_send_status_batch_no_body(self): fake_request = self.mox.CreateMockAnything() + fake_request.GET = {'service': 'nova'} fake_request.method = 'PUT' fake_request.body = None self.mox.ReplayAll() @@ -670,6 +722,7 @@ class DBAPITestCase(StacktachBaseTestCase): def test_send_status_batch_bad_body(self): fake_request = self.mox.CreateMockAnything() fake_request.method = 'PUT' + fake_request.GET = {'service': 'nova'} body_dict = {'bad': 'body'} fake_request.body = json.dumps(body_dict) self.mox.ReplayAll() diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index 6b24486..d1f0127 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -545,6 +545,7 @@ class GlanceExistsNotificationTestCase(StacktachBaseTestCase): "event_type": "image.exists", "timestamp": "2013-06-20 18:31:57.939614", "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "message_id": "d14cfa51-6a0e-4cf8-9130-804738be96d2", "payload": { "audit_period_beginning": audit_period_beginning, "audit_period_ending": audit_period_ending, @@ -587,7 +588,8 @@ class GlanceExistsNotificationTestCase(StacktachBaseTestCase): audit_period_ending=utils.str_time_to_unix(audit_period_ending), size=size, uuid=uuid, - usage=None).AndReturn(raw) + usage=None, + message_id="d14cfa51-6a0e-4cf8-9130-804738be96d2").AndReturn(raw) self.mox.ReplayAll() @@ -609,6 +611,7 @@ class GlanceExistsNotificationTestCase(StacktachBaseTestCase): "event_type": "image.exists", "timestamp": "2013-06-20 18:31:57.939614", "publisher_id": "glance-api01-r2961.global.preprod-ord.ohthree.com", + "message_id": "d14cfa51-6a0e-4cf8-9130-804738be96d2", "payload": { "audit_period_beginning": audit_period_beginning, "audit_period_ending": audit_period_ending, @@ -654,7 +657,8 @@ class GlanceExistsNotificationTestCase(StacktachBaseTestCase): uuid=uuid, usage=None, delete=delete, - deleted_at=utils.str_time_to_unix(deleted_at)).AndReturn(raw) + deleted_at=utils.str_time_to_unix(deleted_at), + message_id="d14cfa51-6a0e-4cf8-9130-804738be96d2").AndReturn(raw) self.mox.ReplayAll() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 9ad5809..2d4f402 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -41,6 +41,7 @@ DECIMAL_DUMMY_TIME = dt.dt_to_decimal(DUMMY_TIME) MESSAGE_ID_1 = "7f28f81b-29a2-43f2-9ba1-ccb3e53ab6c8" MESSAGE_ID_2 = "4d596126-0f04-4329-865f-7b9a7bd69bcf" +MESSAGE_ID_3 = "4d596126-0f04-4329-865f-797387adf45c" BANDWIDTH_PUBLIC_OUTBOUND = 1697240969