From 6ec0ae3015eadb60e8833ad63f80f04f0328ae55 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Mon, 13 May 2013 14:24:01 -0400 Subject: [PATCH] Continuing reconciler work --- stacktach/reconciler.py | 113 ++++++++---- tests/unit/test_reconciler.py | 321 ++++++++++++++++++++++++++++++++++ 2 files changed, 402 insertions(+), 32 deletions(-) create mode 100644 tests/unit/test_reconciler.py diff --git a/stacktach/reconciler.py b/stacktach/reconciler.py index c9e18e4..9446ea8 100644 --- a/stacktach/reconciler.py +++ b/stacktach/reconciler.py @@ -1,40 +1,47 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import json + +from novaclient.exceptions import NotFound from novaclient.v1_1 import client from stacktach import models +from stacktach import utils -reconciler_config = { - 'nova':{ - 'DFW':{ - 'username': 'm0lt3n', - 'project_id': '724740', - 'api_key': '', - 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', - 'auth_system': 'rackspace', - }, - 'ORD':{ - 'username': 'm0lt3n', - 'project_id': '724740', - 'api_key': '', - 'auth_url': 'https://identity.api.rackspacecloud.com/v2.0', - 'auth_system': 'rackspace', - }, - - }, - 'region_mapping_loc': '/etc/stacktach/region_mapping.json' -} - -region_mapping = { - 'x': 'DFW' -} +TERMINATED_AT_KEY = 'OS-INST-USG:terminated_at' class Reconciler(object): - def __init__(self, config): - self.config = reconciler_config - self.region_mapping = region_mapping + def __init__(self, config, region_mapping=None): + self.config = config + self.region_mapping = (region_mapping or + Reconciler._load_region_mapping(config)) self.nova_clients = {} + @classmethod + def _load_region_mapping(cls, config): + with open(config['region_mapping_loc']) as f: + return json.load(f) + def _get_nova(self, region): if region in self.nova_clients: return self.nova_clients[region] @@ -56,12 +63,54 @@ class Reconciler(object): if raws.count() == 0: return False raw = raws[0] - return self.region_mapping[str(raw.deployment.name)] + deployment_name = str(raw.deployment.name) + if deployment_name in self.region_mapping: + return self.region_mapping[deployment_name] + else: + return False + + def _reconcile_from_api(self, launch, server): + terminated_at = server._info[TERMINATED_AT_KEY] + terminated_at = utils.str_time_to_unix(terminated_at) + values = { + 'instance': server.id, + 'launched_at': launch.launched_at, + 'deleted_at': terminated_at, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:nova_api', + } + models.InstanceReconcile(**values).save() + + def _reconcile_from_api_not_found(self, launch): + values = { + 'instance': launch.instance, + 'launched_at': launch.launched_at, + 'deleted_at': 1, + 'instance_type_id': launch.instance_type_id, + 'source': 'reconciler:nova_api:not_found', + } + models.InstanceReconcile(**values).save() def missing_exists_for_instance(self, launched_id, - period_beginning, - period_ending): - launch = models.InstanceUsage.objects.get(id=launched_id) + period_beginning): + reconciled = False + launch = models.InstanceUsage.objects.get(launched_id) region = self._region_for_launch(launch) nova = self._get_nova(region) - server = nova.servers.get(launch.instance) + try: + server = nova.servers.get(launch.instance) + if TERMINATED_AT_KEY in server._info: + # Check to see if instance has been deleted + terminated_at = server._info[TERMINATED_AT_KEY] + terminated_at = utils.str_time_to_unix(terminated_at) + + if terminated_at < period_beginning: + # Check to see if instance was deleted before period. + # If so, we shouldn't expect an exists. + self._reconcile_from_api(launch, server) + reconciled = True + except NotFound: + self._reconcile_from_api_not_found(launch) + reconciled = True + + return reconciled diff --git a/tests/unit/test_reconciler.py b/tests/unit/test_reconciler.py new file mode 100644 index 0000000..cf65427 --- /dev/null +++ b/tests/unit/test_reconciler.py @@ -0,0 +1,321 @@ +# Copyright (c) 2013 - Rackspace Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +import unittest + +import mox +from novaclient.exceptions import NotFound +from novaclient.v1_1 import client as nova_client + +from stacktach import models +from stacktach import reconciler +import utils +from utils import INSTANCE_ID_1 +from utils import REQUEST_ID_1 + + +config = { + 'nova': { + 'RegionOne': { + 'username': 'demo', + 'project_id': '111111', + 'api_key': 'some_key', + 'auth_url': 'https://identity.example.com/v2.0', + 'auth_system': 'keystone', + }, + 'RegionTwo': { + 'username': 'demo', + 'project_id': '111111', + 'api_key': 'some_key', + 'auth_url': 'https://identity.example.com/v2.0', + 'auth_system': 'keystone', + }, + + }, + 'region_mapping_loc': '/etc/stacktach/region_mapping.json', + 'flavor_mapping_loc': '/etc/stacktach/flavor_mapping.json', +} + +region_mapping = { + 'RegionOne.prod.cell1': 'RegionOne', + 'RegionTwo.prod.cell1': 'RegionTwo', +} + + +class ReconcilerTestCase(unittest.TestCase): + def setUp(self): + self.reconciler = reconciler.Reconciler(config, + region_mapping=region_mapping) + self.mox = mox.Mox() + self.mox.StubOutWithMock(models, 'RawData', use_mock_anything=True) + models.RawData.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Deployment', use_mock_anything=True) + models.Deployment.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Lifecycle', use_mock_anything=True) + models.Lifecycle.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'Timing', use_mock_anything=True) + models.Timing.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'RequestTracker', + use_mock_anything=True) + models.RequestTracker.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceUsage', + use_mock_anything=True) + models.InstanceUsage.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceReconcile', + use_mock_anything=True) + models.InstanceReconcile.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceDeletes', + use_mock_anything=True) + models.InstanceDeletes.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'InstanceExists', + use_mock_anything=True) + models.InstanceExists.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(models, 'JsonReport', use_mock_anything=True) + models.JsonReport.objects = self.mox.CreateMockAnything() + self.mox.StubOutWithMock(nova_client, 'Client', use_mock_anything=True) + + def tearDown(self): + self.mox.UnsetStubs() + + def _mocked_nova_client(self): + nova = self.mox.CreateMockAnything() + nova.servers = self.mox.CreateMockAnything() + return nova + + def test_region_for_launch(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(1) + raw = self.mox.CreateMockAnything() + raw.deployment = self.mox.CreateMockAnything() + raw.deployment.name = 'RegionOne.prod.cell1' + result[0].AndReturn(raw) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertEqual('RegionOne', region) + self.mox.VerifyAll() + + def test_region_for_launch_no_mapping(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(1) + raw = self.mox.CreateMockAnything() + raw.deployment = self.mox.CreateMockAnything() + raw.deployment.name = 'RegionOne.prod.cell2' + result[0].AndReturn(raw) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_region_for_launch_no_raws(self): + launch = self.mox.CreateMockAnything() + launch.request_id = REQUEST_ID_1 + result = self.mox.CreateMockAnything() + models.RawData.objects.filter(request_id=REQUEST_ID_1)\ + .AndReturn(result) + result.count().AndReturn(0) + self.mox.ReplayAll() + region = self.reconciler._region_for_launch(launch) + self.assertFalse(region) + self.mox.VerifyAll() + + def test_get_nova(self): + expected_client = self._mocked_nova_client + nova_client.Client('demo', 'some_key', '111111', + auth_url='https://identity.example.com/v2.0', + auth_system='keystone').AndReturn(expected_client) + self.mox.ReplayAll() + client = self.reconciler._get_nova('RegionOne') + self.assertEqual(expected_client, client) + self.mox.VerifyAll() + + def test_get_nova_already_created(self): + expected_client = self.mox.CreateMockAnything() + nova_client.Client('demo', 'some_key', '111111', + auth_url='https://identity.example.com/v2.0', + auth_system='keystone').AndReturn(expected_client) + self.mox.ReplayAll() + self.reconciler._get_nova('RegionOne') + client = self.reconciler._get_nova('RegionOne') + self.assertEqual(expected_client, client) + self.mox.VerifyAll() + + def test_reconcile_from_api(self): + deleted_at = datetime.datetime.utcnow() + launched_at = deleted_at - datetime.timedelta(hours=4) + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = utils.decimal_utc(launched_at) + launch.instance_type_id = 1 + server = self.mox.CreateMockAnything() + server.id = INSTANCE_ID_1 + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at), + } + values = { + 'instance': INSTANCE_ID_1, + 'instance_type_id': 1, + 'launched_at': utils.decimal_utc(launched_at), + 'deleted_at': utils.decimal_utc(deleted_at), + 'source': 'reconciler:nova_api' + } + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**values).AndReturn(result) + result.save() + self.mox.ReplayAll() + self.reconciler._reconcile_from_api(launch, server) + self.mox.VerifyAll() + + def test_reconcile_from_api_not_found(self): + deleted_at = datetime.datetime.utcnow() + launched_at = deleted_at - datetime.timedelta(hours=4) + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + launch.launched_at = utils.decimal_utc(launched_at) + launch.instance_type_id = 1 + values = { + 'instance': INSTANCE_ID_1, + 'instance_type_id': 1, + 'launched_at': utils.decimal_utc(launched_at), + 'deleted_at': 1, + 'source': 'reconciler:nova_api:not_found' + } + result = self.mox.CreateMockAnything() + models.InstanceReconcile(**values).AndReturn(result) + result.save() + self.mox.ReplayAll() + self.reconciler._reconcile_from_api_not_found(launch) + self.mox.VerifyAll() + + def test_missing_exists_for_instance(self): + now = datetime.datetime.utcnow() + deleted_at_dt = now - datetime.timedelta(days=2) + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at_dt), + } + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + self.reconciler._reconcile_from_api(launch, server) + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertTrue(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_deleted_too_soon(self): + now = datetime.datetime.utcnow() + deleted_at_dt = now - datetime.timedelta(hours=4) + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = { + 'OS-INST-USG:terminated_at': str(deleted_at_dt), + } + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_not_deleted(self): + now = datetime.datetime.utcnow() + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + server = self.mox.CreateMockAnything() + server._info = {} + nova.servers.get(INSTANCE_ID_1).AndReturn(server) + + self.mox.StubOutWithMock(self.reconciler, '_reconcile_from_api') + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertFalse(result) + self.mox.VerifyAll() + + def test_missing_exists_for_instance_not_found(self): + now = datetime.datetime.utcnow() + beginning_dt = now - datetime.timedelta(days=1) + beginning_dec = utils.decimal_utc(beginning_dt) + + launch = self.mox.CreateMockAnything() + launch.instance = INSTANCE_ID_1 + models.InstanceUsage.objects.get(1).AndReturn(launch) + self.mox.StubOutWithMock(self.reconciler, '_region_for_launch') + self.reconciler._region_for_launch(launch).AndReturn('RegionOne') + + self.mox.StubOutWithMock(self.reconciler, '_get_nova') + nova = self._mocked_nova_client() + self.reconciler._get_nova('RegionOne').AndReturn(nova) + + nova.servers.get(INSTANCE_ID_1).AndRaise(NotFound(404)) + + self.mox.StubOutWithMock(self.reconciler, + '_reconcile_from_api_not_found') + self.reconciler._reconcile_from_api_not_found(launch) + + self.mox.ReplayAll() + result = self.reconciler.missing_exists_for_instance(1, beginning_dec) + self.assertTrue(result) + self.mox.VerifyAll()