From 743eaf78cc9fde82472d0cd5634702512cc01126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Thu, 13 Oct 2016 13:30:27 -0400 Subject: [PATCH] Add support for multiple storage backends * New storage drivers can be implemented in addition to MongoDB, for example: MySQL or Postgres, etc. * The default database driver still MongoDB * New storage drivers must implement the interface defined by the class BaseDriver * Flexmock and hamcrest are deprected for new unit tests Change-Id: I1cf73f28d469d2f22ecbaf345e53b9596cc0c2f6 --- almanach/api/main.py | 8 +- almanach/collector/main.py | 8 +- almanach/core/exception.py | 4 + almanach/storage/drivers/__init__.py | 0 almanach/storage/drivers/base_driver.py | 93 +++++++++++++++++++ .../mongodb_driver.py} | 5 +- almanach/storage/storage_driver.py | 32 +++++++ doc/source/index.rst | 4 +- tests/core/test_controller.py | 4 +- tests/storage/drivers/__init__.py | 0 .../drivers/test_mongodb_driver.py} | 8 +- tests/storage/test_storage_driver.py | 32 +++++++ 12 files changed, 180 insertions(+), 18 deletions(-) create mode 100644 almanach/storage/drivers/__init__.py create mode 100644 almanach/storage/drivers/base_driver.py rename almanach/storage/{database_adapter.py => drivers/mongodb_driver.py} (97%) create mode 100644 almanach/storage/storage_driver.py create mode 100644 tests/storage/drivers/__init__.py rename tests/{core/test_database_adapter.py => storage/drivers/test_mongodb_driver.py} (98%) create mode 100644 tests/storage/test_storage_driver.py diff --git a/almanach/api/main.py b/almanach/api/main.py index 70124da..b50406a 100644 --- a/almanach/api/main.py +++ b/almanach/api/main.py @@ -22,7 +22,7 @@ from almanach.api import auth_adapter from almanach.api.v1 import routes from almanach.core import controller from almanach.core import opts -from almanach.storage import database_adapter +from almanach.storage import storage_driver LOG = log.getLogger(__name__) @@ -35,10 +35,10 @@ def main(): opts.CONF(sys.argv[1:]) config = opts.CONF - storage = database_adapter.DatabaseAdapter(config) - storage.connect() + database_driver = storage_driver.StorageDriver(config).get_database_driver() + database_driver.connect() - routes.controller = controller.Controller(config, storage) + routes.controller = controller.Controller(config, database_driver) routes.auth_adapter = auth_adapter.AuthenticationAdapter(config).get_authentication_adapter() LOG.info('Listening on %s:%d', config.api.bind_ip, config.api.bind_port) diff --git a/almanach/collector/main.py b/almanach/collector/main.py index cda05ae..79c667b 100644 --- a/almanach/collector/main.py +++ b/almanach/collector/main.py @@ -20,7 +20,7 @@ from almanach.collector import bus_adapter from almanach.collector import retry_adapter from almanach.core import controller from almanach.core import opts -from almanach.storage import database_adapter +from almanach.storage import storage_driver LOG = log.getLogger(__name__) @@ -30,10 +30,10 @@ def main(): opts.CONF(sys.argv[1:]) config = opts.CONF - storage = database_adapter.DatabaseAdapter(config) - storage.connect() + database_driver = storage_driver.StorageDriver(config).get_database_driver() + database_driver.connect() - application_controller = controller.Controller(config, storage) + application_controller = controller.Controller(config, database_driver) connection = kombu.Connection(hostname=config.collector.rabbit_host, port=config.collector.rabbit_port, userid=config.collector.rabbit_username, diff --git a/almanach/core/exception.py b/almanach/core/exception.py index de5b0b4..22714a0 100644 --- a/almanach/core/exception.py +++ b/almanach/core/exception.py @@ -57,3 +57,7 @@ class VolumeTypeNotFoundException(AlmanachException): message = "Unable to find volume_type id '{volume_type_id}'".format(volume_type_id=volume_type_id) super(VolumeTypeNotFoundException, self).__init__(message) + + +class DatabaseDriverNotSupportedException(AlmanachException): + pass diff --git a/almanach/storage/drivers/__init__.py b/almanach/storage/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/almanach/storage/drivers/base_driver.py b/almanach/storage/drivers/base_driver.py new file mode 100644 index 0000000..a55cb5b --- /dev/null +++ b/almanach/storage/drivers/base_driver.py @@ -0,0 +1,93 @@ +# Copyright 2016 Internap. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + + +class BaseDriver(object): + + def __init__(self, config): + self.config = config + + @abc.abstractmethod + def connect(self): + pass + + @abc.abstractmethod + def get_active_entity(self, entity_id): + pass + + @abc.abstractmethod + def count_entities(self): + pass + + @abc.abstractmethod + def count_active_entities(self): + pass + + @abc.abstractmethod + def count_entity_entries(self, entity_id): + pass + + @abc.abstractmethod + def has_active_entity(self, entity_id): + pass + + @abc.abstractmethod + def list_entities(self, project_id, start, end, entity_type=None): + pass + + @abc.abstractmethod + def get_all_entities_by_id(self, entity_id): + pass + + @abc.abstractmethod + def list_entities_by_id(self, entity_id, start, end): + pass + + @abc.abstractmethod + def update_closed_entity(self, entity, data): + pass + + @abc.abstractmethod + def insert_entity(self, entity): + pass + + @abc.abstractmethod + def close_active_entity(self, entity_id, end): + pass + + @abc.abstractmethod + def update_active_entity(self, entity): + pass + + @abc.abstractmethod + def delete_active_entity(self, entity_id): + pass + + @abc.abstractmethod + def insert_volume_type(self, volume_type): + pass + + @abc.abstractmethod + def get_volume_type(self, volume_type_id): + pass + + @abc.abstractmethod + def delete_volume_type(self, volume_type_id): + pass + + @abc.abstractmethod + def list_volume_types(self): + pass diff --git a/almanach/storage/database_adapter.py b/almanach/storage/drivers/mongodb_driver.py similarity index 97% rename from almanach/storage/database_adapter.py rename to almanach/storage/drivers/mongodb_driver.py index 7ef7320..26b2ac9 100644 --- a/almanach/storage/database_adapter.py +++ b/almanach/storage/drivers/mongodb_driver.py @@ -18,14 +18,15 @@ import pymongo from almanach.core import exception from almanach.core import model from almanach.core.model import build_entity_from_dict +from almanach.storage.drivers import base_driver LOG = log.getLogger(__name__) -class DatabaseAdapter(object): +class MongoDbDriver(base_driver.BaseDriver): def __init__(self, config, db=None): + super(MongoDbDriver, self).__init__(config) self.db = db - self.config = config def connect(self): connection = pymongo.MongoClient(self.config.database.connection_url, tz_aware=True) diff --git a/almanach/storage/storage_driver.py b/almanach/storage/storage_driver.py new file mode 100644 index 0000000..fd803f7 --- /dev/null +++ b/almanach/storage/storage_driver.py @@ -0,0 +1,32 @@ +# Copyright 2016 Internap. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log + +from almanach.core import exception +from almanach.storage.drivers import mongodb_driver + +LOG = log.getLogger(__name__) + + +class StorageDriver(object): + def __init__(self, config): + self.config = config + + def get_database_driver(self): + if self.config.database.driver == 'mongodb': + LOG.info('Loading MongoDb storage driver') + return mongodb_driver.MongoDbDriver(self.config) + raise exception.DatabaseDriverNotSupportedException( + 'Unknown database driver {}'.format(self.config.database.driver)) diff --git a/doc/source/index.rst b/doc/source/index.rst index a7a70c4..3e5d032 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -121,8 +121,8 @@ For example with Nova, add the topic "almanach" in the config file :code:`/etc/n notification_topics=almanach -Database configuration ----------------------- +MongoDB configuration +--------------------- Almanach requires a specific user to connect to the database. To create a new user, open a new MongoDB shell: diff --git a/tests/core/test_controller.py b/tests/core/test_controller.py index b104c0e..47f6b6d 100644 --- a/tests/core/test_controller.py +++ b/tests/core/test_controller.py @@ -27,7 +27,7 @@ import pytz from almanach.core import controller from almanach.core import exception from almanach.core import model -from almanach.storage import database_adapter +from almanach.storage.drivers import base_driver from tests import base from tests.builder import a @@ -40,7 +40,7 @@ class ControllerTest(base.BaseTestCase): def setUp(self): super(ControllerTest, self).setUp() - self.database_adapter = flexmock(database_adapter.DatabaseAdapter) + self.database_adapter = flexmock(base_driver.BaseDriver) self.controller = controller.Controller(self.config, self.database_adapter) def test_instance_created(self): diff --git a/tests/storage/drivers/__init__.py b/tests/storage/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_database_adapter.py b/tests/storage/drivers/test_mongodb_driver.py similarity index 98% rename from tests/core/test_database_adapter.py rename to tests/storage/drivers/test_mongodb_driver.py index 28cfca6..e93caed 100644 --- a/tests/core/test_database_adapter.py +++ b/tests/storage/drivers/test_mongodb_driver.py @@ -24,7 +24,7 @@ from hamcrest import contains_inanyorder from almanach.core import exception from almanach.core import model -from almanach.storage.database_adapter import DatabaseAdapter +from almanach.storage.drivers import mongodb_driver from tests import base from tests.builder import a @@ -33,12 +33,12 @@ from tests.builder import volume from tests.builder import volume_type -class DatabaseAdapterTest(base.BaseTestCase): +class MongoDbDriverTest(base.BaseTestCase): def setUp(self): - super(DatabaseAdapterTest, self).setUp() + super(MongoDbDriverTest, self).setUp() mongo_connection = mongomock.Connection() self.db = mongo_connection['almanach'] - self.adapter = DatabaseAdapter(self.config, self.db) + self.adapter = mongodb_driver.MongoDbDriver(self.config, self.db) flexmock(pymongo.MongoClient).new_instances(mongo_connection) def test_insert_instance(self): diff --git a/tests/storage/test_storage_driver.py b/tests/storage/test_storage_driver.py new file mode 100644 index 0000000..118bf04 --- /dev/null +++ b/tests/storage/test_storage_driver.py @@ -0,0 +1,32 @@ +# Copyright 2016 Internap. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from almanach.core import exception +from almanach.storage.drivers import mongodb_driver +from almanach.storage import storage_driver + +from tests import base + + +class StorageDriverTest(base.BaseTestCase): + def setUp(self): + super(StorageDriverTest, self).setUp() + self.storage_driver = storage_driver.StorageDriver(self.config) + + def test_get_default_database_adapter(self): + self.assertIsInstance(self.storage_driver.get_database_driver(), mongodb_driver.MongoDbDriver) + + def test_get_unknown_database_adapter(self): + self.config_fixture.config(driver='foobar', group='database') + self.assertRaises(exception.DatabaseDriverNotSupportedException, self.storage_driver.get_database_driver)