diff --git a/bin/refstack-api b/bin/refstack-api new file mode 100755 index 00000000..1ec8d434 --- /dev/null +++ b/bin/refstack-api @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Command-line launcher for Refstack API +""" + +import sys + +from pecan.commands import serve + +from refstack.api import config as api_config + + +def get_pecan_config(): + """ Get path to pecan configuration file """ + filename = api_config.__file__.replace('.pyc', '.py') + return filename + + +if __name__ == '__main__': + config_path = get_pecan_config() + sys.argv.append(config_path) + serve.gunicorn_run() diff --git a/bin/refstack-manage b/bin/refstack-manage new file mode 100755 index 00000000..073c377d --- /dev/null +++ b/bin/refstack-manage @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Command-line utility for database manage +""" + +import sys + +from oslo.config import cfg + +from refstack.db import migration + +CONF = cfg.CONF + + +class DatabaseManager(object): + + def version(self): + print(migration.version()) + + def upgrade(self): + migration.upgrade(CONF.command.revision) + + def downgrade(self): + migration.downgrade(CONF.command.revision) + + def stamp(self): + migration.stamp(CONF.command.revision) + + def revision(self): + migration.revision(CONF.command.message, CONF.command.autogenerate) + + +def add_command_parsers(subparsers): + db_manager = DatabaseManager() + + parser = subparsers.add_parser('version', + help='show current database version') + parser.set_defaults(func=db_manager.version) + + parser = subparsers.add_parser('upgrade', + help='upgrade database to ' + 'the specified version') + parser.set_defaults(func=db_manager.upgrade) + parser.add_argument('--revision', nargs='?', + help='desired database version') + + parser = subparsers.add_parser('downgrade', + help='downgrade database ' + 'to the specified version') + parser.set_defaults(func=db_manager.downgrade) + parser.add_argument('--revision', nargs='?', + help='desired database version') + + parser = subparsers.add_parser('stamp', + help='stamp database with provided ' + 'revision. Don\'t run any migrations') + parser.add_argument('--revision', nargs='?', + help='should match one from repository or head - ' + 'to stamp database with most recent revision') + parser.set_defaults(func=db_manager.stamp) + + parser = subparsers.add_parser('revision', + help='create template for migration') + parser.add_argument('-m', '--message', + help='text that will be used for migration title') + parser.add_argument('--autogenerate', action='store_true', + help='if True - generates diff based ' + 'on current database state (True by default)') + parser.set_defaults(func=db_manager.revision) + +command_opt = cfg.SubCommandOpt('command', + title='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + +if __name__ == '__main__': + CONF(sys.argv[1:], project='refstack') + CONF.command.func() diff --git a/doc/refstack.md b/doc/refstack.md index 1d0973fc..4e7b21af 100755 --- a/doc/refstack.md +++ b/doc/refstack.md @@ -1,8 +1,5 @@ Refstack Quickstart =================== - -Instruction to run refstack for development or behind your firewall. - ####Install dependencies (on ubuntu 14.x).. - `sudo apt-get install git python-dev libssl-dev python-setuptools` @@ -46,40 +43,44 @@ Instruction to run refstack for development or behind your firewall. - `cd refstack` -- Update the db connection strings in following files to the correct - information of your environment. - - - The `sqlalchemy.url = mysql://root:passw0rd@127.0.0.1/refstack` string - in the `./refstack/db/migrations/alembic.ini` file. - - - The `'db_url': 'mysql://root:passw0rd@127.0.0.1/refstack'` string in the - `./refstack/api/config.py` file. - - - NOTE: You may need to also update the `'debug': False` string in the - `./refstack/api/config.py` file for development. - - Creare virtual environment: `virtualenv .venv --system-site-package` - Source to virtual environment: `source .venv/bin/activate` -- Install refstack: `python setup.py install` -- Create tables in the refstack database. +####Install Refstack application (on ubuntu 14.x).. - - `cd ./refstack/db/migrations/` +- `python setup.py install` - - `alembic upgrade head` +####Configuration file preparation - - `cd ../../..` +- Make a copy of the sample config and update it with the correct information of your environment. Example of config file available in etc directory. -Plug this bad boy into your server infrastructure. +####Database sync -We use nginx and gunicorn, you may use something else if you so desire. +- Check current revision: + + `refstack-manage --config-file /path/to/refstack.conf version` + + The response will show the current database revision. If the revision is `None` (indicating a clear database), the following command should be performed to upgrade the database to the latest revision: + + - Upgrade database to latest revision: + + `refstack-manage --config-file /path/to/refstack.conf upgrade --revision head` + + - Check current revision: + + `refstack-manage --config-file /path/to/refstack.conf version` + + Now it should be `42278d6179b9`. + + +####Start Refstack For the most basic setup that you can try right now, just kick off gunicorn: -`gunicorn_pecan --debug refstack/api/config.py` +- `refstack-api --env REFSTACK_OSLO_CONFIG=/path/to/refstack.conf` Now available http://localhost:8000/ with JSON response {'Root': 'OK'} -and http://localhost:8000/v1/results/ with JSON response {'Results': 'OK'}. \ No newline at end of file +and http://localhost:8000/v1/results/ with JSON response {'Results': 'OK'}. diff --git a/etc/refstack.conf.sample b/etc/refstack.conf.sample new file mode 100644 index 00000000..a375f5ec --- /dev/null +++ b/etc/refstack.conf.sample @@ -0,0 +1,2 @@ +[DEFAULT] +sql_connection = mysql://refstack:@127.0.0.1/refstack diff --git a/refstack/api/app.py b/refstack/api/app.py index e77d3dd7..15929337 100644 --- a/refstack/api/app.py +++ b/refstack/api/app.py @@ -17,31 +17,18 @@ import json import logging +import os +from oslo.config import cfg import pecan from pecan import hooks import webob -from refstack import backend +from refstack import utils logger = logging.getLogger(__name__) - -class BackendHook(hooks.PecanHook): - - """Pecan Hook for providing backend functionality.""" - - def __init__(self, app_config): - """Hook init.""" - self.global_backend = backend.Backend(app_config) - - def before(self, state): - """Before request.""" - state.request.backend = self.global_backend.create_local() - - def after(self, state): - """After request.""" - pass +CONF = cfg.CONF class JSONErrorHook(hooks.PecanHook): @@ -85,9 +72,28 @@ def setup_app(config): app_conf.pop('root'), logging=getattr(config, 'logging', {}), hooks=[JSONErrorHook(app_conf), hooks.RequestViewerHook( - {'items': ['status', 'method', 'controller', 'path']} - ), BackendHook(app_conf)], + {'items': ['status', 'method', 'controller', 'path', 'body']}, + headers=False, writer=utils.LogWriter(logger, logging.DEBUG) + )], **app_conf ) + # By default we expect path to oslo config file in environment variable + # REFSTACK_OSLO_CONFIG (option for testing and development) + # If it is empty we look up those config files + # in the following directories: + # ~/.${project}/ + # ~/ + # /etc/${project}/ + # /etc/ + default_config_files = ((os.getenv('REFSTACK_OSLO_CONFIG'), ) + if os.getenv('REFSTACK_OSLO_CONFIG') + else cfg.find_config_files('refstack')) + + CONF('', + project='refstack', + default_config_files=default_config_files) + + CONF.log_opt_values(logger, logging.DEBUG) + return app diff --git a/refstack/api/config.py b/refstack/api/config.py index 88324522..5fa9b583 100644 --- a/refstack/api/config.py +++ b/refstack/api/config.py @@ -34,7 +34,6 @@ server = { app = { 'root': 'refstack.api.controllers.root.RootController', 'modules': ['refstack.api'], - 'db_url': 'mysql://root:r00t@127.0.0.1/refstack', 'static_root': '%(confdir)s/../static', 'template_path': '%(confdir)s/../templates', # The 'debug' option should be false in production servers, but needs to be diff --git a/refstack/api/controllers/v1.py b/refstack/api/controllers/v1.py index eba5ee65..bb5672da 100644 --- a/refstack/api/controllers/v1.py +++ b/refstack/api/controllers/v1.py @@ -19,6 +19,8 @@ import logging import pecan from pecan import rest +from refstack import db + logger = logging.getLogger(__name__) @@ -37,11 +39,11 @@ class ResultsController(rest.RestController): :param test_id: ID of the test to get the JSON for. """ - test_info = pecan.request.backend.get_test(test_id) + test_info = db.get_test(test_id) if not test_info: pecan.abort(404) - test_list = pecan.request.backend.get_test_results(test_id) + test_list = db.get_test_results(test_id) test_name_list = [test_dict[0] for test_dict in test_list] return {"cpid": test_info.cpid, "created_at": test_info.created_at, @@ -58,7 +60,7 @@ class ResultsController(rest.RestController): detail='Request body \'%s\' could not ' 'be decoded as JSON.' '' % pecan.request.body) - test_id = pecan.request.backend.store_results(results) + test_id = db.store_results(results) return {'test_id': test_id} diff --git a/refstack/backend.py b/refstack/backend.py deleted file mode 100644 index dc8cb627..00000000 --- a/refstack/backend.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Backend provider.""" - -import logging -import uuid - -import sqlalchemy as sa -from sqlalchemy import orm - -from refstack import models - -logger = logging.getLogger(__name__) - - -class Backend(object): - - """Global backend provider.""" - - def __init__(self, app_config): - """Backend factory.""" - engine = sa.create_engine(app_config['db_url']) - self.session_maker = orm.sessionmaker() - self.session_maker.configure(bind=engine) - - def create_local(self): - """Create request-local Backend instance.""" - return LocalBackend(self) - - -class LocalBackend(object): - - """Request-local backend provider.""" - - def __init__(self, global_backend): - """Request-local backend instance.""" - self.db_session = global_backend.session_maker() - - def store_results(self, results): - """Storing results into database. - - :param results: Dict describes test results. - """ - session = self.db_session - - test_id = str(uuid.uuid4()) - test = models.Test(id=test_id, cpid=results.get('cpid'), - duration_seconds=results.get('duration_seconds')) - test_results = results.get('results', []) - for result in test_results: - session.add(models.TestResults( - test_id=test_id, name=result['name'], - uid=result.get('uid', None) - )) - session.add(test) - session.commit() - return test_id - - def get_test(self, test_id): - """Get test information from the database. - - :param test_id: The ID of the test. - """ - test_info = self.db_session.query(models.Test).\ - filter_by(id=test_id).first() - return test_info - - def get_test_results(self, test_id): - """Get all passed tempest tests for a particular test. - - :param test_id: The ID of the test. - """ - results = self.db_session.query(models.TestResults.name).filter_by( - test_id=test_id).all() - return results diff --git a/refstack/db/__init__.py b/refstack/db/__init__.py index e69de29b..0ea4f423 100644 --- a/refstack/db/__init__.py +++ b/refstack/db/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +DB abstraction for Refstack +""" + +from refstack.db.api import * # noqa diff --git a/refstack/db/api.py b/refstack/db/api.py new file mode 100644 index 00000000..f3c48601 --- /dev/null +++ b/refstack/db/api.py @@ -0,0 +1,63 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Defines interface for DB access. + +Functions in this module are imported into the refstack.db namespace. +Call these functions from refstack.db namespace, not the refstack.db.api +namespace. + +""" +from oslo.config import cfg +from oslo.db import api as db_api + + +db_opts = [ + cfg.StrOpt('db_backend', + default='sqlalchemy', + help='The backend to use for database.'), +] + +CONF = cfg.CONF +CONF.register_opts(db_opts) + +_BACKEND_MAPPING = {'sqlalchemy': 'refstack.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +################### +def store_results(results): + """Storing results into database. + + :param results: Dict describes test results. + """ + return IMPL.store_results(results) + + +def get_test(test_id): + """Get test information from the database. + + :param test_id: The ID of the test. + """ + return IMPL.get_test(test_id) + + +def get_test_results(test_id): + """Get all passed tempest tests for a particular test. + + :param test_id: The ID of the test. + """ + return IMPL.get_test_results(test_id) diff --git a/refstack/db/migration.py b/refstack/db/migration.py new file mode 100644 index 00000000..53a4d652 --- /dev/null +++ b/refstack/db/migration.py @@ -0,0 +1,47 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +from refstack.db import utils as db_utils + + +IMPL = db_utils.PluggableBackend( + 'db_backend', sqlalchemy='refstack.db.migrations.alembic.migration') + + +def version(): + """Display the current database version.""" + return IMPL.version() + + +def upgrade(version): + """Upgrade database to 'version' or the most recent version.""" + return IMPL.upgrade(version) + + +def downgrade(version): + """Downgrade database to 'version' or to initial state.""" + return IMPL.downgrade(version) + + +def stamp(version): + """Stamp database with 'version' or the most recent version.""" + return IMPL.stamp(version) + + +def revision(message, autogenerate): + """Generate new migration script.""" + return IMPL.revision(message, autogenerate) diff --git a/refstack/db/migrations/alembic.ini b/refstack/db/migrations/alembic.ini index 0692d945..c0213ab5 100644 --- a/refstack/db/migrations/alembic.ini +++ b/refstack/db/migrations/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = alembic +script_location = %(here)s/alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s @@ -12,7 +12,6 @@ script_location = alembic # revision_environment = false #sqlalchemy.url = driver://user:pass@127.0.0.1/dbname -sqlalchemy.url = mysql://root:r00t@127.0.0.1/refstack # Logging configuration [loggers] diff --git a/refstack/db/migrations/alembic/__init__.py b/refstack/db/migrations/alembic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/refstack/db/migrations/alembic/env.py b/refstack/db/migrations/alembic/env.py index af2ae607..85c4ef59 100755 --- a/refstack/db/migrations/alembic/env.py +++ b/refstack/db/migrations/alembic/env.py @@ -17,54 +17,21 @@ from __future__ import with_statement from alembic import context -from sqlalchemy import engine_from_config, pool -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# add your model's MetaData object here -# for 'autogenerate' support -from refstack.models import Base -target_metadata = Base.metadata -# target_metadata = None - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url) - - with context.begin_transaction(): - context.run_migrations() +from refstack.db.sqlalchemy import api as db_api +from refstack.db.sqlalchemy import models as db_models def run_migrations_online(): + """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context.""" - engine = engine_from_config( - config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) + engine = db_api.get_engine() connection = engine.connect() + target_metadata = db_models.RefStackBase.metadata context.configure( connection=connection, target_metadata=target_metadata) @@ -76,7 +43,4 @@ def run_migrations_online(): connection.close() -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() +run_migrations_online() diff --git a/refstack/db/migrations/alembic/migration.py b/refstack/db/migrations/alembic/migration.py new file mode 100644 index 00000000..1fb29a09 --- /dev/null +++ b/refstack/db/migrations/alembic/migration.py @@ -0,0 +1,87 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Implementation of Alembic commands. +""" +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo.config import cfg + +from refstack.db.sqlalchemy import api as db_api + +CONF = cfg.CONF + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), os.pardir, 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def version(): + """Current database version. + + :returns: Database version + :type: string + """ + engine = db_api.get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(revision): + """Upgrade database. + + :param version: Desired database version + :type version: string + """ + return alembic.command.upgrade(_alembic_config(), revision or 'head') + + +def downgrade(revision): + """Downgrade database. + + :param version: Desired database version + :type version: string + """ + return alembic.command.downgrade(_alembic_config(), revision or 'base') + + +def stamp(revision): + """Stamp database with provided revision. + + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + return alembic.command.stamp(_alembic_config(), revision or 'head') + + +def revision(message=None, autogenerate=False): + """Create template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + return alembic.command.revision(_alembic_config(), message, autogenerate) diff --git a/refstack/db/migrations/alembic/versions/42278d6179b9_init.py b/refstack/db/migrations/alembic/versions/42278d6179b9_init.py index a24fe1d1..1c8ff788 100644 --- a/refstack/db/migrations/alembic/versions/42278d6179b9_init.py +++ b/refstack/db/migrations/alembic/versions/42278d6179b9_init.py @@ -18,6 +18,9 @@ def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.create_table( 'test', + sa.Column('updated_at', sa.DateTime()), + sa.Column('deleted_at', sa.DateTime()), + sa.Column('deleted', sa.Integer, default=0), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('created_at', sa.DateTime(), nullable=False), sa.Column('cpid', sa.String(length=128), nullable=False), @@ -26,7 +29,11 @@ def upgrade(): ) op.create_table( 'meta', + sa.Column('updated_at', sa.DateTime()), + sa.Column('deleted_at', sa.DateTime()), + sa.Column('deleted', sa.Integer, default=0), sa.Column('_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), sa.Column('test_id', sa.String(length=36), nullable=False), sa.Column('meta_key', sa.String(length=64), nullable=False), sa.Column('value', sa.Text(), nullable=True), @@ -36,7 +43,11 @@ def upgrade(): ) op.create_table( 'results', + sa.Column('updated_at', sa.DateTime()), + sa.Column('deleted_at', sa.DateTime()), + sa.Column('deleted', sa.Integer, default=0), sa.Column('_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), sa.Column('test_id', sa.String(length=36), nullable=False), sa.Column('name', sa.String(length=512), nullable=True), sa.Column('uid', sa.String(length=36), nullable=True), diff --git a/refstack/db/sqlalchemy/__init__.py b/refstack/db/sqlalchemy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/refstack/db/sqlalchemy/api.py b/refstack/db/sqlalchemy/api.py new file mode 100644 index 00000000..77c5df9a --- /dev/null +++ b/refstack/db/sqlalchemy/api.py @@ -0,0 +1,96 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Implementation of SQLAlchemy backend. +""" +import sys +import uuid + +from oslo.config import cfg +from oslo.db import options as db_options +from oslo.db.sqlalchemy import session as db_session + +from refstack.db.sqlalchemy import models + + +CONF = cfg.CONF + +_FACADE = None + +_DEFAULT_SQL_CONNECTION = 'sqlite://' +db_options.set_defaults(cfg.CONF, + connection=_DEFAULT_SQL_CONNECTION) + + +def _create_facade_lazily(): + global _FACADE + if _FACADE is None: + _FACADE = db_session.EngineFacade.from_config(CONF) + return _FACADE + + +def get_engine(): + facade = _create_facade_lazily() + return facade.get_engine() + + +def get_session(**kwargs): + facade = _create_facade_lazily() + return facade.get_session(**kwargs) + + +def get_backend(): + """The backend is this module itself.""" + + return sys.modules[__name__] + + +################### + + +def store_results(results): + test = models.Test() + test_id = str(uuid.uuid4()) + test.id = test_id + test.cpid = results.get('cpid') + test.duration_seconds = results.get('duration_seconds') + + received_test_results = results.get('results', []) + session = get_session() + with session.begin(): + test.save(session) + for result in received_test_results: + test_result = models.TestResults() + test_result.test_id = test_id + test_result.name = result['name'] + test_result.uid = result.get('uid', None) + test_result.save(session) + return test_id + + +def get_test(test_id): + session = get_session() + test_info = session.query(models.Test).\ + filter_by(id=test_id).\ + first() + return test_info + + +def get_test_results(test_id): + session = get_session() + results = session.query(models.TestResults.name).\ + filter_by(test_id=test_id).\ + all() + return results diff --git a/refstack/models.py b/refstack/db/sqlalchemy/models.py old mode 100755 new mode 100644 similarity index 73% rename from refstack/models.py rename to refstack/db/sqlalchemy/models.py index efd900db..c91b9128 --- a/refstack/models.py +++ b/refstack/db/sqlalchemy/models.py @@ -13,34 +13,56 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -"""DB models""" +""" +SQLAlchemy models for Refstack data. +""" import datetime +from oslo.config import cfg +from oslo.db.sqlalchemy import models +from oslo.utils import timeutils import sqlalchemy as sa from sqlalchemy import orm from sqlalchemy.ext.declarative import declarative_base -Base = declarative_base() +CONF = cfg.CONF +BASE = declarative_base() -class Test(Base): +class RefStackBase(models.ModelBase, models.TimestampMixin): + + """Base class for RefStack Models.""" + + __table_args__ = {'mysql_engine': 'InnoDB'} + created_at = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow, + nullable=False) + updated_at = sa.Column(sa.DateTime()) + deleted_at = sa.Column(sa.DateTime) + deleted = sa.Column(sa.Integer, default=0) + metadata = None + + def delete(self, session=None): + """Delete this object.""" + self.deleted = self.id + self.deleted_at = timeutils.utcnow() + self.save(session=session) + + +class Test(BASE, RefStackBase): """Test.""" __tablename__ = 'test' id = sa.Column(sa.String(36), primary_key=True) - created_at = sa.Column(sa.DateTime(), default=datetime.datetime.utcnow, - nullable=False) cpid = sa.Column(sa.String(128), index=True, nullable=False) duration_seconds = sa.Column(sa.Integer, nullable=False) results = orm.relationship('TestResults', backref='test') meta = orm.relationship('TestMeta', backref='test') -class TestResults(Base): +class TestResults(BASE, RefStackBase): """Test results.""" @@ -56,7 +78,7 @@ class TestResults(Base): uid = sa.Column(sa.String(36)) -class TestMeta(Base): +class TestMeta(BASE, RefStackBase): """Test metadata.""" diff --git a/refstack/db/utils.py b/refstack/db/utils.py new file mode 100644 index 00000000..83c24643 --- /dev/null +++ b/refstack/db/utils.py @@ -0,0 +1,54 @@ +# Copyright (c) 2015 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utilities for database.""" + +import logging + +from oslo.config import cfg + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class PluggableBackend(object): + """A pluggable backend loaded lazily based on some value.""" + + def __init__(self, pivot, **backends): + self.__backends = backends + self.__pivot = pivot + self.__backend = None + + def __get_backend(self): + if not self.__backend: + backend_name = CONF[self.__pivot] + if backend_name not in self.__backends: + raise Exception('Invalid backend: %s' % backend_name) + + backend = self.__backends[backend_name] + if isinstance(backend, tuple): + name = backend[0] + fromlist = backend[1] + else: + name = backend + fromlist = backend + + self.__backend = __import__(name, None, None, fromlist) + LOG.debug('backend %s', self.__backend) + return self.__backend + + def __getattr__(self, key): + backend = self.__get_backend() + return getattr(backend, key) diff --git a/refstack/tests/api/__init__.py b/refstack/tests/api/__init__.py index 91591da2..47b0a56c 100644 --- a/refstack/tests/api/__init__.py +++ b/refstack/tests/api/__init__.py @@ -19,14 +19,16 @@ import os import alembic import alembic.config -from pecan import set_config -from pecan.testing import load_test_app +from oslo.config import cfg import sqlalchemy as sa import sqlalchemy.exc from unittest import TestCase +from webtest import TestApp import refstack -from refstack.models import Base +from refstack.api import app + +CONF = cfg.CONF class FunctionalTest(TestCase): @@ -39,33 +41,32 @@ class FunctionalTest(TestCase): def setUp(self): """Test setup.""" - self.config = { - 'app': { + class TestConfig(object): + app = { 'root': 'refstack.api.controllers.root.RootController', - 'db_url': os.environ.get( - 'TEST_DB_URL', - 'mysql://root:r00t@127.0.0.1/refstack_test' - ), 'modules': ['refstack.api'], 'static_root': '%(confdir)s/public', 'template_path': '%(confdir)s/${package}/templates', } - } + + test_config = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'refstack.test.conf' + ) + os.environ['REFSTACK_OSLO_CONFIG'] = test_config self.project_path = os.path.abspath( os.path.join(inspect.getabsfile(refstack), '..', '..')) - + self.app = TestApp(app.setup_app(TestConfig())) self.prepare_test_db() self.migrate_test_db() - self.app = load_test_app(self.config) - def tearDown(self): """Test teardown.""" - set_config({}, overwrite=True) + self.app.reset() def prepare_test_db(self): """Create/clear test database.""" - db_url = self.config['app']['db_url'] + db_url = CONF.database.connection db_name = db_url.split('/')[-1] short_db_url = '/'.join(db_url.split('/')[0:-1]) try: @@ -83,21 +84,16 @@ class FunctionalTest(TestCase): conn.execute('create database %s' % db_name) conn.close() - engine = sa.create_engine(db_url) - conn = engine.connect() - conn.execute('commit') - for tbl in reversed(Base.metadata.sorted_tables): - if engine.has_table(tbl.name): - conn.execute('drop table %s' % tbl.name) - conn.close() - def migrate_test_db(self): """Apply migrations to test database.""" alembic_cfg = alembic.config.Config() - alembic_cfg.set_main_option("script_location", - os.path.join(self.project_path, 'alembic')) + alembic_cfg.set_main_option( + "script_location", + os.path.join(self.project_path, 'refstack', 'db', + 'migrations', 'alembic') + ) alembic_cfg.set_main_option("sqlalchemy.url", - self.config['app']['db_url']) + CONF.database.connection) alembic.command.upgrade(alembic_cfg, 'head') def get_json(self, url, headers=None, extra_environ=None, diff --git a/refstack/tests/api/refstack.test.conf b/refstack/tests/api/refstack.test.conf new file mode 100644 index 00000000..f1dff688 --- /dev/null +++ b/refstack/tests/api/refstack.test.conf @@ -0,0 +1,2 @@ +[DEFAULT] +sql_connection = mysql://root:passw0rd@127.0.0.1/refstack diff --git a/refstack/utils.py b/refstack/utils.py index 3fef52b1..3692edc7 100755 --- a/refstack/utils.py +++ b/refstack/utils.py @@ -20,6 +20,7 @@ from datetime import datetime import os import random +import re import string @@ -60,6 +61,18 @@ SEX_TYPE = { STRING_LEN = 64 +class LogWriter(object): + """Stream-like API to logger""" + + def __init__(self, logger, level): + self.logger = logger + self.level = level + + def write(self, s): + if re.sub('[\n ]', '', s): + self.logger.log(self.level, '\n' + s) + + def get_current_time(): return datetime.utcnow() diff --git a/requirements.txt b/requirements.txt index df75748e..7b9bc3eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,9 @@ SQLAlchemy==0.8.3 alembic==0.5.0 -gunicorn==0.17.4 +#gunicorn 19.1.1 has a bug with threading module +gunicorn==18 +oslo.config>=1.6.0 # Apache-2.0 +oslo.db>=1.4.1 # Apache-2.0 pecan>=0.8.2 pyOpenSSL==0.13 pycrypto==2.6 diff --git a/setup.cfg b/setup.cfg index 98125ecd..2f6f762e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,10 @@ classifier = packages = refstack +scripts = + bin/refstack-manage + bin/refstack-api + [global] setup-hooks = pbr.hooks.setup_hook - diff --git a/tox.ini b/tox.ini index f29a4aa6..f534bb59 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,9 @@ commands = python -m unittest discover ./refstack/tests/api distribute = false [testenv:pep8] -commands = flake8 +commands = + flake8 {posargs} + flake8 --filename=refstack* bin distribute = false [testenv:venv]