From 982c025c90543f0cdc906e9bc6f5ad0d3b73f418 Mon Sep 17 00:00:00 2001 From: John Bresnahan Date: Fri, 16 Aug 2013 12:49:46 -1000 Subject: [PATCH] Adding a plug-in module for nova compute This new python package will allow a nova compute deployment to install a staccato download plugin module. This will allow nova compute to use staccato for downloads. Change-Id: I35257f37c8d92ad96298c10ce1be15b7f53ffaef blueprint: staccato-plugin-for-nova --- nova_plugin/README.rst | 6 + nova_plugin/requirements.txt | 3 + nova_plugin/setup.cfg | 34 +++ nova_plugin/setup.py | 7 + .../staccato_nova_download/__init__.py | 144 ++++++++++ .../staccato_nova_download/tests/__init__.py | 1 + .../staccato_nova_download/tests/base.py | 19 ++ .../tests/unit/__init__.py | 0 .../tests/unit/test_basic.py | 269 ++++++++++++++++++ nova_plugin/test-requirements.txt | 0 staccato/common/config.py | 1 + staccato/tests/unit/v1/test_api.py | 1 - 12 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 nova_plugin/README.rst create mode 100644 nova_plugin/requirements.txt create mode 100644 nova_plugin/setup.cfg create mode 100644 nova_plugin/setup.py create mode 100644 nova_plugin/staccato_nova_download/__init__.py create mode 100644 nova_plugin/staccato_nova_download/tests/__init__.py create mode 100644 nova_plugin/staccato_nova_download/tests/base.py create mode 100644 nova_plugin/staccato_nova_download/tests/unit/__init__.py create mode 100644 nova_plugin/staccato_nova_download/tests/unit/test_basic.py create mode 100644 nova_plugin/test-requirements.txt diff --git a/nova_plugin/README.rst b/nova_plugin/README.rst new file mode 100644 index 0000000..5db8bbb --- /dev/null +++ b/nova_plugin/README.rst @@ -0,0 +1,6 @@ +OpenStack Nova Staccato Plugin +============================== + +This plugin will be installed into the python environment as an entry point. +Nova can then load it to manage transfers. This must be installed in the +python environment which nova compute uses. \ No newline at end of file diff --git a/nova_plugin/requirements.txt b/nova_plugin/requirements.txt new file mode 100644 index 0000000..24f789a --- /dev/null +++ b/nova_plugin/requirements.txt @@ -0,0 +1,3 @@ +d2to1>=0.2.10,<0.3 +pbr>=0.5.16,<0.6 +nova \ No newline at end of file diff --git a/nova_plugin/setup.cfg b/nova_plugin/setup.cfg new file mode 100644 index 0000000..630bbd9 --- /dev/null +++ b/nova_plugin/setup.cfg @@ -0,0 +1,34 @@ +[metadata] +name = staccato_nova_download +version = 2013.2 +summary = A plugin for nova that will handle image downloads via staccato +description-file = README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[files] +packages = staccato_nova_download + +[entry_points] +nova.image.download.modules = + staccato = staccato_nova_download + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 diff --git a/nova_plugin/setup.py b/nova_plugin/setup.py new file mode 100644 index 0000000..47eceb6 --- /dev/null +++ b/nova_plugin/setup.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import setuptools + +setuptools.setup( + setup_requires=['d2to1>=0.2.10,<0.3', 'pbr>=0.5,<0.6'], + d2to1=True) diff --git a/nova_plugin/staccato_nova_download/__init__.py b/nova_plugin/staccato_nova_download/__init__.py new file mode 100644 index 0000000..9f210ff --- /dev/null +++ b/nova_plugin/staccato_nova_download/__init__.py @@ -0,0 +1,144 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import httplib +import json +import logging + +from oslo.config import cfg + +from nova import exception +import nova.image.download.base as xfer_base +from nova.openstack.common.gettextutils import _ + + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +opt_groups = [cfg.StrOpt(name='hostname', default='127.0.0.1', + help=_('The hostname of the staccato service.')), + cfg.IntOpt(name='port', default=5309, + help=_('The port where the staccato service is ' + 'listening.')), + cfg.IntOpt(name='poll_interval', default=1, + help=_('The amount of time in second to poll for ' + 'transfer completion')) + ] + +CONF.register_opts(opt_groups, group="staccato_nova_download_module") + + +class StaccatoTransfer(xfer_base.TransferBase): + + def __init__(self): + self.conf_group = CONF['staccato_nova_download_module'] + self.client = httplib.HTTPConnection(self.conf_group.hostname, + self.conf_group.port) + + def _delete(self, xfer_id, headers): + path = '/v1/transfers/%s' % xfer_id + self.client.request('DELETE', path, headers=headers) + response = self.client.getresponse() + if response.status != 204: + msg = _('Error deleting transfer %s') % response.read() + LOG.error(msg) + raise exception.ImageDownloadModuleError( + {'reason': msg, 'module': unicode(self)}) + + def _wait_for_complete(self, xfer_id, headers): + error_states = ['STATE_CANCELED', 'STATE_ERROR', 'STATE_DELETED'] + + path = '/v1/transfers/%s' % xfer_id + while True: + self.client.request('GET', path, headers=headers) + response = self.client.getresponse() + if response.status != 200: + msg = _('Error requesting a new transfer %s') % response.read() + LOG.error(msg) + try: + self._delete(xfer_id, headers) + except Exception as ex: + LOG.error(ex) + raise exception.ImageDownloadModuleError( + {'reason': msg, 'module': unicode(self)}) + + body = response.read() + response_dict = json.loads(body) + if response_dict['status'] == 'STATE_COMPLETE': + break + + if response_dict['status'] in error_states: + try: + self._delete(xfer_id, headers) + except Exception as ex: + LOG.error(ex) + msg = (_('The transfer could not be completed in state %s') + % response_dict['status']) + raise exception.ImageDownloadModuleError( + {'reason': msg, 'module': unicode(self)}) + + def download(self, url_parts, dst_file, metadata, **kwargs): + LOG.debug((_('Attemption to use %(module)s to download %(url)s')) % + {'module': unicode(self), 'url': url_parts.geturl()}) + + headers = {'Content-Type': 'application/json'} + if CONF.auth_strategy == 'keystone': + context = kwargs['context'] + headers['X-Auth-Token'] = getattr(context, 'auth_token', None) + headers['X-User-Id'] = getattr(context, 'user', None) + headers['X-Tenant-Id'] = getattr(context, 'tenant', None) + + data = {'source_url': url_parts.geturl(), + 'destination_url': 'file://%s' % dst_file} + try: + self.client.request('POST', '/v1/transfers', + headers=headers, body=data) + response = self.client.getresponse() + if response.status != 201: + msg = _('Error requesting a new transfer %s') % response.read() + LOG.error(msg) + raise exception.ImageDownloadModuleError( + {'reason': msg, 'module': unicode(self)}) + body = response.read() + response_dict = json.loads(body) + + self._wait_for_complete(response_dict['id'], headers) + except exception.ImageDownloadModuleError: + raise + except Exception as ex: + msg = unicode(ex.message) + LOG.error(msg) + raise exception.ImageDownloadModuleError( + {'reason': msg, 'module': u'StaccatoTransfer'}) + +def get_download_handler(**kwargs): + return StaccatoTransfer() + + +def get_schemes(): + conf_group = CONF['staccato_nova_download_module'] + try: + client = httplib.HTTPConnection(conf_group.hostname, conf_group.port) + response = client.request('GET', '/') + body = response.read() + version_json = json.loads(body) + return version_json['protocols'] + except Exception as ex: + reason = unicode(ex.message) + LOG.error(reason) + raise exception.ImageDownloadModuleError({'reason': reason, + 'module': u'staccato'}) diff --git a/nova_plugin/staccato_nova_download/tests/__init__.py b/nova_plugin/staccato_nova_download/tests/__init__.py new file mode 100644 index 0000000..7fcc62c --- /dev/null +++ b/nova_plugin/staccato_nova_download/tests/__init__.py @@ -0,0 +1 @@ +__author__ = 'jbresnah' diff --git a/nova_plugin/staccato_nova_download/tests/base.py b/nova_plugin/staccato_nova_download/tests/base.py new file mode 100644 index 0000000..fb60a0a --- /dev/null +++ b/nova_plugin/staccato_nova_download/tests/base.py @@ -0,0 +1,19 @@ +from oslo.config import cfg + +import testtools + + +CONF = cfg.CONF + + +class BaseTest(testtools.TestCase): + def setUp(self): + super(BaseTest, self).setUp() + + def tearDown(self): + super(BaseTest, self).tearDown() + + def config(self, **kw): + group = kw.pop('group', None) + for k, v in kw.iteritems(): + CONF.set_override(k, v, group) diff --git a/nova_plugin/staccato_nova_download/tests/unit/__init__.py b/nova_plugin/staccato_nova_download/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nova_plugin/staccato_nova_download/tests/unit/test_basic.py b/nova_plugin/staccato_nova_download/tests/unit/test_basic.py new file mode 100644 index 0000000..451b99d --- /dev/null +++ b/nova_plugin/staccato_nova_download/tests/unit/test_basic.py @@ -0,0 +1,269 @@ +import httplib +import json +import urlparse + +import mox +from nova import exception +from oslo.config import cfg + +from staccato.common import config +import staccato_nova_download + +import staccato_nova_download.tests.base as base + + +CONF = cfg.CONF + +CONF.import_opt('auth_strategy', 'nova.api.auth') + + +class TestBasic(base.BaseTest): + + def setUp(self): + super(TestBasic, self).setUp() + self.mox = mox.Mox() + + def tearDown(self): + super(TestBasic, self).tearDown() + self.mox.UnsetStubs() + + def test_get_schemes(self): + start_protocols = ["file", "http", "somethingelse"] + version_info_back = {'protocols': start_protocols} + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + response = self.mox.CreateMockAnything() + http_obj.request('GET', '/').AndReturn(response) + response.read().AndReturn(json.dumps(version_info_back)) + + self.mox.ReplayAll() + protocols = staccato_nova_download.get_schemes() + self.mox.VerifyAll() + self.assertEqual(start_protocols, protocols) + + def test_get_schemes_failed_connection(self): + start_protocols = ["file", "http", "somethingelse"] + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + http_obj.request('GET', '/').AndRaise(Exception("message")) + + self.mox.ReplayAll() + self.assertRaises(exception.ImageDownloadModuleError, + staccato_nova_download.get_schemes) + self.mox.VerifyAll() + + def test_successfull_download(self): + class FakeResponse(object): + def __init__(self, status, reply): + self.status = status + self.reply = reply + + def read(self): + return json.dumps(self.reply) + + self.config(auth_strategy='notkeystone') + + xfer_id = 'someidstring' + src_url = 'file:///etc/group' + dst_url = 'file:///tmp/group' + data = {'source_url': src_url, 'destination_url': dst_url} + + headers = {'Content-Type': 'application/json'} + + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + + http_obj.request('POST', '/v1/transfers', + headers=headers, body=data) + http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id})) + + path = '/v1/transfers/%s' % xfer_id + http_obj.request('GET', path, headers=headers) + http_obj.getresponse().AndReturn( + FakeResponse(200, {'status': 'STATE_COMPLETE'})) + + self.mox.ReplayAll() + st_plugin = staccato_nova_download.StaccatoTransfer() + + url_parts = urlparse.urlparse(src_url) + dst_url_parts = urlparse.urlparse(dst_url) + st_plugin.download(url_parts, dst_url_parts.path, {}) + self.mox.VerifyAll() + + def test_successful_download_with_keystone(self): + class FakeContext(object): + auth_token = 'sdfsdf' + user = 'buzztroll' + tenant = 'staccato' + + class FakeResponse(object): + def __init__(self, status, reply): + self.status = status + self.reply = reply + + def read(self): + return json.dumps(self.reply) + + self.config(auth_strategy='keystone') + + xfer_id = 'someidstring' + src_url = 'file:///etc/group' + dst_url = 'file:///tmp/group' + data = {'source_url': src_url, 'destination_url': dst_url} + + context = FakeContext() + headers = {'Content-Type': 'application/json', + 'X-Auth-Token': context.auth_token, + 'X-User-Id': context.user, + 'X-Tenant-Id': context.tenant} + + + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + + http_obj.request('POST', '/v1/transfers', + headers=headers, body=data) + http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id})) + + path = '/v1/transfers/%s' % xfer_id + http_obj.request('GET', path, headers=headers) + http_obj.getresponse().AndReturn( + FakeResponse(200, {'status': 'STATE_COMPLETE'})) + + self.mox.ReplayAll() + st_plugin = staccato_nova_download.StaccatoTransfer() + + url_parts = urlparse.urlparse(src_url) + dst_url_parts = urlparse.urlparse(dst_url) + st_plugin.download(url_parts, dst_url_parts.path, + {}, context=context) + self.mox.VerifyAll() + + def test_download_post_error(self): + class FakeResponse(object): + def __init__(self, status, reply): + self.status = status + self.reply = reply + + def read(self): + return json.dumps(self.reply) + + self.config(auth_strategy='notkeystone') + + xfer_id = 'someidstring' + src_url = 'file:///etc/group' + dst_url = 'file:///tmp/group' + data = {'source_url': src_url, 'destination_url': dst_url} + + headers = {'Content-Type': 'application/json'} + + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + + http_obj.request('POST', '/v1/transfers', + headers=headers, body=data) + http_obj.getresponse().AndReturn(FakeResponse(400, {'id': xfer_id})) + + self.mox.ReplayAll() + st_plugin = staccato_nova_download.StaccatoTransfer() + + url_parts = urlparse.urlparse(src_url) + dst_url_parts = urlparse.urlparse(dst_url) + + self.assertRaises(exception.ImageDownloadModuleError, + st_plugin.download, + url_parts, + dst_url_parts.path, + {}) + + self.mox.VerifyAll() + + def test_successful_error_case(self): + class FakeResponse(object): + def __init__(self, status, reply): + self.status = status + self.reply = reply + + def read(self): + return json.dumps(self.reply) + + self.config(auth_strategy='notkeystone') + + xfer_id = 'someidstring' + src_url = 'file:///etc/group' + dst_url = 'file:///tmp/group' + data = {'source_url': src_url, 'destination_url': dst_url} + + headers = {'Content-Type': 'application/json'} + + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + + http_obj.request('POST', '/v1/transfers', + headers=headers, body=data) + http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id})) + + path = '/v1/transfers/%s' % xfer_id + http_obj.request('GET', path, headers=headers) + http_obj.getresponse().AndReturn( + FakeResponse(200, {'status': 'STATE_ERROR'})) + path = '/v1/transfers/%s' % xfer_id + http_obj.request('DELETE', path, headers=headers) + http_obj.getresponse() + + self.mox.ReplayAll() + st_plugin = staccato_nova_download.StaccatoTransfer() + + url_parts = urlparse.urlparse(src_url) + dst_url_parts = urlparse.urlparse(dst_url) + self.assertRaises(exception.ImageDownloadModuleError, + st_plugin.download, + url_parts, + dst_url_parts.path, + {}) + self.mox.VerifyAll() + + def test_status_error_case(self): + class FakeResponse(object): + def __init__(self, status, reply): + self.status = status + self.reply = reply + + def read(self): + return json.dumps(self.reply) + + self.config(auth_strategy='notkeystone') + + xfer_id = 'someidstring' + src_url = 'file:///etc/group' + dst_url = 'file:///tmp/group' + data = {'source_url': src_url, 'destination_url': dst_url} + + headers = {'Content-Type': 'application/json'} + + self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection') + http_obj = httplib.HTTPConnection('127.0.0.1', 5309) + + http_obj.request('POST', '/v1/transfers', + headers=headers, body=data) + http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id})) + + path = '/v1/transfers/%s' % xfer_id + http_obj.request('GET', path, headers=headers) + http_obj.getresponse().AndReturn( + FakeResponse(500, {'status': 'STATE_COMPLETE'})) + + path = '/v1/transfers/%s' % xfer_id + http_obj.request('DELETE', path, headers=headers) + http_obj.getresponse() + self.mox.ReplayAll() + st_plugin = staccato_nova_download.StaccatoTransfer() + + url_parts = urlparse.urlparse(src_url) + dst_url_parts = urlparse.urlparse(dst_url) + self.assertRaises(exception.ImageDownloadModuleError, + st_plugin.download, + url_parts, + dst_url_parts.path, + {}) + self.mox.VerifyAll() diff --git a/nova_plugin/test-requirements.txt b/nova_plugin/test-requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/staccato/common/config.py b/staccato/common/config.py index 889465e..486f496 100644 --- a/staccato/common/config.py +++ b/staccato/common/config.py @@ -53,6 +53,7 @@ common_opts = [ cfg.StrOpt('admin_user_id', default='admin', help='The user ID of the staccato admin'), ] + bind_opts = [ cfg.StrOpt('bind_host', default='0.0.0.0', help=_('Address to bind the server. Useful when ' diff --git a/staccato/tests/unit/v1/test_api.py b/staccato/tests/unit/v1/test_api.py index 12523f5..5286504 100644 --- a/staccato/tests/unit/v1/test_api.py +++ b/staccato/tests/unit/v1/test_api.py @@ -1,6 +1,5 @@ import json import mox -import testtools import uuid import webob.exc