stackforge/staccato
Code Issues Proposed changes

Adding a plug-in module for nova compute

Browse Source 
This new python package will allow a nova compute deployment to install
a staccato download plugin module.  This will allow nova compute to
use staccato for downloads.

Change-Id: I35257f37c8d92ad96298c10ce1be15b7f53ffaef
blueprint: staccato-plugin-for-nova
This commit is contained in:
John Bresnahan 2013-08-16 12:49:46 -10:00
parent 93bed1bf3e
commit 982c025c90
12 changed files with 484 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nova_plugin/README.rst Normal file
View File

@ -0,0 +1,6 @@
OpenStack Nova Staccato Plugin
==============================
This plugin will be installed into the python environment as an entry point.
Nova can then load it to manage transfers. This must be installed in the
python environment which nova compute uses.

3
nova_plugin/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
d2to1>=0.2.10,<0.3
pbr>=0.5.16,<0.6
nova

34
nova_plugin/setup.cfg Normal file
View File

@ -0,0 +1,34 @@
[metadata]
name = staccato_nova_download
version = 2013.2
summary = A plugin for nova that will handle image downloads via staccato
description-file = README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
[global]
setup-hooks =
pbr.hooks.setup_hook
[files]
packages = staccato_nova_download
[entry_points]
nova.image.download.modules =
staccato = staccato_nova_download
[egg_info]
tag_build =
tag_date = 0
tag_svn_revision = 0

7
nova_plugin/setup.py Normal file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env python
import setuptools
setuptools.setup(
setup_requires=['d2to1>=0.2.10,<0.3', 'pbr>=0.5,<0.6'],
d2to1=True)

144
nova_plugin/staccato_nova_download/__init__.py Normal file
View File

@ -0,0 +1,144 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import httplib
import json
import logging
from oslo.config import cfg
from nova import exception
import nova.image.download.base as xfer_base
from nova.openstack.common.gettextutils import _
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
opt_groups = [cfg.StrOpt(name='hostname', default='127.0.0.1',
help=_('The hostname of the staccato service.')),
cfg.IntOpt(name='port', default=5309,
help=_('The port where the staccato service is '
'listening.')),
cfg.IntOpt(name='poll_interval', default=1,
help=_('The amount of time in second to poll for '
'transfer completion'))
]
CONF.register_opts(opt_groups, group="staccato_nova_download_module")
class StaccatoTransfer(xfer_base.TransferBase):
def __init__(self):
self.conf_group = CONF['staccato_nova_download_module']
self.client = httplib.HTTPConnection(self.conf_group.hostname,
self.conf_group.port)
def _delete(self, xfer_id, headers):
path = '/v1/transfers/%s' % xfer_id
self.client.request('DELETE', path, headers=headers)
response = self.client.getresponse()
if response.status != 204:
msg = _('Error deleting transfer %s') % response.read()
LOG.error(msg)
raise exception.ImageDownloadModuleError(
{'reason': msg, 'module': unicode(self)})
def _wait_for_complete(self, xfer_id, headers):
error_states = ['STATE_CANCELED', 'STATE_ERROR', 'STATE_DELETED']
path = '/v1/transfers/%s' % xfer_id
while True:
self.client.request('GET', path, headers=headers)
response = self.client.getresponse()
if response.status != 200:
msg = _('Error requesting a new transfer %s') % response.read()
LOG.error(msg)
try:
self._delete(xfer_id, headers)
except Exception as ex:
LOG.error(ex)
raise exception.ImageDownloadModuleError(
{'reason': msg, 'module': unicode(self)})
body = response.read()
response_dict = json.loads(body)
if response_dict['status'] == 'STATE_COMPLETE':
break
if response_dict['status'] in error_states:
try:
self._delete(xfer_id, headers)
except Exception as ex:
LOG.error(ex)
msg = (_('The transfer could not be completed in state %s')
% response_dict['status'])
raise exception.ImageDownloadModuleError(
{'reason': msg, 'module': unicode(self)})
def download(self, url_parts, dst_file, metadata, **kwargs):
LOG.debug((_('Attemption to use %(module)s to download %(url)s')) %
{'module': unicode(self), 'url': url_parts.geturl()})
headers = {'Content-Type': 'application/json'}
if CONF.auth_strategy == 'keystone':
context = kwargs['context']
headers['X-Auth-Token'] = getattr(context, 'auth_token', None)
headers['X-User-Id'] = getattr(context, 'user', None)
headers['X-Tenant-Id'] = getattr(context, 'tenant', None)
data = {'source_url': url_parts.geturl(),
'destination_url': 'file://%s' % dst_file}
try:
self.client.request('POST', '/v1/transfers',
headers=headers, body=data)
response = self.client.getresponse()
if response.status != 201:
msg = _('Error requesting a new transfer %s') % response.read()
LOG.error(msg)
raise exception.ImageDownloadModuleError(
{'reason': msg, 'module': unicode(self)})
body = response.read()
response_dict = json.loads(body)
self._wait_for_complete(response_dict['id'], headers)
except exception.ImageDownloadModuleError:
raise
except Exception as ex:
msg = unicode(ex.message)
LOG.error(msg)
raise exception.ImageDownloadModuleError(
{'reason': msg, 'module': u'StaccatoTransfer'})
def get_download_handler(**kwargs):
return StaccatoTransfer()
def get_schemes():
conf_group = CONF['staccato_nova_download_module']
try:
client = httplib.HTTPConnection(conf_group.hostname, conf_group.port)
response = client.request('GET', '/')
body = response.read()
version_json = json.loads(body)
return version_json['protocols']
except Exception as ex:
reason = unicode(ex.message)
LOG.error(reason)
raise exception.ImageDownloadModuleError({'reason': reason,
'module': u'staccato'})

1
nova_plugin/staccato_nova_download/tests/__init__.py Normal file
View File

@ -0,0 +1 @@
__author__ = 'jbresnah'

19
nova_plugin/staccato_nova_download/tests/base.py Normal file
View File

@ -0,0 +1,19 @@
from oslo.config import cfg
import testtools
CONF = cfg.CONF
class BaseTest(testtools.TestCase):
def setUp(self):
super(BaseTest, self).setUp()
def tearDown(self):
super(BaseTest, self).tearDown()
def config(self, **kw):
group = kw.pop('group', None)
for k, v in kw.iteritems():
CONF.set_override(k, v, group)

0
nova_plugin/staccato_nova_download/tests/unit/__init__.py Normal file
View File

269
nova_plugin/staccato_nova_download/tests/unit/test_basic.py Normal file
View File

@ -0,0 +1,269 @@
import httplib
import json
import urlparse
import mox
from nova import exception
from oslo.config import cfg
from staccato.common import config
import staccato_nova_download
import staccato_nova_download.tests.base as base
CONF = cfg.CONF
CONF.import_opt('auth_strategy', 'nova.api.auth')
class TestBasic(base.BaseTest):
def setUp(self):
super(TestBasic, self).setUp()
self.mox = mox.Mox()
def tearDown(self):
super(TestBasic, self).tearDown()
self.mox.UnsetStubs()
def test_get_schemes(self):
start_protocols = ["file", "http", "somethingelse"]
version_info_back = {'protocols': start_protocols}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
response = self.mox.CreateMockAnything()
http_obj.request('GET', '/').AndReturn(response)
response.read().AndReturn(json.dumps(version_info_back))
self.mox.ReplayAll()
protocols = staccato_nova_download.get_schemes()
self.mox.VerifyAll()
self.assertEqual(start_protocols, protocols)
def test_get_schemes_failed_connection(self):
start_protocols = ["file", "http", "somethingelse"]
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('GET', '/').AndRaise(Exception("message"))
self.mox.ReplayAll()
self.assertRaises(exception.ImageDownloadModuleError,
staccato_nova_download.get_schemes)
self.mox.VerifyAll()
def test_successfull_download(self):
class FakeResponse(object):
def __init__(self, status, reply):
self.status = status
self.reply = reply
def read(self):
return json.dumps(self.reply)
self.config(auth_strategy='notkeystone')
xfer_id = 'someidstring'
src_url = 'file:///etc/group'
dst_url = 'file:///tmp/group'
data = {'source_url': src_url, 'destination_url': dst_url}
headers = {'Content-Type': 'application/json'}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('POST', '/v1/transfers',
headers=headers, body=data)
http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('GET', path, headers=headers)
http_obj.getresponse().AndReturn(
FakeResponse(200, {'status': 'STATE_COMPLETE'}))
self.mox.ReplayAll()
st_plugin = staccato_nova_download.StaccatoTransfer()
url_parts = urlparse.urlparse(src_url)
dst_url_parts = urlparse.urlparse(dst_url)
st_plugin.download(url_parts, dst_url_parts.path, {})
self.mox.VerifyAll()
def test_successful_download_with_keystone(self):
class FakeContext(object):
auth_token = 'sdfsdf'
user = 'buzztroll'
tenant = 'staccato'
class FakeResponse(object):
def __init__(self, status, reply):
self.status = status
self.reply = reply
def read(self):
return json.dumps(self.reply)
self.config(auth_strategy='keystone')
xfer_id = 'someidstring'
src_url = 'file:///etc/group'
dst_url = 'file:///tmp/group'
data = {'source_url': src_url, 'destination_url': dst_url}
context = FakeContext()
headers = {'Content-Type': 'application/json',
'X-Auth-Token': context.auth_token,
'X-User-Id': context.user,
'X-Tenant-Id': context.tenant}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('POST', '/v1/transfers',
headers=headers, body=data)
http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('GET', path, headers=headers)
http_obj.getresponse().AndReturn(
FakeResponse(200, {'status': 'STATE_COMPLETE'}))
self.mox.ReplayAll()
st_plugin = staccato_nova_download.StaccatoTransfer()
url_parts = urlparse.urlparse(src_url)
dst_url_parts = urlparse.urlparse(dst_url)
st_plugin.download(url_parts, dst_url_parts.path,
{}, context=context)
self.mox.VerifyAll()
def test_download_post_error(self):
class FakeResponse(object):
def __init__(self, status, reply):
self.status = status
self.reply = reply
def read(self):
return json.dumps(self.reply)
self.config(auth_strategy='notkeystone')
xfer_id = 'someidstring'
src_url = 'file:///etc/group'
dst_url = 'file:///tmp/group'
data = {'source_url': src_url, 'destination_url': dst_url}
headers = {'Content-Type': 'application/json'}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('POST', '/v1/transfers',
headers=headers, body=data)
http_obj.getresponse().AndReturn(FakeResponse(400, {'id': xfer_id}))
self.mox.ReplayAll()
st_plugin = staccato_nova_download.StaccatoTransfer()
url_parts = urlparse.urlparse(src_url)
dst_url_parts = urlparse.urlparse(dst_url)
self.assertRaises(exception.ImageDownloadModuleError,
st_plugin.download,
url_parts,
dst_url_parts.path,
{})
self.mox.VerifyAll()
def test_successful_error_case(self):
class FakeResponse(object):
def __init__(self, status, reply):
self.status = status
self.reply = reply
def read(self):
return json.dumps(self.reply)
self.config(auth_strategy='notkeystone')
xfer_id = 'someidstring'
src_url = 'file:///etc/group'
dst_url = 'file:///tmp/group'
data = {'source_url': src_url, 'destination_url': dst_url}
headers = {'Content-Type': 'application/json'}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('POST', '/v1/transfers',
headers=headers, body=data)
http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('GET', path, headers=headers)
http_obj.getresponse().AndReturn(
FakeResponse(200, {'status': 'STATE_ERROR'}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('DELETE', path, headers=headers)
http_obj.getresponse()
self.mox.ReplayAll()
st_plugin = staccato_nova_download.StaccatoTransfer()
url_parts = urlparse.urlparse(src_url)
dst_url_parts = urlparse.urlparse(dst_url)
self.assertRaises(exception.ImageDownloadModuleError,
st_plugin.download,
url_parts,
dst_url_parts.path,
{})
self.mox.VerifyAll()
def test_status_error_case(self):
class FakeResponse(object):
def __init__(self, status, reply):
self.status = status
self.reply = reply
def read(self):
return json.dumps(self.reply)
self.config(auth_strategy='notkeystone')
xfer_id = 'someidstring'
src_url = 'file:///etc/group'
dst_url = 'file:///tmp/group'
data = {'source_url': src_url, 'destination_url': dst_url}
headers = {'Content-Type': 'application/json'}
self.mox.StubOutClassWithMocks(httplib, 'HTTPConnection')
http_obj = httplib.HTTPConnection('127.0.0.1', 5309)
http_obj.request('POST', '/v1/transfers',
headers=headers, body=data)
http_obj.getresponse().AndReturn(FakeResponse(201, {'id': xfer_id}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('GET', path, headers=headers)
http_obj.getresponse().AndReturn(
FakeResponse(500, {'status': 'STATE_COMPLETE'}))
path = '/v1/transfers/%s' % xfer_id
http_obj.request('DELETE', path, headers=headers)
http_obj.getresponse()
self.mox.ReplayAll()
st_plugin = staccato_nova_download.StaccatoTransfer()
url_parts = urlparse.urlparse(src_url)
dst_url_parts = urlparse.urlparse(dst_url)
self.assertRaises(exception.ImageDownloadModuleError,
st_plugin.download,
url_parts,
dst_url_parts.path,
{})
self.mox.VerifyAll()

0
nova_plugin/test-requirements.txt Normal file
View File

1
staccato/common/config.py
View File

@ -53,6 +53,7 @@ common_opts = [
cfg.StrOpt('admin_user_id', default='admin',
help='The user ID of the staccato admin'),
]
bind_opts = [
cfg.StrOpt('bind_host', default='0.0.0.0',
help=_('Address to bind the server. Useful when '

1
staccato/tests/unit/v1/test_api.py
View File

@ -1,6 +1,5 @@
import json
import mox
import testtools
import uuid
import webob.exc