openstack/openstacksdk
Code Issues Proposed changes

Merge "Support uploading image from data and stdin"

Browse Source
This commit is contained in:
Zuul 2019-12-17 20:34:06 +00:00 committed by Gerrit Code Review
commit 1c5e096ecf
8 changed files with 341 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
openstack/cloud/_object_store.py
View File

@ -219,14 +219,11 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
if file_key not in self._file_hash_cache:
self.log.debug(
'Calculating hashes for %(filename)s', {'filename': filename})
md5 = hashlib.md5()
sha256 = hashlib.sha256()
(md5, sha256) = (None, None)
with open(filename, 'rb') as file_obj:
for chunk in iter(lambda: file_obj.read(8192), b''):
md5.update(chunk)
sha256.update(chunk)
(md5, sha256) = self._calculate_data_hashes(file_obj)
self._file_hash_cache[file_key] = dict(
md5=md5.hexdigest(), sha256=sha256.hexdigest())
md5=md5, sha256=sha256)
self.log.debug(
"Image file %(filename)s md5:%(md5)s sha256:%(sha256)s",
{'filename': filename,
@ -235,6 +232,19 @@ class ObjectStoreCloudMixin(_normalize.Normalizer):
return (self._file_hash_cache[file_key]['md5'],
self._file_hash_cache[file_key]['sha256'])
def _calculate_data_hashes(self, data):
md5 = hashlib.md5()
sha256 = hashlib.sha256()
if hasattr(data, 'read'):
for chunk in iter(lambda: data.read(8192), b''):
md5.update(chunk)
sha256.update(chunk)
else:
md5.update(data)
sha256.update(data)
return (md5.hexdigest(), sha256.hexdigest())
@_utils.cache_on_arguments()
def get_object_capabilities(self):
"""Get infomation about the object-storage service

43
openstack/image/_base_proxy.py
View File

@ -14,6 +14,7 @@ import os
import six
from openstack import exceptions
from openstack import proxy
@ -40,7 +41,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
disable_vendor_agent=True,
allow_duplicates=False, meta=None,
wait=False, timeout=3600,
validate_checksum=True,
data=None, validate_checksum=True,
**kwargs):
"""Upload an image.
@ -49,6 +50,8 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
basename of the path.
:param str filename: The path to the file to upload, if needed.
(optional, defaults to None)
:param data: Image data (string or file-like object). It is mutually
exclusive with filename
:param str container: Name of the container in swift where images
should be uploaded for import if the cloud requires such a thing.
(optional, defaults to 'images')
@ -103,23 +106,34 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
# https://docs.openstack.org/image-guide/image-formats.html
container_format = 'bare'
if data and filename:
raise exceptions.SDKException(
'Passing filename and data simultaneously is not supported')
# If there is no filename, see if name is actually the filename
if not filename:
if not filename and not data:
name, filename = self._get_name_and_filename(
name, self._connection.config.config['image_format'])
if not (md5 or sha256):
(md5, sha256) = self._connection._get_file_hashes(filename)
if validate_checksum and data and not isinstance(data, bytes):
raise exceptions.SDKException(
'Validating checksum is not possible when data is not a '
'direct binary object')
if not (md5 or sha256) and validate_checksum:
if filename:
(md5, sha256) = self._connection._get_file_hashes(filename)
elif data and isinstance(data, bytes):
(md5, sha256) = self._connection._calculate_data_hashes(data)
if allow_duplicates:
current_image = None
else:
current_image = self._connection.get_image(name)
current_image = self.find_image(name)
if current_image:
md5_key = current_image.get(
props = current_image.get('properties', {})
md5_key = props.get(
self._IMAGE_MD5_KEY,
current_image.get(self._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = current_image.get(
props.get(self._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = props.get(
self._IMAGE_SHA256_KEY,
current_image.get(self._SHADE_IMAGE_SHA256_KEY, ''))
props.get(self._SHADE_IMAGE_SHA256_KEY, ''))
up_to_date = self._connection._hashes_up_to_date(
md5=md5, sha256=sha256,
md5_key=md5_key, sha256_key=sha256_key)
@ -128,6 +142,11 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
"image %(name)s exists and is up to date",
{'name': name})
return current_image
else:
self.log.debug(
"image %(name)s exists, but contains different "
"checksums. Updating.",
{'name': name})
if disable_vendor_agent:
kwargs.update(
@ -147,9 +166,9 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
if container_format:
image_kwargs['container_format'] = container_format
if filename:
if filename or data:
image = self._upload_image(
name, filename=filename, meta=meta,
name, filename=filename, data=data, meta=meta,
wait=wait, timeout=timeout,
validate_checksum=validate_checksum,
**image_kwargs)
@ -163,7 +182,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
pass
@abc.abstractmethod
def _upload_image(self, name, filename, meta, wait, timeout,
def _upload_image(self, name, filename, data, meta, wait, timeout,
validate_checksum=True,
**image_kwargs):
pass

7
openstack/image/v1/_proxy.py
View File

@ -42,10 +42,13 @@ class Proxy(_base_proxy.BaseImageProxy):
return self._create(_image.Image, **attrs)
def _upload_image(
self, name, filename, meta, wait, timeout, **image_kwargs):
self, name, filename, data, meta, wait, timeout, **image_kwargs):
# NOTE(mordred) wait and timeout parameters are unused, but
# are present for ease at calling site.
image_data = open(filename, 'rb')
if filename and not data:
image_data = open(filename, 'rb')
else:
image_data = data
image_kwargs['properties'].update(meta)
image_kwargs['name'] = name

55
openstack/image/v1/image.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from openstack.image import _download
from openstack import exceptions
from openstack import resource
@ -29,6 +30,11 @@ class Image(resource.Resource, _download.DownloadMixin):
# Remotely they would be still in the resource root
_store_unknown_attrs_as_properties = True
_query_mapping = resource.QueryParameters(
'name', 'container_format', 'disk_format',
'status', 'size_min', 'size_max'
)
#: Hash of the image data used. The Image service uses this value
#: for verification.
checksum = resource.Body('checksum')
@ -73,3 +79,52 @@ class Image(resource.Resource, _download.DownloadMixin):
status = resource.Body('status')
#: The timestamp when this image was last updated.
updated_at = resource.Body('updated_at')
@classmethod
def find(cls, session, name_or_id, ignore_missing=True, **params):
"""Find a resource by its name or id.
:param session: The session to use for making this request.
:type session: :class:`~keystoneauth1.adapter.Adapter`
:param name_or_id: This resource's identifier, if needed by
the request. The default is ``None``.
:param bool ignore_missing: When set to ``False``
:class:`~openstack.exceptions.ResourceNotFound` will be
raised when the resource does not exist.
When set to ``True``, None will be returned when
attempting to find a nonexistent resource.
:param dict params: Any additional parameters to be passed into
underlying methods, such as to
:meth:`~openstack.resource.Resource.existing`
in order to pass on URI parameters.
:return: The :class:`Resource` object matching the given name or id
or None if nothing matches.
:raises: :class:`openstack.exceptions.DuplicateResource` if more
than one resource is found for this request.
:raises: :class:`openstack.exceptions.ResourceNotFound` if nothing
is found and ignore_missing is ``False``.
"""
session = cls._get_session(session)
# Try to short-circuit by looking directly for a matching ID.
try:
match = cls.existing(
id=name_or_id,
connection=session._get_connection(),
**params)
return match.fetch(session, **params)
except exceptions.NotFoundException:
pass
params['name'] = name_or_id
data = cls.list(session, base_path='/images/detail', **params)
result = cls._get_one_match(name_or_id, data)
if result is not None:
return result
if ignore_missing:
return None
raise exceptions.ResourceNotFound(
"No %s found for %s" % (cls.__name__, name_or_id))

17
openstack/image/v2/_proxy.py
View File

@ -148,7 +148,7 @@ class Proxy(_base_proxy.BaseImageProxy):
return img
def _upload_image(self, name, filename=None, meta=None,
def _upload_image(self, name, filename=None, data=None, meta=None,
wait=False, timeout=None, validate_checksum=True,
**kwargs):
# We can never have nice things. Glance v1 took "is_public" as a
@ -166,11 +166,11 @@ class Proxy(_base_proxy.BaseImageProxy):
# This makes me want to die inside
if self._connection.image_api_use_tasks:
return self._upload_image_task(
name, filename, meta=meta,
name, filename, data=data, meta=meta,
wait=wait, timeout=timeout, **kwargs)
else:
return self._upload_image_put(
name, filename, meta=meta,
name, filename, data=data, meta=meta,
validate_checksum=validate_checksum,
**kwargs)
except exceptions.SDKException:
@ -196,8 +196,12 @@ class Proxy(_base_proxy.BaseImageProxy):
return ret
def _upload_image_put(
self, name, filename, meta, validate_checksum, **image_kwargs):
image_data = open(filename, 'rb')
self, name, filename, data, meta,
validate_checksum, **image_kwargs):
if filename and not data:
image_data = open(filename, 'rb')
else:
image_data = data
properties = image_kwargs.pop('properties', {})
@ -232,7 +236,7 @@ class Proxy(_base_proxy.BaseImageProxy):
return image
def _upload_image_task(
self, name, filename,
self, name, filename, data,
wait, timeout, meta, **image_kwargs):
if not self._connection.has_service('object-store'):
@ -251,6 +255,7 @@ class Proxy(_base_proxy.BaseImageProxy):
self._connection.create_object(
container, name, filename,
md5=md5, sha256=sha256,
data=data,
metadata={self._connection._OBJECT_AUTOCREATE_KEY: 'true'},
**{'content-type': 'application/octet-stream',
'x-delete-after': str(24 * 60 * 60)})

133
openstack/tests/unit/cloud/test_image.py
View File

@ -321,7 +321,21 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri=self.get_mock_url(
@ -356,6 +370,7 @@ class TestImage(BaseTestImage):
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
complete_qs=True,
json=self.fake_search_return)
])
@ -365,7 +380,7 @@ class TestImage(BaseTestImage):
is_public=False)
self.assert_calls()
self.assertEqual(self.adapter.request_history[5].text.read(),
self.assertEqual(self.adapter.request_history[7].text.read(),
self.output)
def test_create_image_task(self):
@ -390,7 +405,21 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='HEAD',
uri='{endpoint}/{container}'.format(
@ -517,6 +546,7 @@ class TestImage(BaseTestImage):
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
complete_qs=True,
json=self.fake_search_return)
])
@ -686,7 +716,11 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v1/images/detail',
uri='https://image.example.com/v1/images/' + self.image_name,
status_code=404),
dict(method='GET',
uri='https://image.example.com/v1/images/detail?name='
+ self.image_name,
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v1/images',
@ -726,7 +760,11 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v1/images/detail',
uri='https://image.example.com/v1/images/' + self.image_name,
status_code=404),
dict(method='GET',
uri='https://image.example.com/v1/images/detail?name='
+ self.image_name,
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v1/images',
@ -792,7 +830,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -828,10 +881,6 @@ class TestImage(BaseTestImage):
fake_image['owner_specified.openstack.sha256'] = 'b'
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
json={'images': []}),
dict(method='POST',
uri=self.get_mock_url(
'image', append=['images'], base_url_append='v2'),
@ -870,7 +919,8 @@ class TestImage(BaseTestImage):
exceptions.SDKException,
self.cloud.create_image,
self.image_name, self.imagefile.name,
is_public=False, md5='a', sha256='b'
is_public=False, md5='a', sha256='b',
allow_duplicates=True
)
self.assert_calls()
@ -878,15 +928,10 @@ class TestImage(BaseTestImage):
def test_create_image_put_bad_int(self):
self.cloud.image_api_use_tasks = False
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
json={'images': []}),
])
self.assertRaises(
exc.OpenStackCloudException,
self._call_create_image, self.image_name,
allow_duplicates=True,
min_disk='fish', min_ram=0)
self.assert_calls()
@ -910,7 +955,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -931,6 +991,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])
@ -959,7 +1020,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -980,6 +1056,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])
@ -1009,7 +1086,22 @@ class TestImage(BaseTestImage):
self.register_uris([
dict(method='GET',
uri='https://image.example.com/v2/images',
uri=self.get_mock_url(
'image', append=['images', self.image_name],
base_url_append='v2'),
status_code=404),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['name=' + self.image_name]),
validate=dict(),
json={'images': []}),
dict(method='GET',
uri=self.get_mock_url(
'image', append=['images'],
base_url_append='v2',
qs_elements=['os_hidden=True']),
json={'images': []}),
dict(method='POST',
uri='https://image.example.com/v2/images',
@ -1030,6 +1122,7 @@ class TestImage(BaseTestImage):
json=ret),
dict(method='GET',
uri='https://image.example.com/v2/images',
complete_qs=True,
json={'images': [ret]}),
])

106
openstack/tests/unit/image/v2/test_proxy.py
View File

@ -11,6 +11,7 @@
# under the License.
import mock
import io
import requests
from openstack import exceptions
@ -41,6 +42,7 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
def setUp(self):
super(TestImageProxy, self).setUp()
self.proxy = _proxy.Proxy(self.session)
self.proxy._connection = self.cloud
def test_image_import_no_required_attrs(self):
# container_format and disk_format are required attrs of the image
@ -57,6 +59,110 @@ class TestImageProxy(test_proxy_base.TestProxyBase):
expected_kwargs={"method": "method", "store": None,
"uri": "uri"})
def test_image_create_conflict(self):
self.assertRaises(
exceptions.SDKException, self.proxy.create_image,
name='fake', filename='fake', data='fake',
container='bare', disk_format='raw'
)
def test_image_create_checksum_match(self):
fake_image = image.Image(
id="fake", properties={
self.proxy._IMAGE_MD5_KEY: 'fake_md5',
self.proxy._IMAGE_SHA256_KEY: 'fake_sha256'
})
self.proxy.find_image = mock.Mock(return_value=fake_image)
self.proxy._upload_image = mock.Mock()
res = self.proxy.create_image(
name='fake',
md5='fake_md5', sha256='fake_sha256'
)
self.assertEqual(fake_image, res)
self.proxy._upload_image.assert_not_called()
def test_image_create_checksum_mismatch(self):
fake_image = image.Image(
id="fake", properties={
self.proxy._IMAGE_MD5_KEY: 'fake_md5',
self.proxy._IMAGE_SHA256_KEY: 'fake_sha256'
})
self.proxy.find_image = mock.Mock(return_value=fake_image)
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake',
md5='fake2_md5', sha256='fake2_sha256'
)
self.proxy._upload_image.assert_called()
def test_image_create_allow_duplicates_find_not_called(self):
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake', allow_duplicates=True,
)
self.proxy.find_image.assert_not_called()
def test_image_create_validate_checksum_data_binary(self):
""" Pass real data as binary"""
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
self.proxy.create_image(
name='fake', data=b'fake', validate_checksum=True,
container='bare', disk_format='raw'
)
self.proxy.find_image.assert_called_with('fake')
self.proxy._upload_image.assert_called_with(
'fake', container_format='bare', disk_format='raw',
filename=None, data=b'fake', meta={},
properties={
self.proxy._IMAGE_MD5_KEY: '144c9defac04969c7bfad8efaa8ea194',
self.proxy._IMAGE_SHA256_KEY: 'b5d54c39e66671c9731b9f471e585'
'd8262cd4f54963f0c93082d8dcf33'
'4d4c78',
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=True, wait=False)
def test_image_create_validate_checksum_data_not_binary(self):
self.assertRaises(
exceptions.SDKException, self.proxy.create_image,
name='fake', data=io.StringIO(), validate_checksum=True,
container='bare', disk_format='raw'
)
def test_image_create_data_binary(self):
"""Pass binary file-like object"""
self.proxy.find_image = mock.Mock()
self.proxy._upload_image = mock.Mock()
data = io.BytesIO(b'\0\0')
self.proxy.create_image(
name='fake', data=data, validate_checksum=False,
container='bare', disk_format='raw'
)
self.proxy._upload_image.assert_called_with(
'fake', container_format='bare', disk_format='raw',
filename=None, data=data, meta={},
properties={
self.proxy._IMAGE_MD5_KEY: '',
self.proxy._IMAGE_SHA256_KEY: '',
self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'},
timeout=3600, validate_checksum=False, wait=False)
def test_image_upload_no_args(self):
# container_format and disk_format are required args
self.assertRaises(exceptions.InvalidRequest, self.proxy.upload_image)

4
releasenotes/notes/support_stdin_image_upload-305c04fb2daeb32c.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Add support for creating image from STDIN (i.e. from OSC). When creating from STDIN however, no checksum verification is possible, and thus validate_checksum must be also set to False.