diff --git a/openstack/cloud/_object_store.py b/openstack/cloud/_object_store.py index 5e85bbd68..6bb6fc68d 100644 --- a/openstack/cloud/_object_store.py +++ b/openstack/cloud/_object_store.py @@ -219,14 +219,11 @@ class ObjectStoreCloudMixin(_normalize.Normalizer): if file_key not in self._file_hash_cache: self.log.debug( 'Calculating hashes for %(filename)s', {'filename': filename}) - md5 = hashlib.md5() - sha256 = hashlib.sha256() + (md5, sha256) = (None, None) with open(filename, 'rb') as file_obj: - for chunk in iter(lambda: file_obj.read(8192), b''): - md5.update(chunk) - sha256.update(chunk) + (md5, sha256) = self._calculate_data_hashes(file_obj) self._file_hash_cache[file_key] = dict( - md5=md5.hexdigest(), sha256=sha256.hexdigest()) + md5=md5, sha256=sha256) self.log.debug( "Image file %(filename)s md5:%(md5)s sha256:%(sha256)s", {'filename': filename, @@ -235,6 +232,19 @@ class ObjectStoreCloudMixin(_normalize.Normalizer): return (self._file_hash_cache[file_key]['md5'], self._file_hash_cache[file_key]['sha256']) + def _calculate_data_hashes(self, data): + md5 = hashlib.md5() + sha256 = hashlib.sha256() + + if hasattr(data, 'read'): + for chunk in iter(lambda: data.read(8192), b''): + md5.update(chunk) + sha256.update(chunk) + else: + md5.update(data) + sha256.update(data) + return (md5.hexdigest(), sha256.hexdigest()) + @_utils.cache_on_arguments() def get_object_capabilities(self): """Get infomation about the object-storage service diff --git a/openstack/image/_base_proxy.py b/openstack/image/_base_proxy.py index 93ee70446..499a2e983 100644 --- a/openstack/image/_base_proxy.py +++ b/openstack/image/_base_proxy.py @@ -14,6 +14,7 @@ import os import six +from openstack import exceptions from openstack import proxy @@ -40,7 +41,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): disable_vendor_agent=True, allow_duplicates=False, meta=None, wait=False, timeout=3600, - validate_checksum=True, + data=None, validate_checksum=True, **kwargs): """Upload an image. @@ -49,6 +50,8 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): basename of the path. :param str filename: The path to the file to upload, if needed. (optional, defaults to None) + :param data: Image data (string or file-like object). It is mutually + exclusive with filename :param str container: Name of the container in swift where images should be uploaded for import if the cloud requires such a thing. (optional, defaults to 'images') @@ -103,23 +106,34 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): # https://docs.openstack.org/image-guide/image-formats.html container_format = 'bare' + if data and filename: + raise exceptions.SDKException( + 'Passing filename and data simultaneously is not supported') # If there is no filename, see if name is actually the filename - if not filename: + if not filename and not data: name, filename = self._get_name_and_filename( name, self._connection.config.config['image_format']) - if not (md5 or sha256): - (md5, sha256) = self._connection._get_file_hashes(filename) + if validate_checksum and data and not isinstance(data, bytes): + raise exceptions.SDKException( + 'Validating checksum is not possible when data is not a ' + 'direct binary object') + if not (md5 or sha256) and validate_checksum: + if filename: + (md5, sha256) = self._connection._get_file_hashes(filename) + elif data and isinstance(data, bytes): + (md5, sha256) = self._connection._calculate_data_hashes(data) if allow_duplicates: current_image = None else: - current_image = self._connection.get_image(name) + current_image = self.find_image(name) if current_image: - md5_key = current_image.get( + props = current_image.get('properties', {}) + md5_key = props.get( self._IMAGE_MD5_KEY, - current_image.get(self._SHADE_IMAGE_MD5_KEY, '')) - sha256_key = current_image.get( + props.get(self._SHADE_IMAGE_MD5_KEY, '')) + sha256_key = props.get( self._IMAGE_SHA256_KEY, - current_image.get(self._SHADE_IMAGE_SHA256_KEY, '')) + props.get(self._SHADE_IMAGE_SHA256_KEY, '')) up_to_date = self._connection._hashes_up_to_date( md5=md5, sha256=sha256, md5_key=md5_key, sha256_key=sha256_key) @@ -128,6 +142,11 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): "image %(name)s exists and is up to date", {'name': name}) return current_image + else: + self.log.debug( + "image %(name)s exists, but contains different " + "checksums. Updating.", + {'name': name}) if disable_vendor_agent: kwargs.update( @@ -147,9 +166,9 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): if container_format: image_kwargs['container_format'] = container_format - if filename: + if filename or data: image = self._upload_image( - name, filename=filename, meta=meta, + name, filename=filename, data=data, meta=meta, wait=wait, timeout=timeout, validate_checksum=validate_checksum, **image_kwargs) @@ -163,7 +182,7 @@ class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)): pass @abc.abstractmethod - def _upload_image(self, name, filename, meta, wait, timeout, + def _upload_image(self, name, filename, data, meta, wait, timeout, validate_checksum=True, **image_kwargs): pass diff --git a/openstack/image/v1/_proxy.py b/openstack/image/v1/_proxy.py index d7a0706a8..b808b10f5 100644 --- a/openstack/image/v1/_proxy.py +++ b/openstack/image/v1/_proxy.py @@ -42,10 +42,13 @@ class Proxy(_base_proxy.BaseImageProxy): return self._create(_image.Image, **attrs) def _upload_image( - self, name, filename, meta, wait, timeout, **image_kwargs): + self, name, filename, data, meta, wait, timeout, **image_kwargs): # NOTE(mordred) wait and timeout parameters are unused, but # are present for ease at calling site. - image_data = open(filename, 'rb') + if filename and not data: + image_data = open(filename, 'rb') + else: + image_data = data image_kwargs['properties'].update(meta) image_kwargs['name'] = name diff --git a/openstack/image/v1/image.py b/openstack/image/v1/image.py index a92c4362d..9fc7e7c08 100644 --- a/openstack/image/v1/image.py +++ b/openstack/image/v1/image.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. from openstack.image import _download +from openstack import exceptions from openstack import resource @@ -29,6 +30,11 @@ class Image(resource.Resource, _download.DownloadMixin): # Remotely they would be still in the resource root _store_unknown_attrs_as_properties = True + _query_mapping = resource.QueryParameters( + 'name', 'container_format', 'disk_format', + 'status', 'size_min', 'size_max' + ) + #: Hash of the image data used. The Image service uses this value #: for verification. checksum = resource.Body('checksum') @@ -73,3 +79,52 @@ class Image(resource.Resource, _download.DownloadMixin): status = resource.Body('status') #: The timestamp when this image was last updated. updated_at = resource.Body('updated_at') + + @classmethod + def find(cls, session, name_or_id, ignore_missing=True, **params): + """Find a resource by its name or id. + + :param session: The session to use for making this request. + :type session: :class:`~keystoneauth1.adapter.Adapter` + :param name_or_id: This resource's identifier, if needed by + the request. The default is ``None``. + :param bool ignore_missing: When set to ``False`` + :class:`~openstack.exceptions.ResourceNotFound` will be + raised when the resource does not exist. + When set to ``True``, None will be returned when + attempting to find a nonexistent resource. + :param dict params: Any additional parameters to be passed into + underlying methods, such as to + :meth:`~openstack.resource.Resource.existing` + in order to pass on URI parameters. + + :return: The :class:`Resource` object matching the given name or id + or None if nothing matches. + :raises: :class:`openstack.exceptions.DuplicateResource` if more + than one resource is found for this request. + :raises: :class:`openstack.exceptions.ResourceNotFound` if nothing + is found and ignore_missing is ``False``. + """ + session = cls._get_session(session) + # Try to short-circuit by looking directly for a matching ID. + try: + match = cls.existing( + id=name_or_id, + connection=session._get_connection(), + **params) + return match.fetch(session, **params) + except exceptions.NotFoundException: + pass + + params['name'] = name_or_id + + data = cls.list(session, base_path='/images/detail', **params) + + result = cls._get_one_match(name_or_id, data) + if result is not None: + return result + + if ignore_missing: + return None + raise exceptions.ResourceNotFound( + "No %s found for %s" % (cls.__name__, name_or_id)) diff --git a/openstack/image/v2/_proxy.py b/openstack/image/v2/_proxy.py index 9f9d56226..7961ef31d 100644 --- a/openstack/image/v2/_proxy.py +++ b/openstack/image/v2/_proxy.py @@ -148,7 +148,7 @@ class Proxy(_base_proxy.BaseImageProxy): return img - def _upload_image(self, name, filename=None, meta=None, + def _upload_image(self, name, filename=None, data=None, meta=None, wait=False, timeout=None, validate_checksum=True, **kwargs): # We can never have nice things. Glance v1 took "is_public" as a @@ -166,11 +166,11 @@ class Proxy(_base_proxy.BaseImageProxy): # This makes me want to die inside if self._connection.image_api_use_tasks: return self._upload_image_task( - name, filename, meta=meta, + name, filename, data=data, meta=meta, wait=wait, timeout=timeout, **kwargs) else: return self._upload_image_put( - name, filename, meta=meta, + name, filename, data=data, meta=meta, validate_checksum=validate_checksum, **kwargs) except exceptions.SDKException: @@ -196,8 +196,12 @@ class Proxy(_base_proxy.BaseImageProxy): return ret def _upload_image_put( - self, name, filename, meta, validate_checksum, **image_kwargs): - image_data = open(filename, 'rb') + self, name, filename, data, meta, + validate_checksum, **image_kwargs): + if filename and not data: + image_data = open(filename, 'rb') + else: + image_data = data properties = image_kwargs.pop('properties', {}) @@ -232,7 +236,7 @@ class Proxy(_base_proxy.BaseImageProxy): return image def _upload_image_task( - self, name, filename, + self, name, filename, data, wait, timeout, meta, **image_kwargs): if not self._connection.has_service('object-store'): @@ -251,6 +255,7 @@ class Proxy(_base_proxy.BaseImageProxy): self._connection.create_object( container, name, filename, md5=md5, sha256=sha256, + data=data, metadata={self._connection._OBJECT_AUTOCREATE_KEY: 'true'}, **{'content-type': 'application/octet-stream', 'x-delete-after': str(24 * 60 * 60)}) diff --git a/openstack/tests/unit/cloud/test_image.py b/openstack/tests/unit/cloud/test_image.py index 12aec72fd..e2c280ac0 100644 --- a/openstack/tests/unit/cloud/test_image.py +++ b/openstack/tests/unit/cloud/test_image.py @@ -321,7 +321,21 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', uri=self.get_mock_url( - 'image', append=['images'], base_url_append='v2'), + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='POST', uri=self.get_mock_url( @@ -356,6 +370,7 @@ class TestImage(BaseTestImage): dict(method='GET', uri=self.get_mock_url( 'image', append=['images'], base_url_append='v2'), + complete_qs=True, json=self.fake_search_return) ]) @@ -365,7 +380,7 @@ class TestImage(BaseTestImage): is_public=False) self.assert_calls() - self.assertEqual(self.adapter.request_history[5].text.read(), + self.assertEqual(self.adapter.request_history[7].text.read(), self.output) def test_create_image_task(self): @@ -390,7 +405,21 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', uri=self.get_mock_url( - 'image', append=['images'], base_url_append='v2'), + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='HEAD', uri='{endpoint}/{container}'.format( @@ -517,6 +546,7 @@ class TestImage(BaseTestImage): dict(method='GET', uri=self.get_mock_url( 'image', append=['images'], base_url_append='v2'), + complete_qs=True, json=self.fake_search_return) ]) @@ -686,7 +716,11 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v1/images/detail', + uri='https://image.example.com/v1/images/' + self.image_name, + status_code=404), + dict(method='GET', + uri='https://image.example.com/v1/images/detail?name=' + + self.image_name, json={'images': []}), dict(method='POST', uri='https://image.example.com/v1/images', @@ -726,7 +760,11 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v1/images/detail', + uri='https://image.example.com/v1/images/' + self.image_name, + status_code=404), + dict(method='GET', + uri='https://image.example.com/v1/images/detail?name=' + + self.image_name, json={'images': []}), dict(method='POST', uri='https://image.example.com/v1/images', @@ -792,7 +830,22 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v2/images', + uri=self.get_mock_url( + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='POST', uri='https://image.example.com/v2/images', @@ -828,10 +881,6 @@ class TestImage(BaseTestImage): fake_image['owner_specified.openstack.sha256'] = 'b' self.register_uris([ - dict(method='GET', - uri=self.get_mock_url( - 'image', append=['images'], base_url_append='v2'), - json={'images': []}), dict(method='POST', uri=self.get_mock_url( 'image', append=['images'], base_url_append='v2'), @@ -870,7 +919,8 @@ class TestImage(BaseTestImage): exceptions.SDKException, self.cloud.create_image, self.image_name, self.imagefile.name, - is_public=False, md5='a', sha256='b' + is_public=False, md5='a', sha256='b', + allow_duplicates=True ) self.assert_calls() @@ -878,15 +928,10 @@ class TestImage(BaseTestImage): def test_create_image_put_bad_int(self): self.cloud.image_api_use_tasks = False - self.register_uris([ - dict(method='GET', - uri='https://image.example.com/v2/images', - json={'images': []}), - ]) - self.assertRaises( exc.OpenStackCloudException, self._call_create_image, self.image_name, + allow_duplicates=True, min_disk='fish', min_ram=0) self.assert_calls() @@ -910,7 +955,22 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v2/images', + uri=self.get_mock_url( + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='POST', uri='https://image.example.com/v2/images', @@ -931,6 +991,7 @@ class TestImage(BaseTestImage): json=ret), dict(method='GET', uri='https://image.example.com/v2/images', + complete_qs=True, json={'images': [ret]}), ]) @@ -959,7 +1020,22 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v2/images', + uri=self.get_mock_url( + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='POST', uri='https://image.example.com/v2/images', @@ -980,6 +1056,7 @@ class TestImage(BaseTestImage): json=ret), dict(method='GET', uri='https://image.example.com/v2/images', + complete_qs=True, json={'images': [ret]}), ]) @@ -1009,7 +1086,22 @@ class TestImage(BaseTestImage): self.register_uris([ dict(method='GET', - uri='https://image.example.com/v2/images', + uri=self.get_mock_url( + 'image', append=['images', self.image_name], + base_url_append='v2'), + status_code=404), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['name=' + self.image_name]), + validate=dict(), + json={'images': []}), + dict(method='GET', + uri=self.get_mock_url( + 'image', append=['images'], + base_url_append='v2', + qs_elements=['os_hidden=True']), json={'images': []}), dict(method='POST', uri='https://image.example.com/v2/images', @@ -1030,6 +1122,7 @@ class TestImage(BaseTestImage): json=ret), dict(method='GET', uri='https://image.example.com/v2/images', + complete_qs=True, json={'images': [ret]}), ]) diff --git a/openstack/tests/unit/image/v2/test_proxy.py b/openstack/tests/unit/image/v2/test_proxy.py index 76038ad15..be7693f69 100644 --- a/openstack/tests/unit/image/v2/test_proxy.py +++ b/openstack/tests/unit/image/v2/test_proxy.py @@ -11,6 +11,7 @@ # under the License. import mock +import io import requests from openstack import exceptions @@ -41,6 +42,7 @@ class TestImageProxy(test_proxy_base.TestProxyBase): def setUp(self): super(TestImageProxy, self).setUp() self.proxy = _proxy.Proxy(self.session) + self.proxy._connection = self.cloud def test_image_import_no_required_attrs(self): # container_format and disk_format are required attrs of the image @@ -57,6 +59,110 @@ class TestImageProxy(test_proxy_base.TestProxyBase): expected_kwargs={"method": "method", "store": None, "uri": "uri"}) + def test_image_create_conflict(self): + self.assertRaises( + exceptions.SDKException, self.proxy.create_image, + name='fake', filename='fake', data='fake', + container='bare', disk_format='raw' + ) + + def test_image_create_checksum_match(self): + fake_image = image.Image( + id="fake", properties={ + self.proxy._IMAGE_MD5_KEY: 'fake_md5', + self.proxy._IMAGE_SHA256_KEY: 'fake_sha256' + }) + self.proxy.find_image = mock.Mock(return_value=fake_image) + + self.proxy._upload_image = mock.Mock() + + res = self.proxy.create_image( + name='fake', + md5='fake_md5', sha256='fake_sha256' + ) + self.assertEqual(fake_image, res) + self.proxy._upload_image.assert_not_called() + + def test_image_create_checksum_mismatch(self): + fake_image = image.Image( + id="fake", properties={ + self.proxy._IMAGE_MD5_KEY: 'fake_md5', + self.proxy._IMAGE_SHA256_KEY: 'fake_sha256' + }) + self.proxy.find_image = mock.Mock(return_value=fake_image) + + self.proxy._upload_image = mock.Mock() + + self.proxy.create_image( + name='fake', data=b'fake', + md5='fake2_md5', sha256='fake2_sha256' + ) + self.proxy._upload_image.assert_called() + + def test_image_create_allow_duplicates_find_not_called(self): + self.proxy.find_image = mock.Mock() + + self.proxy._upload_image = mock.Mock() + + self.proxy.create_image( + name='fake', data=b'fake', allow_duplicates=True, + ) + + self.proxy.find_image.assert_not_called() + + def test_image_create_validate_checksum_data_binary(self): + """ Pass real data as binary""" + self.proxy.find_image = mock.Mock() + + self.proxy._upload_image = mock.Mock() + + self.proxy.create_image( + name='fake', data=b'fake', validate_checksum=True, + container='bare', disk_format='raw' + ) + + self.proxy.find_image.assert_called_with('fake') + + self.proxy._upload_image.assert_called_with( + 'fake', container_format='bare', disk_format='raw', + filename=None, data=b'fake', meta={}, + properties={ + self.proxy._IMAGE_MD5_KEY: '144c9defac04969c7bfad8efaa8ea194', + self.proxy._IMAGE_SHA256_KEY: 'b5d54c39e66671c9731b9f471e585' + 'd8262cd4f54963f0c93082d8dcf33' + '4d4c78', + self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'}, + timeout=3600, validate_checksum=True, wait=False) + + def test_image_create_validate_checksum_data_not_binary(self): + self.assertRaises( + exceptions.SDKException, self.proxy.create_image, + name='fake', data=io.StringIO(), validate_checksum=True, + container='bare', disk_format='raw' + ) + + def test_image_create_data_binary(self): + """Pass binary file-like object""" + self.proxy.find_image = mock.Mock() + + self.proxy._upload_image = mock.Mock() + + data = io.BytesIO(b'\0\0') + + self.proxy.create_image( + name='fake', data=data, validate_checksum=False, + container='bare', disk_format='raw' + ) + + self.proxy._upload_image.assert_called_with( + 'fake', container_format='bare', disk_format='raw', + filename=None, data=data, meta={}, + properties={ + self.proxy._IMAGE_MD5_KEY: '', + self.proxy._IMAGE_SHA256_KEY: '', + self.proxy._IMAGE_OBJECT_KEY: 'bare/fake'}, + timeout=3600, validate_checksum=False, wait=False) + def test_image_upload_no_args(self): # container_format and disk_format are required args self.assertRaises(exceptions.InvalidRequest, self.proxy.upload_image) diff --git a/releasenotes/notes/support_stdin_image_upload-305c04fb2daeb32c.yaml b/releasenotes/notes/support_stdin_image_upload-305c04fb2daeb32c.yaml new file mode 100644 index 000000000..0d315e9f8 --- /dev/null +++ b/releasenotes/notes/support_stdin_image_upload-305c04fb2daeb32c.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for creating image from STDIN (i.e. from OSC). When creating from STDIN however, no checksum verification is possible, and thus validate_checksum must be also set to False.