openstack/openstacksdk
Code Issues Proposed changes

Move image methods to sdk image proxy

Browse Source 
We have excellent image uploading code - people should get to use it
whether they use the abstraction layer or not.

The sdk is version specific, so we can split v1 and v2 using those
classes. Make a base class for both proxies so that we can define a
general interface and handle some of the argument normalization
and processing.

NOTE: This is very unfinished. The proxy methods should be transformed
to using the Resource layer. There are many places where calls back in
to the Connection haven't had self._connection pre-pended to them.

The wait logic needs to be reworked. We should make a v2.ImageTask
resource (I think) with a wait method - and a v2.Image with a
wait method so that we can have a proxy wait_for_image method that will
work fully for put and task. Then we should remove the wait loops from
the shade layer and have it call self.image.wait_for_image(image) if
wait/timeout have been passed.

At the end of this, create_image in shade should basically be:

  if volume:
    self.block_storage.create_image()
  else:
    self.image.create_image()
  if wait:
    self.image.wait_for_image(wait, timeout)

This is also a straw man for a general approach to shifting important
logic into the sdk layer so that it can be shared, but also keep things
like the wait/timeout and "call image or block-storage api calls"
in shade.

The block_storage.create_image is going to be interesting - because it
realy needs to return an Image resource. I think the existing code is
racey/buggy - because for not-wait it returns get_image(image_id) - but
I'm pretty sure that can't possibly be guaranteed to exist that instant.
However, with Image resource we can just create a blank Image object
with image_id filled in, and that blank object can be used as a
parameter to wait_for_image.

Change-Id: Idfeb25e8d6b20d7f5ea218aaf05af9a52fb1cfb8
This commit is contained in:
Artem Goncharov 2019-02-25 11:56:02 +01:00 committed by Monty Taylor
parent 1ba0e30ebc
commit 232553daf7
10 changed files with 578 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
openstack/block_storage/_base_proxy.py Normal file
View File

@ -0,0 +1,50 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
from openstack import exceptions
from openstack import proxy
class BaseBlockStorageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
def create_image(
self, name, volume, allow_duplicates,
container_format, disk_format, wait, timeout):
if not disk_format:
disk_format = self._connection.config.config['image_format']
if not container_format:
# https://docs.openstack.org/image-guide/image-formats.html
container_format = 'bare'
if 'id' in volume:
volume_id = volume['id']
else:
volume_obj = self.get_volume(volume)
if not volume_obj:
raise exceptions.SDKException(
"Volume {volume} given to create_image could"
" not be found".format(volume=volume))
volume_id = volume_obj['id']
data = self.post(
'/volumes/{id}/action'.format(id=volume_id),
json={
'os-volume_upload_image': {
'force': allow_duplicates,
'image_name': name,
'container_format': container_format,
'disk_format': disk_format}})
response = self._connection._get_and_munchify(
'os-volume_upload_image', data)
return self._connection.image._existing_image(id=response['image_id'])

4
openstack/block_storage/v2/_proxy.py
View File

@ -10,17 +10,17 @@
# License for the specific language governing permissions and limitations
# under the License.
from openstack.block_storage import _base_proxy
from openstack.block_storage.v2 import backup as _backup
from openstack.block_storage.v2 import snapshot as _snapshot
from openstack.block_storage.v2 import stats as _stats
from openstack.block_storage.v2 import type as _type
from openstack.block_storage.v2 import volume as _volume
from openstack import exceptions
from openstack import proxy
from openstack import resource
class Proxy(proxy.Proxy):
class Proxy(_base_proxy.BaseBlockStorageProxy):
def get_snapshot(self, snapshot):
"""Get a single snapshot

4
openstack/block_storage/v3/_proxy.py
View File

@ -10,17 +10,17 @@
# License for the specific language governing permissions and limitations
# under the License.
from openstack.block_storage import _base_proxy
from openstack.block_storage.v3 import backup as _backup
from openstack.block_storage.v3 import snapshot as _snapshot
from openstack.block_storage.v3 import stats as _stats
from openstack.block_storage.v3 import type as _type
from openstack.block_storage.v3 import volume as _volume
from openstack import exceptions
from openstack import proxy
from openstack import resource
class Proxy(proxy.Proxy):
class Proxy(_base_proxy.BaseBlockStorageProxy):
def get_snapshot(self, snapshot):
"""Get a single snapshot

379
openstack/cloud/openstackcloud.py
View File

@ -54,8 +54,6 @@ import openstack.config
import openstack.config.defaults
from openstack import utils
# Rackspace returns this for intermittent import errors
IMAGE_ERROR_396 = "Image cannot be imported. Error code: '396'"
DEFAULT_OBJECT_SEGMENT_SIZE = 1073741824 # 1GB
# This halves the current default for Swift
DEFAULT_MAX_FILE_SIZE = (5 * 1024 * 1024 * 1024 + 2) / 2
@ -4738,84 +4736,22 @@ class _OpenStackCloudMixin(_normalize.Normalizer):
:raises: OpenStackCloudException if there are problems uploading
"""
if container is None:
container = self._OBJECT_AUTOCREATE_CONTAINER
if not meta:
meta = {}
if not disk_format:
disk_format = self.config.config['image_format']
if not container_format:
# https://docs.openstack.org/image-guide/image-formats.html
container_format = 'bare'
if volume:
if 'id' in volume:
volume_id = volume['id']
else:
volume_obj = self.get_volume(volume)
if not volume_obj:
raise exc.OpenStackCloudException(
"Volume {volume} given to create_image could"
" not be foud".format(volume=volume))
volume_id = volume_obj['id']
return self._upload_image_from_volume(
name=name, volume_id=volume_id,
image = self.block_storage.create_image(
name=name, volume=volume,
allow_duplicates=allow_duplicates,
container_format=container_format, disk_format=disk_format,
wait=wait, timeout=timeout)
# If there is no filename, see if name is actually the filename
if not filename:
name, filename = self._get_name_and_filename(name)
if not (md5 or sha256):
(md5, sha256) = self._get_file_hashes(filename)
if allow_duplicates:
current_image = None
else:
current_image = self.get_image(name)
if current_image:
md5_key = current_image.get(
self._IMAGE_MD5_KEY,
current_image.get(self._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = current_image.get(
self._IMAGE_SHA256_KEY,
current_image.get(self._SHADE_IMAGE_SHA256_KEY, ''))
up_to_date = self._hashes_up_to_date(
md5=md5, sha256=sha256,
md5_key=md5_key, sha256_key=sha256_key)
if up_to_date:
self.log.debug(
"image %(name)s exists and is up to date",
{'name': name})
return current_image
kwargs[self._IMAGE_MD5_KEY] = md5 or ''
kwargs[self._IMAGE_SHA256_KEY] = sha256 or ''
kwargs[self._IMAGE_OBJECT_KEY] = '/'.join([container, name])
if disable_vendor_agent:
kwargs.update(self.config.config['disable_vendor_agent'])
# If a user used the v1 calling format, they will have
# passed a dict called properties along
properties = kwargs.pop('properties', {})
kwargs.update(properties)
image_kwargs = dict(properties=kwargs)
if disk_format:
image_kwargs['disk_format'] = disk_format
if container_format:
image_kwargs['container_format'] = container_format
if self._is_client_version('image', 2):
image = self._upload_image_v2(
name, filename,
image = self.image.create_image(
name, filename=filename,
container=container,
md5=sha256, sha256=sha256,
disk_format=disk_format, container_format=container_format,
disable_vendor_agent=disable_vendor_agent,
wait=wait, timeout=timeout,
meta=meta, **image_kwargs)
else:
image = self._upload_image_v1(
name, filename,
wait=wait, timeout=timeout,
meta=meta, **image_kwargs)
allow_duplicates=allow_duplicates, meta=meta, **kwargs)
self._get_cache(None).invalidate()
if not wait:
return image
@ -4832,300 +4768,11 @@ class _OpenStackCloudMixin(_normalize.Normalizer):
self.delete_image(image.id, wait=True)
raise
def _upload_image_v2(
self, name, filename=None,
wait=False, timeout=3600,
meta=None, **kwargs):
# We can never have nice things. Glance v1 took "is_public" as a
# boolean. Glance v2 takes "visibility". If the user gives us
# is_public, we know what they mean. If they give us visibility, they
# know that they mean.
if 'is_public' in kwargs['properties']:
is_public = kwargs['properties'].pop('is_public')
if is_public:
kwargs['visibility'] = 'public'
else:
kwargs['visibility'] = 'private'
try:
# This makes me want to die inside
if self.image_api_use_tasks:
return self._upload_image_task(
name, filename,
wait=wait, timeout=timeout,
meta=meta, **kwargs)
else:
return self._upload_image_put_v2(
name, filename, meta=meta,
**kwargs)
except exc.OpenStackCloudException:
self.log.debug("Image creation failed", exc_info=True)
raise
except Exception as e:
raise exc.OpenStackCloudException(
"Image creation failed: {message}".format(message=str(e)))
def _make_v2_image_params(self, meta, properties):
ret = {}
for k, v in iter(properties.items()):
if k in ('min_disk', 'min_ram', 'size', 'virtual_size'):
ret[k] = int(v)
elif k == 'protected':
ret[k] = v
else:
if v is None:
ret[k] = None
else:
ret[k] = str(v)
ret.update(meta)
return ret
def _upload_image_from_volume(
self, name, volume_id, allow_duplicates,
container_format, disk_format, wait, timeout):
data = self._volume_client.post(
'/volumes/{id}/action'.format(id=volume_id),
json={
'os-volume_upload_image': {
'force': allow_duplicates,
'image_name': name,
'container_format': container_format,
'disk_format': disk_format}})
response = self._get_and_munchify('os-volume_upload_image', data)
if not wait:
return self.get_image(response['image_id'])
try:
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the image to finish."):
image_obj = self.get_image(response['image_id'])
if image_obj and image_obj.status not in ('queued', 'saving'):
return image_obj
except exc.OpenStackCloudTimeout:
self.log.debug(
"Timeout waiting for image to become ready. Deleting.")
self.delete_image(response['image_id'], wait=True)
raise
def _upload_image_put_v2(self, name, filename, meta, **image_kwargs):
image_data = open(filename, 'rb')
properties = image_kwargs.pop('properties', {})
image_kwargs.update(self._make_v2_image_params(meta, properties))
image_kwargs['name'] = name
data = self._image_client.post('/images', json=image_kwargs)
image = self._get_and_munchify(key=None, data=data)
try:
self._image_client.put(
'/images/{id}/file'.format(id=image.id),
headers={'Content-Type': 'application/octet-stream'},
data=image_data)
except Exception:
self.log.debug("Deleting failed upload of image %s", name)
try:
self._image_client.delete(
'/images/{id}'.format(id=image.id))
except exc.OpenStackCloudHTTPError:
# We're just trying to clean up - if it doesn't work - shrug
self.log.debug(
"Failed deleting image after we failed uploading it.",
exc_info=True)
raise
return self._normalize_image(image)
def _upload_image_v1(
self, name, filename,
wait=False, timeout=3600,
meta=None, **image_kwargs):
# NOTE(mordred) wait and timeout parameters are unused, but
# are present for ease at calling site.
image_data = open(filename, 'rb')
image_kwargs['properties'].update(meta)
image_kwargs['name'] = name
image = self._get_and_munchify(
'image',
self._image_client.post('/images', json=image_kwargs))
checksum = image_kwargs['properties'].get(self._IMAGE_MD5_KEY, '')
try:
# Let us all take a brief moment to be grateful that this
# is not actually how OpenStack APIs work anymore
headers = {
'x-glance-registry-purge-props': 'false',
}
if checksum:
headers['x-image-meta-checksum'] = checksum
image = self._get_and_munchify(
'image',
self._image_client.put(
'/images/{id}'.format(id=image.id),
headers=headers, data=image_data))
except exc.OpenStackCloudHTTPError:
self.log.debug("Deleting failed upload of image %s", name)
try:
self._image_client.delete(
'/images/{id}'.format(id=image.id))
except exc.OpenStackCloudHTTPError:
# We're just trying to clean up - if it doesn't work - shrug
self.log.debug(
"Failed deleting image after we failed uploading it.",
exc_info=True)
raise
return self._normalize_image(image)
def _upload_image_task(
self, name, filename,
wait, timeout, meta, **image_kwargs):
properties = image_kwargs.pop('properties', {})
md5 = properties[self._IMAGE_MD5_KEY]
sha256 = properties[self._IMAGE_SHA256_KEY]
container = properties[self._IMAGE_OBJECT_KEY].split('/', 1)[0]
image_kwargs.update(properties)
image_kwargs.pop('disk_format', None)
image_kwargs.pop('container_format', None)
self.create_container(container)
self.create_object(
container, name, filename,
md5=md5, sha256=sha256,
metadata={self._OBJECT_AUTOCREATE_KEY: 'true'},
**{'content-type': 'application/octet-stream'})
# TODO(mordred): Can we do something similar to what nodepool does
# using glance properties to not delete then upload but instead make a
# new "good" image and then mark the old one as "bad"
task_args = dict(
type='import', input=dict(
import_from='{container}/{name}'.format(
container=container, name=name),
image_properties=dict(name=name)))
data = self._image_client.post('/tasks', json=task_args)
glance_task = self._get_and_munchify(key=None, data=data)
self.list_images.invalidate(self)
if wait:
start = time.time()
image_id = None
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the image to import."):
try:
if image_id is None:
status = self._image_client.get(
'/tasks/{id}'.format(id=glance_task.id))
except exc.OpenStackCloudHTTPError as e:
if e.response.status_code == 503:
# Clear the exception so that it doesn't linger
# and get reported as an Inner Exception later
_utils._exc_clear()
# Intermittent failure - catch and try again
continue
raise
if status['status'] == 'success':
image_id = status['result']['image_id']
try:
image = self.get_image(image_id)
except exc.OpenStackCloudHTTPError as e:
if e.response.status_code == 503:
# Clear the exception so that it doesn't linger
# and get reported as an Inner Exception later
_utils._exc_clear()
# Intermittent failure - catch and try again
continue
raise
if image is None:
continue
self.update_image_properties(
image=image, meta=meta, **image_kwargs)
self.log.debug(
"Image Task %s imported %s in %s",
glance_task.id, image_id, (time.time() - start))
# Clean up after ourselves. The object we created is not
# needed after the import is done.
self.delete_object(container, name)
return self.get_image(image_id)
elif status['status'] == 'failure':
if status['message'] == IMAGE_ERROR_396:
glance_task = self._image_client.post(
'/tasks', data=task_args)
self.list_images.invalidate(self)
else:
# Clean up after ourselves. The image did not import
# and this isn't a 'just retry' error - glance didn't
# like the content. So we don't want to keep it for
# next time.
self.delete_object(container, name)
raise exc.OpenStackCloudException(
"Image creation failed: {message}".format(
message=status['message']),
extra_data=status)
else:
return glance_task
def update_image_properties(
self, image=None, name_or_id=None, meta=None, **properties):
if image is None:
image = self.get_image(name_or_id)
if not meta:
meta = {}
img_props = {}
for k, v in iter(properties.items()):
if v and k in ['ramdisk', 'kernel']:
v = self.get_image_id(v)
k = '{0}_id'.format(k)
img_props[k] = v
# This makes me want to die inside
if self._is_client_version('image', 2):
return self._update_image_properties_v2(image, meta, img_props)
else:
return self._update_image_properties_v1(image, meta, img_props)
def _update_image_properties_v2(self, image, meta, properties):
img_props = image.properties.copy()
for k, v in iter(self._make_v2_image_params(meta, properties).items()):
if image.get(k, None) != v:
img_props[k] = v
if not img_props:
return False
headers = {
'Content-Type': 'application/openstack-images-v2.1-json-patch'}
patch = sorted(list(jsonpatch.JsonPatch.from_diff(
image.properties, img_props)), key=operator.itemgetter('value'))
# No need to fire an API call if there is an empty patch
if patch:
self._image_client.patch(
'/images/{id}'.format(id=image.id),
headers=headers,
data=json.dumps(patch))
self.list_images.invalidate(self)
return True
def _update_image_properties_v1(self, image, meta, properties):
properties.update(meta)
img_props = {}
for k, v in iter(properties.items()):
if image.properties.get(k, None) != v:
img_props['x-image-meta-{key}'.format(key=k)] = v
if not img_props:
return False
self._image_client.put(
'/images/{id}'.format(id=image.id), headers=img_props)
self.list_images.invalidate(self)
return True
image = image or name_or_id
return self.image.update_image_properties(
image=image, meta=meta, **properties)
def create_volume(
self, size,

180
openstack/image/_base_proxy.py Normal file
View File

@ -0,0 +1,180 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
from openstack import proxy
class BaseImageProxy(six.with_metaclass(abc.ABCMeta, proxy.Proxy)):
def create_image(
self, name, filename=None,
container=None,
md5=None, sha256=None,
disk_format=None, container_format=None,
disable_vendor_agent=True,
allow_duplicates=False, meta=None,
wait=False, timeout=3600,
**kwargs):
"""Upload an image.
:param str name: Name of the image to create. If it is a pathname
of an image, the name will be constructed from the
extensionless basename of the path.
:param str filename: The path to the file to upload, if needed.
(optional, defaults to None)
:param str container: Name of the container in swift where images
should be uploaded for import if the cloud
requires such a thing. (optiona, defaults to
'images')
:param str md5: md5 sum of the image file. If not given, an md5 will
be calculated.
:param str sha256: sha256 sum of the image file. If not given, an md5
will be calculated.
:param str disk_format: The disk format the image is in. (optional,
defaults to the os-client-config config value
for this cloud)
:param str container_format: The container format the image is in.
(optional, defaults to the
os-client-config config value for this
cloud)
:param bool disable_vendor_agent: Whether or not to append metadata
flags to the image to inform the
cloud in question to not expect a
vendor agent to be runing.
(optional, defaults to True)
:param allow_duplicates: If true, skips checks that enforce unique
image name. (optional, defaults to False)
:param meta: A dict of key/value pairs to use for metadata that
bypasses automatic type conversion.
:param bool wait: If true, waits for image to be created. Defaults to
true - however, be aware that one of the upload
methods is always synchronous.
:param timeout: Seconds to wait for image creation. None is forever.
Additional kwargs will be passed to the image creation as additional
metadata for the image and will have all values converted to string
except for min_disk, min_ram, size and virtual_size which will be
converted to int.
If you are sure you have all of your data types correct or have an
advanced need to be explicit, use meta. If you are just a normal
consumer, using kwargs is likely the right choice.
If a value is in meta and kwargs, meta wins.
:returns: A ``munch.Munch`` of the Image object
:raises: OpenStackCloudException if there are problems uploading
"""
if container is None:
container = self._connection._OBJECT_AUTOCREATE_CONTAINER
if not meta:
meta = {}
if not disk_format:
disk_format = self._connection.config.config['image_format']
if not container_format:
# https://docs.openstack.org/image-guide/image-formats.html
container_format = 'bare'
# If there is no filename, see if name is actually the filename
if not filename:
name, filename = self._connection._get_name_and_filename(name)
if not (md5 or sha256):
(md5, sha256) = self._connection._get_file_hashes(filename)
if allow_duplicates:
current_image = None
else:
current_image = self._connection.get_image(name)
if current_image:
md5_key = current_image.get(
self._connection._IMAGE_MD5_KEY,
current_image.get(
self._connection._SHADE_IMAGE_MD5_KEY, ''))
sha256_key = current_image.get(
self._connection._IMAGE_SHA256_KEY,
current_image.get(
self._connection._SHADE_IMAGE_SHA256_KEY, ''))
up_to_date = self._connection._hashes_up_to_date(
md5=md5, sha256=sha256,
md5_key=md5_key, sha256_key=sha256_key)
if up_to_date:
self._connection.log.debug(
"image %(name)s exists and is up to date",
{'name': name})
return current_image
kwargs[self._connection._IMAGE_MD5_KEY] = md5 or ''
kwargs[self._connection._IMAGE_SHA256_KEY] = sha256 or ''
kwargs[self._connection._IMAGE_OBJECT_KEY] = '/'.join(
[container, name])
if disable_vendor_agent:
kwargs.update(
self._connection.config.config['disable_vendor_agent'])
# If a user used the v1 calling format, they will have
# passed a dict called properties along
properties = kwargs.pop('properties', {})
kwargs.update(properties)
image_kwargs = dict(properties=kwargs)
if disk_format:
image_kwargs['disk_format'] = disk_format
if container_format:
image_kwargs['container_format'] = container_format
image = self._upload_image(
name, filename,
wait=wait, timeout=timeout,
meta=meta, **image_kwargs)
self._connection._get_cache(None).invalidate()
return image
@abc.abstractmethod
def _upload_image(self, name, filename, meta, **image_kwargs):
pass
@abc.abstractmethod
def _update_image_properties(self, image, meta, properties):
pass
def update_image_properties(
self, image=None, meta=None, **kwargs):
"""
Update the properties of an existing image.
:param image: Name or id of an image or an Image object.
:param meta: A dict of key/value pairs to use for metadata that
bypasses automatic type conversion.
Additional kwargs will be passed to the image creation as additional
metadata for the image and will have all values converted to string
except for min_disk, min_ram, size and virtual_size which will be
converted to int.
"""
if image is None:
image = self._connection.get_image(image)
if not meta:
meta = {}
img_props = {}
for k, v in iter(kwargs.items()):
if v and k in ['ramdisk', 'kernel']:
v = self._connection.get_image_id(v)
k = '{0}_id'.format(k)
img_props[k] = v
return self._update_image_properties(image, meta, img_props)

71
openstack/image/v1/_proxy.py
View File

@ -9,16 +9,23 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
from openstack.cloud import exc
from openstack.image import _base_proxy
from openstack.image.v1 import image as _image
from openstack import proxy
class Proxy(proxy.Proxy):
class Proxy(_base_proxy.BaseImageProxy):
def upload_image(self, **attrs):
"""Upload a new image from attributes
.. warning:
This method is deprecated - and also doesn't work very well.
Please stop using it immediately and switch to
`create_image`.
:param dict attrs: Keyword arguments which will be used to create
a :class:`~openstack.image.v1.image.Image`,
comprised of the properties on the Image class.
@ -26,8 +33,68 @@ class Proxy(proxy.Proxy):
:returns: The results of image creation
:rtype: :class:`~openstack.image.v1.image.Image`
"""
warnings.warn("upload_image is deprecated. Use create_image instead.")
return self._create(_image.Image, **attrs)
def _upload_image(
self, name, filename, meta, wait, timeout, **image_kwargs):
# NOTE(mordred) wait and timeout parameters are unused, but
# are present for ease at calling site.
image_data = open(filename, 'rb')
image_kwargs['properties'].update(meta)
image_kwargs['name'] = name
# TODO(mordred) Convert this to use image Resource
image = self._connection._get_and_munchify(
'image',
self.post('/images', json=image_kwargs))
checksum = image_kwargs['properties'].get(
self._connection._IMAGE_MD5_KEY, '')
try:
# Let us all take a brief moment to be grateful that this
# is not actually how OpenStack APIs work anymore
headers = {
'x-glance-registry-purge-props': 'false',
}
if checksum:
headers['x-image-meta-checksum'] = checksum
image = self._connection._get_and_munchify(
'image',
self.put(
'/images/{id}'.format(id=image.id),
headers=headers, data=image_data))
except exc.OpenStackCloudHTTPError:
self._connection.log.debug(
"Deleting failed upload of image %s", name)
try:
self.delete('/images/{id}'.format(id=image.id))
except exc.OpenStackCloudHTTPError:
# We're just trying to clean up - if it doesn't work - shrug
self._connection.log.warning(
"Failed deleting image after we failed uploading it.",
exc_info=True)
raise
return self._connection._normalize_image(image)
def _update_image_properties(self, image, meta, properties):
properties.update(meta)
img_props = {}
for k, v in iter(properties.items()):
if image.properties.get(k, None) != v:
img_props['x-image-meta-{key}'.format(key=k)] = v
if not img_props:
return False
self.put(
'/images/{id}'.format(id=image.id), headers=img_props)
self._connection.list_images.invalidate(self._connection)
return True
def _existing_image(self, **kwargs):
return _image.Image.existing(connection=self._connection, **kwargs)
def delete_image(self, image, ignore_missing=True):
"""Delete an image

225
openstack/image/v2/_proxy.py
View File

@ -10,21 +10,39 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import jsonpatch
import operator
import time
import warnings
from openstack.cloud import exc
from openstack.cloud import _utils
from openstack import exceptions
from openstack.image import _base_proxy
from openstack.image.v2 import image as _image
from openstack.image.v2 import member as _member
from openstack.image.v2 import schema as _schema
from openstack.image.v2 import task as _task
from openstack import proxy
from openstack import resource
from openstack import utils
# Rackspace returns this for intermittent import errors
_IMAGE_ERROR_396 = "Image cannot be imported. Error code: '396'"
_INT_PROPERTIES = ('min_disk', 'min_ram', 'size', 'virtual_size')
class Proxy(proxy.Proxy):
class Proxy(_base_proxy.BaseImageProxy):
def upload_image(self, container_format=None, disk_format=None,
data=None, **attrs):
"""Upload a new image from attributes
.. warning:
This method is deprecated - and also doesn't work very well.
Please stop using it immediately and switch to
`create_image`.
:param container_format: Format of the container.
A valid value is ami, ari, aki, bare,
ovf, ova, or docker.
@ -38,6 +56,7 @@ class Proxy(proxy.Proxy):
:returns: The results of image creation
:rtype: :class:`~openstack.image.v2.image.Image`
"""
warnings.warn("upload_image is deprecated. Use create_image instead.")
# container_format and disk_format are required to be set
# on the image by the time upload_image is called, but they're not
# required by the _create call. Enforce them here so that we don't
@ -62,6 +81,208 @@ class Proxy(proxy.Proxy):
return img
def _upload_image(
self, name, filename=None,
meta=None, **kwargs):
# We can never have nice things. Glance v1 took "is_public" as a
# boolean. Glance v2 takes "visibility". If the user gives us
# is_public, we know what they mean. If they give us visibility, they
# know that they mean.
if 'is_public' in kwargs['properties']:
is_public = kwargs['properties'].pop('is_public')
if is_public:
kwargs['visibility'] = 'public'
else:
kwargs['visibility'] = 'private'
try:
# This makes me want to die inside
if self._connection.image_api_use_tasks:
return self._upload_image_task(
name, filename,
meta=meta, **kwargs)
else:
return self._upload_image_put(
name, filename, meta=meta,
**kwargs)
except exc.OpenStackCloudException:
self._connection.log.debug("Image creation failed", exc_info=True)
raise
except Exception as e:
raise exc.OpenStackCloudException(
"Image creation failed: {message}".format(message=str(e)))
def _make_v2_image_params(self, meta, properties):
ret = {}
for k, v in iter(properties.items()):
if k in _INT_PROPERTIES:
ret[k] = int(v)
elif k == 'protected':
ret[k] = v
else:
if v is None:
ret[k] = None
else:
ret[k] = str(v)
ret.update(meta)
return ret
def _upload_image_put(
self, name, filename, meta, wait, timeout, **image_kwargs):
image_data = open(filename, 'rb')
properties = image_kwargs.pop('properties', {})
image_kwargs.update(self._make_v2_image_params(meta, properties))
image_kwargs['name'] = name
data = self.post('/images', json=image_kwargs)
image = self._connection._get_and_munchify(key=None, data=data)
try:
response = self.put(
'/images/{id}/file'.format(id=image.id),
headers={'Content-Type': 'application/octet-stream'},
data=image_data)
exceptions.raise_from_response(response)
except Exception:
self._connection.log.debug(
"Deleting failed upload of image %s", name)
try:
response = self.delete(
'/images/{id}'.format(id=image.id))
exceptions.raise_from_response(response)
except exc.OpenStackCloudHTTPError:
# We're just trying to clean up - if it doesn't work - shrug
self._connection.log.warning(
"Failed deleting image after we failed uploading it.",
exc_info=True)
raise
return self._connection._normalize_image(image)
def _upload_image_task(
self, name, filename,
wait, timeout, meta, **image_kwargs):
if not self._connection.has_service('object-store'):
raise exc.OpenStackCloudException(
"The cloud {cloud} is configured to use tasks for image"
" upload, but no object-store service is available."
" Aborting.".format(cloud=self._connection.config.name))
properties = image_kwargs.pop('properties', {})
md5 = properties[self._connection._IMAGE_MD5_KEY]
sha256 = properties[self._connection._IMAGE_SHA256_KEY]
container = properties[
self._connection._IMAGE_OBJECT_KEY].split('/', 1)[0]
image_kwargs.update(properties)
image_kwargs.pop('disk_format', None)
image_kwargs.pop('container_format', None)
self._connection.create_container(container)
self._connection.create_object(
container, name, filename,
md5=md5, sha256=sha256,
metadata={self._connection._OBJECT_AUTOCREATE_KEY: 'true'},
**{'content-type': 'application/octet-stream'})
# TODO(mordred): Can we do something similar to what nodepool does
# using glance properties to not delete then upload but instead make a
# new "good" image and then mark the old one as "bad"
task_args = dict(
type='import', input=dict(
import_from='{container}/{name}'.format(
container=container, name=name),
image_properties=dict(name=name)))
data = self.post('/tasks', json=task_args)
glance_task = self._connection._get_and_munchify(key=None, data=data)
self._connection.list_images.invalidate(self)
if wait:
start = time.time()
image_id = None
for count in utils.iterate_timeout(
timeout,
"Timeout waiting for the image to import."):
try:
if image_id is None:
response = self.get(
'/tasks/{id}'.format(id=glance_task.id))
status = self._connection._get_and_munchify(
key=None, data=response)
except exc.OpenStackCloudHTTPError as e:
if e.response.status_code == 503:
# Clear the exception so that it doesn't linger
# and get reported as an Inner Exception later
_utils._exc_clear()
# Intermittent failure - catch and try again
continue
raise
if status['status'] == 'success':
image_id = status['result']['image_id']
try:
image = self._connection.get_image(image_id)
except exc.OpenStackCloudHTTPError as e:
if e.response.status_code == 503:
# Clear the exception so that it doesn't linger
# and get reported as an Inner Exception later
_utils._exc_clear()
# Intermittent failure - catch and try again
continue
raise
if image is None:
continue
self.update_image_properties(
image=image, meta=meta, **image_kwargs)
self._connection.log.debug(
"Image Task %s imported %s in %s",
glance_task.id, image_id, (time.time() - start))
# Clean up after ourselves. The object we created is not
# needed after the import is done.
self._connection.delete_object(container, name)
return self._connection.get_image(image_id)
elif status['status'] == 'failure':
if status['message'] == _IMAGE_ERROR_396:
glance_task = self.post('/tasks', data=task_args)
self._connection.list_images.invalidate(self)
else:
# Clean up after ourselves. The image did not import
# and this isn't a 'just retry' error - glance didn't
# like the content. So we don't want to keep it for
# next time.
self._connection.delete_object(container, name)
raise exc.OpenStackCloudException(
"Image creation failed: {message}".format(
message=status['message']),
extra_data=status)
else:
return glance_task
def _update_image_properties(self, image, meta, properties):
img_props = image.properties.copy()
for k, v in iter(self._make_v2_image_params(meta, properties).items()):
if image.get(k, None) != v:
img_props[k] = v
if not img_props:
return False
headers = {
'Content-Type': 'application/openstack-images-v2.1-json-patch'}
patch = sorted(list(jsonpatch.JsonPatch.from_diff(
image.properties, img_props)), key=operator.itemgetter('value'))
# No need to fire an API call if there is an empty patch
if patch:
self.patch(
'/images/{id}'.format(id=image.id),
headers=headers,
data=json.dumps(patch))
self._connection.list_images.invalidate(self._connection)
return True
def _existing_image(self, **kwargs):
return _image.Image.existing(connection=self._connection, **kwargs)
def download_image(self, image, stream=False):
"""Download an image

9
openstack/tests/unit/base.py
View File

@ -440,6 +440,15 @@ class TestCase(base.TestCase):
config=self.cloud_config, strict=self.strict_cloud)
self.addCleanup(self.cloud.task_manager.stop)
def get_cinder_discovery_mock_dict(
self,
block_storage_version_json='block-storage-version.json',
block_storage_discovery_url='https://volume.example.com/'):
discovery_fixture = os.path.join(
self.fixtures_directory, block_storage_version_json)
return dict(method='GET', uri=block_storage_discovery_url,
text=open(discovery_fixture, 'r').read())
def get_glance_discovery_mock_dict(
self,
image_version_json='image-version.json',

2
openstack/tests/unit/cloud/test_image.py
View File

@ -986,6 +986,7 @@ class TestImageVolume(BaseTestImage):
def test_create_image_volume(self):
self.register_uris([
self.get_cinder_discovery_mock_dict(),
dict(method='POST',
uri=self.get_mock_url(
'volumev2', append=['volumes', self.volume_id, 'action']),
@ -1017,6 +1018,7 @@ class TestImageVolume(BaseTestImage):
def test_create_image_volume_duplicate(self):
self.register_uris([
self.get_cinder_discovery_mock_dict(),
dict(method='POST',
uri=self.get_mock_url(
'volumev2', append=['volumes', self.volume_id, 'action']),

28
openstack/tests/unit/fixtures/block-storage-version.json Normal file
View File

@ -0,0 +1,28 @@
{
"versions": [
{
"status": "CURRENT",
"updated": "2017-02-25T12:00:00Z",
"links": [
{
"href": "https://docs.openstack.org/",
"type": "text/html",
"rel": "describedby"
},
{
"href": "https://volume.example.com/v2/",
"rel": "self"
}
],
"min_version": "",
"version": "",
"media-types": [
{
"base": "application/json",
"type": "application/vnd.openstack.volume+json;version=2"
}
],
"id": "v2.0"
}
]
}