From 8595026f464b0dff8a41e8297d27e212db0a5a73 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Wed, 14 Aug 2019 13:51:42 -0700
Subject: [PATCH] Retry more operations
This adds a generic retry handler that takes a callable, and uses
that not only for the exsting retries around the POST call, but
also the actions when creating the container.
Change-Id: I910b8e886f107d4fe38a9334ba836f010f92557c
---
.../library/zuul_swift_upload.py | 90 ++++++++++---------
1 file changed, 50 insertions(+), 40 deletions(-)
diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py
index 4a3420a7e..345644c97 100755
--- a/roles/upload-logs-swift/library/zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/zuul_swift_upload.py
@@ -141,6 +141,18 @@ def get_cloud(cloud):
return openstack.connect(cloud=cloud)
+def retry_function(func):
+ for attempt in range(1, POST_ATTEMPTS + 1):
+ try:
+ return func()
+ except Exception:
+ if attempt >= POST_ATTEMPTS:
+ raise
+ else:
+ logging.exception("Error on attempt %d" % attempt)
+ time.sleep(attempt * 10)
+
+
def sizeof_fmt(num, suffix='B'):
# From http://stackoverflow.com/questions/1094841/
# reusable-library-to-get-human-readable-version-of-file-size
@@ -491,30 +503,36 @@ class Uploader():
cdn_url = None
if not self.cloud.get_container(self.container):
- self.cloud.create_container(name=self.container, public=public)
- self.cloud.update_container(
- name=self.container,
- headers={'X-Container-Meta-Web-Index': 'index.html',
- 'X-Container-Meta-Access-Control-Allow-Origin': '*'})
+ retry_function(
+ lambda: self.cloud.create_container(
+ name=self.container, public=public))
+ headers = {'X-Container-Meta-Web-Index': 'index.html',
+ 'X-Container-Meta-Access-Control-Allow-Origin': '*'}
+ retry_function(
+ lambda: self.cloud.update_container(
+ name=self.container,
+ headers=headers))
# 'X-Container-Meta-Web-Listings': 'true'
# The ceph radosgw swift implementation requires an
# index.html at the root in order for any other indexes to
# work.
index_headers = {'access-control-allow-origin': '*'}
- self.cloud.create_object(self.container,
- name='index.html',
- data='',
- content_type='text/html',
- **index_headers)
+ retry_function(
+ lambda: self.cloud.create_object(self.container,
+ name='index.html',
+ data='',
+ content_type='text/html',
+ **index_headers))
# Enable the CDN in rax
if cdn_url:
- self.cloud.session.put(cdn_url)
+ retry_function(lambda: self.cloud.session.put(cdn_url))
if cdn_url:
- endpoint = (self.cloud.session.head(cdn_url)
- .headers['X-Cdn-Ssl-Uri'])
+ endpoint = retry_function(
+ lambda: self.cloud.session.head(
+ cdn_url).headers['X-Cdn-Ssl-Uri'])
container = endpoint
else:
endpoint = self.cloud.object_store.get_endpoint()
@@ -546,7 +564,7 @@ class Uploader():
logging.debug("%s: processing job %s",
threading.current_thread(),
file_detail)
- self._post_file(file_detail)
+ retry_function(lambda: self._post_file(file_detail))
except requests.exceptions.RequestException:
# Do our best to attempt to upload all the files
logging.exception("Error posting file after multiple attempts")
@@ -584,32 +602,24 @@ class Uploader():
# This is required for Rackspace CDN
headers['access-control-allow-origin'] = '*'
- for attempt in range(1, POST_ATTEMPTS + 1):
- try:
- if not file_detail.folder:
- if (file_detail.encoding is None and
- self._is_text_type(file_detail.mimetype)):
- headers['content-encoding'] = 'deflate'
- data = DeflateFilter(open(file_detail.full_path, 'rb'))
- else:
- if file_detail.encoding:
- headers['content-encoding'] = file_detail.encoding
- data = open(file_detail.full_path, 'rb')
- else:
- data = ''
- relative_path = relative_path.rstrip('/')
- if relative_path == '':
- relative_path = '/'
- self.cloud.create_object(self.container,
- name=relative_path,
- data=data,
- **headers)
- break
- except requests.exceptions.RequestException:
- logging.exception(
- "File posting error on attempt %d" % attempt)
- if attempt >= POST_ATTEMPTS:
- raise
+ if not file_detail.folder:
+ if (file_detail.encoding is None and
+ self._is_text_type(file_detail.mimetype)):
+ headers['content-encoding'] = 'deflate'
+ data = DeflateFilter(open(file_detail.full_path, 'rb'))
+ else:
+ if file_detail.encoding:
+ headers['content-encoding'] = file_detail.encoding
+ data = open(file_detail.full_path, 'rb')
+ else:
+ data = ''
+ relative_path = relative_path.rstrip('/')
+ if relative_path == '':
+ relative_path = '/'
+ self.cloud.create_object(self.container,
+ name=relative_path,
+ data=data,
+ **headers)
def run(cloud, container, files,