stackforge/swiftsync
Code Issues Proposed changes

Handle error from swiftclient.

Browse Source 
- Those are not very important and we don't better with retry since we
are syncronizing the rest after when it will be lest busy on the second
pass.
This commit is contained in:
Chmouel Boudjnah 2013-04-09 09:23:04 +02:00
parent 4657adb90f
commit 77e239042f
3 changed files with 57 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
swsync/accounts.py
View File

@ -57,15 +57,20 @@ class Accounts(object):
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
orig_stats, orig_containers = (
swiftclient.get_account(None, orig_token,
http_conn=orig_storage_cnx,
full_listing=True))
try:
orig_stats, orig_containers = (
swiftclient.get_account(None, orig_token,
http_conn=orig_storage_cnx,
full_listing=True))
dest_stats, dest_containers = (
swiftclient.get_account(None, dest_token,
http_conn=dest_storage_cnx,
full_listing=True))
dest_stats, dest_containers = (
swiftclient.get_account(None, dest_token,
http_conn=dest_storage_cnx,
full_listing=True))
except(swiftclient.client.ClientException), e:
logging.info("error getting containeaccount: %s, %s" % (
orig_storage_url, e.http_reason))
return
if int(dest_stats['x-account-container-count']) > \
int(orig_stats['x-account-container-count']):
self.container_cls.delete_container(dest_storage_cnx,

47
swsync/containers.py
View File

@ -41,9 +41,15 @@ class Containers(object):
pool = eventlet.GreenPool(size=self.max_gthreads)
pile = eventlet.GreenPile(pool)
for container in delete_diff:
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container, http_conn=dest_storage_cnx,
)
try:
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container, http_conn=dest_storage_cnx,
)
except(swiftclient.client.ClientException), e:
logging.info("error getting container: %s, %s" % (
container, e.http_reason))
continue
for obj in dest_objects:
logging.info("deleting obj: %s ts:%s", obj['name'],
obj['last_modified'])
@ -62,27 +68,42 @@ class Containers(object):
orig_token, dest_storage_cnx, dest_storage_url, dest_token,
container_name):
orig_container_stats, orig_objects = swiftclient.get_container(
None, orig_token, container_name, http_conn=orig_storage_cnx,
)
try:
orig_container_stats, orig_objects = swiftclient.get_container(
None, orig_token, container_name, http_conn=orig_storage_cnx,
)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: getting container: %s, %s" % (
container_name, e.http_reason))
return
try:
swiftclient.head_container(
"", dest_token, container_name, http_conn=dest_storage_cnx
)
except(swiftclient.client.ClientException):
except(swiftclient.client.ClientException), e:
container_headers = orig_container_stats.copy()
for h in ('x-container-object-count', 'x-trans-id',
'x-container-bytes-used'):
del container_headers[h]
p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
swiftclient.put_container(url,
dest_token, container_name,
headers=container_headers)
try:
swiftclient.put_container(url,
dest_token, container_name,
headers=container_headers)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (
container_name, e.http_reason))
return
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container_name, http_conn=dest_storage_cnx,
)
try:
dest_container_stats, dest_objects = swiftclient.get_container(
None, dest_token, container_name, http_conn=dest_storage_cnx,
)
except(swiftclient.client.ClientException), e:
logging.info("ERROR: creating container: %s, %s" % (
container_name, e.http_reason))
return
set1 = set((x['last_modified'], x['name']) for x in orig_objects)
set2 = set((x['last_modified'], x['name']) for x in dest_objects)

13
swsync/objects.py
View File

@ -14,6 +14,8 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import eventlet
import swift.common.bufferedhttp
import swift.common.http
@ -58,6 +60,7 @@ def get_object(storage_url, token,
if not swift.common.http.is_success(resp.status):
resp.read()
#TODO: logging
raise swiftclient.ClientException(
'status %s %s' % (resp.status, resp.reason))
@ -105,6 +108,10 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url,
post_headers['x-auth-token'] = dest_token
sync_to = dest_storage_url + "/" + container_name
iterlike = swift.container.sync._Iter2FileLikeObject
swiftclient.put_object(sync_to, name=object_name,
headers=post_headers,
contents=iterlike(orig_body))
try:
swiftclient.put_object(sync_to, name=object_name,
headers=post_headers,
contents=iterlike(orig_body))
except(swiftclient.ClientException), e:
logging.info("error sync object: %s, %s" % (
object_name, e.http_reason))