stackforge/swiftsync
Code Issues Proposed changes

Fix pep8+hacking compliance.

Browse Source
This commit is contained in:
Chmouel Boudjnah 2013-04-08 12:18:38 +02:00
parent cfb94a6bb9
commit bbe753971c
5 changed files with 49 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bin/swsync
View File

@ -15,11 +15,11 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import sys
import optparse import optparse
import sys
import swsync.accounts import swsync.accounts
from swsync.utils import parse_ini, ConfigurationError, set_logging import swsync.utils
class Main(object): class Main(object):
@ -36,15 +36,15 @@ class Main(object):
help='Number of containers to distribute objects among') help='Number of containers to distribute objects among')
self.options, args = parser.parse_args() self.options, args = parser.parse_args()
if args: if args:
conf = parse_ini(args[0]) conf = swsync.utils.parse_ini(args[0])
else: else:
try: try:
conf = parse_ini() conf = swsync.utils.parse_ini()
except(ConfigurationError): except(swsync.utils.ConfigurationError):
parser.print_help() parser.print_help()
sys.exit(1) sys.exit(1)
set_logging(self.options.log_level.lower()) swsync.utils.set_logging(self.options.log_level.lower())
#beurk #beurk
swsync.utils.CONFIG = conf swsync.utils.CONFIG = conf
swsync.accounts.main() swsync.accounts.main()

13
swsync/containers.py
View File

@ -16,19 +16,20 @@
# under the License. # under the License.
import logging import logging
import swiftclient
import eventlet import eventlet
import swiftclient
from swsync.objects import sync_object, delete_object import swsync.objects
from swsync.utils import get_config import swsync.utils
class Containers(object): class Containers(object):
"""Containers sync.""" """Containers sync."""
def __init__(self): def __init__(self):
self.max_gthreads = int(get_config("sync", "max_gthreads")) self.max_gthreads = int(swsync.utils.get_config("sync",
self.sync_object = sync_object "max_gthreads"))
self.delete_object = delete_object self.sync_object = swsync.objects.sync_object
self.delete_object = swsync.objects.delete_object
def delete_container(self, dest_storage_cnx, dest_token, def delete_container(self, dest_storage_cnx, dest_token,
orig_containers, orig_containers,

28
swsync/objects.py
View File

@ -14,22 +14,23 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import eventlet
import swift.common.bufferedhttp
import swift.common.http
import swift.container.sync
from swiftclient import client as swiftclient from swiftclient import client as swiftclient
from swift.common.bufferedhttp import http_connect_raw import urllib
from swift.common.http import is_success
from swift.container.sync import _Iter2FileLikeObject
from eventlet import Timeout
import urllib2 import urllib2
from urllib import quote as urllib_quote
def quote(value, safe='/'): def quote(value, safe='/'):
""" """Patched version of urllib.quote.
Patched version of urllib.quote that encodes utf-8 strings before quoting
Encodes utf-8 strings before quoting.
""" """
if isinstance(value, unicode): if isinstance(value, unicode):
value = value.encode('utf-8') value = value.encode('utf-8')
return urllib_quote(value, safe) return urllib.quote(value, safe)
def get_object(storage_url, token, def get_object(storage_url, token,
@ -43,8 +44,8 @@ def get_object(storage_url, token,
path = x.path + '/' + container_name + '/' + object_name path = x.path + '/' + container_name + '/' + object_name
path = quote(path) path = quote(path)
with Timeout(conn_timeout): with eventlet.Timeout(conn_timeout):
conn = http_connect_raw( conn = swift.common.bufferedhttp.http_connect_raw(
x.hostname, x.hostname,
x.port, x.port,
'GET', 'GET',
@ -52,10 +53,10 @@ def get_object(storage_url, token,
headers=headers, headers=headers,
ssl=False) ssl=False)
with Timeout(response_timeout): with eventlet.Timeout(response_timeout):
resp = conn.getresponse() resp = conn.getresponse()
if not is_success(resp.status): if not swift.common.http.is_success(resp.status):
resp.read() resp.read()
raise swiftclient.ClientException( raise swiftclient.ClientException(
'status %s %s' % (resp.status, resp.reason)) 'status %s %s' % (resp.status, resp.reason))
@ -103,6 +104,7 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url,
post_headers = orig_headers post_headers = orig_headers
post_headers['x-auth-token'] = dest_token post_headers['x-auth-token'] = dest_token
sync_to = dest_storage_url + "/" + container_name sync_to = dest_storage_url + "/" + container_name
iterlike = swift.container.sync._Iter2FileLikeObject
swiftclient.put_object(sync_to, name=object_name, swiftclient.put_object(sync_to, name=object_name,
headers=post_headers, headers=post_headers,
contents=_Iter2FileLikeObject(orig_body)) contents=iterlike(orig_body))

4
swsync/utils.py
View File

@ -14,9 +14,9 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import os
import ConfigParser import ConfigParser
import logging import logging
import os
CONFIG = None CONFIG = None
@ -61,7 +61,7 @@ def parse_ini(inicfg=None):
def get_config(section, option, default=None, _config=None): def get_config(section, option, default=None, _config=None):
"""Get section/option from ConfigParser or print default if specified""" """Get section/option from ConfigParser or print default if specified."""
global CONFIG global CONFIG
if _config: if _config:
CONFIG = _config CONFIG = _config

28
tests/units/test_objects.py
View File

@ -15,6 +15,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import swiftclient import swiftclient
import swift
from eventlet import sleep, Timeout from eventlet import sleep, Timeout
import base as test_base import base as test_base
@ -67,7 +68,8 @@ class TestObject(test_base.TestCase):
def test_get_object_not_found(self): def test_get_object_not_found(self):
new_connect = fake_http_connect(404) new_connect = fake_http_connect(404)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
self.assertRaises(swiftclient.ClientException, self.assertRaises(swiftclient.ClientException,
swobjects.get_object, swobjects.get_object,
@ -76,12 +78,14 @@ class TestObject(test_base.TestCase):
def test_sync_object(self): def test_sync_object(self):
body = ("X" * 3) * 1024 body = ("X" * 3) * 1024
new_connect = fake_http_connect(200, body) new_connect = fake_http_connect(200, body)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
def put_object(url, name=None, headers=None, contents=None): def put_object(url, name=None, headers=None, contents=None):
self.assertEqual('obj1', name) self.assertEqual('obj1', name)
self.assertIn('x-auth-token', headers) self.assertIn('x-auth-token', headers)
self.assertIsInstance(contents, swobjects._Iter2FileLikeObject) self.assertIsInstance(contents,
swift.container.sync._Iter2FileLikeObject)
contents_read = contents.read() contents_read = contents.read()
self.assertEqual(len(contents_read), len(body)) self.assertEqual(len(contents_read), len(body))
@ -95,7 +99,8 @@ class TestObject(test_base.TestCase):
utf_obj = "யாமறிந்த" utf_obj = "யாமறிந்த"
body = "FOO" body = "FOO"
new_connect = fake_http_connect(200, body) new_connect = fake_http_connect(200, body)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
def put_object(url, name=None, headers=None, contents=None): def put_object(url, name=None, headers=None, contents=None):
# Container is Quoted # Container is Quoted
@ -114,7 +119,8 @@ class TestObject(test_base.TestCase):
body = ("X" * expected_chunk_time) * chunk_size body = ("X" * expected_chunk_time) * chunk_size
new_connect = fake_http_connect(200, body) new_connect = fake_http_connect(200, body)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
headers, gen = swobjects.get_object(self.orig_storage_url, headers, gen = swobjects.get_object(self.orig_storage_url,
"token", "cont1", "obj1", "token", "cont1", "obj1",
@ -126,7 +132,8 @@ class TestObject(test_base.TestCase):
def test_get_object_full(self): def test_get_object_full(self):
new_connect = fake_http_connect(200, body='foobar') new_connect = fake_http_connect(200, body='foobar')
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
headers, body = swobjects.get_object(self.orig_storage_url, headers, body = swobjects.get_object(self.orig_storage_url,
"token", "cont1", "obj1", "token", "cont1", "obj1",
@ -136,7 +143,8 @@ class TestObject(test_base.TestCase):
def test_get_headers(self): def test_get_headers(self):
headers = {'X-FOO': 'BaR'}.items() headers = {'X-FOO': 'BaR'}.items()
new_connect = fake_http_connect(200, headers=headers) new_connect = fake_http_connect(200, headers=headers)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
headers, gen = swobjects.get_object(self.orig_storage_url, headers, gen = swobjects.get_object(self.orig_storage_url,
"token", "token",
@ -147,7 +155,8 @@ class TestObject(test_base.TestCase):
def test_get_object_over_conn_timeout(self): def test_get_object_over_conn_timeout(self):
new_connect = fake_http_connect(200, connect_waitfor=2) new_connect = fake_http_connect(200, connect_waitfor=2)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
self.assertRaises(Timeout, self.assertRaises(Timeout,
swobjects.get_object, swobjects.get_object,
self.orig_storage_url, "token", "cont1", "obj1", self.orig_storage_url, "token", "cont1", "obj1",
@ -155,7 +164,8 @@ class TestObject(test_base.TestCase):
def test_get_object_over_resp_timeout(self): def test_get_object_over_resp_timeout(self):
new_connect = fake_http_connect(200, resp_waitfor=2) new_connect = fake_http_connect(200, resp_waitfor=2)
self.stubs.Set(swobjects, 'http_connect_raw', new_connect) self.stubs.Set(swift.common.bufferedhttp,
'http_connect_raw', new_connect)
self.assertRaises(Timeout, self.assertRaises(Timeout,
swobjects.get_object, swobjects.get_object,
self.orig_storage_url, "token", "cont1", "obj1", self.orig_storage_url, "token", "cont1", "obj1",