stackforge/swiftsync
Code Issues Proposed changes

Merge pull request #63 from TristanCacqueray/master

Browse Source 
Maintenance fixes
This commit is contained in:
Joe H. Rahme 2013-11-12 02:27:49 -08:00
commit 4523c3ae55
9 changed files with 53 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -49,8 +49,13 @@ setup.py install.
$ virtualenv $HOME/venv $ virtualenv $HOME/venv
$ . $HOME/venv/bin/activate $ . $HOME/venv/bin/activate
$ pip install -r tools/pip-requires
$ python setup.py install $ python setup.py install
Note, without the manual pip install, the installation might failed with
this error: 'TypeError: dist must be a Distribution instance'
ref: https://bugs.launchpad.net/swift/+bug/1217288
swfiller usage swfiller usage
-------------- --------------

5
bin/swfiller
View File

@ -53,9 +53,10 @@ def main():
if args.config and os.path.isfile(args.config): if args.config and os.path.isfile(args.config):
try: try:
conf = utils.parse_ini(args.config) conf = utils.parse_ini(args.config)
except Exception, exc:
logging.info('Unable to parse provided conf file') logging.info('Unable to parse provided conf file')
except Exception: logging.error(exc)
pass sys.exit(1)
else: else:
try: try:
conf = utils.parse_ini() conf = utils.parse_ini()

6
swsync/containers.py
View File

@ -96,7 +96,11 @@ class Containers(object):
container_headers = orig_container_headers.copy() container_headers = orig_container_headers.copy()
for h in ('x-container-object-count', 'x-trans-id', for h in ('x-container-object-count', 'x-trans-id',
'x-container-bytes-used'): 'x-container-bytes-used'):
del container_headers[h] try:
del container_headers[h]
except KeyError:
# Nov2013: swift server does not set x-trans-id header
pass
p = dest_storage_cnx[0] p = dest_storage_cnx[0]
url = "%s://%s%s" % (p.scheme, p.netloc, p.path) url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
try: try:

6
swsync/filler.py
View File

@ -86,6 +86,10 @@ def create_swift_user(client, account_name, account_id, user_amount):
# Get swift_operator_role id # Get swift_operator_role id
roleid = [role.id for role in client.roles.list() roleid = [role.id for role in client.roles.list()
if role.name == get_config('filler', 'swift_operator_role')] if role.name == get_config('filler', 'swift_operator_role')]
if not roleid:
logging.error('Could not find swift_operator_role %s in keystone' %
get_config('filler', 'swift_operator_role'))
sys.exit(1)
roleid = roleid[0] roleid = roleid[0]
# Add tenant/user in swift operator role/group # Add tenant/user in swift operator role/group
client.roles.add_user_role(uid.id, roleid, account_id) client.roles.add_user_role(uid.id, roleid, account_id)
@ -235,7 +239,7 @@ def create_containers(cnx, acc, c_amount, index_containers=None):
try: try:
cnx.put_container(container_name, headers=copy.copy(meta)) cnx.put_container(container_name, headers=copy.copy(meta))
containers_d[container_name] = {'meta': meta, 'objects': []} containers_d[container_name] = {'meta': meta, 'objects': []}
except ClientException, e: except(ClientException), e:
logging.warning("Unable to create container %s due to %s" % logging.warning("Unable to create container %s due to %s" %
(container_name.encode('ascii', 'ignore'), (container_name.encode('ascii', 'ignore'),
e)) e))

10
swsync/objects.py
View File

@ -19,7 +19,12 @@ import logging
import eventlet import eventlet
import swift.common.bufferedhttp import swift.common.bufferedhttp
import swift.common.http import swift.common.http
import swift.container.sync try:
from swift.container.sync import _Iter2FileLikeObject as FileLikeIter
except ImportError:
# Nov2013: swift.common.utils now include a more generic object
from swift.common.utils import FileLikeIter
from swiftclient import client as swiftclient from swiftclient import client as swiftclient
import urllib import urllib
import urllib2 import urllib2
@ -107,11 +112,10 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url,
post_headers = orig_headers post_headers = orig_headers
post_headers['x-auth-token'] = dest_token post_headers['x-auth-token'] = dest_token
sync_to = dest_storage_url + "/" + container_name sync_to = dest_storage_url + "/" + container_name
iterlike = swift.container.sync._Iter2FileLikeObject
try: try:
swiftclient.put_object(sync_to, name=object_name, swiftclient.put_object(sync_to, name=object_name,
headers=post_headers, headers=post_headers,
contents=iterlike(orig_body)) contents=FileLikeIter(orig_body))
except(swiftclient.ClientException), e: except(swiftclient.ClientException), e:
logging.info("error sync object: %s, %s" % ( logging.info("error sync object: %s, %s" % (
object_name, e.http_reason)) object_name, e.http_reason))

26
tests/units/test_accounts.py
View File

@ -101,8 +101,8 @@ class TestAccountSyncMetadata(TestAccountBase):
post_account_called=post_account_called, post_account_called=post_account_called,
get_account_called=get_account_called) get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1) self.assertEqual(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2) self.assertEqual(len(get_account_called), 2)
self.assertTrue(info_called) self.assertTrue(info_called)
self.assertIn('x-account-meta-life', self.assertIn('x-account-meta-life',
@ -134,8 +134,8 @@ class TestAccountSyncMetadata(TestAccountBase):
post_account_called=post_account_called, post_account_called=post_account_called,
get_account_called=get_account_called) get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1) self.assertEqual(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2) self.assertEqual(len(get_account_called), 2)
self.assertTrue(info_called) self.assertTrue(info_called)
self.assertIn('x-account-meta-life', self.assertIn('x-account-meta-life',
@ -167,8 +167,8 @@ class TestAccountSyncMetadata(TestAccountBase):
post_account_called=post_account_called, post_account_called=post_account_called,
get_account_called=get_account_called) get_account_called=get_account_called)
self.assertEquals(len(sync_container_called), 1) self.assertEqual(len(sync_container_called), 1)
self.assertEquals(len(get_account_called), 2) self.assertEqual(len(get_account_called), 2)
self.assertTrue(info_called) self.assertTrue(info_called)
self.assertIn('x-account-meta-life', self.assertIn('x-account-meta-life',
@ -212,7 +212,7 @@ class TestAccountSync(TestAccountBase):
ret = self.accounts_cls.get_swift_auth( ret = self.accounts_cls.get_swift_auth(
"http://test.com", tenant_name, "user", "password") "http://test.com", tenant_name, "user", "password")
tenant_id = fakes.TENANTS_LIST[tenant_name]['id'] tenant_id = fakes.TENANTS_LIST[tenant_name]['id']
self.assertEquals(ret[0], "%s/v1/AUTH_%s" % (fakes.STORAGE_DEST, self.assertEqual(ret[0], "%s/v1/AUTH_%s" % (fakes.STORAGE_DEST,
tenant_id)) tenant_id))
def test_get_ks_auth_orig(self): def test_get_ks_auth_orig(self):
@ -220,11 +220,11 @@ class TestAccountSync(TestAccountBase):
k = fakes.CONFIGDICT['auth']['keystone_origin_admin_credentials'] k = fakes.CONFIGDICT['auth']['keystone_origin_admin_credentials']
tenant_name, username, password = k.split(':') tenant_name, username, password = k.split(':')
self.assertEquals(kwargs['tenant_name'], tenant_name) self.assertEqual(kwargs['tenant_name'], tenant_name)
self.assertEquals(kwargs['username'], username) self.assertEqual(kwargs['username'], username)
self.assertEquals(kwargs['password'], password) self.assertEqual(kwargs['password'], password)
k = fakes.CONFIGDICT['auth']['keystone_origin'] k = fakes.CONFIGDICT['auth']['keystone_origin']
self.assertEquals(k, kwargs['auth_url']) self.assertEqual(k, kwargs['auth_url'])
def test_process(self): def test_process(self):
ret = [] ret = []
@ -240,7 +240,7 @@ class TestAccountSync(TestAccountBase):
for x in fakes.TENANTS_LIST) for x in fakes.TENANTS_LIST)
ret_orig_storage_id = sorted( ret_orig_storage_id = sorted(
x[0][x[0].find('AUTH_') + 5:] for x in ret) x[0][x[0].find('AUTH_') + 5:] for x in ret)
self.assertEquals(tenant_list_ids, ret_orig_storage_id) self.assertEqual(tenant_list_ids, ret_orig_storage_id)
[self.assertTrue(y[1].startswith(fakes.STORAGE_DEST)) for y in ret] [self.assertTrue(y[1].startswith(fakes.STORAGE_DEST)) for y in ret]
def test_sync_account(self): def test_sync_account(self):
@ -270,7 +270,7 @@ class TestAccountSync(TestAccountBase):
ret_container_list = sorted(x[7] for x in ret) ret_container_list = sorted(x[7] for x in ret)
default_container_list = sorted(x[0]['name'] default_container_list = sorted(x[0]['name']
for x in fakes.CONTAINERS_LIST) for x in fakes.CONTAINERS_LIST)
self.assertEquals(ret_container_list, default_container_list) self.assertEqual(ret_container_list, default_container_list)
def test_sync_exception_get_account(self): def test_sync_exception_get_account(self):
called = [] called = []

8
tests/units/test_containers.py
View File

@ -104,8 +104,8 @@ class TestContainersSyncMetadata(TestContainersBase):
post_called, info_called) post_called, info_called)
self.assertEqual(len(get_called), 2) self.assertEqual(len(get_called), 2)
self.assertEqual(len(post_called), 1) self.assertEqual(len(post_called), 1)
self.assertEquals(post_called[0]['x-container-meta-psg'], '') self.assertEqual(post_called[0]['x-container-meta-psg'], '')
self.assertEquals(post_called[0]['x-container-meta-om'], 'enkl') self.assertEqual(post_called[0]['x-container-meta-om'], 'enkl')
self.assertIn('HEADER: sync headers: cont1', info_called) self.assertIn('HEADER: sync headers: cont1', info_called)
def test_sync_containers_metada_added_on_orig(self): def test_sync_containers_metada_added_on_orig(self):
@ -130,7 +130,7 @@ class TestContainersSyncMetadata(TestContainersBase):
self.assertIn('HEADER: sync headers: cont1', info_called) self.assertIn('HEADER: sync headers: cont1', info_called)
self.assertEqual(len(get_called), 2) self.assertEqual(len(get_called), 2)
self.assertEqual(len(post_called), 1) self.assertEqual(len(post_called), 1)
self.assertEquals(post_called[0]['x-container-meta-om'], 'enkl') self.assertEqual(post_called[0]['x-container-meta-om'], 'enkl')
def test_sync_containers_metada_changed(self): def test_sync_containers_metada_changed(self):
get_called = [] get_called = []
@ -153,7 +153,7 @@ class TestContainersSyncMetadata(TestContainersBase):
post_called, info_called) post_called, info_called)
self.assertEqual(len(get_called), 2) self.assertEqual(len(get_called), 2)
self.assertEqual(len(post_called), 1) self.assertEqual(len(post_called), 1)
self.assertEquals(post_called[0]['x-container-meta-psg'], 'magic') self.assertEqual(post_called[0]['x-container-meta-psg'], 'magic')
self.assertIn('HEADER: sync headers: cont1', info_called) self.assertIn('HEADER: sync headers: cont1', info_called)
def test_sync_containers_metadata_raise_client(self): def test_sync_containers_metadata_raise_client(self):

6
tests/units/test_middleware_lm.py
View File

@ -53,11 +53,11 @@ class TestLastModifiedMiddleware(unittest.TestCase):
def test_denied_method_conf(self): def test_denied_method_conf(self):
app = FakeApp() app = FakeApp()
test = middleware.filter_factory({})(app) test = middleware.filter_factory({})(app)
self.assertEquals(test.key_name, 'Last-Modified') self.assertEqual(test.key_name, 'Last-Modified')
test = middleware.filter_factory({'key_name': "Last Modified"})(app) test = middleware.filter_factory({'key_name': "Last Modified"})(app)
self.assertEquals(test.key_name, 'Last-Modified') self.assertEqual(test.key_name, 'Last-Modified')
test = middleware.filter_factory({'key_name': "Custom Key"})(app) test = middleware.filter_factory({'key_name': "Custom Key"})(app)
self.assertEquals(test.key_name, 'Custom-Key') self.assertEqual(test.key_name, 'Custom-Key')
def test_PUT_on_container(self): def test_PUT_on_container(self):
self.called = False self.called = False

11
tests/units/test_objects.py
View File

@ -16,6 +16,12 @@
# under the License. # under the License.
import eventlet import eventlet
import swift import swift
try:
from swift.container.sync import _Iter2FileLikeObject as FileLikeIter
except ImportError:
# Nov2013: swift.common.utils now include a more generic object
from swift.common.utils import FileLikeIter
import swiftclient import swiftclient
import swsync.objects as swobjects import swsync.objects as swobjects
@ -86,8 +92,7 @@ class TestObject(test_base.TestCase):
def put_object(url, name=None, headers=None, contents=None): def put_object(url, name=None, headers=None, contents=None):
self.assertEqual('obj1', name) self.assertEqual('obj1', name)
self.assertIn('x-auth-token', headers) self.assertIn('x-auth-token', headers)
self.assertIsInstance(contents, self.assertIsInstance(contents, FileLikeIter)
swift.container.sync._Iter2FileLikeObject)
contents_read = contents.read() contents_read = contents.read()
self.assertEqual(len(contents_read), len(body)) self.assertEqual(len(contents_read), len(body))
@ -153,7 +158,7 @@ class TestObject(test_base.TestCase):
"cont1", "cont1",
"obj1") "obj1")
self.assertIn('x-foo', headers) self.assertIn('x-foo', headers)
self.assertEquals(headers['x-foo'], 'BaR') self.assertEqual(headers['x-foo'], 'BaR')
def test_get_object_over_conn_timeout(self): def test_get_object_over_conn_timeout(self):
new_connect = fake_http_connect(200, connect_waitfor=2) new_connect = fake_http_connect(200, connect_waitfor=2)