stackforge/swiftsync
Code Issues Proposed changes

Fix issue #26 + PEP8 + add unittest for error catching

Browse Source
This commit is contained in:
Fabien Boucher 2013-04-08 16:59:47 +02:00
parent 3c80224405
commit 88ee16846d
2 changed files with 75 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
swsync/filler.py
View File

@ -37,6 +37,7 @@ import string
import StringIO
from swiftclient import client as sclient
from swiftclient.client import ClientException
import eventlet
@ -186,17 +187,22 @@ def create_objects(cnx, acc, o_amount, fmax, index_containers):
map(get_rand_str, ('meta_v_',) * 3)]
meta = dict(zip(meta_keys, meta_values))
data = f_object.read()
logging.info("Put data for container %s "
"(filename: %s,\tsize: %.3f KB)" %
(container,
object_name.encode('ascii', 'ignore'),
float(len(data))/1024))
etag = cnx.put_object(container, object_name,
data, headers=copy.copy(meta))
f_object.close()
obj_info = {'object_info':
(object_name, etag, len(data)), 'meta': meta}
containers_d[container]['objects'].append(obj_info)
try:
etag = cnx.put_object(container, object_name,
data, headers=copy.copy(meta))
logging.info("Put data for container %s "
"(filename: %s,\tsize: %.3f KB)" %
(container,
object_name.encode('ascii', 'ignore'),
float(len(data))/1024))
obj_info = {'object_info':
(object_name, etag, len(data)), 'meta': meta}
containers_d[container]['objects'].append(obj_info)
except ClientException:
logging.warning('Unable to put object %s in container %s' % (
object_name.encode('ascii', 'ignore'),
container.encode('ascii', 'ignore')))
def create_containers(cnx, acc, c_amount, index_containers=None):
@ -214,8 +220,12 @@ def create_containers(cnx, acc, c_amount, index_containers=None):
meta = dict(zip(meta_keys, meta_values))
logging.info("Create container %s" %
container_name.encode('ascii', 'ignore'))
cnx.put_container(container_name, headers=copy.copy(meta))
containers_d[container_name] = {'meta': meta, 'objects': []}
try:
cnx.put_container(container_name, headers=copy.copy(meta))
containers_d[container_name] = {'meta': meta, 'objects': []}
except ClientException:
logging.warning("Unable to create container %s" %
container_name.encode('ascii', 'ignore'))
def fill_swift(pool, created_account, c_amount,

61
tests/units/test_filler.py
View File

@ -18,13 +18,17 @@
import eventlet
import swiftclient
from swsync.utils import get_config
from fakes import FakeKSClient
from fakes import FakeKSUser
from fakes import FakeSWConnection
from tests.units import base
from swsync import filler
import base as test_base
from fakes import FakeSWConnection, FakeKSClient, FakeKSUser
from swsync import utils
class TestFiller(test_base.TestCase):
class TestFiller(base.TestCase):
def setUp(self):
super(TestFiller, self).setUp()
self._stubs()
@ -34,7 +38,7 @@ class TestFiller(test_base.TestCase):
FakeSWConnection)
def get_connection(self, *args):
return swiftclient.client.Connection(get_config(
return swiftclient.client.Connection(utils.get_config(
'auth', 'keystone_origin'),
'test', 'password',
tenant_name='test')
@ -55,6 +59,25 @@ class TestFiller(test_base.TestCase):
meta_amount = len(return_dict_ref['test'].values())
self.assertEqual(meta_amount, 3)
def test_create_containers_fail(self):
get_containers_created = []
return_dict_ref = {}
self.attempts = 0
def put_container(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_containers_created.append(args[1])
self.stubs.Set(FakeSWConnection, 'put_container', put_container)
cnx = self.get_connection()
filler.create_containers(cnx, 'test', 3, return_dict_ref)
self.assertEqual(len(get_containers_created), 2)
def test_create_objects(self):
get_object_created = []
return_dict_ref = {'test': {'container_a': {'objects': []},
@ -73,6 +96,27 @@ class TestFiller(test_base.TestCase):
self.assertEqual(len(objects_ca), 2)
self.assertEqual(len(objects_cb), 2)
def test_create_objects_fail(self):
get_object_created = []
return_dict_ref = {'test': {'container_a': {'objects': []}}}
self.attempts = 0
def put_object(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_object_created.append(args[1:])
self.stubs.Set(FakeSWConnection,
'put_object',
put_object)
cnx = self.get_connection()
filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
objects_ca = return_dict_ref['test']['container_a']['objects']
self.assertEqual(len(objects_ca), 1)
def test_fill_swift(self):
self.cont_cnt = 0
self.obj_cnt = 0
@ -91,7 +135,7 @@ class TestFiller(test_base.TestCase):
self.stubs.Set(filler, 'create_objects', create_objects)
self.stubs.Set(filler, 'create_containers', create_containers)
concurrency = int(get_config('filler', 'concurrency'))
concurrency = int(utils.get_config('filler', 'concurrency'))
pool = eventlet.GreenPool(concurrency)
created = {('account1', 'account1_id'): ['test', 'test_id', 'role_id'],
@ -111,7 +155,8 @@ class TestFiller(test_base.TestCase):
def add_user_role(*args, **kargs):
self.role_cnt += 1
co = get_config('auth', 'keystone_origin_admin_credentials').split(':')
co = utils.get_config('auth',
'keystone_origin_admin_credentials').split(':')
tenant_name, username, password = co
client = FakeKSClient()
client.roles.add_user_role = add_user_role
@ -130,7 +175,7 @@ class TestFiller(test_base.TestCase):
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(get_config('filler', 'concurrency'))
concurrency = int(utils.get_config('filler', 'concurrency'))
pile = eventlet.GreenPile(concurrency)
client = FakeKSClient()
filler.create_swift_account(client, pile, 1, 1, self.ret_index)