Merge pull request #53 from chmouel/account-metadata-sync
Add account metadata sync.
This commit is contained in:
Fabien Boucher
commit
85490fbbe5
@ -16,6 +16,7 @@
|
||||
# under the License.
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import dateutil.relativedelta
|
||||
@ -51,35 +52,80 @@ class Accounts(object):
|
||||
password=password,
|
||||
tenant_name=tenant_name)
|
||||
|
||||
def account_headers_clean(self, account_headers, to_null=False):
|
||||
ret = {}
|
||||
for key, value in account_headers.iteritems():
|
||||
if key.startswith('x-account-meta'):
|
||||
if to_null:
|
||||
value = ''
|
||||
ret[key] = value
|
||||
return ret
|
||||
|
||||
def sync_account(self, orig_storage_url, orig_token,
|
||||
dest_storage_url, dest_token):
|
||||
"""Sync a single account with url/tok to dest_url/dest_tok."""
|
||||
orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
|
||||
dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
|
||||
account_id = os.path.basename(orig_storage_url.replace("AUTH_", ''))
|
||||
|
||||
try:
|
||||
orig_stats, orig_containers = (
|
||||
orig_account_headers, orig_containers = (
|
||||
swiftclient.get_account(None, orig_token,
|
||||
http_conn=orig_storage_cnx,
|
||||
full_listing=True))
|
||||
|
||||
dest_stats, dest_containers = (
|
||||
dest_account_headers, dest_containers = (
|
||||
swiftclient.get_account(None, dest_token,
|
||||
http_conn=dest_storage_cnx,
|
||||
full_listing=True))
|
||||
except(swiftclient.client.ClientException), e:
|
||||
logging.info("error getting containeaccount: %s, %s" % (
|
||||
orig_storage_url, e.http_reason))
|
||||
logging.info("error getting account: %s, %s" % (
|
||||
account_id, e.http_reason))
|
||||
return
|
||||
if int(dest_stats['x-account-container-count']) > \
|
||||
int(orig_stats['x-account-container-count']):
|
||||
|
||||
if int(dest_account_headers['x-account-container-count']) > \
|
||||
int(orig_account_headers['x-account-container-count']):
|
||||
self.container_cls.delete_container(dest_storage_cnx,
|
||||
dest_token,
|
||||
orig_containers,
|
||||
dest_containers)
|
||||
|
||||
do_headers = False
|
||||
if len(dest_account_headers) != len(orig_account_headers):
|
||||
do_headers = True
|
||||
else:
|
||||
for k, v in orig_account_headers.iteritems():
|
||||
if not k.startswith('x-account-meta'):
|
||||
continue
|
||||
if k not in dest_account_headers:
|
||||
do_headers = True
|
||||
elif dest_account_headers[k] != v:
|
||||
do_headers = True
|
||||
|
||||
if do_headers:
|
||||
orig_metadata_headers = self.account_headers_clean(
|
||||
orig_account_headers)
|
||||
dest_metadata_headers = self.account_headers_clean(
|
||||
dest_account_headers, to_null=True)
|
||||
|
||||
new_headers = dict(dest_metadata_headers.items() +
|
||||
orig_metadata_headers.items())
|
||||
try:
|
||||
swiftclient.post_account(
|
||||
"", dest_token, new_headers,
|
||||
http_conn=dest_storage_cnx,
|
||||
)
|
||||
logging.info("HEADER: sync headers: %s" % (account_id))
|
||||
except(swiftclient.client.ClientException), e:
|
||||
logging.info("ERROR: updating container metadata: %s, %s" % (
|
||||
account_id, e.http_reason))
|
||||
# We don't pass on because since the server was busy
|
||||
# let's pass it on for the next pass
|
||||
return
|
||||
|
||||
for container in orig_containers:
|
||||
logging.info("Syncronizing %s: %s", container['name'], container)
|
||||
logging.info("Syncronizing container %s: %s",
|
||||
container['name'], container)
|
||||
dt1 = datetime.datetime.fromtimestamp(time.time())
|
||||
self.container_cls.sync(orig_storage_cnx,
|
||||
orig_storage_url,
|
||||
|
||||
@ -24,25 +24,183 @@ import tests.units.base
|
||||
import tests.units.fakes as fakes
|
||||
|
||||
|
||||
class TestAccount(tests.units.base.TestCase):
|
||||
class TestAccountBase(tests.units.base.TestCase):
|
||||
def setUp(self):
|
||||
super(TestAccount, self).setUp()
|
||||
super(TestAccountBase, self).setUp()
|
||||
self.accounts_cls = swsync.accounts.Accounts()
|
||||
self._stubs()
|
||||
|
||||
def get_account(self, *args, **kwargs):
|
||||
return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)},
|
||||
[x[0] for x in fakes.CONTAINERS_LIST])
|
||||
|
||||
def _stubs(self):
|
||||
self.stubs.Set(keystoneclient.v2_0, 'client', fakes.FakeKS)
|
||||
self.stubs.Set(swiftclient.client, 'Connection',
|
||||
fakes.FakeSWConnection)
|
||||
self.stubs.Set(swsync.accounts, 'get_config', fakes.fake_get_config)
|
||||
self.stubs.Set(swiftclient, 'get_account', self.get_account)
|
||||
self.stubs.Set(swiftclient, 'http_connection',
|
||||
fakes.FakeSWClient.http_connection)
|
||||
|
||||
|
||||
class TestAccountSyncMetadata(TestAccountBase):
|
||||
def _base_sync_metadata(self, orig_dict={},
|
||||
dest_dict={},
|
||||
get_account_called=[],
|
||||
post_account_called=[],
|
||||
info_called=[],
|
||||
sync_container_called=[],
|
||||
raise_post_account=False):
|
||||
|
||||
def fake_info(msg, *args):
|
||||
info_called.append(msg)
|
||||
self.stubs.Set(logging, 'info', fake_info)
|
||||
|
||||
def get_account(self, *args, **kwargs):
|
||||
if len(get_account_called) == 0:
|
||||
get_account_called.append(args)
|
||||
return orig_dict
|
||||
else:
|
||||
get_account_called.append(args)
|
||||
return dest_dict
|
||||
self.stubs.Set(swiftclient, 'get_account', get_account)
|
||||
|
||||
def post_account(url, token, headers, **kwargs):
|
||||
post_account_called.append(headers)
|
||||
|
||||
if raise_post_account:
|
||||
raise swiftclient.client.ClientException("Error in testing")
|
||||
self.stubs.Set(swiftclient, 'post_account', post_account)
|
||||
|
||||
class Containers(object):
|
||||
def sync(*args, **kwargs):
|
||||
sync_container_called.append(args)
|
||||
self.accounts_cls.container_cls = Containers()
|
||||
self.accounts_cls.sync_account("http://orig", "otoken",
|
||||
"http://dest", "dtoken")
|
||||
|
||||
def test_sync_metadata_delete_dest(self):
|
||||
get_account_called = []
|
||||
sync_container_called = []
|
||||
post_account_called = []
|
||||
info_called = []
|
||||
|
||||
orig_dict = ({'x-account-meta-life': 'beautiful',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
|
||||
dest_dict = ({'x-account-meta-vita': 'bella',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
self._base_sync_metadata(orig_dict,
|
||||
dest_dict,
|
||||
info_called=info_called,
|
||||
sync_container_called=sync_container_called,
|
||||
post_account_called=post_account_called,
|
||||
get_account_called=get_account_called)
|
||||
|
||||
self.assertEquals(len(sync_container_called), 1)
|
||||
self.assertEquals(len(get_account_called), 2)
|
||||
self.assertTrue(info_called)
|
||||
|
||||
self.assertIn('x-account-meta-life',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-life'],
|
||||
'beautiful')
|
||||
self.assertIn('x-account-meta-vita',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-vita'],
|
||||
'')
|
||||
|
||||
def test_sync_metadata_update_dest(self):
|
||||
get_account_called = []
|
||||
sync_container_called = []
|
||||
post_account_called = []
|
||||
info_called = []
|
||||
|
||||
orig_dict = ({'x-account-meta-life': 'beautiful',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
|
||||
dest_dict = ({'x-account-meta-life': 'bella',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
self._base_sync_metadata(orig_dict,
|
||||
dest_dict,
|
||||
info_called=info_called,
|
||||
sync_container_called=sync_container_called,
|
||||
post_account_called=post_account_called,
|
||||
get_account_called=get_account_called)
|
||||
|
||||
self.assertEquals(len(sync_container_called), 1)
|
||||
self.assertEquals(len(get_account_called), 2)
|
||||
self.assertTrue(info_called)
|
||||
|
||||
self.assertIn('x-account-meta-life',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-life'],
|
||||
'beautiful')
|
||||
|
||||
self.assertIn('x-account-meta-life',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-life'],
|
||||
'beautiful')
|
||||
|
||||
def test_sync_metadata_add_to_dest(self):
|
||||
info_called = []
|
||||
get_account_called = []
|
||||
sync_container_called = []
|
||||
post_account_called = []
|
||||
|
||||
orig_dict = ({'x-account-meta-life': 'beautiful',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
|
||||
dest_dict = ({'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
self._base_sync_metadata(orig_dict,
|
||||
dest_dict,
|
||||
info_called=info_called,
|
||||
sync_container_called=sync_container_called,
|
||||
post_account_called=post_account_called,
|
||||
get_account_called=get_account_called)
|
||||
|
||||
self.assertEquals(len(sync_container_called), 1)
|
||||
self.assertEquals(len(get_account_called), 2)
|
||||
self.assertTrue(info_called)
|
||||
|
||||
self.assertIn('x-account-meta-life',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-life'],
|
||||
'beautiful')
|
||||
|
||||
self.assertIn('x-account-meta-life',
|
||||
post_account_called[0])
|
||||
self.assertEqual(post_account_called[0]['x-account-meta-life'],
|
||||
'beautiful')
|
||||
|
||||
def test_sync_metadata_raise(self):
|
||||
info_called = []
|
||||
get_account_called = []
|
||||
sync_container_called = []
|
||||
post_account_called = []
|
||||
|
||||
orig_dict = ({'x-account-meta-life': 'beautiful',
|
||||
'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
|
||||
dest_dict = ({'x-account-container-count': 1},
|
||||
[{'name': 'cont1'}])
|
||||
self._base_sync_metadata(orig_dict,
|
||||
dest_dict,
|
||||
info_called=info_called,
|
||||
sync_container_called=sync_container_called,
|
||||
post_account_called=post_account_called,
|
||||
get_account_called=get_account_called,
|
||||
raise_post_account=True)
|
||||
self.assertTrue(info_called)
|
||||
self.assertIn('ERROR: updating container metadata: orig, ',
|
||||
info_called)
|
||||
self.assertFalse(sync_container_called)
|
||||
|
||||
|
||||
class TestAccountSync(TestAccountBase):
|
||||
def test_get_swift_auth(self):
|
||||
tenant_name = 'foo1'
|
||||
ret = self.accounts_cls.get_swift_auth(
|
||||
@ -82,6 +240,11 @@ class TestAccount(tests.units.base.TestCase):
|
||||
def test_sync_account(self):
|
||||
ret = []
|
||||
|
||||
def get_account(*args, **kwargs):
|
||||
return ({'x-account-container-count': len(fakes.CONTAINERS_LIST)},
|
||||
[x[0] for x in fakes.CONTAINERS_LIST])
|
||||
self.stubs.Set(swiftclient, 'get_account', get_account)
|
||||
|
||||
class Containers(object):
|
||||
def sync(*args, **kwargs):
|
||||
ret.append(args)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user