Add a basic syncer.

- Just basic sync accounts, there is plenty of more works to do.

bin/sync-accounts.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# -*- encoding: utf-8 -*-
+import swiftclient
+
+from sync.containers import sync_container
+from utils import get_config, get_auth
+
+orig_auth_url = get_config('auth', 'keystone_origin')
+orig_tenant, orig_user, orig_password = \
+    get_config('auth', 'keystone_origin_credentials').split(':')
+
+dest_auth_url = get_config('auth', 'keystone_dest')
+dest_tenant, dest_user, dest_password = \
+    get_config('auth', 'keystone_dest_credentials').split(':')
+
+orig_storage_url, orig_token = \
+    get_auth(orig_auth_url, orig_tenant, orig_user, orig_password)
+orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
+
+dest_storage_url, dest_token = \
+    get_auth(dest_auth_url, dest_tenant, dest_user, dest_password)
+dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
+
+
+orig_account_stats, orig_containers = swiftclient.get_account(
+    None, orig_token, http_conn=orig_storage_cnx, full_listing=True
+)
+
+for container in orig_containers:
+    print "Synching %s.." % (container)
+    sync_container(orig_storage_cnx, orig_storage_url, orig_token,
+                   dest_storage_cnx, dest_storage_url, dest_token,
+                   container['name'])

common/utils.py

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+__author__ = "Chmouel Boudjnah <chmouel@chmouel.com>"
+import os
+import ConfigParser
+import swiftclient.client as swclient
+
+
+CONFIG = {}
+curdir = os.path.abspath(os.path.dirname(__file__))
+INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini"))
+
+
+class ConfigurationError(Exception):
+    pass
+
+
+def parse_ini(inifile=INIFILE):
+    if not os.path.exists(inifile):
+        raise ConfigurationError("Error while parsing inifile")
+
+    config = ConfigParser.RawConfigParser()
+    config.read(inifile)
+    return config
+
+
+def get_config(section, option, default=None):
+    """Get section/option from ConfigParser or print default if specified"""
+    global CONFIG
+
+    if not CONFIG:
+        CONFIG = parse_ini()
+
+    if not CONFIG.has_section(section):
+        raise ConfigurationError("Invalid configuration, missing section: %s" %
+                                 section)
+    if CONFIG.has_option(section, option):
+        return CONFIG.get(section, option)
+    elif not default is None:
+        return default
+    else:
+        raise ConfigurationError("Invalid configuration, missing "
+                                 "section/option: %s/%s" % (section, option))
+
+
+def get_auth(auth_url, tenant, user, password):
+    return swclient.Connection(
+        auth_url,
+        '%s:%s' % (tenant, user),
+        password,
+        auth_version=2).get_auth()
+
+if __name__ == '__main__':
+    get_config("foo", "bar")

etc/config.ini

@@ -0,0 +1,5 @@
+[auth]
+keystone_origin = http://vm:5000/v2.0
+keystone_origin_credentials = demo:demo:ADMIN
+keystone_dest = http://vm2:5000/v2.0
+keystone_dest_credentials = demo:demo:ADMIN

sync/__init__.py

@@ -0,0 +1 @@
+# -*- encoding: utf-8 -*-

sync/containers.py

@@ -0,0 +1,58 @@
+import sys
+
+import swiftclient
+import eventlet
+
+from sync.objects import sync_object
+
+
+def sync_container(orig_storage_cnx, orig_storage_url,
+                   orig_token, dest_storage_cnx,
+                   dest_storage_url, dest_token,
+                   container_name):
+    orig_container_stats, orig_objects = swiftclient.get_container(
+        None, orig_token, container_name, http_conn=orig_storage_cnx,
+    )
+
+    try:
+        swiftclient.head_container(
+            "", dest_token, container_name, http_conn=dest_storage_cnx
+        )
+    except(swiftclient.client.ClientException):
+        container_headers = orig_container_stats.copy()
+        for h in ('x-container-object-count', 'x-trans-id',
+                  'x-container-bytes-used'):
+            del container_headers[h]
+        p = dest_storage_cnx[0]
+        url = "%s://%s%s" % (p.scheme, p.netloc, p.path)
+        swiftclient.put_container(url,
+            dest_token, container_name,
+            headers=container_headers)
+
+
+    dest_container_stats, dest_objects = swiftclient.get_container(
+        None, dest_token, container_name, http_conn=dest_storage_cnx,
+    )
+
+    set1 = set((x['hash'], x['name']) for x in orig_objects)
+    set2 = set((x['hash'], x['name']) for x in dest_objects)
+    diff = set1 - set2
+    if not diff:
+        return
+
+    pool = eventlet.GreenPool()
+    cnt = 0
+    for obj in diff:
+        pool.spawn_n(sync_object,
+                     orig_storage_url,
+                     orig_token,
+                     dest_storage_url,
+                     dest_token, container_name,
+                     obj)
+        if cnt == 20:
+            pool.waitall()
+            cnt = 0
+        else:
+            cnt += 1
+
+    pool.waitall()

sync/objects.py

@@ -0,0 +1,69 @@
+# -*- encoding: utf-8 -*-
+from swiftclient import client as swiftclient
+from swift.common.bufferedhttp import http_connect_raw
+from swift.common.http import is_success
+from swift.container.sync import _Iter2FileLikeObject
+from eventlet import Timeout
+import urllib2
+
+
+def get_object(storage_url, token,
+               container_name,
+               object_name,
+               response_timeout=15,
+               conn_timeout=5,
+               resp_chunk_size=65536):
+    headers = {'x-auth-token': token}
+    x = urllib2.urlparse.urlparse(storage_url)
+
+    path = x.path + '/' + container_name + '/' + object_name
+    with Timeout(conn_timeout):
+        conn = http_connect_raw(
+            x.hostname,
+            x.port,
+            'GET',
+            path,
+            headers=headers,
+            ssl=False)
+
+    with Timeout(response_timeout):
+        resp = conn.getresponse()
+
+    if not is_success(resp.status):
+        resp.read()
+        raise swiftclient.ClientException(
+            'status %s %s' % (resp.status, resp.reason))
+
+    if resp_chunk_size:
+        def _object_body():
+            buf = resp.read(resp_chunk_size)
+            while buf:
+                yield buf
+                buf = resp.read(resp_chunk_size)
+        object_body = _object_body()
+    else:
+        object_body = resp.read()
+
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
+
+    return (resp_headers, object_body)
+
+
+def sync_object(orig_storage_url, orig_token, dest_storage_url,
+                dest_token, container_name, object_name_etag):
+    orig_headers, orig_body = get_object(orig_storage_url,
+                                         orig_token,
+                                         container_name,
+                                         object_name_etag[1],
+                                         )
+    post_headers = orig_headers
+    post_headers['x-auth-token'] = dest_token
+    sync_to = dest_storage_url + "/" + container_name
+    swiftclient.put_object(sync_to, name=object_name_etag[1],
+                           headers=post_headers,
+                           contents=_Iter2FileLikeObject(orig_body))
+
+if __name__ == '__main__':
+    pass