Merge lp:~psivaa/uci-engine/data-store-refactor-backport into lp:uci-engine/mthood

[Para Siva]

Back porting vila's commit (451) for better swift connection.

Back porting vila's commit (451) for better swift connection.

[Vincent Ladeuil (vila)]

Given you ran tarmac.sh locally and successfully, this should be good to land.

review: Approve

=== removed directory 'ci-utils/ci_utils/data_store'
=== renamed file 'ci-utils/ci_utils/data_store/__init__.py' => 'ci-utils/ci_utils/data_store.py'
--- ci-utils/ci_utils/data_store/__init__.py	2014-03-16 23:17:08 +0000
+++ ci-utils/ci_utils/data_store.py	2014-05-28 10:09:04 +0000
@@ -32,15 +32,12 @@
 
 
 class DataStore(object):
-    def __init__(self, component, auth_config, identifier=None, public=False):
-        self.container = self._get_container(component, identifier)
+
+    def __init__(self, component, auth_config, public=False):
+        self.container_id = component
         self.auth_config = auth_config
-
-        self._setup_auth()
-
-        self._create_container(self.container)
-        self.change_visibility(public=public)
-        self.container_url = self._get_container_url()
+        self.public = public
+        self.client = None
 
     @staticmethod
     def validate_auth_config(auth_config):
@@ -58,15 +55,38 @@
                 "missing fields: {}".format(', '.join(missing_fields))
             )
 
+    def ensure_swift_client(self):
+        if self.client is not None:
+            return self.client
+
+        self.validate_auth_config(self.auth_config)
+        config = self.auth_config
+        self.client = client.Connection(
+            authurl=config.get('auth_url'),
+            user=config.get('auth_user'),
+            key=config.get('auth_password'),
+            os_options={'tenant_name': config.get('auth_tenant_name'),
+                        'region_name': config.get('auth_region')},
+            auth_version='2.0')
+        try:
+            self._create_container(self.container_id)
+        except client.ClientException:
+            raise DataStoreException("Missing or invalid authentication info.")
+        self.change_visibility(public=self.public)
+        self.container_url = "{}/{}".format(self.client.url, self.container_id)
+
     def list_files(self):
-        hdr, data = client.get_container(*self.auth_bits)
-        return [x['name'] for x in data]
+        self.ensure_swift_client()
+        _, objects = self.client.get_container(self.container_id)
+        return [x['name'] for x in objects]
 
     def put_file(self, filename, contents, content_type=None):
+        self.ensure_swift_client()
         name = _get_file_name(filename)
         try:
-            client.put_object(*self.auth_bits, name=name, contents=contents,
-                              content_type=content_type)
+            self.client.put_object(self.container_id, obj=name,
+                                   contents=contents,
+                                   content_type=content_type)
         except ClientException as e:
             raise DataStoreException(
                 "Failed to upload file: {}, Error: {}".format(filename, e)
@@ -74,48 +94,48 @@
 
         return self.file_path(filename)
 
-    def get_file(self, filename, with_headers=False):
+    def get_file(self, filename):
+        self.ensure_swift_client()
         name = _get_file_name(filename)
 
         contents = None
         try:
-            hdr, contents = client.get_object(*self.auth_bits, name=name)
+            hdr, contents = self.client.get_object(self.container_id, obj=name)
         except ClientException as e:
             raise DataStoreException(
                 "Failed to get file: {}, Error: {}".format(filename, e)
             )
-
-        if with_headers:
-            return hdr, contents
-
         return contents
 
     def change_visibility(self, public=False):
-        read_acl = ''
-
+        self.ensure_swift_client()
         if public:
             read_acl = '.r:*'
-
-        args = self.auth_bits
-        args += ({'X-Container-Read': read_acl},)
-        client.post_container(*args)
+        else:
+            read_acl = ''
+        self.client.post_container(self.container_id,
+                                   {'X-Container-Read': read_acl})
+        self.public = public
 
     def delete_file(self, filename):
+        self.ensure_swift_client()
         name = _get_file_name(filename)
         try:
-            client.delete_object(*self.auth_bits, name=name)
+            self.client.delete_object(self.container_id, obj=name)
         except ClientException as e:
             raise DataStoreException(
                 "Failed to delete file: {}, Error: {}".format(filename, e)
             )
 
     def clear(self):
+        self.ensure_swift_client()
         try:
-            files = client.get_container(*self.auth_bits)[1]
+            files = self.client.get_container(self.container_id)[1]
 
-            for file in files:
+            for f in files:
                 try:
-                    client.delete_object(*self.auth_bits, name=file['name'])
+                    self.client.delete_object(self.container_id,
+                                              obj=f['name'])
                 except ClientException as e:
                     raise DataStoreException(
                         "Failed to delete file: {}".format(e)
@@ -126,11 +146,12 @@
             )
 
     def delete(self, recursive=False):
+        self.ensure_swift_client()
         try:
             if recursive:
                 self.clear()
 
-            client.delete_container(*self.auth_bits)
+            self.client.delete_container(self.container_id)
         except ClientException as e:
             raise DataStoreException(
                 "Failed to delete container: {}".format(e)
@@ -141,55 +162,5 @@
 
         return "{}/{}".format(self.container_url, name)
 
-    # Private functions
-    def _has_file(self, filename):
-        name = _get_file_name(filename)
-        result = True
-
-        try:
-            client.head_object(*self.auth_bits, name=name)
-        except ClientException:
-            result = False
-
-        return result
-
-    def _get_container(self, component, identifier=None):
-        container = component
-
-        if identifier is not None and identifier != "":
-            container = "{}.{}".format(container, identifier)
-
-        return container
-
-    def _setup_auth(self):
-        config = self.auth_config
-
-        self.validate_auth_config(config)
-
-        os_options = {}
-
-        os_options['region_name'] = config.get('auth_region')
-
-        try:
-            self.url, self.token = client.get_auth(
-                config.get('auth_url', ''),
-                config.get('auth_user', ''),
-                config.get('auth_password', ''),
-                auth_version=2,
-                tenant_name=config.get('auth_tenant_name', ''),
-                os_options=os_options,
-            )
-        except client.ClientException:
-            self.url = None
-            self.token = None
-
-        if self.url is None or self.token is None:
-            raise DataStoreException("Missing or invalid authentication info.")
-
-        self.auth_bits = (self.url, self.token, self.container)
-
     def _create_container(self, container):
-        client.put_container(self.url, self.token, container)
-
-    def _get_container_url(self):
-        return "{}/{}".format(self.url, self.container)
+        self.client.put_container(self.container_id)
 
=== modified file 'ci-utils/ci_utils/tests/test_data_store.py'
--- ci-utils/ci_utils/tests/test_data_store.py	2014-03-10 22:25:00 +0000
+++ ci-utils/ci_utils/tests/test_data_store.py	2014-05-28 10:09:04 +0000
@@ -15,28 +15,30 @@
 
 import unittest
 
+
 from ci_utils import data_store
 
-auth_config_valid = {
-    'auth_url': 'http://example.com',
-    'auth_user': 'user',
-    'auth_password': 'pass',
-    'auth_tenant_name': 'tenant',
-    'auth_region': 'region',
-}
-
-auth_config_invalid = {}
 
 filename = 'myproject_1.0-1_source.changes'
 
 
-class TestDataStore(unittest.TestCase):
+class TestDataStoreConfig(unittest.TestCase):
+
     def test_valid_auth_config(self):
-        data_store.DataStore.validate_auth_config(auth_config_valid)
+        data_store.DataStore.validate_auth_config({
+            'auth_url': 'http://example.com',
+            'auth_user': 'user',
+            'auth_password': 'pass',
+            'auth_tenant_name': 'tenant',
+            'auth_region': 'region',
+        })
 
     def test_invalid_auth_config(self):
         with self.assertRaises(data_store.DataStoreException):
-            data_store.DataStore.validate_auth_config(auth_config_invalid)
+            data_store.DataStore.validate_auth_config({})
+
+
+class TestDataStoreFileName(unittest.TestCase):
 
     def test_get_file_name(self):
         path = '/home/ubuntu/myproject/{}'.format(filename)
@@ -53,5 +55,6 @@
         resp = data_store._get_file_name(path)
         self.assertEquals(resp, filename)
 
+
 if __name__ == "__main__":
     unittest.main()
 
=== modified file 'test_runner/tstrun/tests/test_data_store.py'
--- test_runner/tstrun/tests/test_data_store.py	2014-02-17 10:13:29 +0000
+++ test_runner/tstrun/tests/test_data_store.py	2014-05-28 10:09:04 +0000
@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import errno
+import os
 import unittest
 
 
@@ -78,12 +79,11 @@
     def setUp(self):
         super(TestDataStore, self).setUp()
         fixtures.set_uniq_cwd(self)
+        # We create a unique container to isolate ourselves
         self.store = data_store.DataStore(
-            'test-runner', tstrun.get_auth_config(),
-            # FIXME: We need a unique identifier or parallel tests (same tests
-            # but different users) can fail -- vila 2014-02-06
-            identifier='test', public=True)
-        self.addCleanup(self.store.clear)
+            os.path.basename(self.uniq_dir),
+            tstrun.get_auth_config(), public=True)
+        self.addCleanup(self.store.delete, recursive=True)
 
     def test_store_created_empty(self):
         self.assertEqual([], self.store.list_files())