Merge lp:~psivaa/uci-engine/data-store-refactor-backport into lp:uci-engine/mthood

Proposed by Para Siva
Status: Merged
Approved by: Para Siva
Approved revision: 416
Merged at revision: 416
Proposed branch: lp:~psivaa/uci-engine/data-store-refactor-backport
Merge into: lp:uci-engine/mthood
Diff against target: 287 lines (+71/-97)
3 files modified
ci-utils/ci_utils/data_store.py (+51/-80)
ci-utils/ci_utils/tests/test_data_store.py (+15/-12)
test_runner/tstrun/tests/test_data_store.py (+5/-5)
To merge this branch: bzr merge lp:~psivaa/uci-engine/data-store-refactor-backport
Reviewer Review Type Date Requested Status
Vincent Ladeuil (community) Approve
Review via email: mp+221200@code.launchpad.net

Commit message

Back porting vila's commit (451) for better swift connection.

Description of the change

Back porting vila's commit (451) for better swift connection.

To post a comment you must log in.
Revision history for this message
Vincent Ladeuil (vila) wrote :

Given you ran tarmac.sh locally and successfully, this should be good to land.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== removed directory 'ci-utils/ci_utils/data_store'
2=== renamed file 'ci-utils/ci_utils/data_store/__init__.py' => 'ci-utils/ci_utils/data_store.py'
3--- ci-utils/ci_utils/data_store/__init__.py 2014-03-16 23:17:08 +0000
4+++ ci-utils/ci_utils/data_store.py 2014-05-28 10:09:04 +0000
5@@ -32,15 +32,12 @@
6
7
8 class DataStore(object):
9- def __init__(self, component, auth_config, identifier=None, public=False):
10- self.container = self._get_container(component, identifier)
11+
12+ def __init__(self, component, auth_config, public=False):
13+ self.container_id = component
14 self.auth_config = auth_config
15-
16- self._setup_auth()
17-
18- self._create_container(self.container)
19- self.change_visibility(public=public)
20- self.container_url = self._get_container_url()
21+ self.public = public
22+ self.client = None
23
24 @staticmethod
25 def validate_auth_config(auth_config):
26@@ -58,15 +55,38 @@
27 "missing fields: {}".format(', '.join(missing_fields))
28 )
29
30+ def ensure_swift_client(self):
31+ if self.client is not None:
32+ return self.client
33+
34+ self.validate_auth_config(self.auth_config)
35+ config = self.auth_config
36+ self.client = client.Connection(
37+ authurl=config.get('auth_url'),
38+ user=config.get('auth_user'),
39+ key=config.get('auth_password'),
40+ os_options={'tenant_name': config.get('auth_tenant_name'),
41+ 'region_name': config.get('auth_region')},
42+ auth_version='2.0')
43+ try:
44+ self._create_container(self.container_id)
45+ except client.ClientException:
46+ raise DataStoreException("Missing or invalid authentication info.")
47+ self.change_visibility(public=self.public)
48+ self.container_url = "{}/{}".format(self.client.url, self.container_id)
49+
50 def list_files(self):
51- hdr, data = client.get_container(*self.auth_bits)
52- return [x['name'] for x in data]
53+ self.ensure_swift_client()
54+ _, objects = self.client.get_container(self.container_id)
55+ return [x['name'] for x in objects]
56
57 def put_file(self, filename, contents, content_type=None):
58+ self.ensure_swift_client()
59 name = _get_file_name(filename)
60 try:
61- client.put_object(*self.auth_bits, name=name, contents=contents,
62- content_type=content_type)
63+ self.client.put_object(self.container_id, obj=name,
64+ contents=contents,
65+ content_type=content_type)
66 except ClientException as e:
67 raise DataStoreException(
68 "Failed to upload file: {}, Error: {}".format(filename, e)
69@@ -74,48 +94,48 @@
70
71 return self.file_path(filename)
72
73- def get_file(self, filename, with_headers=False):
74+ def get_file(self, filename):
75+ self.ensure_swift_client()
76 name = _get_file_name(filename)
77
78 contents = None
79 try:
80- hdr, contents = client.get_object(*self.auth_bits, name=name)
81+ hdr, contents = self.client.get_object(self.container_id, obj=name)
82 except ClientException as e:
83 raise DataStoreException(
84 "Failed to get file: {}, Error: {}".format(filename, e)
85 )
86-
87- if with_headers:
88- return hdr, contents
89-
90 return contents
91
92 def change_visibility(self, public=False):
93- read_acl = ''
94-
95+ self.ensure_swift_client()
96 if public:
97 read_acl = '.r:*'
98-
99- args = self.auth_bits
100- args += ({'X-Container-Read': read_acl},)
101- client.post_container(*args)
102+ else:
103+ read_acl = ''
104+ self.client.post_container(self.container_id,
105+ {'X-Container-Read': read_acl})
106+ self.public = public
107
108 def delete_file(self, filename):
109+ self.ensure_swift_client()
110 name = _get_file_name(filename)
111 try:
112- client.delete_object(*self.auth_bits, name=name)
113+ self.client.delete_object(self.container_id, obj=name)
114 except ClientException as e:
115 raise DataStoreException(
116 "Failed to delete file: {}, Error: {}".format(filename, e)
117 )
118
119 def clear(self):
120+ self.ensure_swift_client()
121 try:
122- files = client.get_container(*self.auth_bits)[1]
123+ files = self.client.get_container(self.container_id)[1]
124
125- for file in files:
126+ for f in files:
127 try:
128- client.delete_object(*self.auth_bits, name=file['name'])
129+ self.client.delete_object(self.container_id,
130+ obj=f['name'])
131 except ClientException as e:
132 raise DataStoreException(
133 "Failed to delete file: {}".format(e)
134@@ -126,11 +146,12 @@
135 )
136
137 def delete(self, recursive=False):
138+ self.ensure_swift_client()
139 try:
140 if recursive:
141 self.clear()
142
143- client.delete_container(*self.auth_bits)
144+ self.client.delete_container(self.container_id)
145 except ClientException as e:
146 raise DataStoreException(
147 "Failed to delete container: {}".format(e)
148@@ -141,55 +162,5 @@
149
150 return "{}/{}".format(self.container_url, name)
151
152- # Private functions
153- def _has_file(self, filename):
154- name = _get_file_name(filename)
155- result = True
156-
157- try:
158- client.head_object(*self.auth_bits, name=name)
159- except ClientException:
160- result = False
161-
162- return result
163-
164- def _get_container(self, component, identifier=None):
165- container = component
166-
167- if identifier is not None and identifier != "":
168- container = "{}.{}".format(container, identifier)
169-
170- return container
171-
172- def _setup_auth(self):
173- config = self.auth_config
174-
175- self.validate_auth_config(config)
176-
177- os_options = {}
178-
179- os_options['region_name'] = config.get('auth_region')
180-
181- try:
182- self.url, self.token = client.get_auth(
183- config.get('auth_url', ''),
184- config.get('auth_user', ''),
185- config.get('auth_password', ''),
186- auth_version=2,
187- tenant_name=config.get('auth_tenant_name', ''),
188- os_options=os_options,
189- )
190- except client.ClientException:
191- self.url = None
192- self.token = None
193-
194- if self.url is None or self.token is None:
195- raise DataStoreException("Missing or invalid authentication info.")
196-
197- self.auth_bits = (self.url, self.token, self.container)
198-
199 def _create_container(self, container):
200- client.put_container(self.url, self.token, container)
201-
202- def _get_container_url(self):
203- return "{}/{}".format(self.url, self.container)
204+ self.client.put_container(self.container_id)
205
206=== modified file 'ci-utils/ci_utils/tests/test_data_store.py'
207--- ci-utils/ci_utils/tests/test_data_store.py 2014-03-10 22:25:00 +0000
208+++ ci-utils/ci_utils/tests/test_data_store.py 2014-05-28 10:09:04 +0000
209@@ -15,28 +15,30 @@
210
211 import unittest
212
213+
214 from ci_utils import data_store
215
216-auth_config_valid = {
217- 'auth_url': 'http://example.com',
218- 'auth_user': 'user',
219- 'auth_password': 'pass',
220- 'auth_tenant_name': 'tenant',
221- 'auth_region': 'region',
222-}
223-
224-auth_config_invalid = {}
225
226 filename = 'myproject_1.0-1_source.changes'
227
228
229-class TestDataStore(unittest.TestCase):
230+class TestDataStoreConfig(unittest.TestCase):
231+
232 def test_valid_auth_config(self):
233- data_store.DataStore.validate_auth_config(auth_config_valid)
234+ data_store.DataStore.validate_auth_config({
235+ 'auth_url': 'http://example.com',
236+ 'auth_user': 'user',
237+ 'auth_password': 'pass',
238+ 'auth_tenant_name': 'tenant',
239+ 'auth_region': 'region',
240+ })
241
242 def test_invalid_auth_config(self):
243 with self.assertRaises(data_store.DataStoreException):
244- data_store.DataStore.validate_auth_config(auth_config_invalid)
245+ data_store.DataStore.validate_auth_config({})
246+
247+
248+class TestDataStoreFileName(unittest.TestCase):
249
250 def test_get_file_name(self):
251 path = '/home/ubuntu/myproject/{}'.format(filename)
252@@ -53,5 +55,6 @@
253 resp = data_store._get_file_name(path)
254 self.assertEquals(resp, filename)
255
256+
257 if __name__ == "__main__":
258 unittest.main()
259
260=== modified file 'test_runner/tstrun/tests/test_data_store.py'
261--- test_runner/tstrun/tests/test_data_store.py 2014-02-17 10:13:29 +0000
262+++ test_runner/tstrun/tests/test_data_store.py 2014-05-28 10:09:04 +0000
263@@ -14,6 +14,7 @@
264 # along with this program. If not, see <http://www.gnu.org/licenses/>.
265
266 import errno
267+import os
268 import unittest
269
270
271@@ -78,12 +79,11 @@
272 def setUp(self):
273 super(TestDataStore, self).setUp()
274 fixtures.set_uniq_cwd(self)
275+ # We create a unique container to isolate ourselves
276 self.store = data_store.DataStore(
277- 'test-runner', tstrun.get_auth_config(),
278- # FIXME: We need a unique identifier or parallel tests (same tests
279- # but different users) can fail -- vila 2014-02-06
280- identifier='test', public=True)
281- self.addCleanup(self.store.clear)
282+ os.path.basename(self.uniq_dir),
283+ tstrun.get_auth_config(), public=True)
284+ self.addCleanup(self.store.delete, recursive=True)
285
286 def test_store_created_empty(self):
287 self.assertEqual([], self.store.list_files())

Subscribers

People subscribed via source and target branches