Merge lp:~gholt/swift/postcopy into lp:~hudson-openstack/swift/trunk
- postcopy
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | David Goetz |
Approved revision: | 311 |
Merged at revision: | 315 |
Proposed branch: | lp:~gholt/swift/postcopy |
Merge into: | lp:~hudson-openstack/swift/trunk |
Diff against target: |
1104 lines (+479/-138) 11 files modified
doc/source/deployment_guide.rst (+10/-0) etc/proxy-server.conf-sample (+5/-0) swift/common/direct_client.py (+51/-0) swift/obj/server.py (+2/-0) swift/proxy/server.py (+86/-43) test/functional/swift.py (+1/-1) test/functional/tests.py (+2/-2) test/probe/test_account_failures.py (+5/-3) test/probe/test_container_failures.py (+15/-8) test/probe/test_object_handoff.py (+43/-41) test/unit/proxy/test_server.py (+259/-40) |
To merge this branch: | bzr merge lp:~gholt/swift/postcopy |
Related bugs: | |
Related blueprints: |
Offer X-Newest option on object GETs and HEADs
(Undefined)
Have COPY use X-Newest
(Undefined)
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
David Goetz (community) | Approve | ||
John Dickinson | Approve | ||
Review via email: mp+63812@code.launchpad.net |
Commit message
You can specify X-Newest: true on GETs and HEADs to indicate you want Swift to query all backend copies and return the newest version retrieved.
Object COPY requests now always copy the newest object they can find.
Object POSTs are implemented as COPYs now by default (you can revert to previous implementation with conf object_post_as_copy = false)
Account and container GETs and HEADs now shuffle the nodes they use to balance load.
Description of the change
You can specify X-Newest: true on GETs and HEADs to indicate you want Swift to query all backend copies and return the newest version retrieved.
Object COPY requests now always copy the newest object they can find.
Object POSTs are implemented as COPYs now by default (you can revert to previous implementation with conf object_post_as_copy = false)
Account and container GETs and HEADs now shuffle the nodes they use to balance load.
That last part I added because at first I thought it'd be quick and easy; but it ended up having to improve quite a few tests. I can separate it out to another merge if desired; but since it's already here, I left it for the moment.
Preview Diff
1 | === modified file 'doc/source/deployment_guide.rst' |
2 | --- doc/source/deployment_guide.rst 2011-06-05 23:22:35 +0000 |
3 | +++ doc/source/deployment_guide.rst 2011-06-14 22:21:02 +0000 |
4 | @@ -547,6 +547,16 @@ |
5 | node error limited |
6 | allow_account_management false Whether account PUTs and DELETEs |
7 | are even callable |
8 | +object_post_as_copy true Set object_post_as_copy = false |
9 | + to turn on fast posts where only |
10 | + the metadata changes are stored |
11 | + anew and the original data file |
12 | + is kept in place. This makes for |
13 | + quicker posts; but since the |
14 | + container metadata isn't updated |
15 | + in this mode, features like |
16 | + container sync won't be able to |
17 | + sync posts. |
18 | account_autocreate false If set to 'true' authorized |
19 | accounts that do not yet exist |
20 | within the Swift cluster will |
21 | |
22 | === modified file 'etc/proxy-server.conf-sample' |
23 | --- etc/proxy-server.conf-sample 2011-06-05 23:22:35 +0000 |
24 | +++ etc/proxy-server.conf-sample 2011-06-14 22:21:02 +0000 |
25 | @@ -40,6 +40,11 @@ |
26 | # If set to 'true' any authorized user may create and delete accounts; if |
27 | # 'false' no one, even authorized, can. |
28 | # allow_account_management = false |
29 | +# Set object_post_as_copy = false to turn on fast posts where only the metadata |
30 | +# changes are stored anew and the original data file is kept in place. This |
31 | +# makes for quicker posts; but since the container metadata isn't updated in |
32 | +# this mode, features like container sync won't be able to sync posts. |
33 | +# object_post_as_copy = true |
34 | # If set to 'true' authorized accounts that do not yet exist within the Swift |
35 | # cluster will be automatically created. |
36 | # account_autocreate = false |
37 | |
38 | === modified file 'swift/common/direct_client.py' |
39 | --- swift/common/direct_client.py 2011-03-29 02:09:24 +0000 |
40 | +++ swift/common/direct_client.py 2011-06-14 22:21:02 +0000 |
41 | @@ -36,6 +36,57 @@ |
42 | return _quote(value, safe) |
43 | |
44 | |
45 | +def direct_get_account(node, part, account, marker=None, limit=None, |
46 | + prefix=None, delimiter=None, conn_timeout=5, |
47 | + response_timeout=15): |
48 | + """ |
49 | + Get listings directly from the account server. |
50 | + |
51 | + :param node: node dictionary from the ring |
52 | + :param part: partition the account is on |
53 | + :param account: account name |
54 | + :param marker: marker query |
55 | + :param limit: query limit |
56 | + :param prefix: prefix query |
57 | + :param delimeter: delimeter for the query |
58 | + :param conn_timeout: timeout in seconds for establishing the connection |
59 | + :param response_timeout: timeout in seconds for getting the response |
60 | + :returns: a tuple of (response headers, a list of containers) The response |
61 | + headers will be a dict and all header names will be lowercase. |
62 | + """ |
63 | + path = '/' + account |
64 | + qs = 'format=json' |
65 | + if marker: |
66 | + qs += '&marker=%s' % quote(marker) |
67 | + if limit: |
68 | + qs += '&limit=%d' % limit |
69 | + if prefix: |
70 | + qs += '&prefix=%s' % quote(prefix) |
71 | + if delimiter: |
72 | + qs += '&delimiter=%s' % quote(delimiter) |
73 | + with Timeout(conn_timeout): |
74 | + conn = http_connect(node['ip'], node['port'], node['device'], part, |
75 | + 'GET', path, query_string='format=json') |
76 | + with Timeout(response_timeout): |
77 | + resp = conn.getresponse() |
78 | + if resp.status < 200 or resp.status >= 300: |
79 | + resp.read() |
80 | + raise ClientException( |
81 | + 'Account server %s:%s direct GET %s gave status %s' % (node['ip'], |
82 | + node['port'], repr('/%s/%s%s' % (node['device'], part, path)), |
83 | + resp.status), |
84 | + http_host=node['ip'], http_port=node['port'], |
85 | + http_device=node['device'], http_status=resp.status, |
86 | + http_reason=resp.reason) |
87 | + resp_headers = {} |
88 | + for header, value in resp.getheaders(): |
89 | + resp_headers[header.lower()] = value |
90 | + if resp.status == 204: |
91 | + resp.read() |
92 | + return resp_headers, [] |
93 | + return resp_headers, json_loads(resp.read()) |
94 | + |
95 | + |
96 | def direct_head_container(node, part, account, container, conn_timeout=5, |
97 | response_timeout=15): |
98 | """ |
99 | |
100 | === modified file 'swift/obj/server.py' |
101 | --- swift/obj/server.py 2011-05-09 20:21:34 +0000 |
102 | +++ swift/obj/server.py 2011-06-14 22:21:02 +0000 |
103 | @@ -623,6 +623,7 @@ |
104 | file.keep_cache = True |
105 | if 'Content-Encoding' in file.metadata: |
106 | response.content_encoding = file.metadata['Content-Encoding'] |
107 | + response.headers['X-Timestamp'] = file.metadata['X-Timestamp'] |
108 | return request.get_response(response) |
109 | |
110 | def HEAD(self, request): |
111 | @@ -657,6 +658,7 @@ |
112 | response.content_length = file_size |
113 | if 'Content-Encoding' in file.metadata: |
114 | response.content_encoding = file.metadata['Content-Encoding'] |
115 | + response.headers['X-Timestamp'] = file.metadata['X-Timestamp'] |
116 | return response |
117 | |
118 | def DELETE(self, request): |
119 | |
120 | === modified file 'swift/proxy/server.py' |
121 | --- swift/proxy/server.py 2011-06-11 04:57:04 +0000 |
122 | +++ swift/proxy/server.py 2011-06-14 22:21:02 +0000 |
123 | @@ -162,6 +162,7 @@ |
124 | if self.segment > 10: |
125 | sleep(max(self.next_get_time - time.time(), 0)) |
126 | self.next_get_time = time.time() + 1 |
127 | + shuffle(nodes) |
128 | resp = self.controller.GETorHEAD_base(req, _('Object'), partition, |
129 | self.controller.iter_nodes(partition, nodes, |
130 | self.controller.app.object_ring), path, |
131 | @@ -605,6 +606,8 @@ |
132 | statuses = [] |
133 | reasons = [] |
134 | bodies = [] |
135 | + source = None |
136 | + newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES |
137 | for node in nodes: |
138 | if len(statuses) >= attempts: |
139 | break |
140 | @@ -617,23 +620,48 @@ |
141 | headers=req.headers, |
142 | query_string=req.query_string) |
143 | with Timeout(self.app.node_timeout): |
144 | - source = conn.getresponse() |
145 | + possible_source = conn.getresponse() |
146 | except (Exception, TimeoutError): |
147 | self.exception_occurred(node, server_type, |
148 | _('Trying to %(method)s %(path)s') % |
149 | {'method': req.method, 'path': req.path}) |
150 | continue |
151 | - if source.status == 507: |
152 | + if possible_source.status == 507: |
153 | self.error_limit(node) |
154 | continue |
155 | - if 200 <= source.status <= 399: |
156 | + if 200 <= possible_source.status <= 399: |
157 | # 404 if we know we don't have a synced copy |
158 | - if not float(source.getheader('X-PUT-Timestamp', '1')): |
159 | + if not float(possible_source.getheader('X-PUT-Timestamp', 1)): |
160 | statuses.append(404) |
161 | reasons.append('') |
162 | bodies.append('') |
163 | - source.read() |
164 | - continue |
165 | + possible_source.read() |
166 | + continue |
167 | + if (req.method == 'GET' and |
168 | + possible_source.status in (200, 206)) or \ |
169 | + 200 <= possible_source.status <= 399: |
170 | + if newest: |
171 | + ts = 0 |
172 | + if source: |
173 | + ts = float(source.getheader('x-put-timestamp') or |
174 | + source.getheader('x-timestamp') or 0) |
175 | + pts = float(possible_source.getheader('x-put-timestamp') or |
176 | + possible_source.getheader('x-timestamp') or 0) |
177 | + if pts > ts: |
178 | + source = possible_source |
179 | + continue |
180 | + else: |
181 | + source = possible_source |
182 | + break |
183 | + statuses.append(possible_source.status) |
184 | + reasons.append(possible_source.reason) |
185 | + bodies.append(possible_source.read()) |
186 | + if possible_source.status >= 500: |
187 | + self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ |
188 | + 'From %(type)s Server') % |
189 | + {'status': possible_source.status, |
190 | + 'body': bodies[-1][:1024], 'type': server_type}) |
191 | + if source: |
192 | if req.method == 'GET' and source.status in (200, 206): |
193 | res = Response(request=req, conditional_response=True) |
194 | res.bytes_transferred = 0 |
195 | @@ -673,13 +701,6 @@ |
196 | res.charset = None |
197 | res.content_type = source.getheader('Content-Type') |
198 | return res |
199 | - statuses.append(source.status) |
200 | - reasons.append(source.reason) |
201 | - bodies.append(source.read()) |
202 | - if source.status >= 500: |
203 | - self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ |
204 | - 'From %(type)s Server') % {'status': source.status, |
205 | - 'body': bodies[-1][:1024], 'type': server_type}) |
206 | return self.best_response(req, statuses, reasons, bodies, |
207 | '%s %s' % (server_type, req.method)) |
208 | |
209 | @@ -734,6 +755,7 @@ |
210 | lreq = Request.blank('/%s/%s?prefix=%s&format=json&marker=%s' % |
211 | (quote(self.account_name), quote(lcontainer), |
212 | quote(lprefix), quote(marker))) |
213 | + shuffle(lnodes) |
214 | lresp = self.GETorHEAD_base(lreq, _('Container'), lpartition, |
215 | lnodes, lreq.path_info, |
216 | self.app.container_ring.replica_count) |
217 | @@ -861,30 +883,40 @@ |
218 | @delay_denial |
219 | def POST(self, req): |
220 | """HTTP POST request handler.""" |
221 | - error_response = check_metadata(req, 'object') |
222 | - if error_response: |
223 | - return error_response |
224 | - container_partition, containers, _junk, req.acl = \ |
225 | - self.container_info(self.account_name, self.container_name, |
226 | - account_autocreate=self.app.account_autocreate) |
227 | - if 'swift.authorize' in req.environ: |
228 | - aresp = req.environ['swift.authorize'](req) |
229 | - if aresp: |
230 | - return aresp |
231 | - if not containers: |
232 | - return HTTPNotFound(request=req) |
233 | - partition, nodes = self.app.object_ring.get_nodes( |
234 | - self.account_name, self.container_name, self.object_name) |
235 | - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) |
236 | - headers = [] |
237 | - for container in containers: |
238 | - nheaders = dict(req.headers.iteritems()) |
239 | - nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container |
240 | - nheaders['X-Container-Partition'] = container_partition |
241 | - nheaders['X-Container-Device'] = container['device'] |
242 | - headers.append(nheaders) |
243 | - return self.make_requests(req, self.app.object_ring, |
244 | - partition, 'POST', req.path_info, headers) |
245 | + if self.app.object_post_as_copy: |
246 | + req.method = 'PUT' |
247 | + req.path_info = '/%s/%s/%s' % (self.account_name, |
248 | + self.container_name, self.object_name) |
249 | + req.headers['Content-Length'] = 0 |
250 | + req.headers['X-Copy-From'] = '/%s/%s' % (self.container_name, |
251 | + self.object_name) |
252 | + req.headers['X-Fresh-Metadata'] = 'true' |
253 | + return self.PUT(req) |
254 | + else: |
255 | + error_response = check_metadata(req, 'object') |
256 | + if error_response: |
257 | + return error_response |
258 | + container_partition, containers, _junk, req.acl = \ |
259 | + self.container_info(self.account_name, self.container_name, |
260 | + account_autocreate=self.app.account_autocreate) |
261 | + if 'swift.authorize' in req.environ: |
262 | + aresp = req.environ['swift.authorize'](req) |
263 | + if aresp: |
264 | + return aresp |
265 | + if not containers: |
266 | + return HTTPNotFound(request=req) |
267 | + partition, nodes = self.app.object_ring.get_nodes( |
268 | + self.account_name, self.container_name, self.object_name) |
269 | + req.headers['X-Timestamp'] = normalize_timestamp(time.time()) |
270 | + headers = [] |
271 | + for container in containers: |
272 | + nheaders = dict(req.headers.iteritems()) |
273 | + nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container |
274 | + nheaders['X-Container-Partition'] = container_partition |
275 | + nheaders['X-Container-Device'] = container['device'] |
276 | + headers.append(nheaders) |
277 | + return self.make_requests(req, self.app.object_ring, |
278 | + partition, 'POST', req.path_info, headers) |
279 | |
280 | def _send_file(self, conn, path): |
281 | """Method for a file PUT coro""" |
282 | @@ -947,6 +979,7 @@ |
283 | reader = req.environ['wsgi.input'].read |
284 | data_source = iter(lambda: reader(self.app.client_chunk_size), '') |
285 | source_header = req.headers.get('X-Copy-From') |
286 | + source_resp = None |
287 | if source_header: |
288 | source_header = unquote(source_header) |
289 | acct = req.path_info.split('/', 2)[1] |
290 | @@ -962,6 +995,7 @@ |
291 | '<container name>/<object name>') |
292 | source_req = req.copy_get() |
293 | source_req.path_info = source_header |
294 | + source_req.headers['X-Newest'] = 'true' |
295 | orig_obj_name = self.object_name |
296 | orig_container_name = self.container_name |
297 | self.object_name = src_obj_name |
298 | @@ -987,12 +1021,14 @@ |
299 | if not content_type_manually_set: |
300 | new_req.headers['Content-Type'] = \ |
301 | source_resp.headers['Content-Type'] |
302 | - for k, v in source_resp.headers.items(): |
303 | - if k.lower().startswith('x-object-meta-'): |
304 | - new_req.headers[k] = v |
305 | - for k, v in req.headers.items(): |
306 | - if k.lower().startswith('x-object-meta-'): |
307 | - new_req.headers[k] = v |
308 | + if new_req.headers.get('x-fresh-metadata', 'false').lower() \ |
309 | + not in TRUE_VALUES: |
310 | + for k, v in source_resp.headers.items(): |
311 | + if k.lower().startswith('x-object-meta-'): |
312 | + new_req.headers[k] = v |
313 | + for k, v in req.headers.items(): |
314 | + if k.lower().startswith('x-object-meta-'): |
315 | + new_req.headers[k] = v |
316 | req = new_req |
317 | node_iter = self.iter_nodes(partition, nodes, self.app.object_ring) |
318 | pile = GreenPile(len(nodes)) |
319 | @@ -1094,6 +1130,9 @@ |
320 | if source_header: |
321 | resp.headers['X-Copied-From'] = quote( |
322 | source_header.split('/', 2)[2]) |
323 | + if 'last-modified' in source_resp.headers: |
324 | + resp.headers['X-Copied-From-Last-Modified'] = \ |
325 | + source_resp.headers['last-modified'] |
326 | for k, v in req.headers.items(): |
327 | if k.lower().startswith('x-object-meta-'): |
328 | resp.headers[k] = v |
329 | @@ -1187,6 +1226,7 @@ |
330 | return HTTPNotFound(request=req) |
331 | part, nodes = self.app.container_ring.get_nodes( |
332 | self.account_name, self.container_name) |
333 | + shuffle(nodes) |
334 | resp = self.GETorHEAD_base(req, _('Container'), part, nodes, |
335 | req.path_info, self.app.container_ring.replica_count) |
336 | |
337 | @@ -1319,6 +1359,7 @@ |
338 | def GETorHEAD(self, req): |
339 | """Handler for HTTP GET/HEAD requests.""" |
340 | partition, nodes = self.app.account_ring.get_nodes(self.account_name) |
341 | + shuffle(nodes) |
342 | resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, |
343 | req.path_info.rstrip('/'), self.app.account_ring.replica_count) |
344 | if resp.status_int == 404 and self.app.account_autocreate: |
345 | @@ -1449,6 +1490,8 @@ |
346 | int(conf.get('recheck_account_existence', 60)) |
347 | self.allow_account_management = \ |
348 | conf.get('allow_account_management', 'no').lower() in TRUE_VALUES |
349 | + self.object_post_as_copy = \ |
350 | + conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES |
351 | self.resellers_conf = ConfigParser() |
352 | self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) |
353 | self.object_ring = object_ring or \ |
354 | |
355 | === modified file 'test/functional/swift.py' |
356 | --- test/functional/swift.py 2011-02-23 04:25:38 +0000 |
357 | +++ test/functional/swift.py 2011-06-14 22:21:02 +0000 |
358 | @@ -668,7 +668,7 @@ |
359 | |
360 | self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) |
361 | |
362 | - if self.conn.response.status != 202: |
363 | + if self.conn.response.status not in (201, 202): |
364 | raise ResponseError(self.conn.response) |
365 | |
366 | return True |
367 | |
368 | === modified file 'test/functional/tests.py' |
369 | --- test/functional/tests.py 2011-06-10 18:49:41 +0000 |
370 | +++ test/functional/tests.py 2011-06-14 22:21:02 +0000 |
371 | @@ -1032,7 +1032,7 @@ |
372 | self.assert_(file.write()) |
373 | self.assert_status(201) |
374 | self.assert_(file.sync_metadata()) |
375 | - self.assert_status(202) |
376 | + self.assert_status((201, 202)) |
377 | else: |
378 | self.assertRaises(ResponseError, file.write) |
379 | self.assert_status(400) |
380 | @@ -1245,7 +1245,7 @@ |
381 | |
382 | file.metadata = metadata |
383 | self.assert_(file.sync_metadata()) |
384 | - self.assert_status(202) |
385 | + self.assert_status((201, 202)) |
386 | |
387 | file = self.env.container.file(file.name) |
388 | self.assert_(file.initialize()) |
389 | |
390 | === modified file 'test/probe/test_account_failures.py' |
391 | --- test/probe/test_account_failures.py 2011-01-04 23:34:43 +0000 |
392 | +++ test/probe/test_account_failures.py 2011-06-14 22:21:02 +0000 |
393 | @@ -20,7 +20,7 @@ |
394 | from subprocess import Popen |
395 | from time import sleep |
396 | |
397 | -from swift.common import client |
398 | +from swift.common import client, direct_client |
399 | from test.probe.common import get_to_final_state, kill_pids, reset_environment |
400 | |
401 | |
402 | @@ -146,7 +146,8 @@ |
403 | sleep(2) |
404 | # This is the earlier counts and bytes because the first node doesn't |
405 | # have the newest udpates yet. |
406 | - headers, containers = client.get_account(self.url, self.token) |
407 | + headers, containers = \ |
408 | + direct_client.direct_get_account(anodes[0], apart, self.account) |
409 | self.assertEquals(headers['x-account-container-count'], '2') |
410 | self.assertEquals(headers['x-account-object-count'], '1') |
411 | self.assertEquals(headers['x-account-bytes-used'], '4') |
412 | @@ -167,7 +168,8 @@ |
413 | self.assert_(found2) |
414 | |
415 | get_to_final_state() |
416 | - headers, containers = client.get_account(self.url, self.token) |
417 | + headers, containers = \ |
418 | + direct_client.direct_get_account(anodes[0], apart, self.account) |
419 | self.assertEquals(headers['x-account-container-count'], '1') |
420 | self.assertEquals(headers['x-account-object-count'], '2') |
421 | self.assertEquals(headers['x-account-bytes-used'], '9') |
422 | |
423 | === modified file 'test/probe/test_container_failures.py' |
424 | --- test/probe/test_container_failures.py 2011-04-13 17:57:59 +0000 |
425 | +++ test/probe/test_container_failures.py 2011-06-14 22:21:02 +0000 |
426 | @@ -24,7 +24,7 @@ |
427 | import eventlet |
428 | import sqlite3 |
429 | |
430 | -from swift.common import client |
431 | +from swift.common import client, direct_client |
432 | from swift.common.utils import hash_path, readconf |
433 | |
434 | from test.probe.common import get_to_final_state, kill_pids, reset_environment |
435 | @@ -72,7 +72,8 @@ |
436 | # This okay because the first node hasn't got the update that the |
437 | # object was deleted yet. |
438 | self.assert_(object1 in [o['name'] for o in |
439 | - client.get_container(self.url, self.token, container)[1]]) |
440 | + direct_client.direct_get_container(cnodes[0], cpart, |
441 | + self.account, container)[1]]) |
442 | |
443 | # Unfortunately, the following might pass or fail, depending on the |
444 | # position of the account server associated with the first container |
445 | @@ -88,7 +89,8 @@ |
446 | client.put_object(self.url, self.token, container, object2, 'test') |
447 | # First node still doesn't know object1 was deleted yet; this is okay. |
448 | self.assert_(object1 in [o['name'] for o in |
449 | - client.get_container(self.url, self.token, container)[1]]) |
450 | + direct_client.direct_get_container(cnodes[0], cpart, |
451 | + self.account, container)[1]]) |
452 | # And, of course, our new object2 exists. |
453 | self.assert_(object2 in [o['name'] for o in |
454 | client.get_container(self.url, self.token, container)[1]]) |
455 | @@ -150,7 +152,8 @@ |
456 | # server has to indicate the container exists for the put to continue. |
457 | client.put_object(self.url, self.token, container, object2, 'test') |
458 | self.assert_(object1 not in [o['name'] for o in |
459 | - client.get_container(self.url, self.token, container)[1]]) |
460 | + direct_client.direct_get_container(cnodes[0], cpart, |
461 | + self.account, container)[1]]) |
462 | # And, of course, our new object2 exists. |
463 | self.assert_(object2 in [o['name'] for o in |
464 | client.get_container(self.url, self.token, container)[1]]) |
465 | @@ -201,7 +204,8 @@ |
466 | # This okay because the first node hasn't got the update that the |
467 | # object was deleted yet. |
468 | self.assert_(object1 in [o['name'] for o in |
469 | - client.get_container(self.url, self.token, container)[1]]) |
470 | + direct_client.direct_get_container(cnodes[0], cpart, |
471 | + self.account, container)[1]]) |
472 | |
473 | # This fails because all three nodes have to indicate deletion before |
474 | # we tell the user it worked. Since the first node 409s (it hasn't got |
475 | @@ -228,7 +232,8 @@ |
476 | client.put_object(self.url, self.token, container, object2, 'test') |
477 | # First node still doesn't know object1 was deleted yet; this is okay. |
478 | self.assert_(object1 in [o['name'] for o in |
479 | - client.get_container(self.url, self.token, container)[1]]) |
480 | + direct_client.direct_get_container(cnodes[0], cpart, |
481 | + self.account, container)[1]]) |
482 | # And, of course, our new object2 exists. |
483 | self.assert_(object2 in [o['name'] for o in |
484 | client.get_container(self.url, self.token, container)[1]]) |
485 | @@ -277,7 +282,8 @@ |
486 | self.assert_(container in [c['name'] for c in |
487 | client.get_account(self.url, self.token)[1]]) |
488 | self.assert_(object1 not in [o['name'] for o in |
489 | - client.get_container(self.url, self.token, container)[1]]) |
490 | + direct_client.direct_get_container(cnodes[0], cpart, |
491 | + self.account, container)[1]]) |
492 | |
493 | # This fails because all three nodes have to indicate deletion before |
494 | # we tell the user it worked. Since the first node 409s (it hasn't got |
495 | @@ -303,7 +309,8 @@ |
496 | # server has to indicate the container exists for the put to continue. |
497 | client.put_object(self.url, self.token, container, object2, 'test') |
498 | self.assert_(object1 not in [o['name'] for o in |
499 | - client.get_container(self.url, self.token, container)[1]]) |
500 | + direct_client.direct_get_container(cnodes[0], cpart, |
501 | + self.account, container)[1]]) |
502 | # And, of course, our new object2 exists. |
503 | self.assert_(object2 in [o['name'] for o in |
504 | client.get_container(self.url, self.token, container)[1]]) |
505 | |
506 | === modified file 'test/probe/test_object_handoff.py' |
507 | --- test/probe/test_object_handoff.py 2011-01-26 22:31:33 +0000 |
508 | +++ test/probe/test_object_handoff.py 2011-06-14 22:21:02 +0000 |
509 | @@ -124,47 +124,49 @@ |
510 | if not exc: |
511 | raise Exception('Handoff object server still had test object') |
512 | |
513 | - kill(self.pids[self.port2server[onode['port']]], SIGTERM) |
514 | - client.post_object(self.url, self.token, container, obj, |
515 | - headers={'x-object-meta-probe': 'value'}) |
516 | - oheaders = client.head_object(self.url, self.token, container, obj) |
517 | - if oheaders.get('x-object-meta-probe') != 'value': |
518 | - raise Exception('Metadata incorrect, was %s' % repr(oheaders)) |
519 | - exc = False |
520 | - try: |
521 | - direct_client.direct_get_object(another_onode, opart, self.account, |
522 | - container, obj) |
523 | - except Exception: |
524 | - exc = True |
525 | - if not exc: |
526 | - raise Exception('Handoff server claimed it had the object when ' |
527 | - 'it should not have it') |
528 | - self.pids[self.port2server[onode['port']]] = Popen([ |
529 | - 'swift-object-server', |
530 | - '/etc/swift/object-server/%d.conf' % |
531 | - ((onode['port'] - 6000) / 10)]).pid |
532 | - sleep(2) |
533 | - oheaders = direct_client.direct_get_object(onode, opart, self.account, |
534 | - container, obj)[0] |
535 | - if oheaders.get('x-object-meta-probe') == 'value': |
536 | - raise Exception('Previously downed object server had the new ' |
537 | - 'metadata when it should not have it') |
538 | - # Run the extra server last so it'll remove it's extra partition |
539 | - ps = [] |
540 | - for n in onodes: |
541 | - ps.append(Popen(['swift-object-replicator', |
542 | - '/etc/swift/object-server/%d.conf' % |
543 | - ((n['port'] - 6000) / 10), 'once'])) |
544 | - for p in ps: |
545 | - p.wait() |
546 | - call(['swift-object-replicator', |
547 | - '/etc/swift/object-server/%d.conf' % |
548 | - ((another_onode['port'] - 6000) / 10), 'once']) |
549 | - oheaders = direct_client.direct_get_object(onode, opart, self.account, |
550 | - container, obj)[0] |
551 | - if oheaders.get('x-object-meta-probe') != 'value': |
552 | - raise Exception( |
553 | - 'Previously downed object server did not have the new metadata') |
554 | +# Because POST has changed to a COPY by default, POSTs will succeed on all up |
555 | +# nodes now if at least one up node has the object. |
556 | +# kill(self.pids[self.port2server[onode['port']]], SIGTERM) |
557 | +# client.post_object(self.url, self.token, container, obj, |
558 | +# headers={'x-object-meta-probe': 'value'}) |
559 | +# oheaders = client.head_object(self.url, self.token, container, obj) |
560 | +# if oheaders.get('x-object-meta-probe') != 'value': |
561 | +# raise Exception('Metadata incorrect, was %s' % repr(oheaders)) |
562 | +# exc = False |
563 | +# try: |
564 | +# direct_client.direct_get_object(another_onode, opart, self.account, |
565 | +# container, obj) |
566 | +# except Exception: |
567 | +# exc = True |
568 | +# if not exc: |
569 | +# raise Exception('Handoff server claimed it had the object when ' |
570 | +# 'it should not have it') |
571 | +# self.pids[self.port2server[onode['port']]] = Popen([ |
572 | +# 'swift-object-server', |
573 | +# '/etc/swift/object-server/%d.conf' % |
574 | +# ((onode['port'] - 6000) / 10)]).pid |
575 | +# sleep(2) |
576 | +# oheaders = direct_client.direct_get_object(onode, opart, self.account, |
577 | +# container, obj)[0] |
578 | +# if oheaders.get('x-object-meta-probe') == 'value': |
579 | +# raise Exception('Previously downed object server had the new ' |
580 | +# 'metadata when it should not have it') |
581 | +# # Run the extra server last so it'll remove it's extra partition |
582 | +# ps = [] |
583 | +# for n in onodes: |
584 | +# ps.append(Popen(['swift-object-replicator', |
585 | +# '/etc/swift/object-server/%d.conf' % |
586 | +# ((n['port'] - 6000) / 10), 'once'])) |
587 | +# for p in ps: |
588 | +# p.wait() |
589 | +# call(['swift-object-replicator', |
590 | +# '/etc/swift/object-server/%d.conf' % |
591 | +# ((another_onode['port'] - 6000) / 10), 'once']) |
592 | +# oheaders = direct_client.direct_get_object(onode, opart, self.account, |
593 | +# container, obj)[0] |
594 | +# if oheaders.get('x-object-meta-probe') != 'value': |
595 | +# raise Exception( |
596 | +# 'Previously downed object server did not have the new metadata') |
597 | |
598 | kill(self.pids[self.port2server[onode['port']]], SIGTERM) |
599 | client.delete_object(self.url, self.token, container, obj) |
600 | |
601 | === modified file 'test/unit/proxy/test_server.py' |
602 | --- test/unit/proxy/test_server.py 2011-06-11 04:57:04 +0000 |
603 | +++ test/unit/proxy/test_server.py 2011-06-14 22:21:02 +0000 |
604 | @@ -150,7 +150,7 @@ |
605 | |
606 | class FakeConn(object): |
607 | |
608 | - def __init__(self, status, etag=None, body=''): |
609 | + def __init__(self, status, etag=None, body='', timestamp='1'): |
610 | self.status = status |
611 | self.reason = 'Fake' |
612 | self.host = '1.2.3.4' |
613 | @@ -159,6 +159,7 @@ |
614 | self.received = 0 |
615 | self.etag = etag |
616 | self.body = body |
617 | + self.timestamp = timestamp |
618 | |
619 | def getresponse(self): |
620 | if kwargs.get('raise_exc'): |
621 | @@ -173,7 +174,8 @@ |
622 | def getheaders(self): |
623 | headers = {'content-length': len(self.body), |
624 | 'content-type': 'x-application/test', |
625 | - 'x-timestamp': '1', |
626 | + 'x-timestamp': self.timestamp, |
627 | + 'last-modified': self.timestamp, |
628 | 'x-object-meta-test': 'testing', |
629 | 'etag': |
630 | self.etag or '"68b329da9893e34099c7d8ad5cb9c940"', |
631 | @@ -209,6 +211,7 @@ |
632 | def getheader(self, name, default=None): |
633 | return dict(self.getheaders()).get(name.lower(), default) |
634 | |
635 | + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) |
636 | etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) |
637 | x = kwargs.get('missing_container', [False] * len(code_iter)) |
638 | if not isinstance(x, (tuple, list)): |
639 | @@ -226,9 +229,11 @@ |
640 | kwargs['give_connect'](*args, **ckwargs) |
641 | status = code_iter.next() |
642 | etag = etag_iter.next() |
643 | + timestamp = timestamps_iter.next() |
644 | if status == -1: |
645 | raise HTTPException() |
646 | - return FakeConn(status, etag, body=kwargs.get('body', '')) |
647 | + return FakeConn(status, etag, body=kwargs.get('body', ''), |
648 | + timestamp=timestamp) |
649 | |
650 | return connect |
651 | |
652 | @@ -962,6 +967,7 @@ |
653 | |
654 | def test_POST(self): |
655 | with save_globals(): |
656 | + self.app.object_post_as_copy = False |
657 | controller = proxy_server.ObjectController(self.app, 'account', |
658 | 'container', 'object') |
659 | |
660 | @@ -982,6 +988,28 @@ |
661 | test_status_map((200, 200, 404, 500, 500), 503) |
662 | test_status_map((200, 200, 404, 404, 404), 404) |
663 | |
664 | + def test_POST_as_copy(self): |
665 | + with save_globals(): |
666 | + controller = proxy_server.ObjectController(self.app, 'account', |
667 | + 'container', 'object') |
668 | + |
669 | + def test_status_map(statuses, expected): |
670 | + proxy_server.http_connect = fake_http_connect(*statuses) |
671 | + self.app.memcache.store = {} |
672 | + req = Request.blank('/a/c/o', {}, headers={ |
673 | + 'Content-Type': 'foo/bar'}) |
674 | + self.app.update_request(req) |
675 | + res = controller.POST(req) |
676 | + expected = str(expected) |
677 | + self.assertEquals(res.status[:len(expected)], expected) |
678 | + test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) |
679 | + test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202) |
680 | + test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503) |
681 | + test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503) |
682 | + test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404) |
683 | + test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503) |
684 | + test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404) |
685 | + |
686 | def test_DELETE(self): |
687 | with save_globals(): |
688 | controller = proxy_server.ObjectController(self.app, 'account', |
689 | @@ -1028,8 +1056,77 @@ |
690 | test_status_map((404, 404, 500), 404) |
691 | test_status_map((500, 500, 500), 503) |
692 | |
693 | + def test_HEAD_newest(self): |
694 | + with save_globals(): |
695 | + controller = proxy_server.ObjectController(self.app, 'account', |
696 | + 'container', 'object') |
697 | + |
698 | + def test_status_map(statuses, expected, timestamps, |
699 | + expected_timestamp): |
700 | + proxy_server.http_connect = \ |
701 | + fake_http_connect(*statuses, timestamps=timestamps) |
702 | + self.app.memcache.store = {} |
703 | + req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) |
704 | + self.app.update_request(req) |
705 | + res = controller.HEAD(req) |
706 | + self.assertEquals(res.status[:len(str(expected))], |
707 | + str(expected)) |
708 | + self.assertEquals(res.headers.get('last-modified'), |
709 | + expected_timestamp) |
710 | + |
711 | + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3') |
712 | + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3') |
713 | + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3') |
714 | + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') |
715 | + |
716 | + def test_GET_newest(self): |
717 | + with save_globals(): |
718 | + controller = proxy_server.ObjectController(self.app, 'account', |
719 | + 'container', 'object') |
720 | + |
721 | + def test_status_map(statuses, expected, timestamps, |
722 | + expected_timestamp): |
723 | + proxy_server.http_connect = \ |
724 | + fake_http_connect(*statuses, timestamps=timestamps) |
725 | + self.app.memcache.store = {} |
726 | + req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) |
727 | + self.app.update_request(req) |
728 | + res = controller.GET(req) |
729 | + self.assertEquals(res.status[:len(str(expected))], |
730 | + str(expected)) |
731 | + self.assertEquals(res.headers.get('last-modified'), |
732 | + expected_timestamp) |
733 | + |
734 | + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3') |
735 | + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3') |
736 | + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3') |
737 | + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') |
738 | + |
739 | + with save_globals(): |
740 | + controller = proxy_server.ObjectController(self.app, 'account', |
741 | + 'container', 'object') |
742 | + |
743 | + def test_status_map(statuses, expected, timestamps, |
744 | + expected_timestamp): |
745 | + proxy_server.http_connect = \ |
746 | + fake_http_connect(*statuses, timestamps=timestamps) |
747 | + self.app.memcache.store = {} |
748 | + req = Request.blank('/a/c/o', {}) |
749 | + self.app.update_request(req) |
750 | + res = controller.HEAD(req) |
751 | + self.assertEquals(res.status[:len(str(expected))], |
752 | + str(expected)) |
753 | + self.assertEquals(res.headers.get('last-modified'), |
754 | + expected_timestamp) |
755 | + |
756 | + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '1') |
757 | + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '1') |
758 | + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '1') |
759 | + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') |
760 | + |
761 | def test_POST_meta_val_len(self): |
762 | with save_globals(): |
763 | + self.app.object_post_as_copy = False |
764 | controller = proxy_server.ObjectController(self.app, 'account', |
765 | 'container', 'object') |
766 | proxy_server.http_connect = \ |
767 | @@ -1049,8 +1146,30 @@ |
768 | res = controller.POST(req) |
769 | self.assertEquals(res.status_int, 400) |
770 | |
771 | + def test_POST_as_copy_meta_val_len(self): |
772 | + with save_globals(): |
773 | + controller = proxy_server.ObjectController(self.app, 'account', |
774 | + 'container', 'object') |
775 | + proxy_server.http_connect = \ |
776 | + fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202) |
777 | + # acct cont objc objc objc obj obj obj |
778 | + req = Request.blank('/a/c/o', {}, headers={ |
779 | + 'Content-Type': 'foo/bar', |
780 | + 'X-Object-Meta-Foo': 'x' * 256}) |
781 | + self.app.update_request(req) |
782 | + res = controller.POST(req) |
783 | + self.assertEquals(res.status_int, 202) |
784 | + proxy_server.http_connect = fake_http_connect(202, 202, 202) |
785 | + req = Request.blank('/a/c/o', {}, headers={ |
786 | + 'Content-Type': 'foo/bar', |
787 | + 'X-Object-Meta-Foo': 'x' * 257}) |
788 | + self.app.update_request(req) |
789 | + res = controller.POST(req) |
790 | + self.assertEquals(res.status_int, 400) |
791 | + |
792 | def test_POST_meta_key_len(self): |
793 | with save_globals(): |
794 | + self.app.object_post_as_copy = False |
795 | controller = proxy_server.ObjectController(self.app, 'account', |
796 | 'container', 'object') |
797 | proxy_server.http_connect = \ |
798 | @@ -1070,6 +1189,27 @@ |
799 | res = controller.POST(req) |
800 | self.assertEquals(res.status_int, 400) |
801 | |
802 | + def test_POST_as_copy_meta_key_len(self): |
803 | + with save_globals(): |
804 | + controller = proxy_server.ObjectController(self.app, 'account', |
805 | + 'container', 'object') |
806 | + proxy_server.http_connect = \ |
807 | + fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202) |
808 | + # acct cont objc objc objc obj obj obj |
809 | + req = Request.blank('/a/c/o', {}, headers={ |
810 | + 'Content-Type': 'foo/bar', |
811 | + ('X-Object-Meta-' + 'x' * 128): 'x'}) |
812 | + self.app.update_request(req) |
813 | + res = controller.POST(req) |
814 | + self.assertEquals(res.status_int, 202) |
815 | + proxy_server.http_connect = fake_http_connect(202, 202, 202) |
816 | + req = Request.blank('/a/c/o', {}, headers={ |
817 | + 'Content-Type': 'foo/bar', |
818 | + ('X-Object-Meta-' + 'x' * 129): 'x'}) |
819 | + self.app.update_request(req) |
820 | + res = controller.POST(req) |
821 | + self.assertEquals(res.status_int, 400) |
822 | + |
823 | def test_POST_meta_count(self): |
824 | with save_globals(): |
825 | controller = proxy_server.ObjectController(self.app, 'account', |
826 | @@ -1344,7 +1484,8 @@ |
827 | self.assert_status_map(controller.HEAD, (200, 200, 200), 503) |
828 | self.assert_('last_error' in controller.app.object_ring.devs[0]) |
829 | self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503) |
830 | - self.assert_status_map(controller.POST, (200, 202, 202, 202), 503) |
831 | + self.assert_status_map(controller.POST, |
832 | + (200, 200, 200, 200, 202, 202, 202), 503) |
833 | self.assert_status_map(controller.DELETE, |
834 | (200, 204, 204, 204), 503) |
835 | self.app.error_suppression_interval = -300 |
836 | @@ -1437,18 +1578,41 @@ |
837 | |
838 | def test_PUT_POST_requires_container_exist(self): |
839 | with save_globals(): |
840 | - self.app.memcache = FakeMemcacheReturnsNone() |
841 | - controller = proxy_server.ObjectController(self.app, 'account', |
842 | - 'container', 'object') |
843 | - proxy_server.http_connect = \ |
844 | - fake_http_connect(404, 404, 404, 200, 200, 200) |
845 | - req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) |
846 | - self.app.update_request(req) |
847 | - resp = controller.PUT(req) |
848 | - self.assertEquals(resp.status_int, 404) |
849 | - |
850 | - proxy_server.http_connect = \ |
851 | - fake_http_connect(404, 404, 404, 200, 200, 200) |
852 | + self.app.object_post_as_copy = False |
853 | + self.app.memcache = FakeMemcacheReturnsNone() |
854 | + controller = proxy_server.ObjectController(self.app, 'account', |
855 | + 'container', 'object') |
856 | + |
857 | + proxy_server.http_connect = \ |
858 | + fake_http_connect(200, 404, 404, 404, 200, 200, 200) |
859 | + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) |
860 | + self.app.update_request(req) |
861 | + resp = controller.PUT(req) |
862 | + self.assertEquals(resp.status_int, 404) |
863 | + |
864 | + proxy_server.http_connect = \ |
865 | + fake_http_connect(200, 404, 404, 404, 200, 200) |
866 | + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, |
867 | + headers={'Content-Type': 'text/plain'}) |
868 | + self.app.update_request(req) |
869 | + resp = controller.POST(req) |
870 | + self.assertEquals(resp.status_int, 404) |
871 | + |
872 | + def test_PUT_POST_as_copy_requires_container_exist(self): |
873 | + with save_globals(): |
874 | + self.app.memcache = FakeMemcacheReturnsNone() |
875 | + controller = proxy_server.ObjectController(self.app, 'account', |
876 | + 'container', 'object') |
877 | + proxy_server.http_connect = \ |
878 | + fake_http_connect(200, 404, 404, 404, 200, 200, 200) |
879 | + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) |
880 | + self.app.update_request(req) |
881 | + resp = controller.PUT(req) |
882 | + self.assertEquals(resp.status_int, 404) |
883 | + |
884 | + proxy_server.http_connect = \ |
885 | + fake_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, |
886 | + 200) |
887 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, |
888 | headers={'Content-Type': 'text/plain'}) |
889 | self.app.update_request(req) |
890 | @@ -1568,8 +1732,10 @@ |
891 | 'X-Copy-From': 'c/o'}) |
892 | self.app.update_request(req) |
893 | proxy_server.http_connect = \ |
894 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
895 | - # acct cont acct cont objc obj obj obj |
896 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
897 | + 201) |
898 | + # acct cont acct cont objc objc objc obj obj |
899 | + # obj |
900 | self.app.memcache.store = {} |
901 | resp = controller.PUT(req) |
902 | self.assertEquals(resp.status_int, 201) |
903 | @@ -1581,8 +1747,8 @@ |
904 | 'X-Copy-From': 'c/o'}) |
905 | self.app.update_request(req) |
906 | proxy_server.http_connect = \ |
907 | - fake_http_connect(200, 200, 200, 200, 200) |
908 | - # acct cont acct cont objc |
909 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200) |
910 | + # acct cont acct cont objc objc objc |
911 | self.app.memcache.store = {} |
912 | resp = controller.PUT(req) |
913 | self.assertEquals(resp.status_int, 400) |
914 | @@ -1593,8 +1759,10 @@ |
915 | 'X-Copy-From': 'c/o/o2'}) |
916 | req.account = 'a' |
917 | proxy_server.http_connect = \ |
918 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
919 | - # acct cont acct cont objc obj obj obj |
920 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
921 | + 201) |
922 | + # acct cont acct cont objc objc objc obj obj |
923 | + # obj |
924 | self.app.memcache.store = {} |
925 | resp = controller.PUT(req) |
926 | self.assertEquals(resp.status_int, 201) |
927 | @@ -1606,8 +1774,10 @@ |
928 | 'X-Copy-From': 'c/o%20o2'}) |
929 | req.account = 'a' |
930 | proxy_server.http_connect = \ |
931 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
932 | - # acct cont acct cont objc obj obj obj |
933 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
934 | + 201) |
935 | + # acct cont acct cont objc objc objc obj obj |
936 | + # obj |
937 | self.app.memcache.store = {} |
938 | resp = controller.PUT(req) |
939 | self.assertEquals(resp.status_int, 201) |
940 | @@ -1619,8 +1789,10 @@ |
941 | 'X-Copy-From': '/c/o'}) |
942 | self.app.update_request(req) |
943 | proxy_server.http_connect = \ |
944 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
945 | - # acct cont acct cont objc obj obj obj |
946 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
947 | + 201) |
948 | + # acct cont acct cont objc objc objc obj obj |
949 | + # obj |
950 | self.app.memcache.store = {} |
951 | resp = controller.PUT(req) |
952 | self.assertEquals(resp.status_int, 201) |
953 | @@ -1631,8 +1803,10 @@ |
954 | 'X-Copy-From': '/c/o/o2'}) |
955 | req.account = 'a' |
956 | proxy_server.http_connect = \ |
957 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
958 | - # acct cont acct cont objc obj obj obj |
959 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
960 | + 201) |
961 | + # acct cont acct cont objc objc objc obj obj |
962 | + # obj |
963 | self.app.memcache.store = {} |
964 | resp = controller.PUT(req) |
965 | self.assertEquals(resp.status_int, 201) |
966 | @@ -1692,8 +1866,8 @@ |
967 | 'X-Object-Meta-Ours': 'okay'}) |
968 | self.app.update_request(req) |
969 | proxy_server.http_connect = \ |
970 | - fake_http_connect(200, 200, 200, 201, 201, 201) |
971 | - # acct cont objc obj obj obj |
972 | + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
973 | + # acct cont objc objc objc obj obj obj |
974 | self.app.memcache.store = {} |
975 | resp = controller.PUT(req) |
976 | self.assertEquals(resp.status_int, 201) |
977 | @@ -1717,8 +1891,10 @@ |
978 | headers={'Destination': 'c/o'}) |
979 | req.account = 'a' |
980 | proxy_server.http_connect = \ |
981 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
982 | - # acct cont acct cont objc obj obj obj |
983 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
984 | + 201) |
985 | + # acct cont acct cont objc objc objc obj obj |
986 | + # obj |
987 | self.app.memcache.store = {} |
988 | resp = controller.COPY(req) |
989 | self.assertEquals(resp.status_int, 201) |
990 | @@ -1730,8 +1906,10 @@ |
991 | req.account = 'a' |
992 | controller.object_name = 'o/o2' |
993 | proxy_server.http_connect = \ |
994 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
995 | - # acct cont acct cont objc obj obj obj |
996 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
997 | + 201) |
998 | + # acct cont acct cont objc objc objc obj obj |
999 | + # obj |
1000 | self.app.memcache.store = {} |
1001 | resp = controller.COPY(req) |
1002 | self.assertEquals(resp.status_int, 201) |
1003 | @@ -1742,8 +1920,10 @@ |
1004 | req.account = 'a' |
1005 | controller.object_name = 'o' |
1006 | proxy_server.http_connect = \ |
1007 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
1008 | - # acct cont acct cont objc obj obj obj |
1009 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
1010 | + 201) |
1011 | + # acct cont acct cont objc objc objc obj obj |
1012 | + # obj |
1013 | self.app.memcache.store = {} |
1014 | resp = controller.COPY(req) |
1015 | self.assertEquals(resp.status_int, 201) |
1016 | @@ -1755,8 +1935,10 @@ |
1017 | req.account = 'a' |
1018 | controller.object_name = 'o/o2' |
1019 | proxy_server.http_connect = \ |
1020 | - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
1021 | - # acct cont acct cont objc obj obj obj |
1022 | + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, |
1023 | + 201) |
1024 | + # acct cont acct cont objc objc objc obj obj |
1025 | + # obj |
1026 | self.app.memcache.store = {} |
1027 | resp = controller.COPY(req) |
1028 | self.assertEquals(resp.status_int, 201) |
1029 | @@ -1812,8 +1994,8 @@ |
1030 | req.account = 'a' |
1031 | controller.object_name = 'o' |
1032 | proxy_server.http_connect = \ |
1033 | - fake_http_connect(200, 200, 200, 201, 201, 201) |
1034 | - # acct cont objc obj obj obj |
1035 | + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
1036 | + # acct cont objc objc objc obj obj obj |
1037 | self.app.memcache.store = {} |
1038 | resp = controller.COPY(req) |
1039 | self.assertEquals(resp.status_int, 201) |
1040 | @@ -1821,6 +2003,23 @@ |
1041 | 'testing') |
1042 | self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') |
1043 | |
1044 | + def test_COPY_newest(self): |
1045 | + with save_globals(): |
1046 | + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') |
1047 | + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, |
1048 | + headers={'Destination': '/c/o'}) |
1049 | + req.account = 'a' |
1050 | + controller.object_name = 'o' |
1051 | + proxy_server.http_connect = \ |
1052 | + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201, |
1053 | + timestamps=('1', '1', '1', '3', '2', '4', '4', '4')) |
1054 | + # acct cont objc objc objc obj obj obj |
1055 | + self.app.memcache.store = {} |
1056 | + resp = controller.COPY(req) |
1057 | + self.assertEquals(resp.status_int, 201) |
1058 | + self.assertEquals(resp.headers['x-copied-from-last-modified'], |
1059 | + '3') |
1060 | + |
1061 | def test_chunked_put(self): |
1062 | |
1063 | class ChunkedFile(): |
1064 | @@ -2596,6 +2795,7 @@ |
1065 | called[0] = True |
1066 | return HTTPUnauthorized(request=req) |
1067 | with save_globals(): |
1068 | + self.app.object_post_as_copy = False |
1069 | proxy_server.http_connect = \ |
1070 | fake_http_connect(200, 200, 201, 201, 201) |
1071 | controller = proxy_server.ObjectController(self.app, 'account', |
1072 | @@ -2607,6 +2807,24 @@ |
1073 | res = controller.POST(req) |
1074 | self.assert_(called[0]) |
1075 | |
1076 | + def test_POST_as_copy_calls_authorize(self): |
1077 | + called = [False] |
1078 | + |
1079 | + def authorize(req): |
1080 | + called[0] = True |
1081 | + return HTTPUnauthorized(request=req) |
1082 | + with save_globals(): |
1083 | + proxy_server.http_connect = \ |
1084 | + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) |
1085 | + controller = proxy_server.ObjectController(self.app, 'account', |
1086 | + 'container', 'object') |
1087 | + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, |
1088 | + headers={'Content-Length': '5'}, body='12345') |
1089 | + req.environ['swift.authorize'] = authorize |
1090 | + self.app.update_request(req) |
1091 | + res = controller.POST(req) |
1092 | + self.assert_(called[0]) |
1093 | + |
1094 | def test_PUT_calls_authorize(self): |
1095 | called = [False] |
1096 | |
1097 | @@ -2814,6 +3032,7 @@ |
1098 | |
1099 | def test_error_limiting(self): |
1100 | with save_globals(): |
1101 | + proxy_server.shuffle = lambda l: None |
1102 | controller = proxy_server.ContainerController(self.app, 'account', |
1103 | 'container') |
1104 | self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, |
looks go to me