Merge lp:~gholt/swift/lobjects3 into lp:~hudson-openstack/swift/trunk
- lobjects3
- Merge into trunk
Proposed by
gholt
Status: | Rejected |
---|---|
Rejected by: | gholt |
Proposed branch: | lp:~gholt/swift/lobjects3 |
Merge into: | lp:~hudson-openstack/swift/trunk |
Diff against target: |
4643 lines (+3054/-575) 29 files modified
bin/swift-init (+12/-6) bin/swift-object-janitor (+28/-0) doc/source/deployment_guide.rst (+26/-0) doc/source/development_saio.rst (+9/-0) doc/source/index.rst (+1/-0) doc/source/object.rst (+19/-0) doc/source/overview_very_large_objects.rst (+144/-0) etc/object-server.conf-sample (+15/-0) etc/proxy-server.conf-sample (+3/-0) setup.py (+16/-23) swift/common/constraints.py (+11/-6) swift/common/db.py (+1/-2) swift/obj/auditor.py (+9/-15) swift/obj/diskfile.py (+507/-0) swift/obj/janitor.py (+516/-0) swift/obj/replicator.py (+32/-176) swift/obj/server.py (+109/-236) swift/obj/updater.py (+16/-2) swift/proxy/server.py (+391/-26) test/functional/sample.conf (+8/-1) test/functional/tests.py (+1/-1) test/probe/common.py (+1/-1) test/probe/test_object_async_update.py (+1/-1) test/unit/common/test_constraints.py (+6/-8) test/unit/obj/test_diskfile.py (+203/-0) test/unit/obj/test_janitor.py (+433/-0) test/unit/obj/test_server.py (+179/-62) test/unit/obj/test_updater.py (+8/-4) test/unit/proxy/test_server.py (+349/-5) |
To merge this branch: | bzr merge lp:~gholt/swift/lobjects3 |
Related bugs: | |
Related blueprints: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
gholt (community) | Disapprove | ||
Review via email: mp+39792@code.launchpad.net |
Commit message
Very large object support.
Description of the change
Very large object support.
Please read doc/source/
To post a comment you must log in.
Unmerged revisions
- 124. By gholt
-
Merge from trunk
- 123. By gholt
-
Merged from trunk
- 122. By gholt
-
Merged from trunk
- 121. By gholt
-
Added missing SkipTest import
- 120. By gholt
-
Make poor Hudson happier
- 119. By gholt
-
Documentation
- 118. By gholt
-
Tests for proxy server
- 117. By gholt
-
More object server tests
- 116. By gholt
-
More tests for the janitor
- 115. By gholt
-
Working on tests and bugfixes
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bin/swift-init' | |||
2 | --- bin/swift-init 2010-08-19 20:01:44 +0000 | |||
3 | +++ bin/swift-init 2010-11-08 18:51:48 +0000 | |||
4 | @@ -23,10 +23,11 @@ | |||
5 | 23 | import sys | 23 | import sys |
6 | 24 | import time | 24 | import time |
7 | 25 | 25 | ||
9 | 26 | ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', | 26 | ALL_SERVERS = ['account-auditor', 'account-reaper', 'account-replicator', |
10 | 27 | 'account-server', 'auth-server', 'container-auditor', | ||
11 | 27 | 'container-replicator', 'container-server', 'container-updater', | 28 | 'container-replicator', 'container-server', 'container-updater', |
14 | 28 | 'object-auditor', 'object-server', 'object-replicator', 'object-updater', | 29 | 'object-auditor', 'object-janitor', 'object-replicator', 'object-server', |
15 | 29 | 'proxy-server', 'account-replicator', 'auth-server', 'account-reaper'] | 30 | 'object-updater', 'proxy-server'] |
16 | 30 | GRACEFUL_SHUTDOWN_SERVERS = ['account-server', 'container-server', | 31 | GRACEFUL_SHUTDOWN_SERVERS = ['account-server', 'container-server', |
17 | 31 | 'object-server', 'proxy-server', 'auth-server'] | 32 | 'object-server', 'proxy-server', 'auth-server'] |
18 | 32 | MAX_DESCRIPTORS = 32768 | 33 | MAX_DESCRIPTORS = 32768 |
19 | @@ -41,6 +42,7 @@ | |||
20 | 41 | servers = [server] | 42 | servers = [server] |
21 | 42 | command = command.lower() | 43 | command = command.lower() |
22 | 43 | 44 | ||
23 | 45 | |||
24 | 44 | def pid_files(server): | 46 | def pid_files(server): |
25 | 45 | if os.path.exists('/var/run/swift/%s.pid' % server): | 47 | if os.path.exists('/var/run/swift/%s.pid' % server): |
26 | 46 | pid_files = ['/var/run/swift/%s.pid' % server] | 48 | pid_files = ['/var/run/swift/%s.pid' % server] |
27 | @@ -50,6 +52,7 @@ | |||
28 | 50 | pid = int(open(pid_file).read().strip()) | 52 | pid = int(open(pid_file).read().strip()) |
29 | 51 | yield pid_file, pid | 53 | yield pid_file, pid |
30 | 52 | 54 | ||
31 | 55 | |||
32 | 53 | def do_start(server, once=False): | 56 | def do_start(server, once=False): |
33 | 54 | server_type = '-'.join(server.split('-')[:-1]) | 57 | server_type = '-'.join(server.split('-')[:-1]) |
34 | 55 | 58 | ||
35 | @@ -77,7 +80,7 @@ | |||
36 | 77 | os.makedirs(dir) | 80 | os.makedirs(dir) |
37 | 78 | except OSError, err: | 81 | except OSError, err: |
38 | 79 | if err.errno == errno.EACCES: | 82 | if err.errno == errno.EACCES: |
40 | 80 | sys.exit('Unable to create %s. Running as non-root?' % dir) | 83 | sys.exit('Unable to create %s. Running as non-root?' % dir) |
41 | 81 | fp = open(pid_file, 'w') | 84 | fp = open(pid_file, 'w') |
42 | 82 | fp.write('%d\n' % pid) | 85 | fp.write('%d\n' % pid) |
43 | 83 | fp.close() | 86 | fp.close() |
44 | @@ -120,18 +123,21 @@ | |||
45 | 120 | elif os.path.exists('/etc/swift/%s-server/' % server_type): | 123 | elif os.path.exists('/etc/swift/%s-server/' % server_type): |
46 | 121 | # found config directory, searching for config file(s) | 124 | # found config directory, searching for config file(s) |
47 | 122 | launch_args = [] | 125 | launch_args = [] |
49 | 123 | for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): | 126 | for num, ini_file in \ |
50 | 127 | enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): | ||
51 | 124 | pid_file = '/var/run/swift/%s/%d.pid' % (server, num) | 128 | pid_file = '/var/run/swift/%s/%d.pid' % (server, num) |
52 | 125 | # start a server for each ini_file found | 129 | # start a server for each ini_file found |
53 | 126 | launch_args.append((ini_file, pid_file)) | 130 | launch_args.append((ini_file, pid_file)) |
54 | 127 | else: | 131 | else: |
55 | 128 | # maybe there's a config file(s) out there, but I couldn't find it! | 132 | # maybe there's a config file(s) out there, but I couldn't find it! |
57 | 129 | sys.exit('Unable to locate config file for %s. %s does not exist?' % (server, ini_file)) | 133 | sys.exit('Unable to locate config file for %s. %s does not exist?' % |
58 | 134 | (server, ini_file)) | ||
59 | 130 | 135 | ||
60 | 131 | # start all servers | 136 | # start all servers |
61 | 132 | for ini_file, pid_file in launch_args: | 137 | for ini_file, pid_file in launch_args: |
62 | 133 | launch(ini_file, pid_file) | 138 | launch(ini_file, pid_file) |
63 | 134 | 139 | ||
64 | 140 | |||
65 | 135 | def do_stop(server, graceful=False): | 141 | def do_stop(server, graceful=False): |
66 | 136 | if graceful and server in GRACEFUL_SHUTDOWN_SERVERS: | 142 | if graceful and server in GRACEFUL_SHUTDOWN_SERVERS: |
67 | 137 | sig = signal.SIGHUP | 143 | sig = signal.SIGHUP |
68 | 138 | 144 | ||
69 | === added file 'bin/swift-object-janitor' | |||
70 | --- bin/swift-object-janitor 1970-01-01 00:00:00 +0000 | |||
71 | +++ bin/swift-object-janitor 2010-11-08 18:51:48 +0000 | |||
72 | @@ -0,0 +1,28 @@ | |||
73 | 1 | #!/usr/bin/python | ||
74 | 2 | # Copyright (c) 2010 OpenStack, LLC. | ||
75 | 3 | # | ||
76 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
77 | 5 | # you may not use this file except in compliance with the License. | ||
78 | 6 | # You may obtain a copy of the License at | ||
79 | 7 | # | ||
80 | 8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
81 | 9 | # | ||
82 | 10 | # Unless required by applicable law or agreed to in writing, software | ||
83 | 11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
84 | 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
85 | 13 | # implied. | ||
86 | 14 | # See the License for the specific language governing permissions and | ||
87 | 15 | # limitations under the License. | ||
88 | 16 | |||
89 | 17 | import sys | ||
90 | 18 | |||
91 | 19 | from swift.obj.janitor import ObjectJanitor | ||
92 | 20 | from swift.common import utils | ||
93 | 21 | |||
94 | 22 | if __name__ == '__main__': | ||
95 | 23 | if len(sys.argv) < 2: | ||
96 | 24 | print "Usage: swift-object-janitor CONFIG_FILE [once]" | ||
97 | 25 | sys.exit(1) | ||
98 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
99 | 27 | conf = utils.readconf(sys.argv[1], 'object-janitor') | ||
100 | 28 | ObjectJanitor(conf).run(once) | ||
101 | 0 | 29 | ||
102 | === modified file 'doc/source/deployment_guide.rst' | |||
103 | --- doc/source/deployment_guide.rst 2010-10-21 18:59:43 +0000 | |||
104 | +++ doc/source/deployment_guide.rst 2010-11-08 18:51:48 +0000 | |||
105 | @@ -206,6 +206,25 @@ | |||
106 | 206 | object can be reclaimed | 206 | object can be reclaimed |
107 | 207 | ================== ================= ======================================= | 207 | ================== ================= ======================================= |
108 | 208 | 208 | ||
109 | 209 | [object-janitor] | ||
110 | 210 | |||
111 | 211 | =================== ============== ========================================== | ||
112 | 212 | Option Default Description | ||
113 | 213 | ------------------- -------------- ------------------------------------------ | ||
114 | 214 | log_name object-janitor Label used when logging | ||
115 | 215 | log_facility LOG_LOCAL0 Syslog log facility | ||
116 | 216 | log_level INFO Logging level | ||
117 | 217 | interval 300 Minimum time for a pass to take | ||
118 | 218 | concurrency 1 Number of updater workers to spawn | ||
119 | 219 | node_timeout 10 Request timeout to external services | ||
120 | 220 | conn_timeout 0.5 Connection timeout to external services | ||
121 | 221 | slowdown 0.01 Time in seconds to wait between operations | ||
122 | 222 | segment_reclaim_age 604800 Number of seconds before removing orphaned | ||
123 | 223 | object segments | ||
124 | 224 | segments_per_pass 10 Maximum segments per object to remove per | ||
125 | 225 | pass | ||
126 | 226 | =================== ============== ========================================== | ||
127 | 227 | |||
128 | 209 | [object-updater] | 228 | [object-updater] |
129 | 210 | 229 | ||
130 | 211 | ================== ============== ========================================== | 230 | ================== ============== ========================================== |
131 | @@ -438,6 +457,13 @@ | |||
132 | 438 | log_level INFO Log level | 457 | log_level INFO Log level |
133 | 439 | log_headers True If True, log headers in each | 458 | log_headers True If True, log headers in each |
134 | 440 | request | 459 | request |
135 | 460 | max_object_size 107374182400 Maximum size of any object in | ||
136 | 461 | the cluster. Note: The | ||
137 | 462 | segment_size value will cause | ||
138 | 463 | objects larger than that to be | ||
139 | 464 | split into segments | ||
140 | 465 | segment_size 2147483647 Objects will be split into file | ||
141 | 466 | segments no larger than this | ||
142 | 441 | recheck_account_existence 60 Cache timeout in seconds to | 467 | recheck_account_existence 60 Cache timeout in seconds to |
143 | 442 | send memcached for account | 468 | send memcached for account |
144 | 443 | existence | 469 | existence |
145 | 444 | 470 | ||
146 | === modified file 'doc/source/development_saio.rst' | |||
147 | --- doc/source/development_saio.rst 2010-11-02 14:17:25 +0000 | |||
148 | +++ doc/source/development_saio.rst 2010-11-08 18:51:48 +0000 | |||
149 | @@ -440,6 +440,8 @@ | |||
150 | 440 | [object-replicator] | 440 | [object-replicator] |
151 | 441 | vm_test_mode = yes | 441 | vm_test_mode = yes |
152 | 442 | 442 | ||
153 | 443 | [object-janitor] | ||
154 | 444 | |||
155 | 443 | [object-updater] | 445 | [object-updater] |
156 | 444 | 446 | ||
157 | 445 | [object-auditor] | 447 | [object-auditor] |
158 | @@ -461,6 +463,8 @@ | |||
159 | 461 | [object-replicator] | 463 | [object-replicator] |
160 | 462 | vm_test_mode = yes | 464 | vm_test_mode = yes |
161 | 463 | 465 | ||
162 | 466 | [object-janitor] | ||
163 | 467 | |||
164 | 464 | [object-updater] | 468 | [object-updater] |
165 | 465 | 469 | ||
166 | 466 | [object-auditor] | 470 | [object-auditor] |
167 | @@ -482,6 +486,8 @@ | |||
168 | 482 | [object-replicator] | 486 | [object-replicator] |
169 | 483 | vm_test_mode = yes | 487 | vm_test_mode = yes |
170 | 484 | 488 | ||
171 | 489 | [object-janitor] | ||
172 | 490 | |||
173 | 485 | [object-updater] | 491 | [object-updater] |
174 | 486 | 492 | ||
175 | 487 | [object-auditor] | 493 | [object-auditor] |
176 | @@ -503,6 +509,8 @@ | |||
177 | 503 | [object-replicator] | 509 | [object-replicator] |
178 | 504 | vm_test_mode = yes | 510 | vm_test_mode = yes |
179 | 505 | 511 | ||
180 | 512 | [object-janitor] | ||
181 | 513 | |||
182 | 506 | [object-updater] | 514 | [object-updater] |
183 | 507 | 515 | ||
184 | 508 | [object-auditor] | 516 | [object-auditor] |
185 | @@ -571,6 +579,7 @@ | |||
186 | 571 | # Replace devauth with whatever your super_admin key is (recorded in | 579 | # Replace devauth with whatever your super_admin key is (recorded in |
187 | 572 | # /etc/swift/auth-server.conf). | 580 | # /etc/swift/auth-server.conf). |
188 | 573 | swift-auth-recreate-accounts -K devauth | 581 | swift-auth-recreate-accounts -K devauth |
189 | 582 | swift-init object-janitor start | ||
190 | 574 | swift-init object-updater start | 583 | swift-init object-updater start |
191 | 575 | swift-init container-updater start | 584 | swift-init container-updater start |
192 | 576 | swift-init object-replicator start | 585 | swift-init object-replicator start |
193 | 577 | 586 | ||
194 | === modified file 'doc/source/index.rst' | |||
195 | --- doc/source/index.rst 2010-11-04 19:25:23 +0000 | |||
196 | +++ doc/source/index.rst 2010-11-08 18:51:48 +0000 | |||
197 | @@ -26,6 +26,7 @@ | |||
198 | 26 | overview_replication | 26 | overview_replication |
199 | 27 | overview_stats | 27 | overview_stats |
200 | 28 | ratelimit | 28 | ratelimit |
201 | 29 | overview_very_large_objects | ||
202 | 29 | 30 | ||
203 | 30 | Development: | 31 | Development: |
204 | 31 | 32 | ||
205 | 32 | 33 | ||
206 | === modified file 'doc/source/object.rst' | |||
207 | --- doc/source/object.rst 2010-07-19 16:25:18 +0000 | |||
208 | +++ doc/source/object.rst 2010-11-08 18:51:48 +0000 | |||
209 | @@ -24,6 +24,16 @@ | |||
210 | 24 | :undoc-members: | 24 | :undoc-members: |
211 | 25 | :show-inheritance: | 25 | :show-inheritance: |
212 | 26 | 26 | ||
213 | 27 | .. _object-janitor: | ||
214 | 28 | |||
215 | 29 | Object Janitor | ||
216 | 30 | ============== | ||
217 | 31 | |||
218 | 32 | .. automodule:: swift.obj.janitor | ||
219 | 33 | :members: | ||
220 | 34 | :undoc-members: | ||
221 | 35 | :show-inheritance: | ||
222 | 36 | |||
223 | 27 | .. _object-updater: | 37 | .. _object-updater: |
224 | 28 | 38 | ||
225 | 29 | Object Updater | 39 | Object Updater |
226 | @@ -44,3 +54,12 @@ | |||
227 | 44 | :undoc-members: | 54 | :undoc-members: |
228 | 45 | :show-inheritance: | 55 | :show-inheritance: |
229 | 46 | 56 | ||
230 | 57 | .. _object-diskfile: | ||
231 | 58 | |||
232 | 59 | Disk File | ||
233 | 60 | ========= | ||
234 | 61 | |||
235 | 62 | .. automodule:: swift.obj.diskfile | ||
236 | 63 | :members: | ||
237 | 64 | :undoc-members: | ||
238 | 65 | :show-inheritance: | ||
239 | 47 | 66 | ||
240 | === added file 'doc/source/overview_very_large_objects.rst' | |||
241 | --- doc/source/overview_very_large_objects.rst 1970-01-01 00:00:00 +0000 | |||
242 | +++ doc/source/overview_very_large_objects.rst 2010-11-08 18:51:48 +0000 | |||
243 | @@ -0,0 +1,144 @@ | |||
244 | 1 | ========================= | ||
245 | 2 | Very Large Object Support | ||
246 | 3 | ========================= | ||
247 | 4 | |||
248 | 5 | ----- | ||
249 | 6 | Intro | ||
250 | 7 | ----- | ||
251 | 8 | |||
252 | 9 | Supporting very large objects in Swift presented quite a challenge. The main | ||
253 | 10 | problem is storage balance; if you just have a few very large objects in the | ||
254 | 11 | cluster some of the storage nodes will have significantly more data than the | ||
255 | 12 | others. The basic answer is to break these objects up into segments and | ||
256 | 13 | distribute the segments across the cluster evenly. | ||
257 | 14 | |||
258 | 15 | The user could do this themselves, breaking up their very large objects into | ||
259 | 16 | smaller objects and uploading those. But then they'd have to reassemble them | ||
260 | 17 | themselves as well on download. What we do in Swift is essentially emulate this | ||
261 | 18 | behavior for the user, transparently to the user. In this way, we can pick the | ||
262 | 19 | optimal segment size for our cluster, something the user probably wouldn't | ||
263 | 20 | know. This also allows the user to easily do ranged requests on the object | ||
264 | 21 | without having to know how it's split up behind the scenes. | ||
265 | 22 | |||
266 | 23 | ----------------------- | ||
267 | 24 | The Proxy is in Control | ||
268 | 25 | ----------------------- | ||
269 | 26 | |||
270 | 27 | In Swift's implementation, the proxy server is in control of the object | ||
271 | 28 | segmentation. As the user uploads a very large object, the proxy automaticaly | ||
272 | 29 | closes the current segment when it reaches a configurable segment size and | ||
273 | 30 | opens a new segment for more data. Once all the data is uploaded and stored, | ||
274 | 31 | the proxy server then creates a manifest object indicating how the object was | ||
275 | 32 | segmented so that it can be retreived later. The proxy does not spool any data | ||
276 | 33 | to disk, it sends all data directly to the object servers. | ||
277 | 34 | |||
278 | 35 | Because of these segment switchovers occurring in the proxy server, a very | ||
279 | 36 | large object operation will have a higher chance of failure than a normal sized | ||
280 | 37 | object. If at any point the proxy can't communicate with at least a majority of | ||
281 | 38 | the nodes for each and every segment of the object, the whole object operation | ||
282 | 39 | will fail. For instance, a regular 1m object in a 3 replica cluster just needs | ||
283 | 40 | to make at least 2 object server requests to succeed. If the segment size is | ||
284 | 41 | 1G, a 10G object would need 2 object server requests to succeed 11 times during | ||
285 | 42 | the operation (10 segments, 1 manifest). | ||
286 | 43 | |||
287 | 44 | The upside of the proxy being in control is that the client does not need to | ||
288 | 45 | know anything about the segmentation; it's done for them. The downside is that | ||
289 | 46 | failure to upload means starting the whole thing over again. Later we plan to | ||
290 | 47 | implement what we term 'user manifests' where the user can upload several | ||
291 | 48 | objects but download them as if they were one. This would allow the user to | ||
292 | 49 | just reupload a smaller part of the whole in the case of upload failures, but | ||
293 | 50 | would require them to manage the segments themselves. | ||
294 | 51 | |||
295 | 52 | ------------------------- | ||
296 | 53 | How They're Really Stored | ||
297 | 54 | ------------------------- | ||
298 | 55 | |||
299 | 56 | If you have a working knowledge of how Swift stores and locates regular objects | ||
300 | 57 | you'll know that the object name is hashed and that that hash determines the | ||
301 | 58 | storage nodes and disk location for the object's data. Segmented objects work | ||
302 | 59 | much the same way, but they have as many names as they do segments, plus one | ||
303 | 60 | more for the manifest. | ||
304 | 61 | |||
305 | 62 | The manifest is stored at the same location as the object would be if it were a | ||
306 | 63 | normal object. The manifest is the last item stored once all the segments are | ||
307 | 64 | in place, so that only one version of an object is available at any given time. | ||
308 | 65 | This is important because you don't want to be able to retrieve a manifest and | ||
309 | 66 | not be able to find the segments yet. | ||
310 | 67 | |||
311 | 68 | The segments are given the same name as the object with the operation's | ||
312 | 69 | timestamp and the segment number appended. The reason the timestamp is used is | ||
313 | 70 | to keep the segments of multiple uploads of the same very large object from | ||
314 | 71 | colliding. During an overwrite of a very large object, you don't want a | ||
315 | 72 | download of that object to have some new data and some old. Since the segments | ||
316 | 73 | for each version of the object are stored independently, a download before the | ||
317 | 74 | new manifest file is in place will get the old data and a download afterwards | ||
318 | 75 | will get the new data. Also, in the case where the new upload fails in transit, | ||
319 | 76 | the old version will still be available. | ||
320 | 77 | |||
321 | 78 | The storage nodes for each segment are determined the familiar way of hashing | ||
322 | 79 | the segment name (object name + timestamp + segment number as described above) | ||
323 | 80 | and then picking the storage nodes based on the hash. However, there is an | ||
324 | 81 | exception for the first segment. This segment will always be stored on the same | ||
325 | 82 | storage node and disk partition as a regular object of the same name would be, | ||
326 | 83 | though the hash of the segment name determines the exact on disk location. This | ||
327 | 84 | is because with chunked transfer encoded uploads we don't know the object size | ||
328 | 85 | until the upload is complete. We won't realize we need to segment such an | ||
329 | 86 | object until we've already uploaded the first segment's worth to the regular | ||
330 | 87 | object storage nodes. | ||
331 | 88 | |||
332 | 89 | To help keep directory structures smaller, object segments are stored in an | ||
333 | 90 | 'object_segments' directory alongside the usual 'objects' directory. Though all | ||
334 | 91 | these files could be stored in the same directory structure, making this split | ||
335 | 92 | should make replication, reclamation, and other scan-type operations less of a | ||
336 | 93 | jar to the systems. | ||
337 | 94 | |||
338 | 95 | .. note:: | ||
339 | 96 | |||
340 | 97 | Because the first segment is always stored on the same storage nodes and in | ||
341 | 98 | the same partition as a regular file would, you can often see hashes in a | ||
342 | 99 | partition where they don't seem to belong. For instance, a first segment | ||
343 | 100 | name that hashes to 25d40100fd07c8c5fd1a4e31875593e1 might be found in | ||
344 | 101 | partition 568363 (hex 8ac2b) instead of the expected 154944 (hex 25d40) | ||
345 | 102 | because the segment's true object name hashed to | ||
346 | 103 | 8ac2bf59556b61bb5cc521ccb51c200a. You should only see these anomalies in | ||
347 | 104 | the the object_segments directory, however. | ||
348 | 105 | |||
349 | 106 | ----------- | ||
350 | 107 | Cleaning Up | ||
351 | 108 | ----------- | ||
352 | 109 | |||
353 | 110 | Another challenge very large object support brought was in how to keep | ||
354 | 111 | everything cleaned up and reduce the chance of completely orphaned segments | ||
355 | 112 | that would waste a large amount of storage. | ||
356 | 113 | |||
357 | 114 | One such scenario is the common DELETE operation. We don't want to make the | ||
358 | 115 | client wait while we go out and delete all the segments and the manifest from | ||
359 | 116 | all over the cluster. Instead, we write out a file that will start a background | ||
360 | 117 | operation to the cleanup and we just remove the manifest file. An object | ||
361 | 118 | overwrite is much like a delete in that just before placing the new manifest | ||
362 | 119 | file we create the background job. | ||
363 | 120 | |||
364 | 121 | Another scenario is failed uploads. If a user disconnects after uploading 100 | ||
365 | 122 | segments before finishing, we have to clean those up. Also, a proxy could crash | ||
366 | 123 | due to hardware failure right in the middle of such an operation, so we | ||
367 | 124 | wouldn't even be able to make a cleanup job at that point. So, we make a | ||
368 | 125 | pending cleanup job just before starting any segmented upload. That pending | ||
369 | 126 | cleanup job will wait up to a configurable amount of time (1 week by default) | ||
370 | 127 | for the associated manifest to appear. If the manifest appears, the job is | ||
371 | 128 | canceled and deleted. If the manifest never appears, the job goes about | ||
372 | 129 | removing the orphaned segments from the system. | ||
373 | 130 | |||
374 | 131 | Prior to very large object support, we had a background process called object | ||
375 | 132 | async. Object async would send object metadata updates to the container servers | ||
376 | 133 | in the event the container servers couldn't be contacted right away. These | ||
377 | 134 | update jobs were uncommon and quick to complete, so durability wasn't too much | ||
378 | 135 | of a concern. However, these segment cleanup jobs can hang around for a while | ||
379 | 136 | (1 week in the above example of a proxy crash), so durable background jobs | ||
380 | 137 | became important. | ||
381 | 138 | |||
382 | 139 | With this, the object janitor was born. He accomplishes what object async did | ||
383 | 140 | and the segment cleanup operations as well. The janitor jobs are stored just | ||
384 | 141 | like regular object data is, except that they're stored in the object_janitor | ||
385 | 142 | directory structure. Because they're stored in the same way, they can use the | ||
386 | 143 | same object replicator code to keep durability with replicas on multiple | ||
387 | 144 | storage nodes. | ||
388 | 0 | 145 | ||
389 | === modified file 'etc/object-server.conf-sample' | |||
390 | --- etc/object-server.conf-sample 2010-10-19 15:02:36 +0000 | |||
391 | +++ etc/object-server.conf-sample 2010-11-08 18:51:48 +0000 | |||
392 | @@ -44,6 +44,21 @@ | |||
393 | 44 | # The replicator also performs reclamation | 44 | # The replicator also performs reclamation |
394 | 45 | # reclaim_age = 604800 | 45 | # reclaim_age = 604800 |
395 | 46 | 46 | ||
396 | 47 | [object-janitor] | ||
397 | 48 | # log_name = object-janitor | ||
398 | 49 | # interval = 300 | ||
399 | 50 | # concurrency = 1 | ||
400 | 51 | # node_timeout = 10 | ||
401 | 52 | # conn_timeout = 0.5 | ||
402 | 53 | # slowdown will sleep that amount between janitor operations | ||
403 | 54 | # slowdown = 0.01 | ||
404 | 55 | # Number of seconds before assuming a segmented put will never succeed and | ||
405 | 56 | # therefore clean up any orphaned segments from the operation. | ||
406 | 57 | # segment_reclaim_age = 604800 | ||
407 | 58 | # Number of segments to remove per pass when cleaning up after superceded or | ||
408 | 59 | # orphaned segmented put operations. | ||
409 | 60 | # segments_per_pass = 10 | ||
410 | 61 | |||
411 | 47 | [object-updater] | 62 | [object-updater] |
412 | 48 | # log_name = object-updater | 63 | # log_name = object-updater |
413 | 49 | # interval = 300 | 64 | # interval = 300 |
414 | 50 | 65 | ||
415 | === modified file 'etc/proxy-server.conf-sample' | |||
416 | --- etc/proxy-server.conf-sample 2010-11-03 20:17:27 +0000 | |||
417 | +++ etc/proxy-server.conf-sample 2010-11-08 18:51:48 +0000 | |||
418 | @@ -17,6 +17,9 @@ | |||
419 | 17 | # log_facility = LOG_LOCAL0 | 17 | # log_facility = LOG_LOCAL0 |
420 | 18 | # log_level = INFO | 18 | # log_level = INFO |
421 | 19 | # log_headers = False | 19 | # log_headers = False |
422 | 20 | # max_object_size = 107374182400 | ||
423 | 21 | # Files will be split into segments no larger than segment_size | ||
424 | 22 | # segment_size = 2147483647 | ||
425 | 20 | # recheck_account_existence = 60 | 23 | # recheck_account_existence = 60 |
426 | 21 | # recheck_container_existence = 60 | 24 | # recheck_container_existence = 60 |
427 | 22 | # object_chunk_size = 8192 | 25 | # object_chunk_size = 8192 |
428 | 23 | 26 | ||
429 | === modified file 'setup.py' | |||
430 | --- setup.py 2010-11-03 19:50:35 +0000 | |||
431 | +++ setup.py 2010-11-08 18:51:48 +0000 | |||
432 | @@ -21,6 +21,7 @@ | |||
433 | 21 | 21 | ||
434 | 22 | from swift import __version__ as version | 22 | from swift import __version__ as version |
435 | 23 | 23 | ||
436 | 24 | |||
437 | 24 | class local_sdist(sdist): | 25 | class local_sdist(sdist): |
438 | 25 | """Customized sdist hook - builds the ChangeLog file from VC first""" | 26 | """Customized sdist hook - builds the ChangeLog file from VC first""" |
439 | 26 | 27 | ||
440 | @@ -57,29 +58,21 @@ | |||
441 | 57 | 'Environment :: No Input/Output (Daemon)', | 58 | 'Environment :: No Input/Output (Daemon)', |
442 | 58 | ], | 59 | ], |
443 | 59 | install_requires=[], # removed for better compat | 60 | install_requires=[], # removed for better compat |
467 | 60 | scripts=[ | 61 | scripts=['bin/st', 'bin/swift-account-audit', 'bin/swift-account-auditor', |
468 | 61 | 'bin/st', 'bin/swift-account-auditor', | 62 | 'bin/swift-account-reaper', 'bin/swift-account-replicator', |
469 | 62 | 'bin/swift-account-audit', 'bin/swift-account-reaper', | 63 | 'bin/swift-account-server', 'bin/swift-account-stats-logger', |
470 | 63 | 'bin/swift-account-replicator', 'bin/swift-account-server', | 64 | 'bin/swift-auth-add-user', 'bin/swift-auth-recreate-accounts', |
471 | 64 | 'bin/swift-auth-add-user', | 65 | 'bin/swift-auth-server', 'bin/swift-auth-update-reseller-prefixes', |
472 | 65 | 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', | 66 | 'bin/swift-bench', 'bin/swift-container-auditor', |
473 | 66 | 'bin/swift-auth-update-reseller-prefixes', | 67 | 'bin/swift-container-replicator', 'bin/swift-container-server', |
474 | 67 | 'bin/swift-container-auditor', | 68 | 'bin/swift-container-updater', 'bin/swift-drive-audit', |
475 | 68 | 'bin/swift-container-replicator', | 69 | 'bin/swift-get-nodes', 'bin/swift-init', |
476 | 69 | 'bin/swift-container-server', 'bin/swift-container-updater', | 70 | 'bin/swift-log-stats-collector', 'bin/swift-log-uploader', |
477 | 70 | 'bin/swift-drive-audit', 'bin/swift-get-nodes', | 71 | 'bin/swift-object-auditor', 'bin/swift-object-info', |
478 | 71 | 'bin/swift-init', 'bin/swift-object-auditor', | 72 | 'bin/swift-object-janitor', 'bin/swift-object-replicator', |
479 | 72 | 'bin/swift-object-info', | 73 | 'bin/swift-object-server', 'bin/swift-object-updater', |
480 | 73 | 'bin/swift-object-replicator', | 74 | 'bin/swift-proxy-server', 'bin/swift-ring-builder', |
481 | 74 | 'bin/swift-object-server', | 75 | 'bin/swift-stats-populate', 'bin/swift-stats-report'], |
459 | 75 | 'bin/swift-object-updater', 'bin/swift-proxy-server', | ||
460 | 76 | 'bin/swift-ring-builder', 'bin/swift-stats-populate', | ||
461 | 77 | 'bin/swift-stats-report', | ||
462 | 78 | 'bin/swift-bench', | ||
463 | 79 | 'bin/swift-log-uploader', | ||
464 | 80 | 'bin/swift-log-stats-collector', | ||
465 | 81 | 'bin/swift-account-stats-logger', | ||
466 | 82 | ], | ||
482 | 83 | entry_points={ | 76 | entry_points={ |
483 | 84 | 'paste.app_factory': [ | 77 | 'paste.app_factory': [ |
484 | 85 | 'proxy=swift.proxy.server:app_factory', | 78 | 'proxy=swift.proxy.server:app_factory', |
485 | 86 | 79 | ||
486 | === modified file 'swift/common/constraints.py' | |||
487 | --- swift/common/constraints.py 2010-10-26 15:13:14 +0000 | |||
488 | +++ swift/common/constraints.py 2010-11-08 18:51:48 +0000 | |||
489 | @@ -19,8 +19,6 @@ | |||
490 | 19 | HTTPRequestEntityTooLarge | 19 | HTTPRequestEntityTooLarge |
491 | 20 | 20 | ||
492 | 21 | 21 | ||
493 | 22 | #: Max file size allowed for objects | ||
494 | 23 | MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024 + 2 | ||
495 | 24 | #: Max length of the name of a key for metadata | 22 | #: Max length of the name of a key for metadata |
496 | 25 | MAX_META_NAME_LENGTH = 128 | 23 | MAX_META_NAME_LENGTH = 128 |
497 | 26 | #: Max length of the value of a key for metadata | 24 | #: Max length of the value of a key for metadata |
498 | @@ -29,14 +27,18 @@ | |||
499 | 29 | MAX_META_COUNT = 90 | 27 | MAX_META_COUNT = 90 |
500 | 30 | #: Max overall size of metadata | 28 | #: Max overall size of metadata |
501 | 31 | MAX_META_OVERALL_SIZE = 4096 | 29 | MAX_META_OVERALL_SIZE = 4096 |
502 | 30 | #: Max account name length | ||
503 | 31 | MAX_ACCOUNT_NAME_LENGTH = 256 | ||
504 | 32 | #: Max container name length | ||
505 | 33 | MAX_CONTAINER_NAME_LENGTH = 256 | ||
506 | 32 | #: Max object name length | 34 | #: Max object name length |
507 | 33 | MAX_OBJECT_NAME_LENGTH = 1024 | 35 | MAX_OBJECT_NAME_LENGTH = 1024 |
508 | 34 | #: Max object list length of a get request for a container | 36 | #: Max object list length of a get request for a container |
509 | 35 | CONTAINER_LISTING_LIMIT = 10000 | 37 | CONTAINER_LISTING_LIMIT = 10000 |
510 | 36 | #: Max container list length of a get request for an account | 38 | #: Max container list length of a get request for an account |
511 | 37 | ACCOUNT_LISTING_LIMIT = 10000 | 39 | ACCOUNT_LISTING_LIMIT = 10000 |
514 | 38 | MAX_ACCOUNT_NAME_LENGTH = 256 | 40 | #: Default pickle protocol number used for Swift pickles |
515 | 39 | MAX_CONTAINER_NAME_LENGTH = 256 | 41 | PICKLE_PROTOCOL = 2 |
516 | 40 | 42 | ||
517 | 41 | 43 | ||
518 | 42 | def check_metadata(req, target_type): | 44 | def check_metadata(req, target_type): |
519 | @@ -82,19 +84,22 @@ | |||
520 | 82 | return None | 84 | return None |
521 | 83 | 85 | ||
522 | 84 | 86 | ||
524 | 85 | def check_object_creation(req, object_name): | 87 | def check_object_creation(req, object_name, max_object_size=0): |
525 | 86 | """ | 88 | """ |
526 | 87 | Check to ensure that everything is alright about an object to be created. | 89 | Check to ensure that everything is alright about an object to be created. |
527 | 88 | 90 | ||
528 | 89 | :param req: HTTP request object | 91 | :param req: HTTP request object |
529 | 90 | :param object_name: name of object to be created | 92 | :param object_name: name of object to be created |
530 | 93 | :param max_object_size: the maximum object size to check against; 0 if no | ||
531 | 94 | object size checking should be done | ||
532 | 91 | :raises HTTPRequestEntityTooLarge: the object is too large | 95 | :raises HTTPRequestEntityTooLarge: the object is too large |
533 | 92 | :raises HTTPLengthRequered: missing content-length header and not | 96 | :raises HTTPLengthRequered: missing content-length header and not |
534 | 93 | a chunked request | 97 | a chunked request |
535 | 94 | :raises HTTPBadRequest: missing or bad content-type header, or | 98 | :raises HTTPBadRequest: missing or bad content-type header, or |
536 | 95 | bad metadata | 99 | bad metadata |
537 | 96 | """ | 100 | """ |
539 | 97 | if req.content_length and req.content_length > MAX_FILE_SIZE: | 101 | if max_object_size > 0 and req.content_length and \ |
540 | 102 | req.content_length > max_object_size: | ||
541 | 98 | return HTTPRequestEntityTooLarge(body='Your request is too large.', | 103 | return HTTPRequestEntityTooLarge(body='Your request is too large.', |
542 | 99 | request=req, content_type='text/plain') | 104 | request=req, content_type='text/plain') |
543 | 100 | if req.content_length is None and \ | 105 | if req.content_length is None and \ |
544 | 101 | 106 | ||
545 | === modified file 'swift/common/db.py' | |||
546 | --- swift/common/db.py 2010-08-16 22:30:27 +0000 | |||
547 | +++ swift/common/db.py 2010-11-08 18:51:48 +0000 | |||
548 | @@ -32,6 +32,7 @@ | |||
549 | 32 | import simplejson as json | 32 | import simplejson as json |
550 | 33 | import sqlite3 | 33 | import sqlite3 |
551 | 34 | 34 | ||
552 | 35 | from swift.common.constraints import PICKLE_PROTOCOL | ||
553 | 35 | from swift.common.utils import normalize_timestamp, renamer, \ | 36 | from swift.common.utils import normalize_timestamp, renamer, \ |
554 | 36 | mkdirs, lock_parent_directory, fallocate | 37 | mkdirs, lock_parent_directory, fallocate |
555 | 37 | from swift.common.exceptions import LockTimeout | 38 | from swift.common.exceptions import LockTimeout |
556 | @@ -39,8 +40,6 @@ | |||
557 | 39 | 40 | ||
558 | 40 | #: Timeout for trying to connect to a DB | 41 | #: Timeout for trying to connect to a DB |
559 | 41 | BROKER_TIMEOUT = 25 | 42 | BROKER_TIMEOUT = 25 |
560 | 42 | #: Pickle protocol to use | ||
561 | 43 | PICKLE_PROTOCOL = 2 | ||
562 | 44 | #: Max number of pending entries | 43 | #: Max number of pending entries |
563 | 45 | PENDING_CAP = 131072 | 44 | PENDING_CAP = 131072 |
564 | 46 | 45 | ||
565 | 47 | 46 | ||
566 | === modified file 'swift/obj/auditor.py' | |||
567 | --- swift/obj/auditor.py 2010-10-21 18:32:10 +0000 | |||
568 | +++ swift/obj/auditor.py 2010-11-08 18:51:48 +0000 | |||
569 | @@ -18,8 +18,8 @@ | |||
570 | 18 | from hashlib import md5 | 18 | from hashlib import md5 |
571 | 19 | from random import random | 19 | from random import random |
572 | 20 | 20 | ||
575 | 21 | from swift.obj import server as object_server | 21 | from swift.obj.diskfile import DATADIR, DiskFile, invalidate_hash, \ |
576 | 22 | from swift.obj.replicator import invalidate_hash | 22 | read_metadata |
577 | 23 | from swift.common.utils import get_logger, renamer, audit_location_generator | 23 | from swift.common.utils import get_logger, renamer, audit_location_generator |
578 | 24 | from swift.common.exceptions import AuditException | 24 | from swift.common.exceptions import AuditException |
579 | 25 | from swift.common.daemon import Daemon | 25 | from swift.common.daemon import Daemon |
580 | @@ -45,10 +45,8 @@ | |||
581 | 45 | time.sleep(random() * self.interval) | 45 | time.sleep(random() * self.interval) |
582 | 46 | while True: | 46 | while True: |
583 | 47 | begin = time.time() | 47 | begin = time.time() |
588 | 48 | all_locs = audit_location_generator(self.devices, | 48 | all_locs = audit_location_generator(self.devices, DATADIR, |
589 | 49 | object_server.DATADIR, | 49 | mount_check=self.mount_check, logger=self.logger) |
586 | 50 | mount_check=self.mount_check, | ||
587 | 51 | logger=self.logger) | ||
590 | 52 | for path, device, partition in all_locs: | 50 | for path, device, partition in all_locs: |
591 | 53 | self.object_audit(path, device, partition) | 51 | self.object_audit(path, device, partition) |
592 | 54 | if time.time() - reported >= 3600: # once an hour | 52 | if time.time() - reported >= 3600: # once an hour |
593 | @@ -68,10 +66,8 @@ | |||
594 | 68 | """Run the object audit once.""" | 66 | """Run the object audit once.""" |
595 | 69 | self.logger.info('Begin object audit "once" mode') | 67 | self.logger.info('Begin object audit "once" mode') |
596 | 70 | begin = reported = time.time() | 68 | begin = reported = time.time() |
601 | 71 | all_locs = audit_location_generator(self.devices, | 69 | all_locs = audit_location_generator(self.devices, DATADIR, |
602 | 72 | object_server.DATADIR, | 70 | mount_check=self.mount_check, logger=self.logger) |
599 | 73 | mount_check=self.mount_check, | ||
600 | 74 | logger=self.logger) | ||
603 | 75 | for path, device, partition in all_locs: | 71 | for path, device, partition in all_locs: |
604 | 76 | self.object_audit(path, device, partition) | 72 | self.object_audit(path, device, partition) |
605 | 77 | if time.time() - reported >= 3600: # once an hour | 73 | if time.time() - reported >= 3600: # once an hour |
606 | @@ -99,14 +95,12 @@ | |||
607 | 99 | if not path.endswith('.data'): | 95 | if not path.endswith('.data'): |
608 | 100 | return | 96 | return |
609 | 101 | try: | 97 | try: |
611 | 102 | name = object_server.read_metadata(path)['name'] | 98 | name = read_metadata(path)['name'] |
612 | 103 | except Exception, exc: | 99 | except Exception, exc: |
613 | 104 | raise AuditException('Error when reading metadata: %s' % exc) | 100 | raise AuditException('Error when reading metadata: %s' % exc) |
614 | 105 | _, account, container, obj = name.split('/', 3) | 101 | _, account, container, obj = name.split('/', 3) |
619 | 106 | df = object_server.DiskFile(self.devices, device, | 102 | df = DiskFile(self.devices, device, partition, account, container, |
620 | 107 | partition, account, | 103 | obj, keep_data_fp=True) |
617 | 108 | container, obj, | ||
618 | 109 | keep_data_fp=True) | ||
621 | 110 | if df.data_file is None: | 104 | if df.data_file is None: |
622 | 111 | # file is deleted, we found the tombstone | 105 | # file is deleted, we found the tombstone |
623 | 112 | return | 106 | return |
624 | 113 | 107 | ||
625 | === added file 'swift/obj/diskfile.py' | |||
626 | --- swift/obj/diskfile.py 1970-01-01 00:00:00 +0000 | |||
627 | +++ swift/obj/diskfile.py 2010-11-08 18:51:48 +0000 | |||
628 | @@ -0,0 +1,507 @@ | |||
629 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
630 | 2 | # | ||
631 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
632 | 4 | # you may not use this file except in compliance with the License. | ||
633 | 5 | # You may obtain a copy of the License at | ||
634 | 6 | # | ||
635 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
636 | 8 | # | ||
637 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
638 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
639 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
640 | 12 | # implied. | ||
641 | 13 | # See the License for the specific language governing permissions and | ||
642 | 14 | # limitations under the License. | ||
643 | 15 | |||
644 | 16 | from __future__ import with_statement | ||
645 | 17 | import cPickle as pickle | ||
646 | 18 | import errno | ||
647 | 19 | import hashlib | ||
648 | 20 | import os | ||
649 | 21 | from contextlib import contextmanager | ||
650 | 22 | from os.path import basename, dirname, isdir, join, splitext | ||
651 | 23 | from tempfile import mkstemp | ||
652 | 24 | from time import time | ||
653 | 25 | |||
654 | 26 | from eventlet import tpool, sleep | ||
655 | 27 | from xattr import getxattr, setxattr | ||
656 | 28 | |||
657 | 29 | from swift.common.constraints import PICKLE_PROTOCOL | ||
658 | 30 | from swift.common.utils import drop_buffer_cache, hash_path, lock_path, \ | ||
659 | 31 | mkdirs, normalize_timestamp, renamer, split_path, storage_directory | ||
660 | 32 | |||
661 | 33 | |||
662 | 34 | DATADIR = 'objects' | ||
663 | 35 | SEGMENTSDIR = 'object_segments' | ||
664 | 36 | JANITORDIR = 'object_janitor' | ||
665 | 37 | HASH_FILE = 'hashes.pkl' | ||
666 | 38 | METADATA_KEY = 'user.swift.metadata' | ||
667 | 39 | ONE_WEEK = 604800 | ||
668 | 40 | |||
669 | 41 | |||
670 | 42 | def read_metadata(fd): | ||
671 | 43 | """ | ||
672 | 44 | Helper function to read the pickled metadata from an object file. | ||
673 | 45 | |||
674 | 46 | :param fd: file descriptor to load the metadata from | ||
675 | 47 | |||
676 | 48 | :returns: dictionary of metadata | ||
677 | 49 | """ | ||
678 | 50 | metadata = '' | ||
679 | 51 | key = 0 | ||
680 | 52 | try: | ||
681 | 53 | while True: | ||
682 | 54 | metadata += getxattr(fd, '%s%s' % (METADATA_KEY, (key or ''))) | ||
683 | 55 | key += 1 | ||
684 | 56 | except IOError: | ||
685 | 57 | pass | ||
686 | 58 | return pickle.loads(metadata) | ||
687 | 59 | |||
688 | 60 | |||
689 | 61 | def hash_suffix(path, reclaim_age): | ||
690 | 62 | """ | ||
691 | 63 | Performs reclamation and returns an md5 of all (remaining) files. | ||
692 | 64 | |||
693 | 65 | :param reclaim_age: age in seconds at which to remove tombstones | ||
694 | 66 | """ | ||
695 | 67 | md5 = hashlib.md5() | ||
696 | 68 | for hsh in sorted(os.listdir(path)): | ||
697 | 69 | hsh_path = join(path, hsh) | ||
698 | 70 | files = os.listdir(hsh_path) | ||
699 | 71 | if len(files) == 1: | ||
700 | 72 | if files[0].endswith('.ts'): | ||
701 | 73 | # remove tombstones older than reclaim_age | ||
702 | 74 | ts = files[0].rsplit('.', 1)[0] | ||
703 | 75 | if (time() - float(ts)) > reclaim_age: | ||
704 | 76 | os.unlink(join(hsh_path, files[0])) | ||
705 | 77 | files.remove(files[0]) | ||
706 | 78 | elif files: | ||
707 | 79 | files.sort(reverse=True) | ||
708 | 80 | meta = data = tomb = None | ||
709 | 81 | for filename in files: | ||
710 | 82 | if not meta and filename.endswith('.meta'): | ||
711 | 83 | meta = filename | ||
712 | 84 | if not data and filename.endswith('.data'): | ||
713 | 85 | data = filename | ||
714 | 86 | if not tomb and filename.endswith('.ts'): | ||
715 | 87 | tomb = filename | ||
716 | 88 | if (filename < tomb or # any file older than tomb | ||
717 | 89 | filename < data or # any file older than data | ||
718 | 90 | (filename.endswith('.meta') and | ||
719 | 91 | filename < meta)): # old meta | ||
720 | 92 | if filename.endswith('.data'): | ||
721 | 93 | fp = open(join(hsh_path, filename), 'rb') | ||
722 | 94 | metadata = read_metadata(fp) | ||
723 | 95 | if metadata.get('X-Object-Type') == 'manifest': | ||
724 | 96 | manifest = pickle.load(fp) | ||
725 | 97 | partition = dirname(path) | ||
726 | 98 | device = dirname(dirname(partition)) | ||
727 | 99 | devices = dirname(device) | ||
728 | 100 | partition = basename(partition) | ||
729 | 101 | device = basename(device) | ||
730 | 102 | account, container, obj = \ | ||
731 | 103 | split_path(metadata['name'], 3, | ||
732 | 104 | rest_with_last=True) | ||
733 | 105 | df = DiskFile(devices, device, partition, | ||
734 | 106 | 'Segment-Cleanup', manifest['x-timestamp'], | ||
735 | 107 | '%s/%s/%s' % (account, container, obj), | ||
736 | 108 | datadir=JANITORDIR) | ||
737 | 109 | df.store_janitor_segment_cleanup(account, | ||
738 | 110 | container, obj, | ||
739 | 111 | segment_count=(manifest['content-length'] / | ||
740 | 112 | manifest['x-segment-size'] + 1), | ||
741 | 113 | segment_last_deleted=None) | ||
742 | 114 | fp.close() | ||
743 | 115 | os.unlink(join(hsh_path, filename)) | ||
744 | 116 | files.remove(filename) | ||
745 | 117 | if not files: | ||
746 | 118 | os.rmdir(hsh_path) | ||
747 | 119 | for filename in files: | ||
748 | 120 | md5.update(filename) | ||
749 | 121 | try: | ||
750 | 122 | os.rmdir(path) | ||
751 | 123 | except OSError: | ||
752 | 124 | pass | ||
753 | 125 | return md5.hexdigest() | ||
754 | 126 | |||
755 | 127 | |||
756 | 128 | def recalculate_hashes(partition_dir, suffixes, reclaim_age=ONE_WEEK): | ||
757 | 129 | """ | ||
758 | 130 | Recalculates hashes for the given suffixes in the partition and updates | ||
759 | 131 | them in the partition's hashes file. | ||
760 | 132 | |||
761 | 133 | :param partition_dir: directory of the partition in which to recalculate | ||
762 | 134 | :param suffixes: list of suffixes to recalculate | ||
763 | 135 | :param reclaim_age: age in seconds at which tombstones should be removed | ||
764 | 136 | """ | ||
765 | 137 | |||
766 | 138 | def tpool_listdir(partition_dir): | ||
767 | 139 | return dict(((suff, None) for suff in os.listdir(partition_dir) | ||
768 | 140 | if len(suff) == 3 and isdir(join(partition_dir, suff)))) | ||
769 | 141 | hashes_file = join(partition_dir, HASH_FILE) | ||
770 | 142 | with lock_path(partition_dir): | ||
771 | 143 | try: | ||
772 | 144 | with open(hashes_file, 'rb') as fp: | ||
773 | 145 | hashes = pickle.load(fp) | ||
774 | 146 | except Exception: | ||
775 | 147 | hashes = tpool.execute(tpool_listdir, partition_dir) | ||
776 | 148 | for suffix in suffixes: | ||
777 | 149 | suffix_dir = join(partition_dir, suffix) | ||
778 | 150 | if os.path.exists(suffix_dir): | ||
779 | 151 | hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) | ||
780 | 152 | elif suffix in hashes: | ||
781 | 153 | del hashes[suffix] | ||
782 | 154 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
783 | 155 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
784 | 156 | renamer(hashes_file + '.tmp', hashes_file) | ||
785 | 157 | |||
786 | 158 | |||
787 | 159 | def invalidate_hash(suffix_dir): | ||
788 | 160 | """ | ||
789 | 161 | Invalidates the hash for a suffix_dir in the partition's hashes file. | ||
790 | 162 | |||
791 | 163 | :param suffix_dir: absolute path to suffix dir whose hash needs | ||
792 | 164 | invalidating | ||
793 | 165 | """ | ||
794 | 166 | |||
795 | 167 | suffix = os.path.basename(suffix_dir) | ||
796 | 168 | partition_dir = os.path.dirname(suffix_dir) | ||
797 | 169 | hashes_file = join(partition_dir, HASH_FILE) | ||
798 | 170 | with lock_path(partition_dir): | ||
799 | 171 | try: | ||
800 | 172 | with open(hashes_file, 'rb') as fp: | ||
801 | 173 | hashes = pickle.load(fp) | ||
802 | 174 | if suffix in hashes and not hashes[suffix]: | ||
803 | 175 | return | ||
804 | 176 | except Exception: | ||
805 | 177 | return | ||
806 | 178 | hashes[suffix] = None | ||
807 | 179 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
808 | 180 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
809 | 181 | renamer(hashes_file + '.tmp', hashes_file) | ||
810 | 182 | |||
811 | 183 | |||
812 | 184 | def get_hashes(partition_dir, do_listdir=True, reclaim_age=ONE_WEEK): | ||
813 | 185 | """ | ||
814 | 186 | Get a list of hashes for the suffix dir. do_listdir causes it to mistrust | ||
815 | 187 | the hash cache for suffix existence at the (unexpectedly high) cost of a | ||
816 | 188 | listdir. reclaim_age is just passed on to hash_suffix. | ||
817 | 189 | |||
818 | 190 | :param partition_dir: absolute path of partition to get hashes for | ||
819 | 191 | :param do_listdir: force existence check for all hashes in the partition | ||
820 | 192 | :param reclaim_age: age at which to remove tombstones | ||
821 | 193 | |||
822 | 194 | :returns: tuple of (number of suffix dirs hashed, dictionary of hashes) | ||
823 | 195 | """ | ||
824 | 196 | |||
825 | 197 | def tpool_listdir(hashes, partition_dir): | ||
826 | 198 | return dict(((suff, hashes.get(suff, None)) | ||
827 | 199 | for suff in os.listdir(partition_dir) | ||
828 | 200 | if len(suff) == 3 and isdir(join(partition_dir, suff)))) | ||
829 | 201 | hashed = 0 | ||
830 | 202 | hashes_file = join(partition_dir, HASH_FILE) | ||
831 | 203 | with lock_path(partition_dir): | ||
832 | 204 | modified = False | ||
833 | 205 | hashes = {} | ||
834 | 206 | try: | ||
835 | 207 | with open(hashes_file, 'rb') as fp: | ||
836 | 208 | hashes = pickle.load(fp) | ||
837 | 209 | except Exception: | ||
838 | 210 | do_listdir = True | ||
839 | 211 | if do_listdir: | ||
840 | 212 | hashes = tpool.execute(tpool_listdir, hashes, partition_dir) | ||
841 | 213 | modified = True | ||
842 | 214 | for suffix, hash_ in hashes.items(): | ||
843 | 215 | if not hash_: | ||
844 | 216 | suffix_dir = join(partition_dir, suffix) | ||
845 | 217 | if os.path.exists(suffix_dir): | ||
846 | 218 | try: | ||
847 | 219 | hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) | ||
848 | 220 | hashed += 1 | ||
849 | 221 | except OSError: | ||
850 | 222 | logging.exception('Error hashing suffix') | ||
851 | 223 | hashes[suffix] = None | ||
852 | 224 | else: | ||
853 | 225 | del hashes[suffix] | ||
854 | 226 | modified = True | ||
855 | 227 | sleep() | ||
856 | 228 | if modified: | ||
857 | 229 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
858 | 230 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
859 | 231 | renamer(hashes_file + '.tmp', hashes_file) | ||
860 | 232 | return hashed, hashes | ||
861 | 233 | |||
862 | 234 | |||
863 | 235 | class DiskFile(object): | ||
864 | 236 | """ | ||
865 | 237 | Manage files on disk for a single object. | ||
866 | 238 | |||
867 | 239 | :param path: path to devices on the node | ||
868 | 240 | :param device: device name | ||
869 | 241 | :param partition: partition on the device the object lives in | ||
870 | 242 | :param account: account name for the object | ||
871 | 243 | :param container: container name for the object | ||
872 | 244 | :param obj: object name for the object | ||
873 | 245 | :param keep_data_fp: if True, don't close the fp, otherwise close it | ||
874 | 246 | :param disk_chunk_size: size of chunks on file reads | ||
875 | 247 | :param datadir: Sets which directory the root of the data structure is | ||
876 | 248 | named (default: DATADIR) | ||
877 | 249 | :param segment: If set to not None, indicates which segment of an object | ||
878 | 250 | this file represents | ||
879 | 251 | :param segment_timestamp: X-Timestamp of the object's segments (set on the | ||
880 | 252 | PUT, not changed on POSTs), required if segment | ||
881 | 253 | is set to not None | ||
882 | 254 | """ | ||
883 | 255 | |||
884 | 256 | def __init__(self, path, device, partition, account, container, obj, | ||
885 | 257 | keep_data_fp=False, disk_chunk_size=65536, datadir=DATADIR, | ||
886 | 258 | segment=None, segment_timestamp=None): | ||
887 | 259 | self.account = account | ||
888 | 260 | self.container = container | ||
889 | 261 | self.obj = obj | ||
890 | 262 | self.fp = None | ||
891 | 263 | self.disk_chunk_size = disk_chunk_size | ||
892 | 264 | self.name = '/' + '/'.join((account, container, obj)) | ||
893 | 265 | if segment is not None: | ||
894 | 266 | segment_name = '%s/%s/%s' % (obj, segment_timestamp, segment) | ||
895 | 267 | segment_hash = hash_path(account, container, segment_name) | ||
896 | 268 | self.datadir = os.path.join(path, device, | ||
897 | 269 | storage_directory(SEGMENTSDIR, partition, segment_hash)) | ||
898 | 270 | name_hash = hash_path(account, container, obj) | ||
899 | 271 | self.no_longer_segment_datadir = os.path.join(path, device, | ||
900 | 272 | storage_directory(datadir, partition, name_hash)) | ||
901 | 273 | else: | ||
902 | 274 | name_hash = hash_path(account, container, obj) | ||
903 | 275 | self.datadir = self.no_longer_segment_datadir = os.path.join(path, | ||
904 | 276 | device, storage_directory(datadir, partition, name_hash)) | ||
905 | 277 | self.tmpdir = os.path.join(path, device, 'tmp') | ||
906 | 278 | self.metadata = {} | ||
907 | 279 | self.meta_file = None | ||
908 | 280 | self.data_file = None | ||
909 | 281 | if not os.path.exists(self.datadir): | ||
910 | 282 | return | ||
911 | 283 | files = sorted(os.listdir(self.datadir), reverse=True) | ||
912 | 284 | for file in files: | ||
913 | 285 | if file.endswith('.ts'): | ||
914 | 286 | self.data_file = self.meta_file = None | ||
915 | 287 | self.metadata = {'deleted': True} | ||
916 | 288 | return | ||
917 | 289 | if file.endswith('.meta') and not self.meta_file: | ||
918 | 290 | self.meta_file = os.path.join(self.datadir, file) | ||
919 | 291 | if file.endswith('.data') and not self.data_file: | ||
920 | 292 | self.data_file = os.path.join(self.datadir, file) | ||
921 | 293 | break | ||
922 | 294 | if not self.data_file: | ||
923 | 295 | return | ||
924 | 296 | self.fp = open(self.data_file, 'rb') | ||
925 | 297 | self.metadata = read_metadata(self.fp) | ||
926 | 298 | if not keep_data_fp: | ||
927 | 299 | self.close() | ||
928 | 300 | if self.meta_file: | ||
929 | 301 | with open(self.meta_file) as mfp: | ||
930 | 302 | for key in self.metadata.keys(): | ||
931 | 303 | if key.lower() not in ('content-type', 'content-encoding', | ||
932 | 304 | 'deleted', 'content-length', 'etag'): | ||
933 | 305 | del self.metadata[key] | ||
934 | 306 | self.metadata.update(read_metadata(mfp)) | ||
935 | 307 | |||
936 | 308 | def __iter__(self): | ||
937 | 309 | """Returns an iterator over the data file.""" | ||
938 | 310 | try: | ||
939 | 311 | dropped_cache = 0 | ||
940 | 312 | read = 0 | ||
941 | 313 | while True: | ||
942 | 314 | chunk = self.fp.read(self.disk_chunk_size) | ||
943 | 315 | if chunk: | ||
944 | 316 | read += len(chunk) | ||
945 | 317 | if read - dropped_cache > (1024 * 1024): | ||
946 | 318 | drop_buffer_cache(self.fp.fileno(), dropped_cache, | ||
947 | 319 | read - dropped_cache) | ||
948 | 320 | dropped_cache = read | ||
949 | 321 | yield chunk | ||
950 | 322 | else: | ||
951 | 323 | drop_buffer_cache(self.fp.fileno(), dropped_cache, | ||
952 | 324 | read - dropped_cache) | ||
953 | 325 | break | ||
954 | 326 | finally: | ||
955 | 327 | self.close() | ||
956 | 328 | |||
957 | 329 | def app_iter_range(self, start, stop): | ||
958 | 330 | """Returns an iterator over the data file for range (start, stop)""" | ||
959 | 331 | if start: | ||
960 | 332 | self.fp.seek(start) | ||
961 | 333 | if stop is not None: | ||
962 | 334 | length = stop - start | ||
963 | 335 | else: | ||
964 | 336 | length = None | ||
965 | 337 | for chunk in self: | ||
966 | 338 | if length is not None: | ||
967 | 339 | length -= len(chunk) | ||
968 | 340 | if length < 0: | ||
969 | 341 | # Chop off the extra: | ||
970 | 342 | yield chunk[:length] | ||
971 | 343 | break | ||
972 | 344 | yield chunk | ||
973 | 345 | |||
974 | 346 | def close(self): | ||
975 | 347 | """Close the file.""" | ||
976 | 348 | if self.fp: | ||
977 | 349 | self.fp.close() | ||
978 | 350 | self.fp = None | ||
979 | 351 | |||
980 | 352 | def is_deleted(self): | ||
981 | 353 | """ | ||
982 | 354 | Check if the file is deleted. | ||
983 | 355 | |||
984 | 356 | :returns: True if the file doesn't exist or has been flagged as | ||
985 | 357 | deleted. | ||
986 | 358 | """ | ||
987 | 359 | return not self.data_file or 'deleted' in self.metadata | ||
988 | 360 | |||
989 | 361 | @contextmanager | ||
990 | 362 | def mkstemp(self): | ||
991 | 363 | """Context manager to make a temporary file.""" | ||
992 | 364 | if not os.path.exists(self.tmpdir): | ||
993 | 365 | mkdirs(self.tmpdir) | ||
994 | 366 | fd, tmppath = mkstemp(dir=self.tmpdir) | ||
995 | 367 | try: | ||
996 | 368 | yield fd, tmppath | ||
997 | 369 | finally: | ||
998 | 370 | try: | ||
999 | 371 | os.close(fd) | ||
1000 | 372 | except OSError: | ||
1001 | 373 | pass | ||
1002 | 374 | try: | ||
1003 | 375 | os.unlink(tmppath) | ||
1004 | 376 | except OSError: | ||
1005 | 377 | pass | ||
1006 | 378 | |||
1007 | 379 | def put(self, fd, tmppath, metadata, extension='.data', | ||
1008 | 380 | no_longer_segment=False): | ||
1009 | 381 | """ | ||
1010 | 382 | Finalize writing the file on disk, and renames it from the temp file to | ||
1011 | 383 | the real location. This should be called after the data has been | ||
1012 | 384 | written to the temp file. | ||
1013 | 385 | |||
1014 | 386 | :params fd: file descriptor of the temp file | ||
1015 | 387 | :param tmppath: path to the temporary file being used | ||
1016 | 388 | :param metadata: dictionary of metada to be written | ||
1017 | 389 | :param extension: extension to be used when making the file | ||
1018 | 390 | :param no_longer_segment: Set to True if this was originally an object | ||
1019 | 391 | segment but no longer is (case with chunked transfer encoding when | ||
1020 | 392 | the object ends up less than the segment size) | ||
1021 | 393 | """ | ||
1022 | 394 | metadata['name'] = self.name | ||
1023 | 395 | timestamp = normalize_timestamp(metadata['X-Timestamp']) | ||
1024 | 396 | metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) | ||
1025 | 397 | key = 0 | ||
1026 | 398 | while metastr: | ||
1027 | 399 | setxattr(fd, '%s%s' % (METADATA_KEY, key or ''), metastr[:254]) | ||
1028 | 400 | metastr = metastr[254:] | ||
1029 | 401 | key += 1 | ||
1030 | 402 | if 'Content-Length' in metadata: | ||
1031 | 403 | drop_buffer_cache(fd, 0, int(metadata['Content-Length'])) | ||
1032 | 404 | os.fsync(fd) | ||
1033 | 405 | if no_longer_segment: | ||
1034 | 406 | self.datadir = self.no_longer_segment_datadir | ||
1035 | 407 | invalidate_hash(os.path.dirname(self.datadir)) | ||
1036 | 408 | renamer(tmppath, os.path.join(self.datadir, timestamp + extension)) | ||
1037 | 409 | self.metadata = metadata | ||
1038 | 410 | |||
1039 | 411 | def unlinkold(self, timestamp): | ||
1040 | 412 | """ | ||
1041 | 413 | Remove any older versions of the object file. Any file that has an | ||
1042 | 414 | older timestamp than timestamp will be deleted. | ||
1043 | 415 | |||
1044 | 416 | :param timestamp: timestamp to compare with each file | ||
1045 | 417 | """ | ||
1046 | 418 | timestamp = normalize_timestamp(timestamp) | ||
1047 | 419 | for fname in os.listdir(self.datadir): | ||
1048 | 420 | if fname < timestamp: | ||
1049 | 421 | try: | ||
1050 | 422 | os.unlink(os.path.join(self.datadir, fname)) | ||
1051 | 423 | except OSError, err: # pragma: no cover | ||
1052 | 424 | if err.errno != errno.ENOENT: | ||
1053 | 425 | raise | ||
1054 | 426 | |||
1055 | 427 | def tombstone(self, timestamp): | ||
1056 | 428 | """ | ||
1057 | 429 | Creates a tombstone for the DiskFile, indicating any versions older | ||
1058 | 430 | than `timestamp` should be removed. | ||
1059 | 431 | |||
1060 | 432 | :param timestamp: normalized timestamp of the tombstone | ||
1061 | 433 | """ | ||
1062 | 434 | with self.mkstemp() as (fd, tmppath): | ||
1063 | 435 | self.put(fd, tmppath, {'X-Timestamp': timestamp, 'deleted': True}, | ||
1064 | 436 | extension='.ts') | ||
1065 | 437 | self.unlinkold(timestamp) | ||
1066 | 438 | |||
1067 | 439 | def store_janitor_container_update(self, op, account, container, obj, | ||
1068 | 440 | headers, successes): | ||
1069 | 441 | """ | ||
1070 | 442 | Creates a .data file whose contents contain a operation for the | ||
1071 | 443 | object-janitor to send object metadata to the container servers. | ||
1072 | 444 | |||
1073 | 445 | :param op: The operation to send to the container server (usually PUT | ||
1074 | 446 | or DELETE). | ||
1075 | 447 | :param account: The account name for the object. | ||
1076 | 448 | :param container: The container name for the object. | ||
1077 | 449 | :param obj: The object name for the object. | ||
1078 | 450 | :param headers: The headers to include in the requests to the container | ||
1079 | 451 | servers. Should at least contain X-Timestamp indicating | ||
1080 | 452 | the version of the object metadata. | ||
1081 | 453 | :param successes: An array of container node ids that have already | ||
1082 | 454 | received the object metadata update. | ||
1083 | 455 | """ | ||
1084 | 456 | timestamp = normalize_timestamp(time()) | ||
1085 | 457 | with self.mkstemp() as (fd, tmppath): | ||
1086 | 458 | os.write(fd, pickle.dumps({'op': op, 'account': account, | ||
1087 | 459 | 'container': container, 'obj': obj, 'headers': headers, | ||
1088 | 460 | 'successes': successes}, PICKLE_PROTOCOL)) | ||
1089 | 461 | self.put(fd, tmppath, | ||
1090 | 462 | {'X-Op': 'Container-Update', 'X-Timestamp': timestamp}) | ||
1091 | 463 | self.unlinkold(timestamp) | ||
1092 | 464 | |||
1093 | 465 | def store_janitor_segment_cleanup(self, account, container, obj, | ||
1094 | 466 | segment_count, segment_last_deleted): | ||
1095 | 467 | """ | ||
1096 | 468 | Creates a .data file whose contents contain a operation for the | ||
1097 | 469 | object-janitor to send clean up object segments for an object. | ||
1098 | 470 | |||
1099 | 471 | Note that the DiskFile created for this operation is a bit different | ||
1100 | 472 | than most DiskFiles. The DiskFile account name should be | ||
1101 | 473 | 'Segment-Cleanup', the container name should be the segments' | ||
1102 | 474 | timestamp, and the object name should be the full account/container/obj | ||
1103 | 475 | path (no leading /). However, the account, container, obj given in this | ||
1104 | 476 | `store_janitor_segment_cleanup` call should be the usual as related to | ||
1105 | 477 | the actual object. This complexity is so that each set of segments are | ||
1106 | 478 | treated independently. | ||
1107 | 479 | |||
1108 | 480 | For example:: | ||
1109 | 481 | |||
1110 | 482 | df = DiskFile(devices, device, partition, | ||
1111 | 483 | 'Segment-Cleanup', manifest['x-timestamp'], | ||
1112 | 484 | '%s/%s/%s' % (account, container, obj), datadir=JANITORDIR) | ||
1113 | 485 | df.store_janitor_segment_cleanup(account, container, obj, None, | ||
1114 | 486 | None) | ||
1115 | 487 | |||
1116 | 488 | :param account: The account name for the object. | ||
1117 | 489 | :param container: The container name for the object. | ||
1118 | 490 | :param obj: The object name for the object. | ||
1119 | 491 | :param segment_count: The number of segments the object has or None if | ||
1120 | 492 | the number is not known. | ||
1121 | 493 | :param segment_last_deleted: The segment that was last deleted so the | ||
1122 | 494 | next pass of this operation can continue | ||
1123 | 495 | where it left off. Set to None if no | ||
1124 | 496 | segments have been deleted yet. | ||
1125 | 497 | """ | ||
1126 | 498 | timestamp = normalize_timestamp(time()) | ||
1127 | 499 | with self.mkstemp() as (fd, tmppath): | ||
1128 | 500 | os.write(fd, pickle.dumps({'account': account, | ||
1129 | 501 | 'container': container, 'obj': obj, | ||
1130 | 502 | 'segment_count': segment_count, | ||
1131 | 503 | 'segment_last_deleted': segment_last_deleted}, | ||
1132 | 504 | PICKLE_PROTOCOL)) | ||
1133 | 505 | self.put(fd, tmppath, | ||
1134 | 506 | {'X-Op': 'Segment-Cleanup', 'X-Timestamp': timestamp}) | ||
1135 | 507 | self.unlinkold(timestamp) | ||
1136 | 0 | 508 | ||
1137 | === added file 'swift/obj/janitor.py' | |||
1138 | --- swift/obj/janitor.py 1970-01-01 00:00:00 +0000 | |||
1139 | +++ swift/obj/janitor.py 2010-11-08 18:51:48 +0000 | |||
1140 | @@ -0,0 +1,516 @@ | |||
1141 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
1142 | 2 | # | ||
1143 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
1144 | 4 | # you may not use this file except in compliance with the License. | ||
1145 | 5 | # You may obtain a copy of the License at | ||
1146 | 6 | # | ||
1147 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
1148 | 8 | # | ||
1149 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
1150 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
1151 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
1152 | 12 | # implied. | ||
1153 | 13 | # See the License for the specific language governing permissions and | ||
1154 | 14 | # limitations under the License. | ||
1155 | 15 | |||
1156 | 16 | from __future__ import with_statement | ||
1157 | 17 | import cPickle as pickle | ||
1158 | 18 | import os | ||
1159 | 19 | import signal | ||
1160 | 20 | import sys | ||
1161 | 21 | from random import random | ||
1162 | 22 | from time import sleep, time | ||
1163 | 23 | |||
1164 | 24 | from eventlet import patcher, Timeout | ||
1165 | 25 | |||
1166 | 26 | from swift.common.bufferedhttp import http_connect | ||
1167 | 27 | from swift.common.exceptions import ConnectionTimeout | ||
1168 | 28 | from swift.common.ring import Ring | ||
1169 | 29 | from swift.common.utils import get_logger, normalize_timestamp, whataremyips | ||
1170 | 30 | from swift.common.daemon import Daemon | ||
1171 | 31 | from swift.obj.diskfile import DiskFile, JANITORDIR, read_metadata | ||
1172 | 32 | |||
1173 | 33 | |||
1174 | 34 | class ObjectJanitor(Daemon): | ||
1175 | 35 | """ | ||
1176 | 36 | Run background operations for the object server, such as updating container | ||
1177 | 37 | servers with new object metadata and cleaning up discarded segmented | ||
1178 | 38 | objects. | ||
1179 | 39 | """ | ||
1180 | 40 | |||
1181 | 41 | def __init__(self, conf): | ||
1182 | 42 | self.conf = conf | ||
1183 | 43 | self.logger = get_logger(conf, 'object-janitor') | ||
1184 | 44 | self.devices = conf.get('devices', '/srv/node') | ||
1185 | 45 | self.port = int(conf.get('bind_port', 6000)) | ||
1186 | 46 | self.my_node_ids = [] | ||
1187 | 47 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | ||
1188 | 48 | ('true', 't', '1', 'on', 'yes', 'y') | ||
1189 | 49 | swift_dir = conf.get('swift_dir', '/etc/swift') | ||
1190 | 50 | self.interval = int(conf.get('interval', 300)) | ||
1191 | 51 | self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') | ||
1192 | 52 | self.object_ring = None | ||
1193 | 53 | self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') | ||
1194 | 54 | self.container_ring = None | ||
1195 | 55 | self.concurrency = int(conf.get('concurrency', 1)) | ||
1196 | 56 | self.slowdown = float(conf.get('slowdown', 0.01)) | ||
1197 | 57 | self.node_timeout = int(conf.get('node_timeout', 10)) | ||
1198 | 58 | self.conn_timeout = float(conf.get('conn_timeout', 0.5)) | ||
1199 | 59 | self.segment_reclaim_age = int(conf.get('segment_reclaim_age', 604800)) | ||
1200 | 60 | self.segments_per_pass = int(conf.get('segments_per_pass', 10)) | ||
1201 | 61 | self.container_update_successes = 0 | ||
1202 | 62 | self.container_update_failures = 0 | ||
1203 | 63 | self.segment_cleanup_completions = 0 | ||
1204 | 64 | self.segment_cleanup_segments = 0 | ||
1205 | 65 | self.segment_cleanup_failures = 0 | ||
1206 | 66 | |||
1207 | 67 | def get_object_ring(self): | ||
1208 | 68 | """Get the object ring. Load it, if it hasn't been yet.""" | ||
1209 | 69 | if not self.object_ring: | ||
1210 | 70 | self.logger.debug( | ||
1211 | 71 | 'Loading object ring from %s' % self.object_ring_path) | ||
1212 | 72 | self.object_ring = Ring(self.object_ring_path) | ||
1213 | 73 | return self.object_ring | ||
1214 | 74 | |||
1215 | 75 | def get_container_ring(self): | ||
1216 | 76 | """Get the container ring. Load it, if it hasn't been yet.""" | ||
1217 | 77 | if not self.container_ring: | ||
1218 | 78 | self.logger.debug( | ||
1219 | 79 | 'Loading container ring from %s' % self.container_ring_path) | ||
1220 | 80 | self.container_ring = Ring(self.container_ring_path) | ||
1221 | 81 | return self.container_ring | ||
1222 | 82 | |||
1223 | 83 | def run_forever(self): | ||
1224 | 84 | """Run the janitor continuously.""" | ||
1225 | 85 | sleep(random() * self.interval) | ||
1226 | 86 | while True: | ||
1227 | 87 | begin = time() | ||
1228 | 88 | self.full_sweep() | ||
1229 | 89 | elapsed = time() - begin | ||
1230 | 90 | if elapsed < self.interval: | ||
1231 | 91 | sleep(self.interval - elapsed) | ||
1232 | 92 | |||
1233 | 93 | def run_once(self): | ||
1234 | 94 | """Run the janitor once.""" | ||
1235 | 95 | self.full_sweep(fork=False) | ||
1236 | 96 | |||
1237 | 97 | def full_sweep(self, fork=True): | ||
1238 | 98 | """ | ||
1239 | 99 | Run a full sweep of the server for object janitor operations. | ||
1240 | 100 | |||
1241 | 101 | :param fork: If True, subprocesses will be forked for each device up to | ||
1242 | 102 | self.concurrency at any given time. | ||
1243 | 103 | """ | ||
1244 | 104 | self.logger.info('Begin object janitor sweep') | ||
1245 | 105 | begin = time() | ||
1246 | 106 | my_ips = whataremyips() | ||
1247 | 107 | self.my_node_ids = [n['id'] for n in self.get_object_ring().devs | ||
1248 | 108 | if n and n['ip'] in my_ips and n['port'] == self.port] | ||
1249 | 109 | pids = [] | ||
1250 | 110 | for device in os.listdir(self.devices): | ||
1251 | 111 | if self.mount_check and not \ | ||
1252 | 112 | os.path.ismount(os.path.join(self.devices, device)): | ||
1253 | 113 | self.logger.warn('Skipping %s as it is not mounted' % device) | ||
1254 | 114 | continue | ||
1255 | 115 | if fork: | ||
1256 | 116 | while len(pids) >= self.concurrency: | ||
1257 | 117 | pids.remove(os.wait()[0]) | ||
1258 | 118 | # read from rings to ensure they're fresh | ||
1259 | 119 | self.get_object_ring().get_nodes('') | ||
1260 | 120 | self.get_container_ring().get_nodes('') | ||
1261 | 121 | if fork: | ||
1262 | 122 | pid = os.fork() | ||
1263 | 123 | if pid: | ||
1264 | 124 | pids.append(pid) | ||
1265 | 125 | else: | ||
1266 | 126 | signal.signal(signal.SIGTERM, signal.SIG_DFL) | ||
1267 | 127 | patcher.monkey_patch(all=False, socket=True) | ||
1268 | 128 | self.object_sweep(os.path.join(self.devices, device)) | ||
1269 | 129 | sys.exit() | ||
1270 | 130 | else: | ||
1271 | 131 | self.object_sweep(os.path.join(self.devices, device)) | ||
1272 | 132 | if fork: | ||
1273 | 133 | while pids: | ||
1274 | 134 | pids.remove(os.wait()[0]) | ||
1275 | 135 | elapsed = time() - begin | ||
1276 | 136 | self.logger.info('Object janitor sweep completed: %.02fs' % elapsed) | ||
1277 | 137 | |||
1278 | 138 | def object_sweep(self, device): | ||
1279 | 139 | """ | ||
1280 | 140 | If there are janitor pendings on the device, walk each one and update. | ||
1281 | 141 | |||
1282 | 142 | :param device: path to device | ||
1283 | 143 | """ | ||
1284 | 144 | self.container_update_successes = 0 | ||
1285 | 145 | self.container_update_failures = 0 | ||
1286 | 146 | self.segment_cleanup_completions = 0 | ||
1287 | 147 | self.segment_cleanup_segments = 0 | ||
1288 | 148 | self.segment_cleanup_failures = 0 | ||
1289 | 149 | begin = time() | ||
1290 | 150 | janitordir = os.path.join(device, JANITORDIR) | ||
1291 | 151 | try: | ||
1292 | 152 | if not os.path.isdir(janitordir): | ||
1293 | 153 | return | ||
1294 | 154 | for partition in os.listdir(janitordir): | ||
1295 | 155 | partition_path = os.path.join(janitordir, partition) | ||
1296 | 156 | for suffix in os.listdir(partition_path): | ||
1297 | 157 | suffix_path = os.path.join(partition_path, suffix) | ||
1298 | 158 | for janitor in os.listdir(suffix_path): | ||
1299 | 159 | janitor_path = os.path.join(suffix_path, janitor) | ||
1300 | 160 | self.process_object_janitor(device, partition, | ||
1301 | 161 | janitor_path) | ||
1302 | 162 | sleep(self.slowdown) | ||
1303 | 163 | finally: | ||
1304 | 164 | elapsed = time() - begin | ||
1305 | 165 | self.logger.info('Object janitor sweep of %s completed in %.02fs: ' | ||
1306 | 166 | 'container updates: %s successes, %s failures; segment ' | ||
1307 | 167 | 'cleanups: %s completions, %s segments, %s failures' % | ||
1308 | 168 | (device, elapsed, | ||
1309 | 169 | self.container_update_successes, | ||
1310 | 170 | self.container_update_failures, | ||
1311 | 171 | self.segment_cleanup_completions, | ||
1312 | 172 | self.segment_cleanup_segments, | ||
1313 | 173 | self.segment_cleanup_failures)) | ||
1314 | 174 | |||
1315 | 175 | def process_object_janitor(self, device, partition, janitor_path): | ||
1316 | 176 | """ | ||
1317 | 177 | Process the object janitor operation. | ||
1318 | 178 | |||
1319 | 179 | :param device: path to device | ||
1320 | 180 | :param partition: partition for the object | ||
1321 | 181 | :param janitor_path: path to DiskFile for the janitor operation | ||
1322 | 182 | """ | ||
1323 | 183 | data_file = None | ||
1324 | 184 | files = sorted(os.listdir(janitor_path), reverse=True) | ||
1325 | 185 | for file_ in files: | ||
1326 | 186 | if file_.endswith('.ts'): | ||
1327 | 187 | break | ||
1328 | 188 | if file_.endswith('.data'): | ||
1329 | 189 | data_file = os.path.join(janitor_path, file_) | ||
1330 | 190 | break | ||
1331 | 191 | if not data_file: | ||
1332 | 192 | return | ||
1333 | 193 | metadata = read_metadata(data_file) | ||
1334 | 194 | _, account, container, obj = metadata['name'].split('/', 3) | ||
1335 | 195 | if metadata['X-Op'] == 'Container-Update': | ||
1336 | 196 | self.process_container_update(device, partition, account, | ||
1337 | 197 | container, obj) | ||
1338 | 198 | elif metadata['X-Op'] == 'Segment-Cleanup': | ||
1339 | 199 | if self.get_object_ring().get_part_nodes(int(partition))[0]['id'] \ | ||
1340 | 200 | in self.my_node_ids: | ||
1341 | 201 | self.process_segment_cleanup(device, partition, account, | ||
1342 | 202 | container, obj) | ||
1343 | 203 | else: | ||
1344 | 204 | self.logger.error('ERROR: Unknown X-Op: %s' % metadata['X-Op']) | ||
1345 | 205 | return | ||
1346 | 206 | |||
1347 | 207 | def process_container_update(self, device, partition, account, container, | ||
1348 | 208 | obj): | ||
1349 | 209 | """ | ||
1350 | 210 | Process the container update operation, sending the object information | ||
1351 | 211 | to the container server. | ||
1352 | 212 | |||
1353 | 213 | :param device: path to device | ||
1354 | 214 | :param partition: partition for the object | ||
1355 | 215 | :param account: account for the object | ||
1356 | 216 | :param container: container for the object | ||
1357 | 217 | :param obj: name for the object | ||
1358 | 218 | """ | ||
1359 | 219 | disk_file = DiskFile(self.devices, device, partition, account, | ||
1360 | 220 | container, obj, keep_data_fp=True, datadir=JANITORDIR) | ||
1361 | 221 | if disk_file.is_deleted(): | ||
1362 | 222 | return | ||
1363 | 223 | update = pickle.loads(''.join(iter(disk_file))) | ||
1364 | 224 | op = update['op'] | ||
1365 | 225 | account = update['account'] | ||
1366 | 226 | container = update['container'] | ||
1367 | 227 | obj = update['obj'] | ||
1368 | 228 | headers = update['headers'] | ||
1369 | 229 | successes = update.get('successes', []) | ||
1370 | 230 | part, nodes = self.get_container_ring().get_nodes(account, container) | ||
1371 | 231 | path = '/%s/%s/%s' % (account, container, obj) | ||
1372 | 232 | success = True | ||
1373 | 233 | for node in nodes: | ||
1374 | 234 | if node['id'] not in successes: | ||
1375 | 235 | status = self.container_update(node, part, op, path, headers) | ||
1376 | 236 | if not (200 <= status < 300) and status != 404: | ||
1377 | 237 | success = False | ||
1378 | 238 | else: | ||
1379 | 239 | successes.append(node['id']) | ||
1380 | 240 | if success: | ||
1381 | 241 | self.container_update_successes += 1 | ||
1382 | 242 | self.logger.debug('Update sent for %s %s' % | ||
1383 | 243 | (path, disk_file.datadir)) | ||
1384 | 244 | disk_file.tombstone(normalize_timestamp(time())) | ||
1385 | 245 | else: | ||
1386 | 246 | self.container_update_failures += 1 | ||
1387 | 247 | self.logger.debug('Update failed for %s %s' % | ||
1388 | 248 | (path, disk_file.datadir)) | ||
1389 | 249 | if len(update.get('successes', [])) != len(successes): | ||
1390 | 250 | disk_file.store_janitor_container_update(op, account, | ||
1391 | 251 | container, obj, headers, successes) | ||
1392 | 252 | |||
1393 | 253 | def container_update(self, node, part, op, obj, headers): | ||
1394 | 254 | """ | ||
1395 | 255 | Perform the actual container update network operation, sending the | ||
1396 | 256 | object information to the container server. | ||
1397 | 257 | |||
1398 | 258 | :param node: node dictionary from the container ring | ||
1399 | 259 | :param part: partition that holds the container | ||
1400 | 260 | :param op: operation performed (ex: 'POST' or 'DELETE') | ||
1401 | 261 | :param obj: object name being updated | ||
1402 | 262 | :param headers: headers to send with the update | ||
1403 | 263 | """ | ||
1404 | 264 | try: | ||
1405 | 265 | with ConnectionTimeout(self.conn_timeout): | ||
1406 | 266 | conn = http_connect(node['ip'], node['port'], node['device'], | ||
1407 | 267 | part, op, obj, headers) | ||
1408 | 268 | with Timeout(self.node_timeout): | ||
1409 | 269 | resp = conn.getresponse() | ||
1410 | 270 | resp.read() | ||
1411 | 271 | return resp.status | ||
1412 | 272 | except: | ||
1413 | 273 | self.logger.exception('ERROR with remote server ' | ||
1414 | 274 | '%(ip)s:%(port)s/%(device)s' % node) | ||
1415 | 275 | return 500 | ||
1416 | 276 | |||
1417 | 277 | def process_segment_cleanup(self, device, partition, account, container, | ||
1418 | 278 | obj): | ||
1419 | 279 | """ | ||
1420 | 280 | Process the segment cleanup operation, checking to see if the | ||
1421 | 281 | operation should have completed long ago but still has no manifest and | ||
1422 | 282 | therefore should have the orphaned segments removed. | ||
1423 | 283 | |||
1424 | 284 | :param device: path to device | ||
1425 | 285 | :param partition: partition for the operation | ||
1426 | 286 | :param account: account for the operation (should always be | ||
1427 | 287 | "Segment-Cleanup") | ||
1428 | 288 | :param container: container for the operation (actually the timestamp | ||
1429 | 289 | of the original PUT) | ||
1430 | 290 | :param obj: name for the object (actually the full path | ||
1431 | 291 | account/container/object for the original object PUT) | ||
1432 | 292 | """ | ||
1433 | 293 | disk_file = DiskFile(self.devices, device, partition, account, | ||
1434 | 294 | container, obj, keep_data_fp=True, datadir=JANITORDIR) | ||
1435 | 295 | if disk_file.is_deleted(): | ||
1436 | 296 | return | ||
1437 | 297 | disk_file_data = pickle.loads(''.join(iter(disk_file))) | ||
1438 | 298 | segment_timestamp = container | ||
1439 | 299 | # If not None, indicates we have started cleaning up these segments | ||
1440 | 300 | # already and should just continue the operation. | ||
1441 | 301 | segment_last_deleted = disk_file_data.get('segment_last_deleted', None) | ||
1442 | 302 | # Sometimes we'll have no clue how many segments there are, which is | ||
1443 | 303 | # what None means. | ||
1444 | 304 | segment_count = disk_file_data.get('segment_count', None) | ||
1445 | 305 | account = disk_file_data['account'] | ||
1446 | 306 | container = disk_file_data['container'] | ||
1447 | 307 | obj = disk_file_data['obj'] | ||
1448 | 308 | path = '/%s/%s/%s' % (account, container, obj) | ||
1449 | 309 | if segment_last_deleted is not None: | ||
1450 | 310 | self.logger.debug('Continue cleaning up segments for %s/%s/%s %s ' | ||
1451 | 311 | '@%s' % (device, partition, path, segment_timestamp, | ||
1452 | 312 | segment_last_deleted)) | ||
1453 | 313 | self.cleanup_segments(disk_file, account, container, obj, | ||
1454 | 314 | segment_timestamp, segment_count, segment_last_deleted) | ||
1455 | 315 | return | ||
1456 | 316 | part, nodes = self.get_object_ring().get_nodes(account, container, obj) | ||
1457 | 317 | newest_object = None | ||
1458 | 318 | newest_manifest = None | ||
1459 | 319 | responses = 0 | ||
1460 | 320 | for node in nodes: | ||
1461 | 321 | try: | ||
1462 | 322 | with ConnectionTimeout(self.conn_timeout): | ||
1463 | 323 | conn = http_connect(node['ip'], node['port'], | ||
1464 | 324 | node['device'], part, 'GET', path) | ||
1465 | 325 | with Timeout(self.node_timeout): | ||
1466 | 326 | resp = conn.getresponse() | ||
1467 | 327 | responses += 1 | ||
1468 | 328 | if resp.status // 100 != 2: | ||
1469 | 329 | conn.close() | ||
1470 | 330 | continue | ||
1471 | 331 | if resp.getheader('x-object-type', '') != 'manifest': | ||
1472 | 332 | if resp.getheader('x-timestamp') > newest_object: | ||
1473 | 333 | newest_object = resp.getheader('x-timestamp') | ||
1474 | 334 | conn.close() | ||
1475 | 335 | if newest_object > segment_timestamp: | ||
1476 | 336 | # We don't have to talk to the other nodes if we | ||
1477 | 337 | # already know we have a newer object. | ||
1478 | 338 | break | ||
1479 | 339 | continue | ||
1480 | 340 | with Timeout(self.node_timeout): | ||
1481 | 341 | body = resp.read() | ||
1482 | 342 | manifest = pickle.loads(body) | ||
1483 | 343 | if manifest['x-timestamp'] > newest_manifest: | ||
1484 | 344 | newest_manifest = manifest['x-timestamp'] | ||
1485 | 345 | conn.close() | ||
1486 | 346 | if newest_manifest > segment_timestamp: | ||
1487 | 347 | # We don't have to talk to the other nodes if we already | ||
1488 | 348 | # know we have a newer manifest. | ||
1489 | 349 | break | ||
1490 | 350 | except: | ||
1491 | 351 | self.logger.exception('ERROR with remote server ' | ||
1492 | 352 | '%(ip)s:%(port)s/%(device)s' % node) | ||
1493 | 353 | if newest_object > segment_timestamp or \ | ||
1494 | 354 | newest_manifest > segment_timestamp: | ||
1495 | 355 | self.logger.debug('Newer object/manifest; discarding old ' | ||
1496 | 356 | 'segments for %s/%s/%s %s' % (device, partition, path, | ||
1497 | 357 | segment_timestamp)) | ||
1498 | 358 | self.cleanup_segments(disk_file, account, container, obj, | ||
1499 | 359 | segment_timestamp, segment_count, segment_last_deleted) | ||
1500 | 360 | elif newest_manifest == segment_timestamp: | ||
1501 | 361 | self.logger.debug('Exact manifest confirmed; discarding janitor ' | ||
1502 | 362 | 'cleanup operation for %s/%s/%s %s' % (device, partition, | ||
1503 | 363 | path, segment_timestamp)) | ||
1504 | 364 | disk_file.tombstone(normalize_timestamp(time())) | ||
1505 | 365 | elif responses == len(nodes) and \ | ||
1506 | 366 | time() - float(segment_timestamp) > \ | ||
1507 | 367 | self.segment_reclaim_age: | ||
1508 | 368 | self.logger.debug('All nodes agree the manifest still does not ' | ||
1509 | 369 | 'exist after %ss; discarding orphaned segments for ' | ||
1510 | 370 | '%s/%s/%s %s' % (self.segment_reclaim_age, device, | ||
1511 | 371 | partition, path, segment_timestamp)) | ||
1512 | 372 | self.cleanup_segments(disk_file, account, container, obj, | ||
1513 | 373 | segment_timestamp, segment_count, segment_last_deleted) | ||
1514 | 374 | |||
1515 | 375 | def cleanup_segments(self, disk_file, account, container, obj, | ||
1516 | 376 | segment_timestamp, segment_count, segment_last_deleted): | ||
1517 | 377 | """ | ||
1518 | 378 | Issues DELETEs to the segments of the object, up to | ||
1519 | 379 | self.segments_per_pass. If all the segments are not deleted before this | ||
1520 | 380 | function returns, it will update the disk_file with the latest point | ||
1521 | 381 | completed. | ||
1522 | 382 | |||
1523 | 383 | :param disk_file: DiskFile for the Segment Cleanup operation. | ||
1524 | 384 | :param account: Account name for the manifest object to delete segments | ||
1525 | 385 | for. | ||
1526 | 386 | :param container: Container name for the manifest object to delete | ||
1527 | 387 | segments for. | ||
1528 | 388 | :param obj: Object name for the manifest object to delete segments for. | ||
1529 | 389 | :param segment_timestamp: The timestamp for the segments themselves. | ||
1530 | 390 | :param segment_count: The number of segments for the object; may be | ||
1531 | 391 | None if not yet known and this function should | ||
1532 | 392 | try to determine the count. | ||
1533 | 393 | :param segment_last_deleted: The segment number that was last deleted; | ||
1534 | 394 | None of none have yet been deleted. This | ||
1535 | 395 | allows continuation of previous deletion | ||
1536 | 396 | attempts. | ||
1537 | 397 | """ | ||
1538 | 398 | path = '/%s/%s/%s' % (account, container, obj) | ||
1539 | 399 | if segment_last_deleted is None: | ||
1540 | 400 | segment_last_deleted = -1 | ||
1541 | 401 | if segment_count is None: | ||
1542 | 402 | # We need to determine how many segments there are so that if we | ||
1543 | 403 | # crash this run, next run we'll know what range we'll need to work | ||
1544 | 404 | # within. Example: Determine we have 10 segments; delete the first | ||
1545 | 405 | # five and crash; next run we now know we need to delete no more | ||
1546 | 406 | # than 10 segments. With the approach of just deleting segments | ||
1547 | 407 | # until a 404 and not predetermining the count, with such a crash | ||
1548 | 408 | # we'd never delete the last five segments. | ||
1549 | 409 | segment_count = 0 | ||
1550 | 410 | assumptions = 0 | ||
1551 | 411 | while True: | ||
1552 | 412 | if segment_count: | ||
1553 | 413 | ring_obj = \ | ||
1554 | 414 | '%s/%s/%s' % (obj, segment_timestamp, segment_count) | ||
1555 | 415 | else: | ||
1556 | 416 | ring_obj = obj | ||
1557 | 417 | part, nodes = self.get_object_ring().get_nodes(account, | ||
1558 | 418 | container, ring_obj) | ||
1559 | 419 | headers = {'X-Object-Segment': str(segment_count), | ||
1560 | 420 | 'X-Object-Segment-Timestamp': segment_timestamp} | ||
1561 | 421 | status = 0 | ||
1562 | 422 | for node in nodes: | ||
1563 | 423 | try: | ||
1564 | 424 | with ConnectionTimeout(self.conn_timeout): | ||
1565 | 425 | conn = http_connect(node['ip'], node['port'], | ||
1566 | 426 | node['device'], part, 'HEAD', path, | ||
1567 | 427 | headers=headers) | ||
1568 | 428 | with Timeout(self.node_timeout): | ||
1569 | 429 | resp = conn.getresponse() | ||
1570 | 430 | resp.read() | ||
1571 | 431 | if resp.status == 404: | ||
1572 | 432 | status = 404 | ||
1573 | 433 | elif resp.status // 100 == 2: | ||
1574 | 434 | status = 200 | ||
1575 | 435 | break | ||
1576 | 436 | except: | ||
1577 | 437 | self.logger.exception('ERROR with remote server ' | ||
1578 | 438 | '%(ip)s:%(port)s/%(device)s' % node) | ||
1579 | 439 | if status == 404: | ||
1580 | 440 | break | ||
1581 | 441 | segment_count += 1 | ||
1582 | 442 | if not status: | ||
1583 | 443 | # We couldn't determine if a segment existed, so we'll just | ||
1584 | 444 | # assume it did and move to the next one; but we'll only do | ||
1585 | 445 | # this for five consecutive segments before just giving up | ||
1586 | 446 | # completely and trying the next run. | ||
1587 | 447 | assumptions += 1 | ||
1588 | 448 | if assumptions > 5: | ||
1589 | 449 | self.segment_cleanup_failures += 1 | ||
1590 | 450 | return | ||
1591 | 451 | assumptions = 0 | ||
1592 | 452 | disk_file.store_janitor_segment_cleanup(account, container, obj, | ||
1593 | 453 | segment_count, segment_last_deleted) | ||
1594 | 454 | starting_segment_last_deleted = segment_last_deleted | ||
1595 | 455 | try: | ||
1596 | 456 | segments_this_pass = self.segments_per_pass | ||
1597 | 457 | while True: | ||
1598 | 458 | segment = segment_last_deleted + 1 | ||
1599 | 459 | if segment: | ||
1600 | 460 | ring_obj = '%s/%s/%s' % (obj, segment_timestamp, segment) | ||
1601 | 461 | else: | ||
1602 | 462 | ring_obj = obj | ||
1603 | 463 | part, nodes = self.get_object_ring().get_nodes(account, | ||
1604 | 464 | container, ring_obj) | ||
1605 | 465 | headers = {'X-Object-Segment': str(segment), | ||
1606 | 466 | 'X-Object-Segment-Timestamp': segment_timestamp, | ||
1607 | 467 | 'X-Timestamp': normalize_timestamp(time())} | ||
1608 | 468 | not_found_count = 0 | ||
1609 | 469 | success = False | ||
1610 | 470 | for node in nodes: | ||
1611 | 471 | try: | ||
1612 | 472 | with ConnectionTimeout(self.conn_timeout): | ||
1613 | 473 | conn = http_connect(node['ip'], node['port'], | ||
1614 | 474 | node['device'], part, 'DELETE', path, | ||
1615 | 475 | headers=headers) | ||
1616 | 476 | with Timeout(self.node_timeout): | ||
1617 | 477 | resp = conn.getresponse() | ||
1618 | 478 | resp.read() | ||
1619 | 479 | if resp.status == 404: | ||
1620 | 480 | not_found_count += 1 | ||
1621 | 481 | elif resp.status // 100 == 2: | ||
1622 | 482 | # For the sake of this clean up operation, we're | ||
1623 | 483 | # going to consider even one success complete | ||
1624 | 484 | # success. | ||
1625 | 485 | success = True | ||
1626 | 486 | except: | ||
1627 | 487 | self.logger.exception('ERROR with remote server ' | ||
1628 | 488 | '%(ip)s:%(port)s/%(device)s' % node) | ||
1629 | 489 | if not_found_count == len(nodes): | ||
1630 | 490 | success = True | ||
1631 | 491 | if not success: | ||
1632 | 492 | # If we didn't even have one success, we'll have to leave | ||
1633 | 493 | # the janitor operation to retry later. | ||
1634 | 494 | disk_file.store_janitor_segment_cleanup(account, container, | ||
1635 | 495 | obj, segment_count, segment_last_deleted) | ||
1636 | 496 | self.segment_cleanup_failures += 1 | ||
1637 | 497 | return | ||
1638 | 498 | segment_last_deleted = segment | ||
1639 | 499 | self.segment_cleanup_segments += 1 | ||
1640 | 500 | if segment >= segment_count - 1: | ||
1641 | 501 | disk_file.tombstone(normalize_timestamp(time())) | ||
1642 | 502 | self.segment_cleanup_completions += 1 | ||
1643 | 503 | return | ||
1644 | 504 | segments_this_pass -= 1 | ||
1645 | 505 | if segments_this_pass <= 0: | ||
1646 | 506 | disk_file.store_janitor_segment_cleanup(account, container, | ||
1647 | 507 | obj, segment_count, segment_last_deleted) | ||
1648 | 508 | return | ||
1649 | 509 | except: | ||
1650 | 510 | # If we get an unexpected exception, log it and try to update the | ||
1651 | 511 | # disk file to indicate the segment last deleted. | ||
1652 | 512 | self.logger.exception('ERROR unexpected exception with %s:' % path) | ||
1653 | 513 | self.segment_cleanup_failures += 1 | ||
1654 | 514 | if segment_last_deleted != starting_segment_last_deleted: | ||
1655 | 515 | disk_file.store_janitor_segment_cleanup(account, container, | ||
1656 | 516 | obj, segment_count, segment_last_deleted) | ||
1657 | 0 | 517 | ||
1658 | === modified file 'swift/obj/replicator.py' | |||
1659 | --- swift/obj/replicator.py 2010-10-19 01:05:54 +0000 | |||
1660 | +++ swift/obj/replicator.py 2010-11-08 18:51:48 +0000 | |||
1661 | @@ -19,7 +19,6 @@ | |||
1662 | 19 | import shutil | 19 | import shutil |
1663 | 20 | import time | 20 | import time |
1664 | 21 | import logging | 21 | import logging |
1665 | 22 | import hashlib | ||
1666 | 23 | import itertools | 22 | import itertools |
1667 | 24 | import cPickle as pickle | 23 | import cPickle as pickle |
1668 | 25 | 24 | ||
1669 | @@ -29,168 +28,16 @@ | |||
1670 | 29 | from eventlet.support.greenlets import GreenletExit | 28 | from eventlet.support.greenlets import GreenletExit |
1671 | 30 | 29 | ||
1672 | 31 | from swift.common.ring import Ring | 30 | from swift.common.ring import Ring |
1675 | 32 | from swift.common.utils import whataremyips, unlink_older_than, lock_path, \ | 31 | from swift.common.utils import compute_eta, get_logger, unlink_older_than, \ |
1676 | 33 | renamer, compute_eta, get_logger | 32 | whataremyips |
1677 | 34 | from swift.common.bufferedhttp import http_connect | 33 | from swift.common.bufferedhttp import http_connect |
1678 | 35 | from swift.common.daemon import Daemon | 34 | from swift.common.daemon import Daemon |
1679 | 35 | from swift.obj.diskfile import DATADIR, get_hashes, JANITORDIR, \ | ||
1680 | 36 | recalculate_hashes, SEGMENTSDIR | ||
1681 | 37 | |||
1682 | 36 | 38 | ||
1683 | 37 | hubs.use_hub('poll') | 39 | hubs.use_hub('poll') |
1684 | 38 | 40 | ||
1685 | 39 | PICKLE_PROTOCOL = 2 | ||
1686 | 40 | ONE_WEEK = 604800 | ||
1687 | 41 | HASH_FILE = 'hashes.pkl' | ||
1688 | 42 | |||
1689 | 43 | |||
1690 | 44 | def hash_suffix(path, reclaim_age): | ||
1691 | 45 | """ | ||
1692 | 46 | Performs reclamation and returns an md5 of all (remaining) files. | ||
1693 | 47 | |||
1694 | 48 | :param reclaim_age: age in seconds at which to remove tombstones | ||
1695 | 49 | """ | ||
1696 | 50 | md5 = hashlib.md5() | ||
1697 | 51 | for hsh in sorted(os.listdir(path)): | ||
1698 | 52 | hsh_path = join(path, hsh) | ||
1699 | 53 | files = os.listdir(hsh_path) | ||
1700 | 54 | if len(files) == 1: | ||
1701 | 55 | if files[0].endswith('.ts'): | ||
1702 | 56 | # remove tombstones older than reclaim_age | ||
1703 | 57 | ts = files[0].rsplit('.', 1)[0] | ||
1704 | 58 | if (time.time() - float(ts)) > reclaim_age: | ||
1705 | 59 | os.unlink(join(hsh_path, files[0])) | ||
1706 | 60 | files.remove(files[0]) | ||
1707 | 61 | elif files: | ||
1708 | 62 | files.sort(reverse=True) | ||
1709 | 63 | meta = data = tomb = None | ||
1710 | 64 | for filename in files: | ||
1711 | 65 | if not meta and filename.endswith('.meta'): | ||
1712 | 66 | meta = filename | ||
1713 | 67 | if not data and filename.endswith('.data'): | ||
1714 | 68 | data = filename | ||
1715 | 69 | if not tomb and filename.endswith('.ts'): | ||
1716 | 70 | tomb = filename | ||
1717 | 71 | if (filename < tomb or # any file older than tomb | ||
1718 | 72 | filename < data or # any file older than data | ||
1719 | 73 | (filename.endswith('.meta') and | ||
1720 | 74 | filename < meta)): # old meta | ||
1721 | 75 | os.unlink(join(hsh_path, filename)) | ||
1722 | 76 | files.remove(filename) | ||
1723 | 77 | if not files: | ||
1724 | 78 | os.rmdir(hsh_path) | ||
1725 | 79 | for filename in files: | ||
1726 | 80 | md5.update(filename) | ||
1727 | 81 | try: | ||
1728 | 82 | os.rmdir(path) | ||
1729 | 83 | except OSError: | ||
1730 | 84 | pass | ||
1731 | 85 | return md5.hexdigest() | ||
1732 | 86 | |||
1733 | 87 | |||
1734 | 88 | def recalculate_hashes(partition_dir, suffixes, reclaim_age=ONE_WEEK): | ||
1735 | 89 | """ | ||
1736 | 90 | Recalculates hashes for the given suffixes in the partition and updates | ||
1737 | 91 | them in the partition's hashes file. | ||
1738 | 92 | |||
1739 | 93 | :param partition_dir: directory of the partition in which to recalculate | ||
1740 | 94 | :param suffixes: list of suffixes to recalculate | ||
1741 | 95 | :param reclaim_age: age in seconds at which tombstones should be removed | ||
1742 | 96 | """ | ||
1743 | 97 | |||
1744 | 98 | def tpool_listdir(partition_dir): | ||
1745 | 99 | return dict(((suff, None) for suff in os.listdir(partition_dir) | ||
1746 | 100 | if len(suff) == 3 and isdir(join(partition_dir, suff)))) | ||
1747 | 101 | hashes_file = join(partition_dir, HASH_FILE) | ||
1748 | 102 | with lock_path(partition_dir): | ||
1749 | 103 | try: | ||
1750 | 104 | with open(hashes_file, 'rb') as fp: | ||
1751 | 105 | hashes = pickle.load(fp) | ||
1752 | 106 | except Exception: | ||
1753 | 107 | hashes = tpool.execute(tpool_listdir, partition_dir) | ||
1754 | 108 | for suffix in suffixes: | ||
1755 | 109 | suffix_dir = join(partition_dir, suffix) | ||
1756 | 110 | if os.path.exists(suffix_dir): | ||
1757 | 111 | hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) | ||
1758 | 112 | elif suffix in hashes: | ||
1759 | 113 | del hashes[suffix] | ||
1760 | 114 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
1761 | 115 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
1762 | 116 | renamer(hashes_file + '.tmp', hashes_file) | ||
1763 | 117 | |||
1764 | 118 | |||
1765 | 119 | def invalidate_hash(suffix_dir): | ||
1766 | 120 | """ | ||
1767 | 121 | Invalidates the hash for a suffix_dir in the partition's hashes file. | ||
1768 | 122 | |||
1769 | 123 | :param suffix_dir: absolute path to suffix dir whose hash needs | ||
1770 | 124 | invalidating | ||
1771 | 125 | """ | ||
1772 | 126 | |||
1773 | 127 | suffix = os.path.basename(suffix_dir) | ||
1774 | 128 | partition_dir = os.path.dirname(suffix_dir) | ||
1775 | 129 | hashes_file = join(partition_dir, HASH_FILE) | ||
1776 | 130 | with lock_path(partition_dir): | ||
1777 | 131 | try: | ||
1778 | 132 | with open(hashes_file, 'rb') as fp: | ||
1779 | 133 | hashes = pickle.load(fp) | ||
1780 | 134 | if suffix in hashes and not hashes[suffix]: | ||
1781 | 135 | return | ||
1782 | 136 | except Exception: | ||
1783 | 137 | return | ||
1784 | 138 | hashes[suffix] = None | ||
1785 | 139 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
1786 | 140 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
1787 | 141 | renamer(hashes_file + '.tmp', hashes_file) | ||
1788 | 142 | |||
1789 | 143 | |||
1790 | 144 | def get_hashes(partition_dir, do_listdir=True, reclaim_age=ONE_WEEK): | ||
1791 | 145 | """ | ||
1792 | 146 | Get a list of hashes for the suffix dir. do_listdir causes it to mistrust | ||
1793 | 147 | the hash cache for suffix existence at the (unexpectedly high) cost of a | ||
1794 | 148 | listdir. reclaim_age is just passed on to hash_suffix. | ||
1795 | 149 | |||
1796 | 150 | :param partition_dir: absolute path of partition to get hashes for | ||
1797 | 151 | :param do_listdir: force existence check for all hashes in the partition | ||
1798 | 152 | :param reclaim_age: age at which to remove tombstones | ||
1799 | 153 | |||
1800 | 154 | :returns: tuple of (number of suffix dirs hashed, dictionary of hashes) | ||
1801 | 155 | """ | ||
1802 | 156 | |||
1803 | 157 | def tpool_listdir(hashes, partition_dir): | ||
1804 | 158 | return dict(((suff, hashes.get(suff, None)) | ||
1805 | 159 | for suff in os.listdir(partition_dir) | ||
1806 | 160 | if len(suff) == 3 and isdir(join(partition_dir, suff)))) | ||
1807 | 161 | hashed = 0 | ||
1808 | 162 | hashes_file = join(partition_dir, HASH_FILE) | ||
1809 | 163 | with lock_path(partition_dir): | ||
1810 | 164 | modified = False | ||
1811 | 165 | hashes = {} | ||
1812 | 166 | try: | ||
1813 | 167 | with open(hashes_file, 'rb') as fp: | ||
1814 | 168 | hashes = pickle.load(fp) | ||
1815 | 169 | except Exception: | ||
1816 | 170 | do_listdir = True | ||
1817 | 171 | if do_listdir: | ||
1818 | 172 | hashes = tpool.execute(tpool_listdir, hashes, partition_dir) | ||
1819 | 173 | modified = True | ||
1820 | 174 | for suffix, hash_ in hashes.items(): | ||
1821 | 175 | if not hash_: | ||
1822 | 176 | suffix_dir = join(partition_dir, suffix) | ||
1823 | 177 | if os.path.exists(suffix_dir): | ||
1824 | 178 | try: | ||
1825 | 179 | hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) | ||
1826 | 180 | hashed += 1 | ||
1827 | 181 | except OSError: | ||
1828 | 182 | logging.exception('Error hashing suffix') | ||
1829 | 183 | hashes[suffix] = None | ||
1830 | 184 | else: | ||
1831 | 185 | del hashes[suffix] | ||
1832 | 186 | modified = True | ||
1833 | 187 | sleep() | ||
1834 | 188 | if modified: | ||
1835 | 189 | with open(hashes_file + '.tmp', 'wb') as fp: | ||
1836 | 190 | pickle.dump(hashes, fp, PICKLE_PROTOCOL) | ||
1837 | 191 | renamer(hashes_file + '.tmp', hashes_file) | ||
1838 | 192 | return hashed, hashes | ||
1839 | 193 | |||
1840 | 194 | 41 | ||
1841 | 195 | class ObjectReplicator(Daemon): | 42 | class ObjectReplicator(Daemon): |
1842 | 196 | """ | 43 | """ |
1843 | @@ -302,7 +149,7 @@ | |||
1844 | 302 | if not had_any: | 149 | if not had_any: |
1845 | 303 | return False | 150 | return False |
1846 | 304 | args.append(join(rsync_module, node['device'], | 151 | args.append(join(rsync_module, node['device'], |
1848 | 305 | 'objects', job['partition'])) | 152 | job.get('datadir', DATADIR), job['partition'])) |
1849 | 306 | return self._rsync(args) == 0 | 153 | return self._rsync(args) == 0 |
1850 | 307 | 154 | ||
1851 | 308 | def check_ring(self): | 155 | def check_ring(self): |
1852 | @@ -337,12 +184,14 @@ | |||
1853 | 337 | for node in job['nodes']: | 184 | for node in job['nodes']: |
1854 | 338 | success = self.rsync(node, job, suffixes) | 185 | success = self.rsync(node, job, suffixes) |
1855 | 339 | if success: | 186 | if success: |
1856 | 187 | headers = {'Content-Length': '0', | ||
1857 | 188 | 'X-Data-Dir': job.get('datadir', DATADIR)} | ||
1858 | 340 | with Timeout(self.http_timeout): | 189 | with Timeout(self.http_timeout): |
1859 | 341 | http_connect(node['ip'], | 190 | http_connect(node['ip'], |
1860 | 342 | node['port'], | 191 | node['port'], |
1861 | 343 | node['device'], job['partition'], 'REPLICATE', | 192 | node['device'], job['partition'], 'REPLICATE', |
1862 | 344 | '/' + '-'.join(suffixes), | 193 | '/' + '-'.join(suffixes), |
1864 | 345 | headers={'Content-Length': '0'}).getresponse().read() | 194 | headers=headers).getresponse().read() |
1865 | 346 | responses.append(success) | 195 | responses.append(success) |
1866 | 347 | if not suffixes or (len(responses) == \ | 196 | if not suffixes or (len(responses) == \ |
1867 | 348 | self.object_ring.replica_count and all(responses)): | 197 | self.object_ring.replica_count and all(responses)): |
1868 | @@ -374,10 +223,12 @@ | |||
1869 | 374 | node = next(nodes) | 223 | node = next(nodes) |
1870 | 375 | attempts_left -= 1 | 224 | attempts_left -= 1 |
1871 | 376 | try: | 225 | try: |
1872 | 226 | headers = {'Content-Length': '0', | ||
1873 | 227 | 'X-Data-Dir': job.get('datadir', DATADIR)} | ||
1874 | 377 | with Timeout(self.http_timeout): | 228 | with Timeout(self.http_timeout): |
1875 | 378 | resp = http_connect(node['ip'], node['port'], | 229 | resp = http_connect(node['ip'], node['port'], |
1876 | 379 | node['device'], job['partition'], 'REPLICATE', | 230 | node['device'], job['partition'], 'REPLICATE', |
1878 | 380 | '', headers={'Content-Length': '0'}).getresponse() | 231 | '', headers=headers).getresponse() |
1879 | 381 | if resp.status == 507: | 232 | if resp.status == 507: |
1880 | 382 | self.logger.error('%s/%s responded as unmounted' % | 233 | self.logger.error('%s/%s responded as unmounted' % |
1881 | 383 | (node['ip'], node['device'])) | 234 | (node['ip'], node['device'])) |
1882 | @@ -397,11 +248,13 @@ | |||
1883 | 397 | self.rsync(node, job, suffixes) | 248 | self.rsync(node, job, suffixes) |
1884 | 398 | recalculate_hashes(job['path'], suffixes, | 249 | recalculate_hashes(job['path'], suffixes, |
1885 | 399 | reclaim_age=self.reclaim_age) | 250 | reclaim_age=self.reclaim_age) |
1886 | 251 | headers = {'Content-Length': '0', | ||
1887 | 252 | 'X-Data-Dir': job.get('datadir', DATADIR)} | ||
1888 | 400 | with Timeout(self.http_timeout): | 253 | with Timeout(self.http_timeout): |
1889 | 401 | conn = http_connect(node['ip'], node['port'], | 254 | conn = http_connect(node['ip'], node['port'], |
1890 | 402 | node['device'], job['partition'], 'REPLICATE', | 255 | node['device'], job['partition'], 'REPLICATE', |
1891 | 403 | '/' + '-'.join(suffixes), | 256 | '/' + '-'.join(suffixes), |
1893 | 404 | headers={'Content-Length': '0'}) | 257 | headers=headers) |
1894 | 405 | conn.getresponse().read() | 258 | conn.getresponse().read() |
1895 | 406 | self.suffix_sync += len(suffixes) | 259 | self.suffix_sync += len(suffixes) |
1896 | 407 | except (Exception, Timeout): | 260 | except (Exception, Timeout): |
1897 | @@ -489,24 +342,27 @@ | |||
1898 | 489 | dev for dev in self.object_ring.devs | 342 | dev for dev in self.object_ring.devs |
1899 | 490 | if dev and dev['ip'] in ips and dev['port'] == self.port]: | 343 | if dev and dev['ip'] in ips and dev['port'] == self.port]: |
1900 | 491 | dev_path = join(self.devices_dir, local_dev['device']) | 344 | dev_path = join(self.devices_dir, local_dev['device']) |
1901 | 492 | obj_path = join(dev_path, 'objects') | ||
1902 | 493 | tmp_path = join(dev_path, 'tmp') | ||
1903 | 494 | if self.mount_check and not os.path.ismount(dev_path): | 345 | if self.mount_check and not os.path.ismount(dev_path): |
1904 | 495 | self.logger.warn('%s is not mounted' % local_dev['device']) | 346 | self.logger.warn('%s is not mounted' % local_dev['device']) |
1905 | 496 | continue | 347 | continue |
1906 | 348 | tmp_path = join(dev_path, 'tmp') | ||
1907 | 497 | unlink_older_than(tmp_path, time.time() - self.reclaim_age) | 349 | unlink_older_than(tmp_path, time.time() - self.reclaim_age) |
1920 | 498 | if not os.path.exists(obj_path): | 350 | for datadir in (DATADIR, JANITORDIR, SEGMENTSDIR): |
1921 | 499 | continue | 351 | obj_path = join(dev_path, datadir) |
1922 | 500 | for partition in os.listdir(obj_path): | 352 | if os.path.exists(obj_path): |
1923 | 501 | try: | 353 | for partition in os.listdir(obj_path): |
1924 | 502 | nodes = [node for node in | 354 | try: |
1925 | 503 | self.object_ring.get_part_nodes(int(partition)) | 355 | nodes = [node for node in |
1926 | 504 | if node['id'] != local_dev['id']] | 356 | self.object_ring.get_part_nodes( |
1927 | 505 | jobs.append(dict(path=join(obj_path, partition), | 357 | int(partition)) |
1928 | 506 | nodes=nodes, delete=len(nodes) > 2, | 358 | if node['id'] != local_dev['id']] |
1929 | 507 | partition=partition)) | 359 | jobs.append(dict( |
1930 | 508 | except ValueError: | 360 | path=join(obj_path, partition), |
1931 | 509 | continue | 361 | nodes=nodes, delete=len(nodes) > 2, |
1932 | 362 | partition=partition, | ||
1933 | 363 | datadir=datadir)) | ||
1934 | 364 | except ValueError: | ||
1935 | 365 | continue | ||
1936 | 510 | random.shuffle(jobs) | 366 | random.shuffle(jobs) |
1937 | 511 | # Partititons that need to be deleted take priority | 367 | # Partititons that need to be deleted take priority |
1938 | 512 | jobs.sort(key=lambda job: not job['delete']) | 368 | jobs.sort(key=lambda job: not job['delete']) |
1939 | 513 | 369 | ||
1940 | === modified file 'swift/obj/server.py' | |||
1941 | --- swift/obj/server.py 2010-11-01 21:47:48 +0000 | |||
1942 | +++ swift/obj/server.py 2010-11-08 18:51:48 +0000 | |||
1943 | @@ -17,225 +17,28 @@ | |||
1944 | 17 | 17 | ||
1945 | 18 | from __future__ import with_statement | 18 | from __future__ import with_statement |
1946 | 19 | import cPickle as pickle | 19 | import cPickle as pickle |
1947 | 20 | import errno | ||
1948 | 21 | import os | 20 | import os |
1949 | 22 | import time | 21 | import time |
1950 | 23 | import traceback | 22 | import traceback |
1951 | 24 | from datetime import datetime | 23 | from datetime import datetime |
1952 | 25 | from hashlib import md5 | 24 | from hashlib import md5 |
1953 | 26 | from tempfile import mkstemp | ||
1954 | 27 | from urllib import unquote | 25 | from urllib import unquote |
1955 | 28 | from contextlib import contextmanager | ||
1956 | 29 | 26 | ||
1957 | 30 | from webob import Request, Response, UTC | 27 | from webob import Request, Response, UTC |
1958 | 31 | from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ | 28 | from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ |
1959 | 32 | HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ | 29 | HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ |
1960 | 33 | HTTPNotModified, HTTPPreconditionFailed, \ | 30 | HTTPNotModified, HTTPPreconditionFailed, \ |
1961 | 34 | HTTPRequestTimeout, HTTPUnprocessableEntity, HTTPMethodNotAllowed | 31 | HTTPRequestTimeout, HTTPUnprocessableEntity, HTTPMethodNotAllowed |
1962 | 35 | from xattr import getxattr, setxattr | ||
1963 | 36 | from eventlet import sleep, Timeout | 32 | from eventlet import sleep, Timeout |
1964 | 37 | 33 | ||
1968 | 38 | from swift.common.utils import mkdirs, normalize_timestamp, \ | 34 | from swift.common.utils import drop_buffer_cache, fallocate, get_logger, \ |
1969 | 39 | storage_directory, hash_path, renamer, fallocate, \ | 35 | mkdirs, normalize_timestamp, split_path |
1967 | 40 | split_path, drop_buffer_cache, get_logger, write_pickle | ||
1970 | 41 | from swift.common.bufferedhttp import http_connect | 36 | from swift.common.bufferedhttp import http_connect |
1971 | 42 | from swift.common.constraints import check_object_creation, check_mount, \ | 37 | from swift.common.constraints import check_object_creation, check_mount, \ |
1972 | 43 | check_float, check_utf8 | 38 | check_float, check_utf8 |
1973 | 44 | from swift.common.exceptions import ConnectionTimeout | 39 | from swift.common.exceptions import ConnectionTimeout |
2168 | 45 | from swift.obj.replicator import get_hashes, invalidate_hash, \ | 40 | from swift.obj.diskfile import DATADIR, DiskFile, get_hashes, JANITORDIR, \ |
2169 | 46 | recalculate_hashes | 41 | recalculate_hashes |
1976 | 47 | |||
1977 | 48 | |||
1978 | 49 | DATADIR = 'objects' | ||
1979 | 50 | ASYNCDIR = 'async_pending' | ||
1980 | 51 | PICKLE_PROTOCOL = 2 | ||
1981 | 52 | METADATA_KEY = 'user.swift.metadata' | ||
1982 | 53 | MAX_OBJECT_NAME_LENGTH = 1024 | ||
1983 | 54 | |||
1984 | 55 | |||
1985 | 56 | def read_metadata(fd): | ||
1986 | 57 | """ | ||
1987 | 58 | Helper function to read the pickled metadata from an object file. | ||
1988 | 59 | |||
1989 | 60 | :param fd: file descriptor to load the metadata from | ||
1990 | 61 | |||
1991 | 62 | :returns: dictionary of metadata | ||
1992 | 63 | """ | ||
1993 | 64 | metadata = '' | ||
1994 | 65 | key = 0 | ||
1995 | 66 | try: | ||
1996 | 67 | while True: | ||
1997 | 68 | metadata += getxattr(fd, '%s%s' % (METADATA_KEY, (key or ''))) | ||
1998 | 69 | key += 1 | ||
1999 | 70 | except IOError: | ||
2000 | 71 | pass | ||
2001 | 72 | return pickle.loads(metadata) | ||
2002 | 73 | |||
2003 | 74 | |||
2004 | 75 | class DiskFile(object): | ||
2005 | 76 | """ | ||
2006 | 77 | Manage object files on disk. | ||
2007 | 78 | |||
2008 | 79 | :param path: path to devices on the node | ||
2009 | 80 | :param device: device name | ||
2010 | 81 | :param partition: partition on the device the object lives in | ||
2011 | 82 | :param account: account name for the object | ||
2012 | 83 | :param container: container name for the object | ||
2013 | 84 | :param obj: object name for the object | ||
2014 | 85 | :param keep_data_fp: if True, don't close the fp, otherwise close it | ||
2015 | 86 | :param disk_chunk_Size: size of chunks on file reads | ||
2016 | 87 | """ | ||
2017 | 88 | |||
2018 | 89 | def __init__(self, path, device, partition, account, container, obj, | ||
2019 | 90 | keep_data_fp=False, disk_chunk_size=65536): | ||
2020 | 91 | self.disk_chunk_size = disk_chunk_size | ||
2021 | 92 | self.name = '/' + '/'.join((account, container, obj)) | ||
2022 | 93 | name_hash = hash_path(account, container, obj) | ||
2023 | 94 | self.datadir = os.path.join(path, device, | ||
2024 | 95 | storage_directory(DATADIR, partition, name_hash)) | ||
2025 | 96 | self.tmpdir = os.path.join(path, device, 'tmp') | ||
2026 | 97 | self.metadata = {} | ||
2027 | 98 | self.meta_file = None | ||
2028 | 99 | self.data_file = None | ||
2029 | 100 | if not os.path.exists(self.datadir): | ||
2030 | 101 | return | ||
2031 | 102 | files = sorted(os.listdir(self.datadir), reverse=True) | ||
2032 | 103 | for file in files: | ||
2033 | 104 | if file.endswith('.ts'): | ||
2034 | 105 | self.data_file = self.meta_file = None | ||
2035 | 106 | self.metadata = {'deleted': True} | ||
2036 | 107 | return | ||
2037 | 108 | if file.endswith('.meta') and not self.meta_file: | ||
2038 | 109 | self.meta_file = os.path.join(self.datadir, file) | ||
2039 | 110 | if file.endswith('.data') and not self.data_file: | ||
2040 | 111 | self.data_file = os.path.join(self.datadir, file) | ||
2041 | 112 | break | ||
2042 | 113 | if not self.data_file: | ||
2043 | 114 | return | ||
2044 | 115 | self.fp = open(self.data_file, 'rb') | ||
2045 | 116 | self.metadata = read_metadata(self.fp) | ||
2046 | 117 | if not keep_data_fp: | ||
2047 | 118 | self.close() | ||
2048 | 119 | if self.meta_file: | ||
2049 | 120 | with open(self.meta_file) as mfp: | ||
2050 | 121 | for key in self.metadata.keys(): | ||
2051 | 122 | if key.lower() not in ('content-type', 'content-encoding', | ||
2052 | 123 | 'deleted', 'content-length', 'etag'): | ||
2053 | 124 | del self.metadata[key] | ||
2054 | 125 | self.metadata.update(read_metadata(mfp)) | ||
2055 | 126 | |||
2056 | 127 | def __iter__(self): | ||
2057 | 128 | """Returns an iterator over the data file.""" | ||
2058 | 129 | try: | ||
2059 | 130 | dropped_cache = 0 | ||
2060 | 131 | read = 0 | ||
2061 | 132 | while True: | ||
2062 | 133 | chunk = self.fp.read(self.disk_chunk_size) | ||
2063 | 134 | if chunk: | ||
2064 | 135 | read += len(chunk) | ||
2065 | 136 | if read - dropped_cache > (1024 * 1024): | ||
2066 | 137 | drop_buffer_cache(self.fp.fileno(), dropped_cache, | ||
2067 | 138 | read - dropped_cache) | ||
2068 | 139 | dropped_cache = read | ||
2069 | 140 | yield chunk | ||
2070 | 141 | else: | ||
2071 | 142 | drop_buffer_cache(self.fp.fileno(), dropped_cache, | ||
2072 | 143 | read - dropped_cache) | ||
2073 | 144 | break | ||
2074 | 145 | finally: | ||
2075 | 146 | self.close() | ||
2076 | 147 | |||
2077 | 148 | def app_iter_range(self, start, stop): | ||
2078 | 149 | """Returns an iterator over the data file for range (start, stop)""" | ||
2079 | 150 | if start: | ||
2080 | 151 | self.fp.seek(start) | ||
2081 | 152 | if stop is not None: | ||
2082 | 153 | length = stop - start | ||
2083 | 154 | else: | ||
2084 | 155 | length = None | ||
2085 | 156 | for chunk in self: | ||
2086 | 157 | if length is not None: | ||
2087 | 158 | length -= len(chunk) | ||
2088 | 159 | if length < 0: | ||
2089 | 160 | # Chop off the extra: | ||
2090 | 161 | yield chunk[:length] | ||
2091 | 162 | break | ||
2092 | 163 | yield chunk | ||
2093 | 164 | |||
2094 | 165 | def close(self): | ||
2095 | 166 | """Close the file.""" | ||
2096 | 167 | if self.fp: | ||
2097 | 168 | self.fp.close() | ||
2098 | 169 | self.fp = None | ||
2099 | 170 | |||
2100 | 171 | def is_deleted(self): | ||
2101 | 172 | """ | ||
2102 | 173 | Check if the file is deleted. | ||
2103 | 174 | |||
2104 | 175 | :returns: True if the file doesn't exist or has been flagged as | ||
2105 | 176 | deleted. | ||
2106 | 177 | """ | ||
2107 | 178 | return not self.data_file or 'deleted' in self.metadata | ||
2108 | 179 | |||
2109 | 180 | @contextmanager | ||
2110 | 181 | def mkstemp(self): | ||
2111 | 182 | """Contextmanager to make a temporary file.""" | ||
2112 | 183 | if not os.path.exists(self.tmpdir): | ||
2113 | 184 | mkdirs(self.tmpdir) | ||
2114 | 185 | fd, tmppath = mkstemp(dir=self.tmpdir) | ||
2115 | 186 | try: | ||
2116 | 187 | yield fd, tmppath | ||
2117 | 188 | finally: | ||
2118 | 189 | try: | ||
2119 | 190 | os.close(fd) | ||
2120 | 191 | except OSError: | ||
2121 | 192 | pass | ||
2122 | 193 | try: | ||
2123 | 194 | os.unlink(tmppath) | ||
2124 | 195 | except OSError: | ||
2125 | 196 | pass | ||
2126 | 197 | |||
2127 | 198 | def put(self, fd, tmppath, metadata, extension='.data'): | ||
2128 | 199 | """ | ||
2129 | 200 | Finalize writing the file on disk, and renames it from the temp file to | ||
2130 | 201 | the real location. This should be called after the data has been | ||
2131 | 202 | written to the temp file. | ||
2132 | 203 | |||
2133 | 204 | :params fd: file descriptor of the temp file | ||
2134 | 205 | :param tmppath: path to the temporary file being used | ||
2135 | 206 | :param metadata: dictionary of metada to be written | ||
2136 | 207 | :param extention: extension to be used when making the file | ||
2137 | 208 | """ | ||
2138 | 209 | metadata['name'] = self.name | ||
2139 | 210 | timestamp = normalize_timestamp(metadata['X-Timestamp']) | ||
2140 | 211 | metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) | ||
2141 | 212 | key = 0 | ||
2142 | 213 | while metastr: | ||
2143 | 214 | setxattr(fd, '%s%s' % (METADATA_KEY, key or ''), metastr[:254]) | ||
2144 | 215 | metastr = metastr[254:] | ||
2145 | 216 | key += 1 | ||
2146 | 217 | if 'Content-Length' in metadata: | ||
2147 | 218 | drop_buffer_cache(fd, 0, int(metadata['Content-Length'])) | ||
2148 | 219 | os.fsync(fd) | ||
2149 | 220 | invalidate_hash(os.path.dirname(self.datadir)) | ||
2150 | 221 | renamer(tmppath, os.path.join(self.datadir, timestamp + extension)) | ||
2151 | 222 | self.metadata = metadata | ||
2152 | 223 | |||
2153 | 224 | def unlinkold(self, timestamp): | ||
2154 | 225 | """ | ||
2155 | 226 | Remove any older versions of the object file. Any file that has an | ||
2156 | 227 | older timestamp than timestamp will be deleted. | ||
2157 | 228 | |||
2158 | 229 | :param timestamp: timestamp to compare with each file | ||
2159 | 230 | """ | ||
2160 | 231 | timestamp = normalize_timestamp(timestamp) | ||
2161 | 232 | for fname in os.listdir(self.datadir): | ||
2162 | 233 | if fname < timestamp: | ||
2163 | 234 | try: | ||
2164 | 235 | os.unlink(os.path.join(self.datadir, fname)) | ||
2165 | 236 | except OSError, err: # pragma: no cover | ||
2166 | 237 | if err.errno != errno.ENOENT: | ||
2167 | 238 | raise | ||
2170 | 239 | 42 | ||
2171 | 240 | 43 | ||
2172 | 241 | class ObjectController(object): | 44 | class ObjectController(object): |
2173 | @@ -262,7 +65,7 @@ | |||
2174 | 262 | self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 | 65 | self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 |
2175 | 263 | 66 | ||
2176 | 264 | def container_update(self, op, account, container, obj, headers_in, | 67 | def container_update(self, op, account, container, obj, headers_in, |
2178 | 265 | headers_out, objdevice): | 68 | headers_out, objdevice, objpartition): |
2179 | 266 | """ | 69 | """ |
2180 | 267 | Update the container when objects are updated. | 70 | Update the container when objects are updated. |
2181 | 268 | 71 | ||
2182 | @@ -274,6 +77,7 @@ | |||
2183 | 274 | :param headers_out: dictionary of headers to send in the container | 77 | :param headers_out: dictionary of headers to send in the container |
2184 | 275 | request | 78 | request |
2185 | 276 | :param objdevice: device name that the object is in | 79 | :param objdevice: device name that the object is in |
2186 | 80 | :param objpartition: partition that the object is in | ||
2187 | 277 | """ | 81 | """ |
2188 | 278 | host = headers_in.get('X-Container-Host', None) | 82 | host = headers_in.get('X-Container-Host', None) |
2189 | 279 | partition = headers_in.get('X-Container-Partition', None) | 83 | partition = headers_in.get('X-Container-Partition', None) |
2190 | @@ -293,20 +97,16 @@ | |||
2191 | 293 | return | 97 | return |
2192 | 294 | else: | 98 | else: |
2193 | 295 | self.logger.error('ERROR Container update failed (saving ' | 99 | self.logger.error('ERROR Container update failed (saving ' |
2196 | 296 | 'for async update later): %d response from %s:%s/%s' % | 100 | 'for janitor update later): %d response from %s:%s/%s' |
2197 | 297 | (response.status, ip, port, contdevice)) | 101 | % (response.status, ip, port, contdevice)) |
2198 | 298 | except: | 102 | except: |
2199 | 299 | self.logger.exception('ERROR container update failed with ' | 103 | self.logger.exception('ERROR container update failed with ' |
2201 | 300 | '%s:%s/%s transaction %s (saving for async update later)' % | 104 | '%s:%s/%s transaction %s (saving for janitor update later)' % |
2202 | 301 | (ip, port, contdevice, headers_in.get('x-cf-trans-id', '-'))) | 105 | (ip, port, contdevice, headers_in.get('x-cf-trans-id', '-'))) |
2211 | 302 | async_dir = os.path.join(self.devices, objdevice, ASYNCDIR) | 106 | df = DiskFile(self.devices, objdevice, objpartition, account, |
2212 | 303 | ohash = hash_path(account, container, obj) | 107 | container, obj, datadir=JANITORDIR) |
2213 | 304 | write_pickle( | 108 | df.store_janitor_container_update(op, account, container, obj, |
2214 | 305 | {'op': op, 'account': account, 'container': container, | 109 | headers_out, []) |
2207 | 306 | 'obj': obj, 'headers': headers_out}, | ||
2208 | 307 | os.path.join(async_dir, ohash[-3:], ohash + '-' + | ||
2209 | 308 | normalize_timestamp(headers_out['x-timestamp'])), | ||
2210 | 309 | os.path.join(self.devices, objdevice, 'tmp')) | ||
2215 | 310 | 110 | ||
2216 | 311 | def POST(self, request): | 111 | def POST(self, request): |
2217 | 312 | """Handle HTTP POST requests for the Swift Object Server.""" | 112 | """Handle HTTP POST requests for the Swift Object Server.""" |
2218 | @@ -355,7 +155,15 @@ | |||
2219 | 355 | if error_response: | 155 | if error_response: |
2220 | 356 | return error_response | 156 | return error_response |
2221 | 357 | file = DiskFile(self.devices, device, partition, account, container, | 157 | file = DiskFile(self.devices, device, partition, account, container, |
2223 | 358 | obj, disk_chunk_size=self.disk_chunk_size) | 158 | obj, disk_chunk_size=self.disk_chunk_size, keep_data_fp=True, |
2224 | 159 | segment=request.headers.get('x-object-segment'), | ||
2225 | 160 | segment_timestamp=request.headers['x-timestamp']) | ||
2226 | 161 | overwritten_manifest = False | ||
2227 | 162 | if not file.is_deleted() and \ | ||
2228 | 163 | file.metadata.get('X-Object-Type') == 'manifest' and \ | ||
2229 | 164 | 'x-object-segment' not in request.headers: | ||
2230 | 165 | overwritten_manifest = pickle.loads(''.join(iter(file))) | ||
2231 | 166 | file.close() | ||
2232 | 359 | upload_expiration = time.time() + self.max_upload_time | 167 | upload_expiration = time.time() + self.max_upload_time |
2233 | 360 | etag = md5() | 168 | etag = md5() |
2234 | 361 | upload_size = 0 | 169 | upload_size = 0 |
2235 | @@ -397,17 +205,51 @@ | |||
2236 | 397 | if 'content-encoding' in request.headers: | 205 | if 'content-encoding' in request.headers: |
2237 | 398 | metadata['Content-Encoding'] = \ | 206 | metadata['Content-Encoding'] = \ |
2238 | 399 | request.headers['Content-Encoding'] | 207 | request.headers['Content-Encoding'] |
2240 | 400 | file.put(fd, tmppath, metadata) | 208 | if 'x-object-type' in request.headers: |
2241 | 209 | metadata['X-Object-Type'] = request.headers['x-object-type'] | ||
2242 | 210 | if 'x-object-segment' in request.headers: | ||
2243 | 211 | metadata['X-Object-Segment'] = \ | ||
2244 | 212 | request.headers['x-object-segment'] | ||
2245 | 213 | no_longer_segment = False | ||
2246 | 214 | if 'x-object-segment-if-length' in request.headers and \ | ||
2247 | 215 | int(request.headers['x-object-segment-if-length']) != \ | ||
2248 | 216 | os.fstat(fd).st_size: | ||
2249 | 217 | del metadata['X-Object-Type'] | ||
2250 | 218 | del metadata['X-Object-Segment'] | ||
2251 | 219 | no_longer_segment = True | ||
2252 | 220 | elif int(request.headers.get('x-object-segment', -1)) == 0: | ||
2253 | 221 | # Write out a janitor operation to clean up this multi-segment | ||
2254 | 222 | # PUT in the future if it fails. | ||
2255 | 223 | df = DiskFile(self.devices, device, partition, | ||
2256 | 224 | 'Segment-Cleanup', request.headers['x-timestamp'], | ||
2257 | 225 | '%s/%s/%s' % (account, container, obj), | ||
2258 | 226 | datadir=JANITORDIR) | ||
2259 | 227 | df.store_janitor_segment_cleanup(account, container, obj, | ||
2260 | 228 | segment_count=None, segment_last_deleted=None) | ||
2261 | 229 | if overwritten_manifest: | ||
2262 | 230 | # Write out a janitor operation to clean up the overwritten | ||
2263 | 231 | # multi-segment object. | ||
2264 | 232 | df = DiskFile(self.devices, device, partition, | ||
2265 | 233 | 'Segment-Cleanup', overwritten_manifest['x-timestamp'], | ||
2266 | 234 | '%s/%s/%s' % (account, container, obj), datadir=JANITORDIR) | ||
2267 | 235 | df.store_janitor_segment_cleanup(account, container, obj, | ||
2268 | 236 | segment_count=(overwritten_manifest['content-length'] / | ||
2269 | 237 | overwritten_manifest['x-segment-size'] + 1), | ||
2270 | 238 | segment_last_deleted=None) | ||
2271 | 239 | file.put(fd, tmppath, metadata, | ||
2272 | 240 | no_longer_segment=no_longer_segment) | ||
2273 | 401 | file.unlinkold(metadata['X-Timestamp']) | 241 | file.unlinkold(metadata['X-Timestamp']) |
2283 | 402 | self.container_update('PUT', account, container, obj, request.headers, | 242 | if 'X-Object-Segment' not in file.metadata: |
2284 | 403 | {'x-size': file.metadata['Content-Length'], | 243 | self.container_update('PUT', account, container, obj, |
2285 | 404 | 'x-content-type': file.metadata['Content-Type'], | 244 | request.headers, |
2286 | 405 | 'x-timestamp': file.metadata['X-Timestamp'], | 245 | {'x-size': request.headers.get('x-object-length', |
2287 | 406 | 'x-etag': file.metadata['ETag'], | 246 | file.metadata['Content-Length']), |
2288 | 407 | 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, | 247 | 'x-content-type': file.metadata['Content-Type'], |
2289 | 408 | device) | 248 | 'x-timestamp': file.metadata['X-Timestamp'], |
2290 | 409 | resp = HTTPCreated(request=request, etag=etag) | 249 | 'x-etag': file.metadata['ETag'], |
2291 | 410 | return resp | 250 | 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, |
2292 | 251 | device, partition) | ||
2293 | 252 | return HTTPCreated(request=request, etag=etag) | ||
2294 | 411 | 253 | ||
2295 | 412 | def GET(self, request): | 254 | def GET(self, request): |
2296 | 413 | """Handle HTTP GET requests for the Swift Object Server.""" | 255 | """Handle HTTP GET requests for the Swift Object Server.""" |
2297 | @@ -420,7 +262,9 @@ | |||
2298 | 420 | if self.mount_check and not check_mount(self.devices, device): | 262 | if self.mount_check and not check_mount(self.devices, device): |
2299 | 421 | return Response(status='507 %s is not mounted' % device) | 263 | return Response(status='507 %s is not mounted' % device) |
2300 | 422 | file = DiskFile(self.devices, device, partition, account, container, | 264 | file = DiskFile(self.devices, device, partition, account, container, |
2302 | 423 | obj, keep_data_fp=True, disk_chunk_size=self.disk_chunk_size) | 265 | obj, keep_data_fp=True, disk_chunk_size=self.disk_chunk_size, |
2303 | 266 | segment=request.headers.get('x-object-segment'), | ||
2304 | 267 | segment_timestamp=request.headers.get('x-object-segment-timestamp')) | ||
2305 | 424 | if file.is_deleted(): | 268 | if file.is_deleted(): |
2306 | 425 | if request.headers.get('if-match') == '*': | 269 | if request.headers.get('if-match') == '*': |
2307 | 426 | return HTTPPreconditionFailed(request=request) | 270 | return HTTPPreconditionFailed(request=request) |
2308 | @@ -460,7 +304,9 @@ | |||
2309 | 460 | 'application/octet-stream'), app_iter=file, | 304 | 'application/octet-stream'), app_iter=file, |
2310 | 461 | request=request, conditional_response=True) | 305 | request=request, conditional_response=True) |
2311 | 462 | for key, value in file.metadata.iteritems(): | 306 | for key, value in file.metadata.iteritems(): |
2313 | 463 | if key.lower().startswith('x-object-meta-'): | 307 | if key.lower().startswith('x-object-meta-') or \ |
2314 | 308 | key.lower() in ('x-timestamp', 'x-object-type', | ||
2315 | 309 | 'x-object-segment'): | ||
2316 | 464 | response.headers[key] = value | 310 | response.headers[key] = value |
2317 | 465 | response.etag = file.metadata['ETag'] | 311 | response.etag = file.metadata['ETag'] |
2318 | 466 | response.last_modified = float(file.metadata['X-Timestamp']) | 312 | response.last_modified = float(file.metadata['X-Timestamp']) |
2319 | @@ -482,13 +328,17 @@ | |||
2320 | 482 | if self.mount_check and not check_mount(self.devices, device): | 328 | if self.mount_check and not check_mount(self.devices, device): |
2321 | 483 | return Response(status='507 %s is not mounted' % device) | 329 | return Response(status='507 %s is not mounted' % device) |
2322 | 484 | file = DiskFile(self.devices, device, partition, account, container, | 330 | file = DiskFile(self.devices, device, partition, account, container, |
2324 | 485 | obj, disk_chunk_size=self.disk_chunk_size) | 331 | obj, disk_chunk_size=self.disk_chunk_size, |
2325 | 332 | segment=request.headers.get('x-object-segment'), | ||
2326 | 333 | segment_timestamp=request.headers.get('x-object-segment-timestamp')) | ||
2327 | 486 | if file.is_deleted(): | 334 | if file.is_deleted(): |
2328 | 487 | return HTTPNotFound(request=request) | 335 | return HTTPNotFound(request=request) |
2329 | 488 | response = Response(content_type=file.metadata['Content-Type'], | 336 | response = Response(content_type=file.metadata['Content-Type'], |
2330 | 489 | request=request, conditional_response=True) | 337 | request=request, conditional_response=True) |
2331 | 490 | for key, value in file.metadata.iteritems(): | 338 | for key, value in file.metadata.iteritems(): |
2333 | 491 | if key.lower().startswith('x-object-meta-'): | 339 | if key.lower().startswith('x-object-meta-') or \ |
2334 | 340 | key.lower() in ('x-timestamp', 'x-object-type', | ||
2335 | 341 | 'x-object-segment'): | ||
2336 | 492 | response.headers[key] = value | 342 | response.headers[key] = value |
2337 | 493 | response.etag = file.metadata['ETag'] | 343 | response.etag = file.metadata['ETag'] |
2338 | 494 | response.last_modified = float(file.metadata['X-Timestamp']) | 344 | response.last_modified = float(file.metadata['X-Timestamp']) |
2339 | @@ -513,21 +363,43 @@ | |||
2340 | 513 | return Response(status='507 %s is not mounted' % device) | 363 | return Response(status='507 %s is not mounted' % device) |
2341 | 514 | response_class = HTTPNoContent | 364 | response_class = HTTPNoContent |
2342 | 515 | file = DiskFile(self.devices, device, partition, account, container, | 365 | file = DiskFile(self.devices, device, partition, account, container, |
2344 | 516 | obj, disk_chunk_size=self.disk_chunk_size) | 366 | obj, disk_chunk_size=self.disk_chunk_size, keep_data_fp=True, |
2345 | 367 | segment=request.headers.get('x-object-segment'), | ||
2346 | 368 | segment_timestamp=request.headers.get('x-object-segment-timestamp')) | ||
2347 | 369 | deleted_manifest = False | ||
2348 | 517 | if file.is_deleted(): | 370 | if file.is_deleted(): |
2349 | 518 | response_class = HTTPNotFound | 371 | response_class = HTTPNotFound |
2350 | 372 | elif 'x-object-segment' not in request.headers and \ | ||
2351 | 373 | file.metadata.get('X-Object-Type') == 'manifest': | ||
2352 | 374 | deleted_manifest = pickle.loads(''.join(iter(file))) | ||
2353 | 375 | file.close() | ||
2354 | 519 | metadata = { | 376 | metadata = { |
2355 | 520 | 'X-Timestamp': request.headers['X-Timestamp'], 'deleted': True, | 377 | 'X-Timestamp': request.headers['X-Timestamp'], 'deleted': True, |
2356 | 521 | } | 378 | } |
2357 | 522 | with file.mkstemp() as (fd, tmppath): | 379 | with file.mkstemp() as (fd, tmppath): |
2358 | 380 | if deleted_manifest: | ||
2359 | 381 | # Write out a janitor operation to clean up the deleted | ||
2360 | 382 | # multi-segment object. Note that setting the | ||
2361 | 383 | # segment_last_deleted = -1 will cause the object-janitor to | ||
2362 | 384 | # start removing the segments immediately rather than waiting | ||
2363 | 385 | # segment_reclaim_age (otherwise it can't tell the difference | ||
2364 | 386 | # between a deleted manifest and manifest that just hasn't | ||
2365 | 387 | # appeared yet). | ||
2366 | 388 | df = DiskFile(self.devices, device, partition, | ||
2367 | 389 | 'Segment-Cleanup', deleted_manifest['x-timestamp'], | ||
2368 | 390 | '%s/%s/%s' % (account, container, obj), datadir=JANITORDIR) | ||
2369 | 391 | df.store_janitor_segment_cleanup(account, container, obj, | ||
2370 | 392 | segment_count=(deleted_manifest['content-length'] / | ||
2371 | 393 | deleted_manifest['x-segment-size']) + 1, | ||
2372 | 394 | segment_last_deleted=-1) | ||
2373 | 523 | file.put(fd, tmppath, metadata, extension='.ts') | 395 | file.put(fd, tmppath, metadata, extension='.ts') |
2374 | 524 | file.unlinkold(metadata['X-Timestamp']) | 396 | file.unlinkold(metadata['X-Timestamp']) |
2381 | 525 | self.container_update('DELETE', account, container, obj, | 397 | if 'x-object-segment' not in request.headers: |
2382 | 526 | request.headers, {'x-timestamp': metadata['X-Timestamp'], | 398 | self.container_update('DELETE', account, container, obj, |
2383 | 527 | 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, | 399 | request.headers, {'x-timestamp': metadata['X-Timestamp'], |
2384 | 528 | device) | 400 | 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, |
2385 | 529 | resp = response_class(request=request) | 401 | device, partition) |
2386 | 530 | return resp | 402 | return response_class(request=request) |
2387 | 531 | 403 | ||
2388 | 532 | def REPLICATE(self, request): | 404 | def REPLICATE(self, request): |
2389 | 533 | """ | 405 | """ |
2390 | @@ -542,7 +414,8 @@ | |||
2391 | 542 | content_type='text/plain') | 414 | content_type='text/plain') |
2392 | 543 | if self.mount_check and not check_mount(self.devices, device): | 415 | if self.mount_check and not check_mount(self.devices, device): |
2393 | 544 | return Response(status='507 %s is not mounted' % device) | 416 | return Response(status='507 %s is not mounted' % device) |
2395 | 545 | path = os.path.join(self.devices, device, DATADIR, partition) | 417 | path = os.path.join(self.devices, device, |
2396 | 418 | request.headers.get('x-data-dir', DATADIR), partition) | ||
2397 | 546 | if not os.path.exists(path): | 419 | if not os.path.exists(path): |
2398 | 547 | mkdirs(path) | 420 | mkdirs(path) |
2399 | 548 | if suffix: | 421 | if suffix: |
2400 | 549 | 422 | ||
2401 | === modified file 'swift/obj/updater.py' | |||
2402 | --- swift/obj/updater.py 2010-09-23 16:09:30 +0000 | |||
2403 | +++ swift/obj/updater.py 2010-11-08 18:51:48 +0000 | |||
2404 | @@ -27,11 +27,25 @@ | |||
2405 | 27 | from swift.common.ring import Ring | 27 | from swift.common.ring import Ring |
2406 | 28 | from swift.common.utils import get_logger, renamer, write_pickle | 28 | from swift.common.utils import get_logger, renamer, write_pickle |
2407 | 29 | from swift.common.daemon import Daemon | 29 | from swift.common.daemon import Daemon |
2409 | 30 | from swift.obj.server import ASYNCDIR | 30 | |
2410 | 31 | |||
2411 | 32 | # Old-style async pending directory | ||
2412 | 33 | ASYNCDIR = 'async_pending' | ||
2413 | 31 | 34 | ||
2414 | 32 | 35 | ||
2415 | 33 | class ObjectUpdater(Daemon): | 36 | class ObjectUpdater(Daemon): |
2417 | 34 | """Update object information in container listings.""" | 37 | """ |
2418 | 38 | Update object information in container listings based on postponed | ||
2419 | 39 | operations stored in the old-style async pending directory. | ||
2420 | 40 | |||
2421 | 41 | After upgrade, no new operations will be stored in this old-style async | ||
2422 | 42 | pending directory. Once this daemon empties those directories of all | ||
2423 | 43 | operations, the daemon may be disabled and the directories removed. | ||
2424 | 44 | |||
2425 | 45 | In a future release of Swift, this daemon will be removed. | ||
2426 | 46 | |||
2427 | 47 | The new functionality is in swift.obj.janitor. | ||
2428 | 48 | """ | ||
2429 | 35 | 49 | ||
2430 | 36 | def __init__(self, conf): | 50 | def __init__(self, conf): |
2431 | 37 | self.conf = conf | 51 | self.conf = conf |
2432 | 38 | 52 | ||
2433 | === modified file 'swift/proxy/server.py' | |||
2434 | --- swift/proxy/server.py 2010-11-05 14:47:43 +0000 | |||
2435 | +++ swift/proxy/server.py 2010-11-08 18:51:48 +0000 | |||
2436 | @@ -14,21 +14,22 @@ | |||
2437 | 14 | # limitations under the License. | 14 | # limitations under the License. |
2438 | 15 | 15 | ||
2439 | 16 | from __future__ import with_statement | 16 | from __future__ import with_statement |
2440 | 17 | import cPickle as pickle | ||
2441 | 17 | import mimetypes | 18 | import mimetypes |
2442 | 18 | import os | 19 | import os |
2443 | 19 | import time | 20 | import time |
2444 | 20 | import traceback | 21 | import traceback |
2445 | 21 | from ConfigParser import ConfigParser | 22 | from ConfigParser import ConfigParser |
2446 | 23 | from hashlib import md5 | ||
2447 | 22 | from urllib import unquote, quote | 24 | from urllib import unquote, quote |
2448 | 23 | import uuid | 25 | import uuid |
2449 | 24 | import functools | 26 | import functools |
2450 | 25 | 27 | ||
2451 | 26 | from eventlet.timeout import Timeout | 28 | from eventlet.timeout import Timeout |
2457 | 27 | from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \ | 29 | from webob.exc import HTTPBadRequest, HTTPCreated, HTTPInternalServerError, \ |
2458 | 28 | HTTPNotFound, HTTPPreconditionFailed, \ | 30 | HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \ |
2459 | 29 | HTTPRequestTimeout, HTTPServiceUnavailable, \ | 31 | HTTPRequestEntityTooLarge, HTTPRequestTimeout, HTTPServerError, \ |
2460 | 30 | HTTPUnprocessableEntity, HTTPRequestEntityTooLarge, HTTPServerError, \ | 32 | HTTPServiceUnavailable, HTTPUnprocessableEntity, status_map |
2456 | 31 | status_map | ||
2461 | 32 | from webob import Request, Response | 33 | from webob import Request, Response |
2462 | 33 | 34 | ||
2463 | 34 | from swift.common.ring import Ring | 35 | from swift.common.ring import Ring |
2464 | @@ -37,7 +38,7 @@ | |||
2465 | 37 | from swift.common.bufferedhttp import http_connect | 38 | from swift.common.bufferedhttp import http_connect |
2466 | 38 | from swift.common.constraints import check_metadata, check_object_creation, \ | 39 | from swift.common.constraints import check_metadata, check_object_creation, \ |
2467 | 39 | check_utf8, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ | 40 | check_utf8, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ |
2469 | 40 | MAX_FILE_SIZE | 41 | PICKLE_PROTOCOL |
2470 | 41 | from swift.common.exceptions import ChunkReadTimeout, \ | 42 | from swift.common.exceptions import ChunkReadTimeout, \ |
2471 | 42 | ChunkWriteTimeout, ConnectionTimeout | 43 | ChunkWriteTimeout, ConnectionTimeout |
2472 | 43 | 44 | ||
2473 | @@ -89,6 +90,144 @@ | |||
2474 | 89 | return wrapped | 90 | return wrapped |
2475 | 90 | 91 | ||
2476 | 91 | 92 | ||
2477 | 93 | class SegmentedIterable(object): | ||
2478 | 94 | """ | ||
2479 | 95 | Iterable that returns the object contents for a segmented object in Swift. | ||
2480 | 96 | |||
2481 | 97 | In addition to these params, you can also set the `response` attr just | ||
2482 | 98 | after creating the SegmentedIterable and it will update the response's | ||
2483 | 99 | `bytes_transferred` value (used to log the size of the request). | ||
2484 | 100 | |||
2485 | 101 | :param controller: The ObjectController instance to work with. | ||
2486 | 102 | :param content_length: The total length of the object. | ||
2487 | 103 | :param segment_size: The length of each segment (except perhaps the last) | ||
2488 | 104 | of the object. | ||
2489 | 105 | :param timestamp: The X-Timestamp of the object's segments (set on the PUT, | ||
2490 | 106 | not changed on the POSTs). | ||
2491 | 107 | """ | ||
2492 | 108 | |||
2493 | 109 | def __init__(self, controller, content_length, segment_size, timestamp): | ||
2494 | 110 | self.controller = controller | ||
2495 | 111 | self.content_length = content_length | ||
2496 | 112 | self.segment_size = segment_size | ||
2497 | 113 | self.timestamp = timestamp | ||
2498 | 114 | self.position = 0 | ||
2499 | 115 | self.segment = -1 | ||
2500 | 116 | self.segment_iter = None | ||
2501 | 117 | self.response = None | ||
2502 | 118 | |||
2503 | 119 | def load_next_segment(self): | ||
2504 | 120 | """ Loads the self.segment_iter with the next segment's contents. """ | ||
2505 | 121 | self.segment += 1 | ||
2506 | 122 | if self.segment: | ||
2507 | 123 | ring_object_name = '%s/%s/%s' % (self.controller.object_name, | ||
2508 | 124 | self.timestamp, self.segment) | ||
2509 | 125 | else: | ||
2510 | 126 | ring_object_name = self.controller.object_name | ||
2511 | 127 | partition, nodes = self.controller.app.object_ring.get_nodes( | ||
2512 | 128 | self.controller.account_name, self.controller.container_name, | ||
2513 | 129 | ring_object_name) | ||
2514 | 130 | path = '/%s/%s/%s' % (self.controller.account_name, | ||
2515 | 131 | self.controller.container_name, self.controller.object_name) | ||
2516 | 132 | req = Request.blank(path, headers={'X-Object-Segment': self.segment, | ||
2517 | 133 | 'X-Object-Segment-Timestamp': self.timestamp}) | ||
2518 | 134 | resp = self.controller.GETorHEAD_base(req, 'Object', | ||
2519 | 135 | partition, self.controller.iter_nodes(partition, nodes, | ||
2520 | 136 | self.controller.app.object_ring), path, | ||
2521 | 137 | self.controller.app.object_ring.replica_count) | ||
2522 | 138 | if resp.status_int // 100 != 2: | ||
2523 | 139 | raise Exception( | ||
2524 | 140 | 'Could not load segment %s of %s' % (self.segment, path)) | ||
2525 | 141 | self.segment_iter = resp.app_iter | ||
2526 | 142 | |||
2527 | 143 | def __iter__(self): | ||
2528 | 144 | """ Standard iterator function that returns the object's contents. """ | ||
2529 | 145 | while self.position < self.content_length: | ||
2530 | 146 | if not self.segment_iter: | ||
2531 | 147 | self.load_next_segment() | ||
2532 | 148 | while True: | ||
2533 | 149 | with ChunkReadTimeout(self.controller.app.node_timeout): | ||
2534 | 150 | try: | ||
2535 | 151 | chunk = self.segment_iter.next() | ||
2536 | 152 | break | ||
2537 | 153 | except StopIteration: | ||
2538 | 154 | self.load_next_segment() | ||
2539 | 155 | if self.position + len(chunk) > self.content_length: | ||
2540 | 156 | chunk = chunk[:self.content_length - self.position] | ||
2541 | 157 | self.position += len(chunk) | ||
2542 | 158 | if self.response: | ||
2543 | 159 | self.response.bytes_transferred = \ | ||
2544 | 160 | getattr(self.response, 'bytes_transferred', 0) + len(chunk) | ||
2545 | 161 | yield chunk | ||
2546 | 162 | |||
2547 | 163 | def app_iter_range(self, start, stop): | ||
2548 | 164 | """ | ||
2549 | 165 | Non-standard iterator function for use with Webob in serving Range | ||
2550 | 166 | requests more quickly. | ||
2551 | 167 | |||
2552 | 168 | .. note:: | ||
2553 | 169 | |||
2554 | 170 | This currently helps on speed by jumping to the proper segment to | ||
2555 | 171 | start with (and ending without reading the trailing segments, but | ||
2556 | 172 | that already happened technically with __iter__). | ||
2557 | 173 | |||
2558 | 174 | But, what it does not do yet is issue a Range request with the | ||
2559 | 175 | first segment to allow the object server to seek to the segment | ||
2560 | 176 | start point. | ||
2561 | 177 | |||
2562 | 178 | Instead, it just reads and throws away all leading segment data. | ||
2563 | 179 | Since segments are 2G by default, it'll have to transfer the whole | ||
2564 | 180 | 2G from the object server to the proxy server even if it only needs | ||
2565 | 181 | the last byte. In practice, this should happen fairly quickly | ||
2566 | 182 | relative to how long requests take for these very large files; but | ||
2567 | 183 | it's still wasteful. | ||
2568 | 184 | |||
2569 | 185 | Anyway, it shouldn't be too hard to implement, I just want to keep | ||
2570 | 186 | the complexity down for now. | ||
2571 | 187 | |||
2572 | 188 | :param start: The first byte (zero-based) to return. None for 0. | ||
2573 | 189 | :param stop: The last byte (zero-based) to return. None for end. | ||
2574 | 190 | """ | ||
2575 | 191 | if start is None: | ||
2576 | 192 | start = 0 | ||
2577 | 193 | if start: | ||
2578 | 194 | self.segment = (start / self.segment_size) - 1 | ||
2579 | 195 | self.load_next_segment() | ||
2580 | 196 | self.position = self.segment * self.segment_size | ||
2581 | 197 | segment_start = start - (self.segment * self.segment_size) | ||
2582 | 198 | while segment_start: | ||
2583 | 199 | with ChunkReadTimeout(self.controller.app.node_timeout): | ||
2584 | 200 | chunk = self.segment_iter.next() | ||
2585 | 201 | self.position += len(chunk) | ||
2586 | 202 | if len(chunk) > segment_start: | ||
2587 | 203 | chunk = chunk[segment_start:] | ||
2588 | 204 | if self.response: | ||
2589 | 205 | self.response.bytes_transferred = \ | ||
2590 | 206 | getattr(self.response, 'bytes_transferred', 0) + \ | ||
2591 | 207 | len(chunk) | ||
2592 | 208 | yield chunk | ||
2593 | 209 | segment_start = 0 | ||
2594 | 210 | else: | ||
2595 | 211 | segment_start -= len(chunk) | ||
2596 | 212 | if stop is not None: | ||
2597 | 213 | length = stop - start | ||
2598 | 214 | else: | ||
2599 | 215 | length = None | ||
2600 | 216 | for chunk in self: | ||
2601 | 217 | if length is not None: | ||
2602 | 218 | length -= len(chunk) | ||
2603 | 219 | if length < 0: | ||
2604 | 220 | # bytes_transferred had len(chunk) added by __iter__ so we | ||
2605 | 221 | # need to subtract what we aren't going to use of the chunk | ||
2606 | 222 | if self.response: | ||
2607 | 223 | self.response.bytes_transferred = \ | ||
2608 | 224 | getattr(self.response, 'bytes_transferred', | ||
2609 | 225 | length) + length | ||
2610 | 226 | yield chunk[:length] | ||
2611 | 227 | break | ||
2612 | 228 | yield chunk | ||
2613 | 229 | |||
2614 | 230 | |||
2615 | 92 | def get_container_memcache_key(account, container): | 231 | def get_container_memcache_key(account, container): |
2616 | 93 | path = '/%s/%s' % (account, container) | 232 | path = '/%s/%s' % (account, container) |
2617 | 94 | return 'container%s' % path | 233 | return 'container%s' % path |
2618 | @@ -518,11 +657,56 @@ | |||
2619 | 518 | aresp = req.environ['swift.authorize'](req) | 657 | aresp = req.environ['swift.authorize'](req) |
2620 | 519 | if aresp: | 658 | if aresp: |
2621 | 520 | return aresp | 659 | return aresp |
2622 | 660 | # This is bit confusing, so an explanation: | ||
2623 | 661 | # * First we attempt the GET/HEAD normally, as this is the usual case. | ||
2624 | 662 | # * If the request was a Range request and gave us a 416 Unsatisfiable | ||
2625 | 663 | # response, we might be trying to do an invalid Range on a manifest | ||
2626 | 664 | # object, so we try again with no Range. | ||
2627 | 665 | # * If it turns out we have a manifest object, and we had a Range | ||
2628 | 666 | # request originally that actually succeeded or we had a HEAD | ||
2629 | 667 | # request, we have to do the request again as a full GET because | ||
2630 | 668 | # we'll need the whole manifest. | ||
2631 | 669 | # * Finally, if we had a manifest object, we pass it and the request | ||
2632 | 670 | # off to GETorHEAD_segmented; otherwise we just return the response. | ||
2633 | 521 | partition, nodes = self.app.object_ring.get_nodes( | 671 | partition, nodes = self.app.object_ring.get_nodes( |
2634 | 522 | self.account_name, self.container_name, self.object_name) | 672 | self.account_name, self.container_name, self.object_name) |
2636 | 523 | return self.GETorHEAD_base(req, 'Object', partition, | 673 | resp = mresp = self.GETorHEAD_base(req, 'Object', partition, |
2637 | 674 | self.iter_nodes(partition, nodes, self.app.object_ring), | ||
2638 | 675 | req.path_info, self.app.object_ring.replica_count) | ||
2639 | 676 | range_value = None | ||
2640 | 677 | if mresp.status_int == 416: | ||
2641 | 678 | range_value = req.range | ||
2642 | 679 | req.range = None | ||
2643 | 680 | mresp = self.GETorHEAD_base(req, 'Object', partition, | ||
2644 | 524 | self.iter_nodes(partition, nodes, self.app.object_ring), | 681 | self.iter_nodes(partition, nodes, self.app.object_ring), |
2645 | 525 | req.path_info, self.app.object_ring.replica_count) | 682 | req.path_info, self.app.object_ring.replica_count) |
2646 | 683 | if mresp.status_int // 100 != 2: | ||
2647 | 684 | return resp | ||
2648 | 685 | if 'x-object-type' in mresp.headers: | ||
2649 | 686 | if mresp.headers['x-object-type'] == 'manifest': | ||
2650 | 687 | if req.method == 'HEAD': | ||
2651 | 688 | req.method = 'GET' | ||
2652 | 689 | mresp = self.GETorHEAD_base(req, 'Object', partition, | ||
2653 | 690 | self.iter_nodes(partition, nodes, | ||
2654 | 691 | self.app.object_ring), req.path_info, | ||
2655 | 692 | self.app.object_ring.replica_count) | ||
2656 | 693 | if mresp.status_int // 100 != 2: | ||
2657 | 694 | return mresp | ||
2658 | 695 | req.method = 'HEAD' | ||
2659 | 696 | elif req.range: | ||
2660 | 697 | range_value = req.range | ||
2661 | 698 | req.range = None | ||
2662 | 699 | mresp = self.GETorHEAD_base(req, 'Object', partition, | ||
2663 | 700 | self.iter_nodes(partition, nodes, | ||
2664 | 701 | self.app.object_ring), req.path_info, | ||
2665 | 702 | self.app.object_ring.replica_count) | ||
2666 | 703 | if mresp.status_int // 100 != 2: | ||
2667 | 704 | return mresp | ||
2668 | 705 | if range_value: | ||
2669 | 706 | req.range = range_value | ||
2670 | 707 | return self.GETorHEAD_segmented(req, mresp) | ||
2671 | 708 | return HTTPNotFound(request=req) | ||
2672 | 709 | return resp | ||
2673 | 526 | 710 | ||
2674 | 527 | @public | 711 | @public |
2675 | 528 | @delay_denial | 712 | @delay_denial |
2676 | @@ -536,6 +720,32 @@ | |||
2677 | 536 | """Handler for HTTP HEAD requests.""" | 720 | """Handler for HTTP HEAD requests.""" |
2678 | 537 | return self.GETorHEAD(req) | 721 | return self.GETorHEAD(req) |
2679 | 538 | 722 | ||
2680 | 723 | def GETorHEAD_segmented(self, req, mresp): | ||
2681 | 724 | """ | ||
2682 | 725 | Performs a GET for a segmented object. | ||
2683 | 726 | |||
2684 | 727 | :param req: The webob.Request to process. | ||
2685 | 728 | :param mresp: The webob.Response for the original manifest request. | ||
2686 | 729 | :returns: webob.Response object. | ||
2687 | 730 | """ | ||
2688 | 731 | manifest = pickle.loads(''.join(mresp.app_iter)) | ||
2689 | 732 | content_length = int(manifest['content-length']) | ||
2690 | 733 | segment_size = int(manifest['x-segment-size']) | ||
2691 | 734 | headers = dict(mresp.headers) | ||
2692 | 735 | headers.update(manifest) | ||
2693 | 736 | del headers['x-segment-size'] | ||
2694 | 737 | resp = Response(app_iter=SegmentedIterable(self, content_length, | ||
2695 | 738 | segment_size, manifest['x-timestamp']), headers=headers, | ||
2696 | 739 | request=req, conditional_response=True) | ||
2697 | 740 | resp.headers['etag'] = manifest['etag'].strip('"') | ||
2698 | 741 | resp.last_modified = mresp.last_modified | ||
2699 | 742 | resp.content_length = int(manifest['content-length']) | ||
2700 | 743 | resp.content_type = manifest['content-type'] | ||
2701 | 744 | if 'content-encoding' in manifest: | ||
2702 | 745 | resp.content_encoding = manifest['content-encoding'] | ||
2703 | 746 | resp.app_iter.response = req.get_response(resp) | ||
2704 | 747 | return resp.app_iter.response | ||
2705 | 748 | |||
2706 | 539 | @public | 749 | @public |
2707 | 540 | @delay_denial | 750 | @delay_denial |
2708 | 541 | def POST(self, req): | 751 | def POST(self, req): |
2709 | @@ -609,7 +819,8 @@ | |||
2710 | 609 | req.headers['Content-Type'] = 'application/octet-stream' | 819 | req.headers['Content-Type'] = 'application/octet-stream' |
2711 | 610 | else: | 820 | else: |
2712 | 611 | req.headers['Content-Type'] = guessed_type | 821 | req.headers['Content-Type'] = guessed_type |
2714 | 612 | error_response = check_object_creation(req, self.object_name) | 822 | error_response = check_object_creation(req, self.object_name, |
2715 | 823 | self.app.max_object_size) | ||
2716 | 613 | if error_response: | 824 | if error_response: |
2717 | 614 | return error_response | 825 | return error_response |
2718 | 615 | conns = [] | 826 | conns = [] |
2719 | @@ -654,11 +865,50 @@ | |||
2720 | 654 | if k.lower().startswith('x-object-meta-'): | 865 | if k.lower().startswith('x-object-meta-'): |
2721 | 655 | new_req.headers[k] = v | 866 | new_req.headers[k] = v |
2722 | 656 | req = new_req | 867 | req = new_req |
2723 | 868 | if req.headers.get('transfer-encoding') == 'chunked' or \ | ||
2724 | 869 | req.content_length > self.app.segment_size: | ||
2725 | 870 | resp = self.PUT_segmented_object(req, data_source, partition, | ||
2726 | 871 | nodes, container_partition, containers) | ||
2727 | 872 | else: | ||
2728 | 873 | resp = self.PUT_whole_object(req, data_source, partition, nodes, | ||
2729 | 874 | container_partition, containers) | ||
2730 | 875 | if source_header: | ||
2731 | 876 | resp.headers['X-Copied-From'] = quote( | ||
2732 | 877 | source_header.split('/', 2)[2]) | ||
2733 | 878 | for k, v in req.headers.items(): | ||
2734 | 879 | if k.lower().startswith('x-object-meta-'): | ||
2735 | 880 | resp.headers[k] = v | ||
2736 | 881 | # reset the bytes, since the user didn't actually send anything | ||
2737 | 882 | req.bytes_transferred = 0 | ||
2738 | 883 | resp.last_modified = float(req.headers['X-Timestamp']) | ||
2739 | 884 | return resp | ||
2740 | 885 | |||
2741 | 886 | def PUT_whole_object(self, req, data_source, partition, nodes, | ||
2742 | 887 | container_partition=None, containers=None): | ||
2743 | 888 | """ | ||
2744 | 889 | Performs a PUT for a whole object (one with a content-length <= | ||
2745 | 890 | self.app.segment_size). | ||
2746 | 891 | |||
2747 | 892 | :param req: The webob.Request to process. | ||
2748 | 893 | :param data_source: An iterator providing the data to store. | ||
2749 | 894 | :param partition: The object ring partition the object falls on. | ||
2750 | 895 | :param nodes: The object ring nodes the object falls on. | ||
2751 | 896 | :param container_partition: The container ring partition the container | ||
2752 | 897 | for the object falls on, None if the | ||
2753 | 898 | container is not to be updated. | ||
2754 | 899 | :param containers: The container ring nodes the container for the | ||
2755 | 900 | object falls on, None if the container is not to be | ||
2756 | 901 | updated. | ||
2757 | 902 | :returns: webob.Response object. | ||
2758 | 903 | """ | ||
2759 | 904 | conns = [] | ||
2760 | 905 | update_containers = containers is not None | ||
2761 | 657 | for node in self.iter_nodes(partition, nodes, self.app.object_ring): | 906 | for node in self.iter_nodes(partition, nodes, self.app.object_ring): |
2766 | 658 | container = containers.pop() | 907 | if update_containers: |
2767 | 659 | req.headers['X-Container-Host'] = '%(ip)s:%(port)s' % container | 908 | container = containers.pop() |
2768 | 660 | req.headers['X-Container-Partition'] = container_partition | 909 | req.headers['X-Container-Host'] = '%(ip)s:%(port)s' % container |
2769 | 661 | req.headers['X-Container-Device'] = container['device'] | 910 | req.headers['X-Container-Partition'] = container_partition |
2770 | 911 | req.headers['X-Container-Device'] = container['device'] | ||
2771 | 662 | req.headers['Expect'] = '100-continue' | 912 | req.headers['Expect'] = '100-continue' |
2772 | 663 | resp = conn = None | 913 | resp = conn = None |
2773 | 664 | if not self.error_limited(node): | 914 | if not self.error_limited(node): |
2774 | @@ -676,12 +926,14 @@ | |||
2775 | 676 | if conn and resp: | 926 | if conn and resp: |
2776 | 677 | if resp.status == 100: | 927 | if resp.status == 100: |
2777 | 678 | conns.append(conn) | 928 | conns.append(conn) |
2779 | 679 | if not containers: | 929 | if (update_containers and not containers) or \ |
2780 | 930 | len(conns) == len(nodes): | ||
2781 | 680 | break | 931 | break |
2782 | 681 | continue | 932 | continue |
2783 | 682 | elif resp.status == 507: | 933 | elif resp.status == 507: |
2784 | 683 | self.error_limit(node) | 934 | self.error_limit(node) |
2786 | 684 | containers.insert(0, container) | 935 | if update_containers: |
2787 | 936 | containers.insert(0, container) | ||
2788 | 685 | if len(conns) <= len(nodes) / 2: | 937 | if len(conns) <= len(nodes) / 2: |
2789 | 686 | self.app.logger.error( | 938 | self.app.logger.error( |
2790 | 687 | 'Object PUT returning 503, %s/%s required connections, ' | 939 | 'Object PUT returning 503, %s/%s required connections, ' |
2791 | @@ -701,7 +953,7 @@ | |||
2792 | 701 | break | 953 | break |
2793 | 702 | len_chunk = len(chunk) | 954 | len_chunk = len(chunk) |
2794 | 703 | req.bytes_transferred += len_chunk | 955 | req.bytes_transferred += len_chunk |
2796 | 704 | if req.bytes_transferred > MAX_FILE_SIZE: | 956 | if req.bytes_transferred > self.app.max_object_size: |
2797 | 705 | return HTTPRequestEntityTooLarge(request=req) | 957 | return HTTPRequestEntityTooLarge(request=req) |
2798 | 706 | for conn in list(conns): | 958 | for conn in list(conns): |
2799 | 707 | try: | 959 | try: |
2800 | @@ -767,18 +1019,129 @@ | |||
2801 | 767 | statuses.append(503) | 1019 | statuses.append(503) |
2802 | 768 | reasons.append('') | 1020 | reasons.append('') |
2803 | 769 | bodies.append('') | 1021 | bodies.append('') |
2805 | 770 | resp = self.best_response(req, statuses, reasons, bodies, 'Object PUT', | 1022 | return self.best_response(req, statuses, reasons, bodies, 'Object PUT', |
2806 | 771 | etag=etag) | 1023 | etag=etag) |
2817 | 772 | if source_header: | 1024 | |
2818 | 773 | resp.headers['X-Copied-From'] = quote( | 1025 | def PUT_segmented_object(self, req, data_source, partition, nodes, |
2819 | 774 | source_header.split('/', 2)[2]) | 1026 | container_partition, containers): |
2820 | 775 | for k, v in req.headers.items(): | 1027 | """ |
2821 | 776 | if k.lower().startswith('x-object-meta-'): | 1028 | Performs a PUT for a segmented object (one with a content-length > |
2822 | 777 | resp.headers[k] = v | 1029 | self.app.segment_size). |
2823 | 778 | # reset the bytes, since the user didn't actually send anything | 1030 | |
2824 | 779 | req.bytes_transferred = 0 | 1031 | :param req: The webob.Request to process. |
2825 | 780 | resp.last_modified = float(req.headers['X-Timestamp']) | 1032 | :param data_source: An iterator providing the data to store. |
2826 | 781 | return resp | 1033 | :param partition: The object ring partition the object falls on. |
2827 | 1034 | :param nodes: The object ring nodes the object falls on. | ||
2828 | 1035 | :param container_partition: The container ring partition the container | ||
2829 | 1036 | for the object falls on. | ||
2830 | 1037 | :param containers: The container ring nodes the container for the | ||
2831 | 1038 | object falls on. | ||
2832 | 1039 | :returns: webob.Response object. | ||
2833 | 1040 | """ | ||
2834 | 1041 | req.bytes_transferred = 0 | ||
2835 | 1042 | leftover_chunk = [None] | ||
2836 | 1043 | etag = md5() | ||
2837 | 1044 | |||
2838 | 1045 | def segment_iter(): | ||
2839 | 1046 | amount_given = 0 | ||
2840 | 1047 | while amount_given < self.app.segment_size: | ||
2841 | 1048 | if leftover_chunk[0]: | ||
2842 | 1049 | chunk = leftover_chunk[0] | ||
2843 | 1050 | leftover_chunk[0] = None | ||
2844 | 1051 | else: | ||
2845 | 1052 | with ChunkReadTimeout(self.app.client_timeout): | ||
2846 | 1053 | chunk = data_source.next() | ||
2847 | 1054 | req.bytes_transferred += len(chunk) | ||
2848 | 1055 | etag.update(chunk) | ||
2849 | 1056 | if amount_given + len(chunk) > self.app.segment_size: | ||
2850 | 1057 | yield chunk[:self.app.segment_size - amount_given] | ||
2851 | 1058 | leftover_chunk[0] = \ | ||
2852 | 1059 | chunk[self.app.segment_size - amount_given:] | ||
2853 | 1060 | amount_given = self.app.segment_size | ||
2854 | 1061 | else: | ||
2855 | 1062 | yield chunk | ||
2856 | 1063 | amount_given += len(chunk) | ||
2857 | 1064 | |||
2858 | 1065 | def segment_iter_iter(): | ||
2859 | 1066 | while True: | ||
2860 | 1067 | if not leftover_chunk[0]: | ||
2861 | 1068 | with ChunkReadTimeout(self.app.client_timeout): | ||
2862 | 1069 | leftover_chunk[0] = data_source.next() | ||
2863 | 1070 | req.bytes_transferred += len(leftover_chunk[0]) | ||
2864 | 1071 | etag.update(leftover_chunk[0]) | ||
2865 | 1072 | yield segment_iter() | ||
2866 | 1073 | |||
2867 | 1074 | segment_number = 0 | ||
2868 | 1075 | chunked = req.headers.get('transfer-encoding') == 'chunked' | ||
2869 | 1076 | if not chunked: | ||
2870 | 1077 | amount_left = req.content_length | ||
2871 | 1078 | headers = {'X-Timestamp': req.headers['X-Timestamp'], | ||
2872 | 1079 | 'Content-Type': req.headers['content-type'], | ||
2873 | 1080 | 'X-Object-Type': 'segment'} | ||
2874 | 1081 | for segment_source in segment_iter_iter(): | ||
2875 | 1082 | if chunked: | ||
2876 | 1083 | headers['Transfer-Encoding'] = 'chunked' | ||
2877 | 1084 | if segment_number == 0: | ||
2878 | 1085 | headers['X-Object-Segment-If-Length'] = \ | ||
2879 | 1086 | self.app.segment_size | ||
2880 | 1087 | elif amount_left > self.app.segment_size: | ||
2881 | 1088 | headers['Content-Length'] = self.app.segment_size | ||
2882 | 1089 | else: | ||
2883 | 1090 | headers['Content-Length'] = amount_left | ||
2884 | 1091 | headers['X-Object-Segment'] = segment_number | ||
2885 | 1092 | segment_req = Request.blank(req.path_info, | ||
2886 | 1093 | environ={'REQUEST_METHOD': 'PUT'}, headers=headers) | ||
2887 | 1094 | if 'X-Object-Segment-If-Length' in headers: | ||
2888 | 1095 | del headers['X-Object-Segment-If-Length'] | ||
2889 | 1096 | if segment_number: | ||
2890 | 1097 | ring_object_name = '%s/%s/%s' % (self.object_name, | ||
2891 | 1098 | req.headers['x-timestamp'], segment_number) | ||
2892 | 1099 | else: | ||
2893 | 1100 | ring_object_name = self.object_name | ||
2894 | 1101 | segment_partition, segment_nodes = self.app.object_ring.get_nodes( | ||
2895 | 1102 | self.account_name, self.container_name, ring_object_name) | ||
2896 | 1103 | resp = self.PUT_whole_object(segment_req, segment_source, | ||
2897 | 1104 | segment_partition, segment_nodes) | ||
2898 | 1105 | if resp.status_int // 100 == 4: | ||
2899 | 1106 | return resp | ||
2900 | 1107 | elif resp.status_int // 100 != 2: | ||
2901 | 1108 | return HTTPServiceUnavailable(request=req, | ||
2902 | 1109 | body='Unable to complete very large file operation.') | ||
2903 | 1110 | if segment_number == 0 and \ | ||
2904 | 1111 | req.bytes_transferred < self.app.segment_size: | ||
2905 | 1112 | return HTTPCreated(request=req, etag=etag.hexdigest()) | ||
2906 | 1113 | if not chunked: | ||
2907 | 1114 | amount_left -= self.app.segment_size | ||
2908 | 1115 | segment_number += 1 | ||
2909 | 1116 | etag = etag.hexdigest() | ||
2910 | 1117 | if 'etag' in req.headers and req.headers['etag'].lower() != etag: | ||
2911 | 1118 | return HTTPUnprocessableEntity(request=req) | ||
2912 | 1119 | manifest = {'x-timestamp': req.headers['x-timestamp'], | ||
2913 | 1120 | 'content-length': req.bytes_transferred, | ||
2914 | 1121 | 'content-type': req.headers['content-type'], | ||
2915 | 1122 | 'x-segment-size': self.app.segment_size, | ||
2916 | 1123 | 'etag': etag} | ||
2917 | 1124 | if 'content-encoding' in req.headers: | ||
2918 | 1125 | manifest['content-encoding'] = req.headers['content-encoding'] | ||
2919 | 1126 | manifest = pickle.dumps(manifest, protocol=PICKLE_PROTOCOL) | ||
2920 | 1127 | headers = {'X-Timestamp': req.headers['X-Timestamp'], | ||
2921 | 1128 | 'Content-Type': req.headers['content-type'], | ||
2922 | 1129 | 'Content-Length': len(manifest), | ||
2923 | 1130 | 'X-Object-Type': 'manifest', | ||
2924 | 1131 | 'X-Object-Length': req.bytes_transferred} | ||
2925 | 1132 | headers.update(i for i in req.headers.iteritems() | ||
2926 | 1133 | if i[0].lower().startswith('x-object-meta-') and len(i[0]) > 14) | ||
2927 | 1134 | manifest_req = Request.blank(req.path_info, | ||
2928 | 1135 | environ={'REQUEST_METHOD': 'PUT'}, body=manifest, headers=headers) | ||
2929 | 1136 | manifest_source = iter(lambda: | ||
2930 | 1137 | manifest_req.body_file.read(self.app.client_chunk_size), '') | ||
2931 | 1138 | resp = self.PUT_whole_object(manifest_req, manifest_source, partition, | ||
2932 | 1139 | nodes, container_partition=container_partition, | ||
2933 | 1140 | containers=containers) | ||
2934 | 1141 | if resp.status_int // 100 != 2: | ||
2935 | 1142 | return HTTPServiceUnavailable(request=req, | ||
2936 | 1143 | body='Unable to complete very large file operation.') | ||
2937 | 1144 | return HTTPCreated(request=req, etag=etag) | ||
2938 | 782 | 1145 | ||
2939 | 783 | @public | 1146 | @public |
2940 | 784 | @delay_denial | 1147 | @delay_denial |
2941 | @@ -1233,6 +1596,8 @@ | |||
2942 | 1233 | if conf is None: | 1596 | if conf is None: |
2943 | 1234 | conf = {} | 1597 | conf = {} |
2944 | 1235 | swift_dir = conf.get('swift_dir', '/etc/swift') | 1598 | swift_dir = conf.get('swift_dir', '/etc/swift') |
2945 | 1599 | self.max_object_size = int(conf.get('max_object_size', 107374182400)) | ||
2946 | 1600 | self.segment_size = int(conf.get('segment_size', 2147483647)) | ||
2947 | 1236 | self.node_timeout = int(conf.get('node_timeout', 10)) | 1601 | self.node_timeout = int(conf.get('node_timeout', 10)) |
2948 | 1237 | self.conn_timeout = float(conf.get('conn_timeout', 0.5)) | 1602 | self.conn_timeout = float(conf.get('conn_timeout', 0.5)) |
2949 | 1238 | self.client_timeout = int(conf.get('client_timeout', 60)) | 1603 | self.client_timeout = int(conf.get('client_timeout', 60)) |
2950 | 1239 | 1604 | ||
2951 | === modified file 'test/functional/sample.conf' | |||
2952 | --- test/functional/sample.conf 2010-09-09 17:24:25 +0000 | |||
2953 | +++ test/functional/sample.conf 2010-11-08 18:51:48 +0000 | |||
2954 | @@ -1,8 +1,15 @@ | |||
2955 | 1 | # sample config | ||
2956 | 2 | auth_host = 127.0.0.1 | 1 | auth_host = 127.0.0.1 |
2957 | 3 | auth_port = 11000 | 2 | auth_port = 11000 |
2958 | 4 | auth_ssl = no | 3 | auth_ssl = no |
2959 | 5 | 4 | ||
2960 | 5 | # The maximum object size the cluster allows (set in the proxy server's conf) | ||
2961 | 6 | max_object_size = 107374182400 | ||
2962 | 7 | |||
2963 | 8 | # The file segment size for the cluster (set in the proxy server's conf) | ||
2964 | 9 | # Set to 0 for no segment size testing (recommended if the segment size is | ||
2965 | 10 | # quite large and you don't want to spend the time testing it) | ||
2966 | 11 | segment_size = 0 | ||
2967 | 12 | |||
2968 | 6 | # Primary functional test account (needs admin access to the account) | 13 | # Primary functional test account (needs admin access to the account) |
2969 | 7 | account = test | 14 | account = test |
2970 | 8 | username = tester | 15 | username = tester |
2971 | 9 | 16 | ||
2972 | === modified file 'test/functional/tests.py' | |||
2973 | --- test/functional/tests.py 2010-10-29 20:30:34 +0000 | |||
2974 | +++ test/functional/tests.py 2010-11-08 18:51:48 +0000 | |||
2975 | @@ -1092,7 +1092,7 @@ | |||
2976 | 1092 | self.assert_(file.read(hdrs={'Range': r}) == data[0:1000]) | 1092 | self.assert_(file.read(hdrs={'Range': r}) == data[0:1000]) |
2977 | 1093 | 1093 | ||
2978 | 1094 | def testFileSizeLimit(self): | 1094 | def testFileSizeLimit(self): |
2980 | 1095 | limit = 5*2**30 + 2 | 1095 | limit = int(config.get('max_object_size', 107374182400)) |
2981 | 1096 | tsecs = 3 | 1096 | tsecs = 3 |
2982 | 1097 | 1097 | ||
2983 | 1098 | for i in (limit-100, limit-10, limit-1, limit, limit+1, limit+10, | 1098 | for i in (limit-100, limit-10, limit-1, limit, limit+1, limit+10, |
2984 | 1099 | 1099 | ||
2985 | === modified file 'test/probe/common.py' | |||
2986 | --- test/probe/common.py 2010-09-12 00:03:09 +0000 | |||
2987 | +++ test/probe/common.py 2010-11-08 18:51:48 +0000 | |||
2988 | @@ -88,7 +88,7 @@ | |||
2989 | 88 | for p in ps: | 88 | for p in ps: |
2990 | 89 | p.wait() | 89 | p.wait() |
2991 | 90 | ps = [] | 90 | ps = [] |
2993 | 91 | for job in ('container-updater', 'object-updater'): | 91 | for job in ('container-updater', 'object-janitor'): |
2994 | 92 | for n in xrange(1, 5): | 92 | for n in xrange(1, 5): |
2995 | 93 | ps.append(Popen(['swift-%s' % job, | 93 | ps.append(Popen(['swift-%s' % job, |
2996 | 94 | '/etc/swift/%s-server/%d.conf' % | 94 | '/etc/swift/%s-server/%d.conf' % |
2997 | 95 | 95 | ||
2998 | === modified file 'test/probe/test_object_async_update.py' | |||
2999 | --- test/probe/test_object_async_update.py 2010-09-06 04:06:16 +0000 | |||
3000 | +++ test/probe/test_object_async_update.py 2010-11-08 18:51:48 +0000 | |||
3001 | @@ -55,7 +55,7 @@ | |||
3002 | 55 | self.account, container)[1]) | 55 | self.account, container)[1]) |
3003 | 56 | ps = [] | 56 | ps = [] |
3004 | 57 | for n in xrange(1, 5): | 57 | for n in xrange(1, 5): |
3006 | 58 | ps.append(Popen(['swift-object-updater', | 58 | ps.append(Popen(['swift-object-janitor', |
3007 | 59 | '/etc/swift/object-server/%d.conf' % n, 'once'])) | 59 | '/etc/swift/object-server/%d.conf' % n, 'once'])) |
3008 | 60 | for p in ps: | 60 | for p in ps: |
3009 | 61 | p.wait() | 61 | p.wait() |
3010 | 62 | 62 | ||
3011 | === modified file 'test/unit/common/test_constraints.py' | |||
3012 | --- test/unit/common/test_constraints.py 2010-08-16 22:30:27 +0000 | |||
3013 | +++ test/unit/common/test_constraints.py 2010-11-08 18:51:48 +0000 | |||
3014 | @@ -90,22 +90,20 @@ | |||
3015 | 90 | headers=headers), 'object'), HTTPBadRequest)) | 90 | headers=headers), 'object'), HTTPBadRequest)) |
3016 | 91 | 91 | ||
3017 | 92 | def test_check_object_creation_content_length(self): | 92 | def test_check_object_creation_content_length(self): |
3020 | 93 | headers = {'Content-Length': str(constraints.MAX_FILE_SIZE), | 93 | headers = {'Content-Length': '1024', 'Content-Type': 'text/plain'} |
3019 | 94 | 'Content-Type': 'text/plain'} | ||
3021 | 95 | self.assertEquals(constraints.check_object_creation(Request.blank('/', | 94 | self.assertEquals(constraints.check_object_creation(Request.blank('/', |
3025 | 96 | headers=headers), 'object_name'), None) | 95 | headers=headers), 'object_name', 1024), None) |
3026 | 97 | headers = {'Content-Length': str(constraints.MAX_FILE_SIZE + 1), | 96 | headers = {'Content-Length': '1025', 'Content-Type': 'text/plain'} |
3024 | 98 | 'Content-Type': 'text/plain'} | ||
3027 | 99 | self.assert_(isinstance(constraints.check_object_creation( | 97 | self.assert_(isinstance(constraints.check_object_creation( |
3029 | 100 | Request.blank('/', headers=headers), 'object_name'), | 98 | Request.blank('/', headers=headers), 'object_name', 1024), |
3030 | 101 | HTTPRequestEntityTooLarge)) | 99 | HTTPRequestEntityTooLarge)) |
3031 | 102 | headers = {'Transfer-Encoding': 'chunked', | 100 | headers = {'Transfer-Encoding': 'chunked', |
3032 | 103 | 'Content-Type': 'text/plain'} | 101 | 'Content-Type': 'text/plain'} |
3033 | 104 | self.assertEquals(constraints.check_object_creation(Request.blank('/', | 102 | self.assertEquals(constraints.check_object_creation(Request.blank('/', |
3035 | 105 | headers=headers), 'object_name'), None) | 103 | headers=headers), 'object_name', 1024), None) |
3036 | 106 | headers = {'Content-Type': 'text/plain'} | 104 | headers = {'Content-Type': 'text/plain'} |
3037 | 107 | self.assert_(isinstance(constraints.check_object_creation( | 105 | self.assert_(isinstance(constraints.check_object_creation( |
3039 | 108 | Request.blank('/', headers=headers), 'object_name'), | 106 | Request.blank('/', headers=headers), 'object_name', 1024), |
3040 | 109 | HTTPLengthRequired)) | 107 | HTTPLengthRequired)) |
3041 | 110 | 108 | ||
3042 | 111 | def test_check_object_creation_name_length(self): | 109 | def test_check_object_creation_name_length(self): |
3043 | 112 | 110 | ||
3044 | === added file 'test/unit/obj/test_diskfile.py' | |||
3045 | --- test/unit/obj/test_diskfile.py 1970-01-01 00:00:00 +0000 | |||
3046 | +++ test/unit/obj/test_diskfile.py 2010-11-08 18:51:48 +0000 | |||
3047 | @@ -0,0 +1,203 @@ | |||
3048 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
3049 | 2 | # | ||
3050 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
3051 | 4 | # you may not use this file except in compliance with the License. | ||
3052 | 5 | # You may obtain a copy of the License at | ||
3053 | 6 | # | ||
3054 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
3055 | 8 | # | ||
3056 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
3057 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
3058 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
3059 | 12 | # implied. | ||
3060 | 13 | # See the License for the specific language governing permissions and | ||
3061 | 14 | # limitations under the License. | ||
3062 | 15 | |||
3063 | 16 | import cPickle as pickle | ||
3064 | 17 | import os | ||
3065 | 18 | import sys | ||
3066 | 19 | import unittest | ||
3067 | 20 | from nose import SkipTest | ||
3068 | 21 | from shutil import rmtree | ||
3069 | 22 | from StringIO import StringIO | ||
3070 | 23 | from time import time | ||
3071 | 24 | |||
3072 | 25 | from xattr import setxattr | ||
3073 | 26 | |||
3074 | 27 | from swift.common.constraints import PICKLE_PROTOCOL | ||
3075 | 28 | from swift.common.utils import mkdirs, normalize_timestamp | ||
3076 | 29 | from swift.obj.diskfile import DiskFile, hash_suffix, JANITORDIR, \ | ||
3077 | 30 | METADATA_KEY | ||
3078 | 31 | |||
3079 | 32 | |||
3080 | 33 | class TestDiskFile(unittest.TestCase): | ||
3081 | 34 | """ Test swift.obj.diskfile """ | ||
3082 | 35 | |||
3083 | 36 | def setUp(self): | ||
3084 | 37 | """ Set up for testing swift.obj.diskfile """ | ||
3085 | 38 | self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') | ||
3086 | 39 | if not self.path_to_test_xfs or \ | ||
3087 | 40 | not os.path.exists(self.path_to_test_xfs): | ||
3088 | 41 | print >> sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \ | ||
3089 | 42 | 'pointing to a valid directory.\n' \ | ||
3090 | 43 | 'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \ | ||
3091 | 44 | 'system for testing.' | ||
3092 | 45 | self.testdir = '/tmp/SWIFTUNITTEST' | ||
3093 | 46 | else: | ||
3094 | 47 | self.testdir = os.path.join(self.path_to_test_xfs, | ||
3095 | 48 | 'tmp_test_obj_diskfile') | ||
3096 | 49 | mkdirs(self.testdir) | ||
3097 | 50 | rmtree(self.testdir) | ||
3098 | 51 | mkdirs(os.path.join(self.testdir, 'sda1')) | ||
3099 | 52 | mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) | ||
3100 | 53 | |||
3101 | 54 | def tearDown(self): | ||
3102 | 55 | """ Tear down for testing swift.obj.diskfile """ | ||
3103 | 56 | rmtree(self.testdir) | ||
3104 | 57 | |||
3105 | 58 | def test_disk_file_app_iter_corners(self): | ||
3106 | 59 | if not self.path_to_test_xfs: | ||
3107 | 60 | raise SkipTest | ||
3108 | 61 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3109 | 62 | mkdirs(df.datadir) | ||
3110 | 63 | f = open(os.path.join(df.datadir, | ||
3111 | 64 | normalize_timestamp(time()) + '.data'), 'wb') | ||
3112 | 65 | f.write('1234567890') | ||
3113 | 66 | setxattr(f.fileno(), METADATA_KEY, pickle.dumps({}, PICKLE_PROTOCOL)) | ||
3114 | 67 | f.close() | ||
3115 | 68 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3116 | 69 | keep_data_fp=True) | ||
3117 | 70 | it = df.app_iter_range(0, None) | ||
3118 | 71 | sio = StringIO() | ||
3119 | 72 | for chunk in it: | ||
3120 | 73 | sio.write(chunk) | ||
3121 | 74 | self.assertEquals(sio.getvalue(), '1234567890') | ||
3122 | 75 | |||
3123 | 76 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3124 | 77 | keep_data_fp=True) | ||
3125 | 78 | it = df.app_iter_range(5, None) | ||
3126 | 79 | sio = StringIO() | ||
3127 | 80 | for chunk in it: | ||
3128 | 81 | sio.write(chunk) | ||
3129 | 82 | self.assertEquals(sio.getvalue(), '67890') | ||
3130 | 83 | |||
3131 | 84 | def test_disk_file_mkstemp_creates_dir(self): | ||
3132 | 85 | tmpdir = os.path.join(self.testdir, 'sda1', 'tmp') | ||
3133 | 86 | os.rmdir(tmpdir) | ||
3134 | 87 | with DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o').mkstemp(): | ||
3135 | 88 | self.assert_(os.path.exists(tmpdir)) | ||
3136 | 89 | |||
3137 | 90 | def test_hash_suffix_creates_janitor_jobs(self): | ||
3138 | 91 | if not self.path_to_test_xfs: | ||
3139 | 92 | raise SkipTest | ||
3140 | 93 | # Ensure the janitor job we expect to create is not there right now. | ||
3141 | 94 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3142 | 95 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3143 | 96 | self.assert_(df.is_deleted()) | ||
3144 | 97 | # Set up manifest file to be tombstoned | ||
3145 | 98 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3146 | 99 | with df.mkstemp() as (fd, tmppath): | ||
3147 | 100 | os.write(fd, pickle.dumps({'x-timestamp': normalize_timestamp(1), | ||
3148 | 101 | 'content-length': 1234, 'content-type': 'text/plain', | ||
3149 | 102 | 'x-segment-size': 123, | ||
3150 | 103 | 'etag': 'd41d8cd98f00b204e9800998ecf8427e'}, | ||
3151 | 104 | protocol=PICKLE_PROTOCOL)) | ||
3152 | 105 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(2), | ||
3153 | 106 | 'X-Object-Type': 'manifest'}) | ||
3154 | 107 | # Make tombstone DiskFile didn't create, such as one rsynced over. | ||
3155 | 108 | open(os.path.join(df.datadir, normalize_timestamp(3) + '.ts'), 'wb') | ||
3156 | 109 | # Finally, we call what we want to test. | ||
3157 | 110 | hash_suffix(os.path.dirname(df.datadir), 604800) | ||
3158 | 111 | # Ensure the janitor job got created. | ||
3159 | 112 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3160 | 113 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3161 | 114 | self.assert_(not df.is_deleted()) | ||
3162 | 115 | |||
3163 | 116 | def test_segment_info_overrides_datadir(self): | ||
3164 | 117 | if not self.path_to_test_xfs: | ||
3165 | 118 | raise SkipTest | ||
3166 | 119 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3167 | 120 | datadir1 = df.datadir.split('/')[6] | ||
3168 | 121 | df = DiskFile(self.testdir, 'sda1', '1', 'a', 'c', 'o') | ||
3169 | 122 | datadir2 = df.datadir.split('/')[6] | ||
3170 | 123 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', segment=0) | ||
3171 | 124 | datadir3 = df.datadir.split('/')[6] | ||
3172 | 125 | self.assertEquals(datadir1, datadir2) | ||
3173 | 126 | self.assertNotEquals(datadir1, datadir3) | ||
3174 | 127 | |||
3175 | 128 | def test_no_longer_segment(self): | ||
3176 | 129 | if not self.path_to_test_xfs: | ||
3177 | 130 | raise SkipTest | ||
3178 | 131 | # Normal case | ||
3179 | 132 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3180 | 133 | datadir1 = df.datadir | ||
3181 | 134 | with df.mkstemp() as (fd, tmppath): | ||
3182 | 135 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(1)}) | ||
3183 | 136 | self.assert_(os.path.exists(datadir1)) | ||
3184 | 137 | # Normal case with no_longer_segment (doesn't make sense, but shouldn't | ||
3185 | 138 | # blow up) | ||
3186 | 139 | df = DiskFile(self.testdir, 'sda1', '1', 'a', 'c', 'o') | ||
3187 | 140 | datadir1 = df.datadir | ||
3188 | 141 | with df.mkstemp() as (fd, tmppath): | ||
3189 | 142 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(1)}, | ||
3190 | 143 | no_longer_segment=True) | ||
3191 | 144 | self.assert_(os.path.exists(datadir1)) | ||
3192 | 145 | # Segment case | ||
3193 | 146 | df = DiskFile(self.testdir, 'sda1', '2', 'a', 'c', 'o', segment=0) | ||
3194 | 147 | datadir1 = df.datadir | ||
3195 | 148 | with df.mkstemp() as (fd, tmppath): | ||
3196 | 149 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(1)}) | ||
3197 | 150 | self.assert_(os.path.exists(datadir1)) | ||
3198 | 151 | # Segment case with no_longer_segment | ||
3199 | 152 | df = DiskFile(self.testdir, 'sda1', '3', 'a', 'c', 'o') | ||
3200 | 153 | normal_datadir1 = df.datadir | ||
3201 | 154 | self.assert_(not os.path.exists(normal_datadir1)) | ||
3202 | 155 | df = DiskFile(self.testdir, 'sda1', '3', 'a', 'c', 'o', segment=0) | ||
3203 | 156 | datadir1 = df.datadir | ||
3204 | 157 | with df.mkstemp() as (fd, tmppath): | ||
3205 | 158 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(1)}, | ||
3206 | 159 | no_longer_segment=True) | ||
3207 | 160 | self.assert_(not os.path.exists(datadir1)) | ||
3208 | 161 | self.assert_(os.path.exists(normal_datadir1)) | ||
3209 | 162 | |||
3210 | 163 | def test_tombstone(self): | ||
3211 | 164 | if not self.path_to_test_xfs: | ||
3212 | 165 | raise SkipTest | ||
3213 | 166 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3214 | 167 | with df.mkstemp() as (fd, tmppath): | ||
3215 | 168 | df.put(fd, tmppath, {'X-Timestamp': normalize_timestamp(1)}) | ||
3216 | 169 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3217 | 170 | self.assert_(not df.is_deleted()) | ||
3218 | 171 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3219 | 172 | df.tombstone(normalize_timestamp(2)) | ||
3220 | 173 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3221 | 174 | self.assert_(df.is_deleted()) | ||
3222 | 175 | |||
3223 | 176 | def test_store_janitor_container_update(self): | ||
3224 | 177 | if not self.path_to_test_xfs: | ||
3225 | 178 | raise SkipTest | ||
3226 | 179 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3227 | 180 | datadir=JANITORDIR) | ||
3228 | 181 | df.store_janitor_container_update('PUT', 'a', 'c', 'o', | ||
3229 | 182 | {'X-Timestamp': normalize_timestamp(1)}, []) | ||
3230 | 183 | df = DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3231 | 184 | datadir=JANITORDIR) | ||
3232 | 185 | self.assertEquals(pickle.load(open(df.data_file, 'rb')), | ||
3233 | 186 | {'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', | ||
3234 | 187 | 'headers': {'X-Timestamp': '0000000001.00000'}, 'successes': []}) | ||
3235 | 188 | |||
3236 | 189 | def test_store_janitor_segment_cleanup(self): | ||
3237 | 190 | if not self.path_to_test_xfs: | ||
3238 | 191 | raise SkipTest | ||
3239 | 192 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3240 | 193 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3241 | 194 | df.store_janitor_segment_cleanup('a', 'c', 'o', 123, 45) | ||
3242 | 195 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3243 | 196 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3244 | 197 | self.assertEquals(pickle.load(open(df.data_file, 'rb')), | ||
3245 | 198 | {'account': 'a', 'container': 'c', 'obj': 'o', | ||
3246 | 199 | 'segment_count': 123, 'segment_last_deleted': 45}) | ||
3247 | 200 | |||
3248 | 201 | |||
3249 | 202 | if __name__ == '__main__': | ||
3250 | 203 | unittest.main() | ||
3251 | 0 | 204 | ||
3252 | === added file 'test/unit/obj/test_janitor.py' | |||
3253 | --- test/unit/obj/test_janitor.py 1970-01-01 00:00:00 +0000 | |||
3254 | +++ test/unit/obj/test_janitor.py 2010-11-08 18:51:48 +0000 | |||
3255 | @@ -0,0 +1,433 @@ | |||
3256 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
3257 | 2 | # | ||
3258 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
3259 | 4 | # you may not use this file except in compliance with the License. | ||
3260 | 5 | # You may obtain a copy of the License at | ||
3261 | 6 | # | ||
3262 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
3263 | 8 | # | ||
3264 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
3265 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
3266 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
3267 | 12 | # implied. | ||
3268 | 13 | # See the License for the specific language governing permissions and | ||
3269 | 14 | # limitations under the License. | ||
3270 | 15 | |||
3271 | 16 | import cPickle as pickle | ||
3272 | 17 | import os | ||
3273 | 18 | import sys | ||
3274 | 19 | import traceback | ||
3275 | 20 | import unittest | ||
3276 | 21 | from gzip import GzipFile | ||
3277 | 22 | from nose import SkipTest | ||
3278 | 23 | from shutil import rmtree | ||
3279 | 24 | from time import time | ||
3280 | 25 | |||
3281 | 26 | from eventlet import spawn, TimeoutError, listen | ||
3282 | 27 | from eventlet.timeout import Timeout | ||
3283 | 28 | |||
3284 | 29 | from swift.common.constraints import PICKLE_PROTOCOL | ||
3285 | 30 | from swift.common.ring import Ring, RingData | ||
3286 | 31 | from swift.common import utils | ||
3287 | 32 | from swift.common.utils import hash_path, normalize_timestamp, mkdirs | ||
3288 | 33 | from swift.obj import janitor as object_janitor | ||
3289 | 34 | from swift.obj.diskfile import DiskFile, JANITORDIR | ||
3290 | 35 | |||
3291 | 36 | |||
3292 | 37 | class TestObjectJanitor(unittest.TestCase): | ||
3293 | 38 | |||
3294 | 39 | def setUp(self): | ||
3295 | 40 | utils.HASH_PATH_SUFFIX = 'endcap' | ||
3296 | 41 | self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') | ||
3297 | 42 | if not self.path_to_test_xfs or \ | ||
3298 | 43 | not os.path.exists(self.path_to_test_xfs): | ||
3299 | 44 | print >> sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \ | ||
3300 | 45 | 'pointing to a valid directory.\n' \ | ||
3301 | 46 | 'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \ | ||
3302 | 47 | 'system for testing.' | ||
3303 | 48 | self.testdir = '/tmp/SWIFTUNITTEST' | ||
3304 | 49 | else: | ||
3305 | 50 | self.testdir = os.path.join(self.path_to_test_xfs, | ||
3306 | 51 | 'tmp_test_object_server_ObjectController') | ||
3307 | 52 | rmtree(self.testdir, ignore_errors=1) | ||
3308 | 53 | os.mkdir(self.testdir) | ||
3309 | 54 | pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], | ||
3310 | 55 | [{'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', | ||
3311 | 56 | 'zone': 0}, | ||
3312 | 57 | {'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', | ||
3313 | 58 | 'zone': 2}], 30), | ||
3314 | 59 | GzipFile(os.path.join(self.testdir, 'object.ring.gz'), 'wb')) | ||
3315 | 60 | self.object_ring = Ring(os.path.join(self.testdir, 'object.ring.gz')) | ||
3316 | 61 | pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]], | ||
3317 | 62 | [{'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', | ||
3318 | 63 | 'zone': 0}, | ||
3319 | 64 | {'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', | ||
3320 | 65 | 'zone': 2}], 30), | ||
3321 | 66 | GzipFile(os.path.join(self.testdir, 'container.ring.gz'), 'wb')) | ||
3322 | 67 | self.container_ring = \ | ||
3323 | 68 | Ring(os.path.join(self.testdir, 'container.ring.gz')) | ||
3324 | 69 | self.devices_dir = os.path.join(self.testdir, 'devices') | ||
3325 | 70 | os.mkdir(self.devices_dir) | ||
3326 | 71 | self.sda1 = os.path.join(self.devices_dir, 'sda1') | ||
3327 | 72 | os.mkdir(self.sda1) | ||
3328 | 73 | os.mkdir(os.path.join(self.sda1, 'tmp')) | ||
3329 | 74 | |||
3330 | 75 | def tearDown(self): | ||
3331 | 76 | rmtree(self.testdir, ignore_errors=1) | ||
3332 | 77 | |||
3333 | 78 | def test_creation(self): | ||
3334 | 79 | if not self.path_to_test_xfs: | ||
3335 | 80 | raise SkipTest | ||
3336 | 81 | janitor = object_janitor.ObjectJanitor({ | ||
3337 | 82 | 'devices': self.devices_dir, | ||
3338 | 83 | 'mount_check': 'false', | ||
3339 | 84 | 'swift_dir': self.testdir, | ||
3340 | 85 | 'interval': '1', | ||
3341 | 86 | 'concurrency': '2', | ||
3342 | 87 | 'node_timeout': '5', | ||
3343 | 88 | }) | ||
3344 | 89 | self.assert_(hasattr(janitor, 'logger')) | ||
3345 | 90 | self.assert_(janitor.logger is not None) | ||
3346 | 91 | self.assertEquals(janitor.devices, self.devices_dir) | ||
3347 | 92 | self.assertEquals(janitor.interval, 1) | ||
3348 | 93 | self.assertEquals(janitor.concurrency, 2) | ||
3349 | 94 | self.assertEquals(janitor.node_timeout, 5) | ||
3350 | 95 | self.assert_(janitor.get_container_ring() is not None) | ||
3351 | 96 | |||
3352 | 97 | def test_run_once_container_update(self): | ||
3353 | 98 | if not self.path_to_test_xfs: | ||
3354 | 99 | raise SkipTest | ||
3355 | 100 | janitor = object_janitor.ObjectJanitor({ | ||
3356 | 101 | 'devices': self.devices_dir, | ||
3357 | 102 | 'mount_check': 'false', | ||
3358 | 103 | 'swift_dir': self.testdir, | ||
3359 | 104 | 'interval': '1', | ||
3360 | 105 | 'concurrency': '1', | ||
3361 | 106 | 'node_timeout': '15', | ||
3362 | 107 | }) | ||
3363 | 108 | janitor.run_once() | ||
3364 | 109 | janitor_dir = os.path.join(self.sda1, JANITORDIR) | ||
3365 | 110 | os.mkdir(janitor_dir) | ||
3366 | 111 | janitor.run_once() | ||
3367 | 112 | self.assert_(os.path.exists(janitor_dir)) | ||
3368 | 113 | |||
3369 | 114 | disk_file = DiskFile(self.devices_dir, 'sda1', | ||
3370 | 115 | str(self.container_ring.get_nodes('a', 'c', 'o')[0]), 'a', 'c', | ||
3371 | 116 | 'o', datadir=JANITORDIR) | ||
3372 | 117 | ts = normalize_timestamp(1) | ||
3373 | 118 | with disk_file.mkstemp() as (fd, tmppath): | ||
3374 | 119 | os.write(fd, pickle.dumps({'op': 'PUT', | ||
3375 | 120 | 'account': 'a', 'container': 'c', 'obj': 'o', | ||
3376 | 121 | 'headers': {'X-Container-Timestamp': ts}}, | ||
3377 | 122 | PICKLE_PROTOCOL)) | ||
3378 | 123 | disk_file.put(fd, tmppath, | ||
3379 | 124 | {'X-Op': 'Container-Update', 'X-Timestamp': ts}) | ||
3380 | 125 | janitor.run_once() | ||
3381 | 126 | self.assert_(os.path.exists(os.path.join(disk_file.datadir, | ||
3382 | 127 | ts + '.data'))) | ||
3383 | 128 | |||
3384 | 129 | bindsock = listen(('127.0.0.1', 0)) | ||
3385 | 130 | |||
3386 | 131 | def accepter(sock, return_code): | ||
3387 | 132 | try: | ||
3388 | 133 | with Timeout(3): | ||
3389 | 134 | inc = sock.makefile('rb') | ||
3390 | 135 | out = sock.makefile('wb') | ||
3391 | 136 | out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % | ||
3392 | 137 | return_code) | ||
3393 | 138 | out.flush() | ||
3394 | 139 | self.assertEquals(inc.readline(), | ||
3395 | 140 | 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') | ||
3396 | 141 | headers = {} | ||
3397 | 142 | line = inc.readline() | ||
3398 | 143 | while line and line != '\r\n': | ||
3399 | 144 | headers[line.split(':')[0].lower()] = \ | ||
3400 | 145 | line.split(':')[1].strip() | ||
3401 | 146 | line = inc.readline() | ||
3402 | 147 | self.assert_('x-container-timestamp' in headers) | ||
3403 | 148 | except BaseException, err: | ||
3404 | 149 | return err | ||
3405 | 150 | return None | ||
3406 | 151 | |||
3407 | 152 | def accept(return_codes): | ||
3408 | 153 | codes = iter(return_codes) | ||
3409 | 154 | try: | ||
3410 | 155 | events = [] | ||
3411 | 156 | for x in xrange(len(return_codes)): | ||
3412 | 157 | with Timeout(3): | ||
3413 | 158 | sock, addr = bindsock.accept() | ||
3414 | 159 | events.append( | ||
3415 | 160 | spawn(accepter, sock, codes.next())) | ||
3416 | 161 | for event in events: | ||
3417 | 162 | err = event.wait() | ||
3418 | 163 | if err: | ||
3419 | 164 | raise err | ||
3420 | 165 | except BaseException, err: | ||
3421 | 166 | return err | ||
3422 | 167 | return None | ||
3423 | 168 | |||
3424 | 169 | event = spawn(accept, [201, 500]) | ||
3425 | 170 | for dev in janitor.get_container_ring().devs: | ||
3426 | 171 | if dev is not None: | ||
3427 | 172 | dev['port'] = bindsock.getsockname()[1] | ||
3428 | 173 | janitor.run_once() | ||
3429 | 174 | err = event.wait() | ||
3430 | 175 | if err: | ||
3431 | 176 | raise err | ||
3432 | 177 | disk_file = DiskFile(self.devices_dir, 'sda1', | ||
3433 | 178 | str(self.container_ring.get_nodes('a', 'c', 'o')[0]), 'a', 'c', | ||
3434 | 179 | 'o', datadir=JANITORDIR) | ||
3435 | 180 | self.assert_(not disk_file.is_deleted()) | ||
3436 | 181 | event = spawn(accept, [201]) | ||
3437 | 182 | janitor.run_once() | ||
3438 | 183 | err = event.wait() | ||
3439 | 184 | if err: | ||
3440 | 185 | raise err | ||
3441 | 186 | disk_file = DiskFile(self.devices_dir, 'sda1', | ||
3442 | 187 | str(self.container_ring.get_nodes('a', 'c', 'o')[0]), 'a', 'c', | ||
3443 | 188 | 'o', datadir=JANITORDIR) | ||
3444 | 189 | self.assert_(disk_file.is_deleted()) | ||
3445 | 190 | |||
3446 | 191 | def _segment_cleanup_in_progress_helper(self, statuses, expect_success): | ||
3447 | 192 | if not self.path_to_test_xfs: | ||
3448 | 193 | raise SkipTest | ||
3449 | 194 | janitor = object_janitor.ObjectJanitor({'devices': self.devices_dir, | ||
3450 | 195 | 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', | ||
3451 | 196 | 'concurrency': '1', 'node_timeout': '15'}) | ||
3452 | 197 | |||
3453 | 198 | # Quick test of connection refusals | ||
3454 | 199 | df = DiskFile(self.devices_dir, 'sda1', | ||
3455 | 200 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3456 | 201 | 'Segment-Cleanup', normalize_timestamp(1), 'a/c/o', | ||
3457 | 202 | datadir=JANITORDIR) | ||
3458 | 203 | df.store_janitor_segment_cleanup('a', 'c', 'o', 2, 0) | ||
3459 | 204 | janitor.run_once() | ||
3460 | 205 | df = DiskFile(self.devices_dir, 'sda1', | ||
3461 | 206 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3462 | 207 | 'Segment-Cleanup', normalize_timestamp(1), 'a/c/o', | ||
3463 | 208 | datadir=JANITORDIR) | ||
3464 | 209 | self.assert_(not df.is_deleted()) | ||
3465 | 210 | |||
3466 | 211 | bindsock = listen(('127.0.0.1', 0)) | ||
3467 | 212 | janitor.port = bindsock.getsockname()[1] | ||
3468 | 213 | |||
3469 | 214 | def accepter(sock, return_code): | ||
3470 | 215 | try: | ||
3471 | 216 | with Timeout(3): | ||
3472 | 217 | inc = sock.makefile('rb') | ||
3473 | 218 | out = sock.makefile('wb') | ||
3474 | 219 | out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % | ||
3475 | 220 | return_code) | ||
3476 | 221 | out.flush() | ||
3477 | 222 | self.assertEquals(inc.readline(), | ||
3478 | 223 | 'DELETE /sda1/2/a/c/o HTTP/1.1\r\n') | ||
3479 | 224 | headers = {} | ||
3480 | 225 | line = inc.readline() | ||
3481 | 226 | while line and line != '\r\n': | ||
3482 | 227 | headers[line.split(':')[0].lower()] = \ | ||
3483 | 228 | line.split(':')[1].strip() | ||
3484 | 229 | line = inc.readline() | ||
3485 | 230 | self.assert_('x-object-segment-timestamp' in headers) | ||
3486 | 231 | self.assertEquals(headers.get('x-object-segment'), '1') | ||
3487 | 232 | except BaseException, err: | ||
3488 | 233 | return err | ||
3489 | 234 | return None | ||
3490 | 235 | |||
3491 | 236 | def accept(return_codes): | ||
3492 | 237 | codes = iter(return_codes) | ||
3493 | 238 | try: | ||
3494 | 239 | events = [] | ||
3495 | 240 | for x in xrange(len(return_codes)): | ||
3496 | 241 | with Timeout(3): | ||
3497 | 242 | sock, addr = bindsock.accept() | ||
3498 | 243 | events.append( | ||
3499 | 244 | spawn(accepter, sock, codes.next())) | ||
3500 | 245 | for event in events: | ||
3501 | 246 | err = event.wait() | ||
3502 | 247 | if err: | ||
3503 | 248 | raise err | ||
3504 | 249 | except BaseException, err: | ||
3505 | 250 | return err | ||
3506 | 251 | return None | ||
3507 | 252 | |||
3508 | 253 | event = spawn(accept, statuses) | ||
3509 | 254 | for dev in janitor.get_object_ring().devs: | ||
3510 | 255 | if dev is not None: | ||
3511 | 256 | dev['port'] = bindsock.getsockname()[1] | ||
3512 | 257 | janitor.run_once() | ||
3513 | 258 | err = event.wait() | ||
3514 | 259 | if err: | ||
3515 | 260 | raise err | ||
3516 | 261 | df = DiskFile(self.devices_dir, 'sda1', | ||
3517 | 262 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3518 | 263 | 'Segment-Cleanup', normalize_timestamp(1), 'a/c/o', | ||
3519 | 264 | datadir=JANITORDIR) | ||
3520 | 265 | self.assertEquals(df.is_deleted(), expect_success) | ||
3521 | 266 | |||
3522 | 267 | def test_segment_cleanup_in_progress_happy_path(self): | ||
3523 | 268 | self._segment_cleanup_in_progress_helper([204, 204], True) | ||
3524 | 269 | |||
3525 | 270 | def test_segment_cleanup_in_progress_one_failure(self): | ||
3526 | 271 | self._segment_cleanup_in_progress_helper([204, 500], True) | ||
3527 | 272 | |||
3528 | 273 | def test_segment_cleanup_in_progress_all_failures(self): | ||
3529 | 274 | self._segment_cleanup_in_progress_helper([500, 500], False) | ||
3530 | 275 | |||
3531 | 276 | def test_segment_cleanup_in_progress_all_not_found(self): | ||
3532 | 277 | self._segment_cleanup_in_progress_helper([404, 404], True) | ||
3533 | 278 | |||
3534 | 279 | def test_segment_cleanup_in_progress_one_not_found_one_success(self): | ||
3535 | 280 | self._segment_cleanup_in_progress_helper([404, 204], True) | ||
3536 | 281 | |||
3537 | 282 | def test_segment_cleanup_in_progress_one_not_found_one_failure(self): | ||
3538 | 283 | self._segment_cleanup_in_progress_helper([404, 500], False) | ||
3539 | 284 | |||
3540 | 285 | def _segment_cleanup_fresh_start_helper(self, cleanup_timestamp, | ||
3541 | 286 | existing_timestamp, statuses, expect_success): | ||
3542 | 287 | if not self.path_to_test_xfs: | ||
3543 | 288 | raise SkipTest | ||
3544 | 289 | janitor = object_janitor.ObjectJanitor({'devices': self.devices_dir, | ||
3545 | 290 | 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', | ||
3546 | 291 | 'concurrency': '1', 'node_timeout': '15', 'segments_per_pass': 2}) | ||
3547 | 292 | |||
3548 | 293 | # Quick test of connection refusals | ||
3549 | 294 | df = DiskFile(self.devices_dir, 'sda1', | ||
3550 | 295 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3551 | 296 | 'Segment-Cleanup', normalize_timestamp(cleanup_timestamp), 'a/c/o', | ||
3552 | 297 | datadir=JANITORDIR) | ||
3553 | 298 | df.store_janitor_segment_cleanup('a', 'c', 'o', None, None) | ||
3554 | 299 | janitor.run_once() | ||
3555 | 300 | df = DiskFile(self.devices_dir, 'sda1', | ||
3556 | 301 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3557 | 302 | 'Segment-Cleanup', normalize_timestamp(cleanup_timestamp), 'a/c/o', | ||
3558 | 303 | datadir=JANITORDIR) | ||
3559 | 304 | self.assert_(not df.is_deleted()) | ||
3560 | 305 | |||
3561 | 306 | bindsock = listen(('127.0.0.1', 0)) | ||
3562 | 307 | janitor.port = bindsock.getsockname()[1] | ||
3563 | 308 | |||
3564 | 309 | def accepter(sock, return_code): | ||
3565 | 310 | try: | ||
3566 | 311 | with Timeout(3): | ||
3567 | 312 | inc = sock.makefile('rb') | ||
3568 | 313 | out = sock.makefile('wb') | ||
3569 | 314 | request = inc.readline() | ||
3570 | 315 | if request.startswith('GET '): | ||
3571 | 316 | self.assert_(request, 'GET /sda1/0/a/c/o HTTP/1.1\r\n') | ||
3572 | 317 | if return_code == 200: | ||
3573 | 318 | pickl = pickle.dumps({'x-timestamp': | ||
3574 | 319 | normalize_timestamp(existing_timestamp)}) | ||
3575 | 320 | out.write('HTTP/1.1 %d OK\r\nContent-Length: ' | ||
3576 | 321 | '%s\r\nX-Object-Type: manifest\r\n\r\n' % | ||
3577 | 322 | (return_code, len(pickl))) | ||
3578 | 323 | out.write(pickl) | ||
3579 | 324 | out.flush() | ||
3580 | 325 | else: | ||
3581 | 326 | out.write('HTTP/1.1 %d OK\r\nContent-Length: ' | ||
3582 | 327 | '0\r\n\r\n' % return_code) | ||
3583 | 328 | out.flush() | ||
3584 | 329 | return None | ||
3585 | 330 | if request.startswith('HEAD '): | ||
3586 | 331 | self.assert_(request, | ||
3587 | 332 | 'HEAD /sda1/2/a/c/o HTTP/1.1\r\n') | ||
3588 | 333 | out.write('HTTP/1.1 %d OK\r\nContent-Length: ' | ||
3589 | 334 | '0\r\n\r\n' % return_code) | ||
3590 | 335 | out.flush() | ||
3591 | 336 | return None | ||
3592 | 337 | else: | ||
3593 | 338 | self.assert_(request, | ||
3594 | 339 | 'DELETE /sda1/2/a/c/o HTTP/1.1\r\n') | ||
3595 | 340 | out.write( | ||
3596 | 341 | 'HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % | ||
3597 | 342 | return_code) | ||
3598 | 343 | out.flush() | ||
3599 | 344 | headers = {} | ||
3600 | 345 | line = inc.readline() | ||
3601 | 346 | while line and line != '\r\n': | ||
3602 | 347 | headers[line.split(':')[0].lower()] = \ | ||
3603 | 348 | line.split(':')[1].strip() | ||
3604 | 349 | line = inc.readline() | ||
3605 | 350 | self.assert_('x-object-segment' in headers) | ||
3606 | 351 | self.assert_('x-object-segment-timestamp' in headers) | ||
3607 | 352 | except BaseException, err: | ||
3608 | 353 | traceback.print_exc() | ||
3609 | 354 | return err | ||
3610 | 355 | return None | ||
3611 | 356 | |||
3612 | 357 | def accept(return_codes): | ||
3613 | 358 | codes = iter(return_codes) | ||
3614 | 359 | try: | ||
3615 | 360 | events = [] | ||
3616 | 361 | for x in xrange(len(return_codes)): | ||
3617 | 362 | with Timeout(3): | ||
3618 | 363 | sock, addr = bindsock.accept() | ||
3619 | 364 | events.append( | ||
3620 | 365 | spawn(accepter, sock, codes.next())) | ||
3621 | 366 | for event in events: | ||
3622 | 367 | err = event.wait() | ||
3623 | 368 | if err: | ||
3624 | 369 | raise err | ||
3625 | 370 | except BaseException, err: | ||
3626 | 371 | return err | ||
3627 | 372 | return None | ||
3628 | 373 | |||
3629 | 374 | event = spawn(accept, statuses) | ||
3630 | 375 | for dev in janitor.get_object_ring().devs: | ||
3631 | 376 | if dev is not None: | ||
3632 | 377 | dev['port'] = bindsock.getsockname()[1] | ||
3633 | 378 | janitor.run_once() | ||
3634 | 379 | err = event.wait() | ||
3635 | 380 | if err: | ||
3636 | 381 | raise err | ||
3637 | 382 | df = DiskFile(self.devices_dir, 'sda1', | ||
3638 | 383 | str(self.object_ring.get_nodes('a', 'c', 'o')[0]), | ||
3639 | 384 | 'Segment-Cleanup', normalize_timestamp(cleanup_timestamp), 'a/c/o', | ||
3640 | 385 | datadir=JANITORDIR) | ||
3641 | 386 | self.assertEquals(df.is_deleted(), expect_success) | ||
3642 | 387 | |||
3643 | 388 | def test_segment_cleanup_fresh_start_happy_path(self): | ||
3644 | 389 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(1), | ||
3645 | 390 | normalize_timestamp(1), | ||
3646 | 391 | [404, 404, # Check for manifest | ||
3647 | 392 | 200, 200, 404, 404, # Check for segments (2 total) | ||
3648 | 393 | 204, 204, 204, 204], # Delete segments | ||
3649 | 394 | True) | ||
3650 | 395 | |||
3651 | 396 | def test_segment_cleanup_fresh_start_manifest_exists(self): | ||
3652 | 397 | t = time() | ||
3653 | 398 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(t), | ||
3654 | 399 | normalize_timestamp(t), [200, 200], True) | ||
3655 | 400 | |||
3656 | 401 | def test_segment_cleanup_fresh_start_old_manifest_exists(self): | ||
3657 | 402 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(2), | ||
3658 | 403 | normalize_timestamp(1), | ||
3659 | 404 | [200, 200, # Check for manifest | ||
3660 | 405 | 200, 200, 404, 404, # Check for segments (2 total) | ||
3661 | 406 | 204, 204, 204, 204], # Delete segments | ||
3662 | 407 | True) | ||
3663 | 408 | |||
3664 | 409 | def test_segment_cleanup_fresh_start_old_manifest_exists2(self): | ||
3665 | 410 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(time()), | ||
3666 | 411 | normalize_timestamp(1), [200, 200], False) | ||
3667 | 412 | |||
3668 | 413 | def test_segment_cleanup_fresh_start_new_manifest_exists(self): | ||
3669 | 414 | t = time() | ||
3670 | 415 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(t - 1), | ||
3671 | 416 | normalize_timestamp(t), | ||
3672 | 417 | [200, # Check for manifest | ||
3673 | 418 | 200, 200, 404, 404, # Check for segments (2 total) | ||
3674 | 419 | 204, 204, 204, 204], # Delete segments | ||
3675 | 420 | True) | ||
3676 | 421 | |||
3677 | 422 | def test_segment_cleanup_fresh_start_stops_at_segments_per_pass(self): | ||
3678 | 423 | self._segment_cleanup_fresh_start_helper(normalize_timestamp(1), | ||
3679 | 424 | normalize_timestamp(1), | ||
3680 | 425 | [404, 404, # Check for manifest | ||
3681 | 426 | 200, 200, 200, 404, 404, # Check for segments (3 total) | ||
3682 | 427 | 204, 204, 204, 204], # Delete segments (only 2 expected) | ||
3683 | 428 | False) | ||
3684 | 429 | |||
3685 | 430 | |||
3686 | 431 | |||
3687 | 432 | if __name__ == '__main__': | ||
3688 | 433 | unittest.main() | ||
3689 | 0 | 434 | ||
3690 | === modified file 'test/unit/obj/test_server.py' | |||
3691 | --- test/unit/obj/test_server.py 2010-10-13 21:26:43 +0000 | |||
3692 | +++ test/unit/obj/test_server.py 2010-11-08 18:51:48 +0000 | |||
3693 | @@ -13,7 +13,7 @@ | |||
3694 | 13 | # See the License for the specific language governing permissions and | 13 | # See the License for the specific language governing permissions and |
3695 | 14 | # limitations under the License. | 14 | # limitations under the License. |
3696 | 15 | 15 | ||
3698 | 16 | """ Tests for swift.object_server """ | 16 | """ Tests for swift.obj.server """ |
3699 | 17 | 17 | ||
3700 | 18 | import cPickle as pickle | 18 | import cPickle as pickle |
3701 | 19 | import os | 19 | import os |
3702 | @@ -22,23 +22,25 @@ | |||
3703 | 22 | from nose import SkipTest | 22 | from nose import SkipTest |
3704 | 23 | from shutil import rmtree | 23 | from shutil import rmtree |
3705 | 24 | from StringIO import StringIO | 24 | from StringIO import StringIO |
3707 | 25 | from time import gmtime, sleep, strftime, time | 25 | from time import gmtime, strftime, time |
3708 | 26 | 26 | ||
3709 | 27 | from eventlet import sleep, spawn, wsgi, listen | 27 | from eventlet import sleep, spawn, wsgi, listen |
3710 | 28 | from webob import Request | 28 | from webob import Request |
3712 | 29 | from xattr import getxattr, setxattr | 29 | from xattr import getxattr |
3713 | 30 | 30 | ||
3714 | 31 | from test.unit import connect_tcp, readuntil2crlfs | 31 | from test.unit import connect_tcp, readuntil2crlfs |
3715 | 32 | from swift.common.constraints import PICKLE_PROTOCOL | ||
3716 | 32 | from swift.obj import server as object_server | 33 | from swift.obj import server as object_server |
3717 | 34 | from swift.obj.diskfile import DATADIR, DiskFile, JANITORDIR, METADATA_KEY | ||
3718 | 33 | from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ | 35 | from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ |
3719 | 34 | NullLogger, storage_directory | 36 | NullLogger, storage_directory |
3720 | 35 | 37 | ||
3721 | 36 | 38 | ||
3722 | 37 | class TestObjectController(unittest.TestCase): | 39 | class TestObjectController(unittest.TestCase): |
3724 | 38 | """ Test swift.object_server.ObjectController """ | 40 | """ Test swift.obj.server.ObjectController """ |
3725 | 39 | 41 | ||
3726 | 40 | def setUp(self): | 42 | def setUp(self): |
3728 | 41 | """ Set up for testing swift.object_server.ObjectController """ | 43 | """ Set up for testing swift.obj.server.ObjectController """ |
3729 | 42 | self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') | 44 | self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS') |
3730 | 43 | if not self.path_to_test_xfs or \ | 45 | if not self.path_to_test_xfs or \ |
3731 | 44 | not os.path.exists(self.path_to_test_xfs): | 46 | not os.path.exists(self.path_to_test_xfs): |
3732 | @@ -49,7 +51,7 @@ | |||
3733 | 49 | self.testdir = '/tmp/SWIFTUNITTEST' | 51 | self.testdir = '/tmp/SWIFTUNITTEST' |
3734 | 50 | else: | 52 | else: |
3735 | 51 | self.testdir = os.path.join(self.path_to_test_xfs, | 53 | self.testdir = os.path.join(self.path_to_test_xfs, |
3737 | 52 | 'tmp_test_object_server_ObjectController') | 54 | 'tmp_test_obj_server_ObjectController') |
3738 | 53 | mkdirs(self.testdir) | 55 | mkdirs(self.testdir) |
3739 | 54 | rmtree(self.testdir) | 56 | rmtree(self.testdir) |
3740 | 55 | mkdirs(os.path.join(self.testdir, 'sda1')) | 57 | mkdirs(os.path.join(self.testdir, 'sda1')) |
3741 | @@ -59,11 +61,11 @@ | |||
3742 | 59 | self.object_controller.bytes_per_sync = 1 | 61 | self.object_controller.bytes_per_sync = 1 |
3743 | 60 | 62 | ||
3744 | 61 | def tearDown(self): | 63 | def tearDown(self): |
3746 | 62 | """ Tear down for testing swift.object_server.ObjectController """ | 64 | """ Tear down for testing swift.obj.server.ObjectController """ |
3747 | 63 | rmtree(self.testdir) | 65 | rmtree(self.testdir) |
3748 | 64 | 66 | ||
3749 | 65 | def test_POST_update_meta(self): | 67 | def test_POST_update_meta(self): |
3751 | 66 | """ Test swift.object_server.ObjectController.POST """ | 68 | """ Test swift.obj.server.ObjectController.POST """ |
3752 | 67 | if not self.path_to_test_xfs: | 69 | if not self.path_to_test_xfs: |
3753 | 68 | raise SkipTest | 70 | raise SkipTest |
3754 | 69 | timestamp = normalize_timestamp(time()) | 71 | timestamp = normalize_timestamp(time()) |
3755 | @@ -221,12 +223,11 @@ | |||
3756 | 221 | resp = self.object_controller.PUT(req) | 223 | resp = self.object_controller.PUT(req) |
3757 | 222 | self.assertEquals(resp.status_int, 201) | 224 | self.assertEquals(resp.status_int, 201) |
3758 | 223 | objfile = os.path.join(self.testdir, 'sda1', | 225 | objfile = os.path.join(self.testdir, 'sda1', |
3761 | 224 | storage_directory(object_server.DATADIR, 'p', | 226 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3760 | 225 | hash_path('a', 'c', 'o')), | ||
3762 | 226 | timestamp + '.data') | 227 | timestamp + '.data') |
3763 | 227 | self.assert_(os.path.isfile(objfile)) | 228 | self.assert_(os.path.isfile(objfile)) |
3764 | 228 | self.assertEquals(open(objfile).read(), 'VERIFY') | 229 | self.assertEquals(open(objfile).read(), 'VERIFY') |
3766 | 229 | self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), | 230 | self.assertEquals(pickle.loads(getxattr(objfile, METADATA_KEY)), |
3767 | 230 | {'X-Timestamp': timestamp, | 231 | {'X-Timestamp': timestamp, |
3768 | 231 | 'Content-Length': '6', | 232 | 'Content-Length': '6', |
3769 | 232 | 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', | 233 | 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', |
3770 | @@ -253,12 +254,11 @@ | |||
3771 | 253 | resp = self.object_controller.PUT(req) | 254 | resp = self.object_controller.PUT(req) |
3772 | 254 | self.assertEquals(resp.status_int, 201) | 255 | self.assertEquals(resp.status_int, 201) |
3773 | 255 | objfile = os.path.join(self.testdir, 'sda1', | 256 | objfile = os.path.join(self.testdir, 'sda1', |
3776 | 256 | storage_directory(object_server.DATADIR, 'p', | 257 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3775 | 257 | hash_path('a', 'c', 'o')), | ||
3777 | 258 | timestamp + '.data') | 258 | timestamp + '.data') |
3778 | 259 | self.assert_(os.path.isfile(objfile)) | 259 | self.assert_(os.path.isfile(objfile)) |
3779 | 260 | self.assertEquals(open(objfile).read(), 'VERIFY TWO') | 260 | self.assertEquals(open(objfile).read(), 'VERIFY TWO') |
3781 | 261 | self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), | 261 | self.assertEquals(pickle.loads(getxattr(objfile, METADATA_KEY)), |
3782 | 262 | {'X-Timestamp': timestamp, | 262 | {'X-Timestamp': timestamp, |
3783 | 263 | 'Content-Length': '10', | 263 | 'Content-Length': '10', |
3784 | 264 | 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', | 264 | 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', |
3785 | @@ -299,12 +299,11 @@ | |||
3786 | 299 | resp = self.object_controller.PUT(req) | 299 | resp = self.object_controller.PUT(req) |
3787 | 300 | self.assertEquals(resp.status_int, 201) | 300 | self.assertEquals(resp.status_int, 201) |
3788 | 301 | objfile = os.path.join(self.testdir, 'sda1', | 301 | objfile = os.path.join(self.testdir, 'sda1', |
3791 | 302 | storage_directory(object_server.DATADIR, 'p', | 302 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3790 | 303 | hash_path('a', 'c', 'o')), | ||
3792 | 304 | timestamp + '.data') | 303 | timestamp + '.data') |
3793 | 305 | self.assert_(os.path.isfile(objfile)) | 304 | self.assert_(os.path.isfile(objfile)) |
3794 | 306 | self.assertEquals(open(objfile).read(), 'VERIFY THREE') | 305 | self.assertEquals(open(objfile).read(), 'VERIFY THREE') |
3796 | 307 | self.assertEquals(pickle.loads(getxattr(objfile, object_server.METADATA_KEY)), | 306 | self.assertEquals(pickle.loads(getxattr(objfile, METADATA_KEY)), |
3797 | 308 | {'X-Timestamp': timestamp, | 307 | {'X-Timestamp': timestamp, |
3798 | 309 | 'Content-Length': '12', | 308 | 'Content-Length': '12', |
3799 | 310 | 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', | 309 | 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', |
3800 | @@ -375,7 +374,7 @@ | |||
3801 | 375 | object_server.http_connect = old_http_connect | 374 | object_server.http_connect = old_http_connect |
3802 | 376 | 375 | ||
3803 | 377 | def test_HEAD(self): | 376 | def test_HEAD(self): |
3805 | 378 | """ Test swift.object_server.ObjectController.HEAD """ | 377 | """ Test swift.obj.server.ObjectController.HEAD """ |
3806 | 379 | if not self.path_to_test_xfs: | 378 | if not self.path_to_test_xfs: |
3807 | 380 | raise SkipTest | 379 | raise SkipTest |
3808 | 381 | req = Request.blank('/sda1/p/a/c') | 380 | req = Request.blank('/sda1/p/a/c') |
3809 | @@ -410,8 +409,7 @@ | |||
3810 | 410 | self.assertEquals(resp.headers['x-object-meta-two'], 'Two') | 409 | self.assertEquals(resp.headers['x-object-meta-two'], 'Two') |
3811 | 411 | 410 | ||
3812 | 412 | objfile = os.path.join(self.testdir, 'sda1', | 411 | objfile = os.path.join(self.testdir, 'sda1', |
3815 | 413 | storage_directory(object_server.DATADIR, 'p', | 412 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3814 | 414 | hash_path('a', 'c', 'o')), | ||
3816 | 415 | timestamp + '.data') | 413 | timestamp + '.data') |
3817 | 416 | os.unlink(objfile) | 414 | os.unlink(objfile) |
3818 | 417 | req = Request.blank('/sda1/p/a/c/o') | 415 | req = Request.blank('/sda1/p/a/c/o') |
3819 | @@ -442,7 +440,7 @@ | |||
3820 | 442 | self.assertEquals(resp.status_int, 404) | 440 | self.assertEquals(resp.status_int, 404) |
3821 | 443 | 441 | ||
3822 | 444 | def test_GET(self): | 442 | def test_GET(self): |
3824 | 445 | """ Test swift.object_server.ObjectController.GET """ | 443 | """ Test swift.obj.server.ObjectController.GET """ |
3825 | 446 | if not self.path_to_test_xfs: | 444 | if not self.path_to_test_xfs: |
3826 | 447 | raise SkipTest | 445 | raise SkipTest |
3827 | 448 | req = Request.blank('/sda1/p/a/c') | 446 | req = Request.blank('/sda1/p/a/c') |
3828 | @@ -500,8 +498,7 @@ | |||
3829 | 500 | self.assertEquals(resp.headers['content-length'], '2') | 498 | self.assertEquals(resp.headers['content-length'], '2') |
3830 | 501 | 499 | ||
3831 | 502 | objfile = os.path.join(self.testdir, 'sda1', | 500 | objfile = os.path.join(self.testdir, 'sda1', |
3834 | 503 | storage_directory(object_server.DATADIR, 'p', | 501 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3833 | 504 | hash_path('a', 'c', 'o')), | ||
3835 | 505 | timestamp + '.data') | 502 | timestamp + '.data') |
3836 | 506 | os.unlink(objfile) | 503 | os.unlink(objfile) |
3837 | 507 | req = Request.blank('/sda1/p/a/c/o') | 504 | req = Request.blank('/sda1/p/a/c/o') |
3838 | @@ -712,7 +709,7 @@ | |||
3839 | 712 | self.assertEquals(resp.status_int, 200) | 709 | self.assertEquals(resp.status_int, 200) |
3840 | 713 | 710 | ||
3841 | 714 | def test_DELETE(self): | 711 | def test_DELETE(self): |
3843 | 715 | """ Test swift.object_server.ObjectController.DELETE """ | 712 | """ Test swift.obj.server.ObjectController.DELETE """ |
3844 | 716 | if not self.path_to_test_xfs: | 713 | if not self.path_to_test_xfs: |
3845 | 717 | raise SkipTest | 714 | raise SkipTest |
3846 | 718 | req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) | 715 | req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) |
3847 | @@ -751,8 +748,7 @@ | |||
3848 | 751 | resp = self.object_controller.DELETE(req) | 748 | resp = self.object_controller.DELETE(req) |
3849 | 752 | self.assertEquals(resp.status_int, 204) | 749 | self.assertEquals(resp.status_int, 204) |
3850 | 753 | objfile = os.path.join(self.testdir, 'sda1', | 750 | objfile = os.path.join(self.testdir, 'sda1', |
3853 | 754 | storage_directory(object_server.DATADIR, 'p', | 751 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3852 | 755 | hash_path('a', 'c', 'o')), | ||
3854 | 756 | timestamp + '.ts') | 752 | timestamp + '.ts') |
3855 | 757 | self.assert_(os.path.isfile(objfile)) | 753 | self.assert_(os.path.isfile(objfile)) |
3856 | 758 | 754 | ||
3857 | @@ -764,13 +760,12 @@ | |||
3858 | 764 | resp = self.object_controller.DELETE(req) | 760 | resp = self.object_controller.DELETE(req) |
3859 | 765 | self.assertEquals(resp.status_int, 204) | 761 | self.assertEquals(resp.status_int, 204) |
3860 | 766 | objfile = os.path.join(self.testdir, 'sda1', | 762 | objfile = os.path.join(self.testdir, 'sda1', |
3863 | 767 | storage_directory(object_server.DATADIR, 'p', | 763 | storage_directory(DATADIR, 'p', hash_path('a', 'c', 'o')), |
3862 | 768 | hash_path('a', 'c', 'o')), | ||
3864 | 769 | timestamp + '.ts') | 764 | timestamp + '.ts') |
3865 | 770 | self.assert_(os.path.isfile(objfile)) | 765 | self.assert_(os.path.isfile(objfile)) |
3866 | 771 | 766 | ||
3867 | 772 | def test_call(self): | 767 | def test_call(self): |
3869 | 773 | """ Test swift.object_server.ObjectController.__call__ """ | 768 | """ Test swift.obj.server.ObjectController.__call__ """ |
3870 | 774 | inbuf = StringIO() | 769 | inbuf = StringIO() |
3871 | 775 | errbuf = StringIO() | 770 | errbuf = StringIO() |
3872 | 776 | outbuf = StringIO() | 771 | outbuf = StringIO() |
3873 | @@ -886,39 +881,6 @@ | |||
3874 | 886 | resp = self.object_controller.PUT(req) | 881 | resp = self.object_controller.PUT(req) |
3875 | 887 | self.assertEquals(resp.status_int, 400) | 882 | self.assertEquals(resp.status_int, 400) |
3876 | 888 | 883 | ||
3877 | 889 | def test_disk_file_app_iter_corners(self): | ||
3878 | 890 | if not self.path_to_test_xfs: | ||
3879 | 891 | raise SkipTest | ||
3880 | 892 | df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') | ||
3881 | 893 | mkdirs(df.datadir) | ||
3882 | 894 | f = open(os.path.join(df.datadir, | ||
3883 | 895 | normalize_timestamp(time()) + '.data'), 'wb') | ||
3884 | 896 | f.write('1234567890') | ||
3885 | 897 | setxattr(f.fileno(), object_server.METADATA_KEY, | ||
3886 | 898 | pickle.dumps({}, object_server.PICKLE_PROTOCOL)) | ||
3887 | 899 | f.close() | ||
3888 | 900 | df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3889 | 901 | keep_data_fp=True) | ||
3890 | 902 | it = df.app_iter_range(0, None) | ||
3891 | 903 | sio = StringIO() | ||
3892 | 904 | for chunk in it: | ||
3893 | 905 | sio.write(chunk) | ||
3894 | 906 | self.assertEquals(sio.getvalue(), '1234567890') | ||
3895 | 907 | |||
3896 | 908 | df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', | ||
3897 | 909 | keep_data_fp=True) | ||
3898 | 910 | it = df.app_iter_range(5, None) | ||
3899 | 911 | sio = StringIO() | ||
3900 | 912 | for chunk in it: | ||
3901 | 913 | sio.write(chunk) | ||
3902 | 914 | self.assertEquals(sio.getvalue(), '67890') | ||
3903 | 915 | |||
3904 | 916 | def test_disk_file_mkstemp_creates_dir(self): | ||
3905 | 917 | tmpdir = os.path.join(self.testdir, 'sda1', 'tmp') | ||
3906 | 918 | os.rmdir(tmpdir) | ||
3907 | 919 | with object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o').mkstemp(): | ||
3908 | 920 | self.assert_(os.path.exists(tmpdir)) | ||
3909 | 921 | |||
3910 | 922 | def test_max_upload_time(self): | 884 | def test_max_upload_time(self): |
3911 | 923 | if not self.path_to_test_xfs: | 885 | if not self.path_to_test_xfs: |
3912 | 924 | raise SkipTest | 886 | raise SkipTest |
3913 | @@ -1006,6 +968,161 @@ | |||
3914 | 1006 | self.assertEquals(resp.status_int, 200) | 968 | self.assertEquals(resp.status_int, 200) |
3915 | 1007 | self.assertEquals(resp.headers['content-encoding'], 'gzip') | 969 | self.assertEquals(resp.headers['content-encoding'], 'gzip') |
3916 | 1008 | 970 | ||
3917 | 971 | def test_overwritten_manifest(self): | ||
3918 | 972 | """ | ||
3919 | 973 | Ensures a janitor segment cleanup job is created for an overwritten | ||
3920 | 974 | manifest. | ||
3921 | 975 | """ | ||
3922 | 976 | if not self.path_to_test_xfs: | ||
3923 | 977 | raise SkipTest | ||
3924 | 978 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3925 | 979 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3926 | 980 | self.assert_(df.is_deleted()) | ||
3927 | 981 | |||
3928 | 982 | manifest = {'x-timestamp': normalize_timestamp(1), | ||
3929 | 983 | 'content-length': 100, 'content-type': 'text/plain', | ||
3930 | 984 | 'x-segment-size': 10, 'etag': 'd41d8cd98f00b204e9800998ecf8427e'} | ||
3931 | 985 | manifest = pickle.dumps(manifest, protocol=PICKLE_PROTOCOL) | ||
3932 | 986 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
3933 | 987 | headers={'X-Timestamp': normalize_timestamp(1), | ||
3934 | 988 | 'Content-Length': str(len(manifest)), 'Content-Type': 'text/plain', | ||
3935 | 989 | 'X-Object-Type': 'manifest', 'X-Object-Length': '100'}, | ||
3936 | 990 | body=manifest) | ||
3937 | 991 | resp = self.object_controller.PUT(req) | ||
3938 | 992 | self.assertEquals(resp.status_int, 201) | ||
3939 | 993 | |||
3940 | 994 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
3941 | 995 | headers={'X-Timestamp': normalize_timestamp(2), | ||
3942 | 996 | 'Content-Length': '1', 'Content-Type': 'text/plain'}, body=' ') | ||
3943 | 997 | resp = self.object_controller.PUT(req) | ||
3944 | 998 | self.assertEquals(resp.status_int, 201) | ||
3945 | 999 | |||
3946 | 1000 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3947 | 1001 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3948 | 1002 | self.assert_(not df.is_deleted()) | ||
3949 | 1003 | |||
3950 | 1004 | def test_segmented_put(self): | ||
3951 | 1005 | if not self.path_to_test_xfs: | ||
3952 | 1006 | raise SkipTest | ||
3953 | 1007 | |||
3954 | 1008 | # Ensure the janitor cleanup job doesn't exist yet | ||
3955 | 1009 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3956 | 1010 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3957 | 1011 | self.assert_(df.is_deleted()) | ||
3958 | 1012 | |||
3959 | 1013 | # Put the first segment | ||
3960 | 1014 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
3961 | 1015 | headers={'X-Timestamp': normalize_timestamp(1), | ||
3962 | 1016 | 'Content-Length': '1', 'Content-Type': 'text/plain', | ||
3963 | 1017 | 'X-Object-Type': 'segment', 'X-Object-Segment': '0', | ||
3964 | 1018 | 'X-Object-Segment-If-Length': '1'}, body='1') | ||
3965 | 1019 | resp = self.object_controller.PUT(req) | ||
3966 | 1020 | self.assertEquals(resp.status_int, 201) | ||
3967 | 1021 | |||
3968 | 1022 | # Ensure the janitor cleanup job now exists | ||
3969 | 1023 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3970 | 1024 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3971 | 1025 | self.assert_(not df.is_deleted()) | ||
3972 | 1026 | |||
3973 | 1027 | # Second segment would go to a different node | ||
3974 | 1028 | |||
3975 | 1029 | # Put the manifest | ||
3976 | 1030 | manifest = {'x-timestamp': normalize_timestamp(1), | ||
3977 | 1031 | 'content-length': 2, 'content-type': 'text/plain', | ||
3978 | 1032 | 'x-segment-size': 1, 'etag': 'c20ad4d76fe97759aa27a0c99bff6710'} | ||
3979 | 1033 | manifest = pickle.dumps(manifest, protocol=PICKLE_PROTOCOL) | ||
3980 | 1034 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
3981 | 1035 | headers={'X-Timestamp': normalize_timestamp(1), | ||
3982 | 1036 | 'Content-Length': str(len(manifest)), 'Content-Type': 'text/plain', | ||
3983 | 1037 | 'X-Object-Type': 'manifest', 'X-Object-Length': '2'}, | ||
3984 | 1038 | body=manifest) | ||
3985 | 1039 | resp = self.object_controller.PUT(req) | ||
3986 | 1040 | self.assertEquals(resp.status_int, 201) | ||
3987 | 1041 | |||
3988 | 1042 | # The janitor cleanup job should still exist (only the janitor will | ||
3989 | 1043 | # verify the manifest is in place an remove the job). | ||
3990 | 1044 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
3991 | 1045 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
3992 | 1046 | self.assert_(not df.is_deleted()) | ||
3993 | 1047 | |||
3994 | 1048 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'GET'}) | ||
3995 | 1049 | resp = self.object_controller.GET(req) | ||
3996 | 1050 | self.assertEquals(resp.status_int, 200) | ||
3997 | 1051 | self.assertEquals(resp.body, manifest) | ||
3998 | 1052 | |||
3999 | 1053 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'GET'}, | ||
4000 | 1054 | headers={'X-Object-Segment': '0', | ||
4001 | 1055 | 'X-Object-Segment-Timestamp': normalize_timestamp(1)}) | ||
4002 | 1056 | resp = self.object_controller.GET(req) | ||
4003 | 1057 | self.assertEquals(resp.status_int, 200) | ||
4004 | 1058 | self.assertEquals(resp.body, '1') | ||
4005 | 1059 | |||
4006 | 1060 | def test_segmented_put_no_longer(self): | ||
4007 | 1061 | if not self.path_to_test_xfs: | ||
4008 | 1062 | raise SkipTest | ||
4009 | 1063 | |||
4010 | 1064 | # Ensure the janitor cleanup job doesn't exist to begin with | ||
4011 | 1065 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
4012 | 1066 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
4013 | 1067 | self.assert_(df.is_deleted()) | ||
4014 | 1068 | |||
4015 | 1069 | # Put the first segment, that really ends up a whole object | ||
4016 | 1070 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4017 | 1071 | headers={'X-Timestamp': normalize_timestamp(1), | ||
4018 | 1072 | 'Content-Length': '1', 'Content-Type': 'text/plain', | ||
4019 | 1073 | 'X-Object-Type': 'segment', 'X-Object-Segment': '0', | ||
4020 | 1074 | 'X-Object-Segment-If-Length': '2'}, body='1') | ||
4021 | 1075 | resp = self.object_controller.PUT(req) | ||
4022 | 1076 | self.assertEquals(resp.status_int, 201) | ||
4023 | 1077 | |||
4024 | 1078 | # Ensure the janitor cleanup job doesn't exist since we put a whole | ||
4025 | 1079 | # file, not a segment (due to X-Object-Segment-If-Length). | ||
4026 | 1080 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
4027 | 1081 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
4028 | 1082 | self.assert_(df.is_deleted()) | ||
4029 | 1083 | |||
4030 | 1084 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'GET'}) | ||
4031 | 1085 | resp = self.object_controller.GET(req) | ||
4032 | 1086 | self.assertEquals(resp.status_int, 200) | ||
4033 | 1087 | self.assertEquals(resp.body, '1') | ||
4034 | 1088 | |||
4035 | 1089 | def test_deleted_manifest(self): | ||
4036 | 1090 | """ | ||
4037 | 1091 | Ensures a janitor segment cleanup job is created for a deleted | ||
4038 | 1092 | manifest. | ||
4039 | 1093 | """ | ||
4040 | 1094 | if not self.path_to_test_xfs: | ||
4041 | 1095 | raise SkipTest | ||
4042 | 1096 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
4043 | 1097 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR) | ||
4044 | 1098 | self.assert_(df.is_deleted()) | ||
4045 | 1099 | |||
4046 | 1100 | manifest = {'x-timestamp': normalize_timestamp(1), | ||
4047 | 1101 | 'content-length': 100, 'content-type': 'text/plain', | ||
4048 | 1102 | 'x-segment-size': 10, 'etag': 'd41d8cd98f00b204e9800998ecf8427e'} | ||
4049 | 1103 | manifest = pickle.dumps(manifest, protocol=PICKLE_PROTOCOL) | ||
4050 | 1104 | req = Request.blank('/sda1/0/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4051 | 1105 | headers={'X-Timestamp': normalize_timestamp(1), | ||
4052 | 1106 | 'Content-Length': str(len(manifest)), 'Content-Type': 'text/plain', | ||
4053 | 1107 | 'X-Object-Type': 'manifest', 'X-Object-Length': '100'}, | ||
4054 | 1108 | body=manifest) | ||
4055 | 1109 | resp = self.object_controller.PUT(req) | ||
4056 | 1110 | self.assertEquals(resp.status_int, 201) | ||
4057 | 1111 | |||
4058 | 1112 | req = Request.blank('/sda1/0/a/c/o', | ||
4059 | 1113 | environ={'REQUEST_METHOD': 'DELETE'}, | ||
4060 | 1114 | headers={'X-Timestamp': normalize_timestamp(2), | ||
4061 | 1115 | 'Content-Length': '0', 'Content-Type': 'text/plain'}, body='') | ||
4062 | 1116 | resp = self.object_controller.DELETE(req) | ||
4063 | 1117 | self.assertEquals(resp.status_int, 204) | ||
4064 | 1118 | |||
4065 | 1119 | df = DiskFile(self.testdir, 'sda1', '0', 'Segment-Cleanup', | ||
4066 | 1120 | normalize_timestamp(1), 'a/c/o', datadir=JANITORDIR, | ||
4067 | 1121 | keep_data_fp=True) | ||
4068 | 1122 | self.assert_(not df.is_deleted()) | ||
4069 | 1123 | job = pickle.loads(''.join(iter(df))) | ||
4070 | 1124 | self.assertEquals(job['segment_last_deleted'], -1) | ||
4071 | 1125 | |||
4072 | 1009 | 1126 | ||
4073 | 1010 | if __name__ == '__main__': | 1127 | if __name__ == '__main__': |
4074 | 1011 | unittest.main() | 1128 | unittest.main() |
4075 | 1012 | 1129 | ||
4076 | === modified file 'test/unit/obj/test_updater.py' | |||
4077 | --- test/unit/obj/test_updater.py 2010-09-23 16:09:30 +0000 | |||
4078 | +++ test/unit/obj/test_updater.py 2010-11-08 18:51:48 +0000 | |||
4079 | @@ -24,7 +24,7 @@ | |||
4080 | 24 | from eventlet import spawn, TimeoutError, listen | 24 | from eventlet import spawn, TimeoutError, listen |
4081 | 25 | from eventlet.timeout import Timeout | 25 | from eventlet.timeout import Timeout |
4082 | 26 | 26 | ||
4084 | 27 | from swift.obj import updater as object_updater, server as object_server | 27 | from swift.obj import updater as object_updater |
4085 | 28 | from swift.common.ring import RingData | 28 | from swift.common.ring import RingData |
4086 | 29 | from swift.common import utils | 29 | from swift.common import utils |
4087 | 30 | from swift.common.utils import hash_path, normalize_timestamp, mkdirs | 30 | from swift.common.utils import hash_path, normalize_timestamp, mkdirs |
4088 | @@ -48,7 +48,7 @@ | |||
4089 | 48 | os.mkdir(self.devices_dir) | 48 | os.mkdir(self.devices_dir) |
4090 | 49 | self.sda1 = os.path.join(self.devices_dir, 'sda1') | 49 | self.sda1 = os.path.join(self.devices_dir, 'sda1') |
4091 | 50 | os.mkdir(self.sda1) | 50 | os.mkdir(self.sda1) |
4093 | 51 | os.mkdir(os.path.join(self.sda1,'tmp')) | 51 | os.mkdir(os.path.join(self.sda1, 'tmp')) |
4094 | 52 | 52 | ||
4095 | 53 | def tearDown(self): | 53 | def tearDown(self): |
4096 | 54 | rmtree(self.testdir, ignore_errors=1) | 54 | rmtree(self.testdir, ignore_errors=1) |
4097 | @@ -80,7 +80,7 @@ | |||
4098 | 80 | 'node_timeout': '15', | 80 | 'node_timeout': '15', |
4099 | 81 | }) | 81 | }) |
4100 | 82 | cu.run_once() | 82 | cu.run_once() |
4102 | 83 | async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) | 83 | async_dir = os.path.join(self.sda1, object_updater.ASYNCDIR) |
4103 | 84 | os.mkdir(async_dir) | 84 | os.mkdir(async_dir) |
4104 | 85 | cu.run_once() | 85 | cu.run_once() |
4105 | 86 | self.assert_(os.path.exists(async_dir)) | 86 | self.assert_(os.path.exists(async_dir)) |
4106 | @@ -103,6 +103,7 @@ | |||
4107 | 103 | self.assert_(os.path.exists(op_path)) | 103 | self.assert_(os.path.exists(op_path)) |
4108 | 104 | 104 | ||
4109 | 105 | bindsock = listen(('127.0.0.1', 0)) | 105 | bindsock = listen(('127.0.0.1', 0)) |
4110 | 106 | |||
4111 | 106 | def accepter(sock, return_code): | 107 | def accepter(sock, return_code): |
4112 | 107 | try: | 108 | try: |
4113 | 108 | with Timeout(3): | 109 | with Timeout(3): |
4114 | @@ -123,6 +124,7 @@ | |||
4115 | 123 | except BaseException, err: | 124 | except BaseException, err: |
4116 | 124 | return err | 125 | return err |
4117 | 125 | return None | 126 | return None |
4118 | 127 | |||
4119 | 126 | def accept(return_codes): | 128 | def accept(return_codes): |
4120 | 127 | codes = iter(return_codes) | 129 | codes = iter(return_codes) |
4121 | 128 | try: | 130 | try: |
4122 | @@ -139,7 +141,8 @@ | |||
4123 | 139 | except BaseException, err: | 141 | except BaseException, err: |
4124 | 140 | return err | 142 | return err |
4125 | 141 | return None | 143 | return None |
4127 | 142 | event = spawn(accept, [201,500]) | 144 | |
4128 | 145 | event = spawn(accept, [201, 500]) | ||
4129 | 143 | for dev in cu.get_container_ring().devs: | 146 | for dev in cu.get_container_ring().devs: |
4130 | 144 | if dev is not None: | 147 | if dev is not None: |
4131 | 145 | dev['port'] = bindsock.getsockname()[1] | 148 | dev['port'] = bindsock.getsockname()[1] |
4132 | @@ -155,5 +158,6 @@ | |||
4133 | 155 | raise err | 158 | raise err |
4134 | 156 | self.assert_(not os.path.exists(op_path)) | 159 | self.assert_(not os.path.exists(op_path)) |
4135 | 157 | 160 | ||
4136 | 161 | |||
4137 | 158 | if __name__ == '__main__': | 162 | if __name__ == '__main__': |
4138 | 159 | unittest.main() | 163 | unittest.main() |
4139 | 160 | 164 | ||
4140 | === modified file 'test/unit/proxy/test_server.py' | |||
4141 | --- test/unit/proxy/test_server.py 2010-11-05 14:47:43 +0000 | |||
4142 | +++ test/unit/proxy/test_server.py 2010-11-08 18:51:48 +0000 | |||
4143 | @@ -34,7 +34,7 @@ | |||
4144 | 34 | from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen | 34 | from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen |
4145 | 35 | from eventlet.timeout import Timeout | 35 | from eventlet.timeout import Timeout |
4146 | 36 | import simplejson | 36 | import simplejson |
4148 | 37 | from webob import Request | 37 | from webob import Request, Response |
4149 | 38 | from webob.exc import HTTPUnauthorized | 38 | from webob.exc import HTTPUnauthorized |
4150 | 39 | 39 | ||
4151 | 40 | from test.unit import connect_tcp, readuntil2crlfs | 40 | from test.unit import connect_tcp, readuntil2crlfs |
4152 | @@ -44,7 +44,7 @@ | |||
4153 | 44 | from swift.obj import server as object_server | 44 | from swift.obj import server as object_server |
4154 | 45 | from swift.common import ring | 45 | from swift.common import ring |
4155 | 46 | from swift.common.constraints import MAX_META_NAME_LENGTH, \ | 46 | from swift.common.constraints import MAX_META_NAME_LENGTH, \ |
4157 | 47 | MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, MAX_FILE_SIZE | 47 | MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE |
4158 | 48 | from swift.common.utils import mkdirs, normalize_timestamp, NullLogger | 48 | from swift.common.utils import mkdirs, normalize_timestamp, NullLogger |
4159 | 49 | 49 | ||
4160 | 50 | 50 | ||
4161 | @@ -53,7 +53,9 @@ | |||
4162 | 53 | 53 | ||
4163 | 54 | 54 | ||
4164 | 55 | def fake_http_connect(*code_iter, **kwargs): | 55 | def fake_http_connect(*code_iter, **kwargs): |
4165 | 56 | |||
4166 | 56 | class FakeConn(object): | 57 | class FakeConn(object): |
4167 | 58 | |||
4168 | 57 | def __init__(self, status, etag=None, body=''): | 59 | def __init__(self, status, etag=None, body=''): |
4169 | 58 | self.status = status | 60 | self.status = status |
4170 | 59 | self.reason = 'Fake' | 61 | self.reason = 'Fake' |
4171 | @@ -158,6 +160,7 @@ | |||
4172 | 158 | 160 | ||
4173 | 159 | 161 | ||
4174 | 160 | class FakeMemcache(object): | 162 | class FakeMemcache(object): |
4175 | 163 | |||
4176 | 161 | def __init__(self): | 164 | def __init__(self): |
4177 | 162 | self.store = {} | 165 | self.store = {} |
4178 | 163 | 166 | ||
4179 | @@ -212,9 +215,12 @@ | |||
4180 | 212 | class TestProxyServer(unittest.TestCase): | 215 | class TestProxyServer(unittest.TestCase): |
4181 | 213 | 216 | ||
4182 | 214 | def test_unhandled_exception(self): | 217 | def test_unhandled_exception(self): |
4183 | 218 | |||
4184 | 215 | class MyApp(proxy_server.Application): | 219 | class MyApp(proxy_server.Application): |
4185 | 220 | |||
4186 | 216 | def get_controller(self, path): | 221 | def get_controller(self, path): |
4187 | 217 | raise Exception('this shouldnt be caught') | 222 | raise Exception('this shouldnt be caught') |
4188 | 223 | |||
4189 | 218 | app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), | 224 | app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), |
4190 | 219 | container_ring=FakeRing(), object_ring=FakeRing()) | 225 | container_ring=FakeRing(), object_ring=FakeRing()) |
4191 | 220 | req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) | 226 | req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) |
4192 | @@ -323,8 +329,11 @@ | |||
4193 | 323 | test_status_map((200, 200, 204, 500, 404), 503) | 329 | test_status_map((200, 200, 204, 500, 404), 503) |
4194 | 324 | 330 | ||
4195 | 325 | def test_PUT_connect_exceptions(self): | 331 | def test_PUT_connect_exceptions(self): |
4196 | 332 | |||
4197 | 326 | def mock_http_connect(*code_iter, **kwargs): | 333 | def mock_http_connect(*code_iter, **kwargs): |
4198 | 334 | |||
4199 | 327 | class FakeConn(object): | 335 | class FakeConn(object): |
4200 | 336 | |||
4201 | 328 | def __init__(self, status): | 337 | def __init__(self, status): |
4202 | 329 | self.status = status | 338 | self.status = status |
4203 | 330 | self.reason = 'Fake' | 339 | self.reason = 'Fake' |
4204 | @@ -372,8 +381,11 @@ | |||
4205 | 372 | test_status_map((200, 200, 503, 503, -1), 503) | 381 | test_status_map((200, 200, 503, 503, -1), 503) |
4206 | 373 | 382 | ||
4207 | 374 | def test_PUT_send_exceptions(self): | 383 | def test_PUT_send_exceptions(self): |
4208 | 384 | |||
4209 | 375 | def mock_http_connect(*code_iter, **kwargs): | 385 | def mock_http_connect(*code_iter, **kwargs): |
4210 | 386 | |||
4211 | 376 | class FakeConn(object): | 387 | class FakeConn(object): |
4212 | 388 | |||
4213 | 377 | def __init__(self, status): | 389 | def __init__(self, status): |
4214 | 378 | self.status = status | 390 | self.status = status |
4215 | 379 | self.reason = 'Fake' | 391 | self.reason = 'Fake' |
4216 | @@ -430,15 +442,18 @@ | |||
4217 | 430 | controller = proxy_server.ObjectController(self.app, 'account', | 442 | controller = proxy_server.ObjectController(self.app, 'account', |
4218 | 431 | 'container', 'object') | 443 | 'container', 'object') |
4219 | 432 | req = Request.blank('/a/c/o', {}, headers={ | 444 | req = Request.blank('/a/c/o', {}, headers={ |
4221 | 433 | 'Content-Length': str(MAX_FILE_SIZE + 1), | 445 | 'Content-Length': str(self.app.max_object_size + 1), |
4222 | 434 | 'Content-Type': 'foo/bar'}) | 446 | 'Content-Type': 'foo/bar'}) |
4223 | 435 | self.app.update_request(req) | 447 | self.app.update_request(req) |
4224 | 436 | res = controller.PUT(req) | 448 | res = controller.PUT(req) |
4225 | 437 | self.assertEquals(res.status_int, 413) | 449 | self.assertEquals(res.status_int, 413) |
4226 | 438 | 450 | ||
4227 | 439 | def test_PUT_getresponse_exceptions(self): | 451 | def test_PUT_getresponse_exceptions(self): |
4228 | 452 | |||
4229 | 440 | def mock_http_connect(*code_iter, **kwargs): | 453 | def mock_http_connect(*code_iter, **kwargs): |
4230 | 454 | |||
4231 | 441 | class FakeConn(object): | 455 | class FakeConn(object): |
4232 | 456 | |||
4233 | 442 | def __init__(self, status): | 457 | def __init__(self, status): |
4234 | 443 | self.status = status | 458 | self.status = status |
4235 | 444 | self.reason = 'Fake' | 459 | self.reason = 'Fake' |
4236 | @@ -633,6 +648,7 @@ | |||
4237 | 633 | dev['port'] = 1 | 648 | dev['port'] = 1 |
4238 | 634 | 649 | ||
4239 | 635 | class SlowBody(): | 650 | class SlowBody(): |
4240 | 651 | |||
4241 | 636 | def __init__(self): | 652 | def __init__(self): |
4242 | 637 | self.sent = 0 | 653 | self.sent = 0 |
4243 | 638 | 654 | ||
4244 | @@ -680,6 +696,7 @@ | |||
4245 | 680 | dev['port'] = 1 | 696 | dev['port'] = 1 |
4246 | 681 | 697 | ||
4247 | 682 | class SlowBody(): | 698 | class SlowBody(): |
4248 | 699 | |||
4249 | 683 | def __init__(self): | 700 | def __init__(self): |
4250 | 684 | self.sent = 0 | 701 | self.sent = 0 |
4251 | 685 | 702 | ||
4252 | @@ -1334,7 +1351,9 @@ | |||
4253 | 1334 | 1351 | ||
4254 | 1335 | def test_chunked_put(self): | 1352 | def test_chunked_put(self): |
4255 | 1336 | # quick test of chunked put w/o PATH_TO_TEST_XFS | 1353 | # quick test of chunked put w/o PATH_TO_TEST_XFS |
4256 | 1354 | |||
4257 | 1337 | class ChunkedFile(): | 1355 | class ChunkedFile(): |
4258 | 1356 | |||
4259 | 1338 | def __init__(self, bytes): | 1357 | def __init__(self, bytes): |
4260 | 1339 | self.bytes = bytes | 1358 | self.bytes = bytes |
4261 | 1340 | self.read_bytes = 0 | 1359 | self.read_bytes = 0 |
4262 | @@ -1375,12 +1394,13 @@ | |||
4263 | 1375 | req.body_file = ChunkedFile(11) | 1394 | req.body_file = ChunkedFile(11) |
4264 | 1376 | self.app.memcache.store = {} | 1395 | self.app.memcache.store = {} |
4265 | 1377 | self.app.update_request(req) | 1396 | self.app.update_request(req) |
4266 | 1397 | orig_max_object_size = self.app.max_object_size | ||
4267 | 1378 | try: | 1398 | try: |
4269 | 1379 | server.MAX_FILE_SIZE = 10 | 1399 | self.app.max_object_size = 10 |
4270 | 1380 | res = controller.PUT(req) | 1400 | res = controller.PUT(req) |
4271 | 1381 | self.assertEquals(res.status_int, 413) | 1401 | self.assertEquals(res.status_int, 413) |
4272 | 1382 | finally: | 1402 | finally: |
4274 | 1383 | server.MAX_FILE_SIZE = MAX_FILE_SIZE | 1403 | self.app.max_object_size = orig_max_object_size |
4275 | 1384 | 1404 | ||
4276 | 1385 | def test_chunked_put_and_a_bit_more(self): | 1405 | def test_chunked_put_and_a_bit_more(self): |
4277 | 1386 | # Since we're starting up a lot here, we're going to test more than | 1406 | # Since we're starting up a lot here, we're going to test more than |
4278 | @@ -1495,6 +1515,7 @@ | |||
4279 | 1495 | self.assertEquals(headers[:len(exp)], exp) | 1515 | self.assertEquals(headers[:len(exp)], exp) |
4280 | 1496 | # Check unhandled exception | 1516 | # Check unhandled exception |
4281 | 1497 | orig_update_request = prosrv.update_request | 1517 | orig_update_request = prosrv.update_request |
4282 | 1518 | |||
4283 | 1498 | def broken_update_request(env, req): | 1519 | def broken_update_request(env, req): |
4284 | 1499 | raise Exception('fake') | 1520 | raise Exception('fake') |
4285 | 1500 | prosrv.update_request = broken_update_request | 1521 | prosrv.update_request = broken_update_request |
4286 | @@ -1545,6 +1566,7 @@ | |||
4287 | 1545 | # in a test for logging x-forwarded-for (first entry only). | 1566 | # in a test for logging x-forwarded-for (first entry only). |
4288 | 1546 | 1567 | ||
4289 | 1547 | class Logger(object): | 1568 | class Logger(object): |
4290 | 1569 | |||
4291 | 1548 | def info(self, msg): | 1570 | def info(self, msg): |
4292 | 1549 | self.msg = msg | 1571 | self.msg = msg |
4293 | 1550 | orig_logger = prosrv.logger | 1572 | orig_logger = prosrv.logger |
4294 | @@ -1568,6 +1590,7 @@ | |||
4295 | 1568 | # Turn on header logging. | 1590 | # Turn on header logging. |
4296 | 1569 | 1591 | ||
4297 | 1570 | class Logger(object): | 1592 | class Logger(object): |
4298 | 1593 | |||
4299 | 1571 | def info(self, msg): | 1594 | def info(self, msg): |
4300 | 1572 | self.msg = msg | 1595 | self.msg = msg |
4301 | 1573 | orig_logger = prosrv.logger | 1596 | orig_logger = prosrv.logger |
4302 | @@ -1919,6 +1942,188 @@ | |||
4303 | 1919 | res = controller.PUT(req) | 1942 | res = controller.PUT(req) |
4304 | 1920 | self.assert_(called[0]) | 1943 | self.assert_(called[0]) |
4305 | 1921 | 1944 | ||
4306 | 1945 | def test_GETorHEAD1(self): | ||
4307 | 1946 | """ | ||
4308 | 1947 | Ensures we call GETorHEAD_base again without a range if we do a range | ||
4309 | 1948 | request and get a 416 Requested Range Not Satisfiable, just in case the | ||
4310 | 1949 | primary object is a manifest. | ||
4311 | 1950 | """ | ||
4312 | 1951 | called_without_range = [False] | ||
4313 | 1952 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4314 | 1953 | 'container', 'object') | ||
4315 | 1954 | |||
4316 | 1955 | def local_GETorHEAD_base(req, server_type, partition, nodes, path, | ||
4317 | 1956 | attempts): | ||
4318 | 1957 | if req.range: | ||
4319 | 1958 | return Response(status=416) | ||
4320 | 1959 | called_without_range[0] = True | ||
4321 | 1960 | return Response() | ||
4322 | 1961 | |||
4323 | 1962 | controller.GETorHEAD_base = local_GETorHEAD_base | ||
4324 | 1963 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}, | ||
4325 | 1964 | headers={'Range': 'bytes=0-1'}) | ||
4326 | 1965 | controller.GETorHEAD(req) | ||
4327 | 1966 | self.assert_(called_without_range[0]) | ||
4328 | 1967 | |||
4329 | 1968 | def test_GETorHEAD2(self): | ||
4330 | 1969 | """ | ||
4331 | 1970 | Ensures we call GETorHEAD_base again if the first request was a HEAD | ||
4332 | 1971 | and the primary object is a manifest. | ||
4333 | 1972 | """ | ||
4334 | 1973 | called_get = [False] | ||
4335 | 1974 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4336 | 1975 | 'container', 'object') | ||
4337 | 1976 | |||
4338 | 1977 | def local_GETorHEAD_base(req, server_type, partition, nodes, path, | ||
4339 | 1978 | attempts): | ||
4340 | 1979 | if req.method == 'HEAD': | ||
4341 | 1980 | return Response(headers={'x-object-type': 'manifest'}) | ||
4342 | 1981 | elif req.method == 'GET': | ||
4343 | 1982 | called_get[0] = True | ||
4344 | 1983 | return Response(headers={'x-object-type': 'manifest'}, | ||
4345 | 1984 | body=pickle.dumps({'content-length': 0, 'x-segment-size': 1, | ||
4346 | 1985 | 'x-timestamp': normalize_timestamp(2), | ||
4347 | 1986 | 'etag': 'd41d8cd98f00b204e9800998ecf8427e', | ||
4348 | 1987 | 'content-type': 'text/plain'})) | ||
4349 | 1988 | else: | ||
4350 | 1989 | raise Exception('Unexpected method %s' % req.method) | ||
4351 | 1990 | |||
4352 | 1991 | controller.GETorHEAD_base = local_GETorHEAD_base | ||
4353 | 1992 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) | ||
4354 | 1993 | controller.GETorHEAD(req) | ||
4355 | 1994 | self.assert_(called_get[0]) | ||
4356 | 1995 | |||
4357 | 1996 | def test_GETorHEAD3(self): | ||
4358 | 1997 | """ | ||
4359 | 1998 | Ensures we call GETorHEAD_base again without a range if the first | ||
4360 | 1999 | request was a GET with range that succeeded and the primary object is a | ||
4361 | 2000 | manifest. | ||
4362 | 2001 | """ | ||
4363 | 2002 | called_without_range = [False] | ||
4364 | 2003 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4365 | 2004 | 'container', 'object') | ||
4366 | 2005 | |||
4367 | 2006 | def local_GETorHEAD_base(req, server_type, partition, nodes, path, | ||
4368 | 2007 | attempts): | ||
4369 | 2008 | if not req.range: | ||
4370 | 2009 | called_without_range[0] = True | ||
4371 | 2010 | return Response(headers={'x-object-type': 'manifest'}, | ||
4372 | 2011 | body=pickle.dumps({'content-length': 0, 'x-segment-size': 1, | ||
4373 | 2012 | 'x-timestamp': normalize_timestamp(2), | ||
4374 | 2013 | 'etag': 'd41d8cd98f00b204e9800998ecf8427e', | ||
4375 | 2014 | 'content-type': 'text/plain'})) | ||
4376 | 2015 | |||
4377 | 2016 | controller.GETorHEAD_base = local_GETorHEAD_base | ||
4378 | 2017 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}, | ||
4379 | 2018 | headers={'Range': 'bytes=0-1'}) | ||
4380 | 2019 | controller.GETorHEAD(req) | ||
4381 | 2020 | self.assert_(called_without_range[0]) | ||
4382 | 2021 | |||
4383 | 2022 | def test_PUT_segmented_object1(self): | ||
4384 | 2023 | with save_globals(): | ||
4385 | 2024 | proxy_server.http_connect = \ | ||
4386 | 2025 | fake_http_connect(200, 200, # account, container checks | ||
4387 | 2026 | 201, 201, 201, # segment 0 | ||
4388 | 2027 | 201, 201, 201, # segment 1 | ||
4389 | 2028 | 201, 201, 201, # segment 2 | ||
4390 | 2029 | 201, 201, 201) # manifest | ||
4391 | 2030 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4392 | 2031 | 'container', 'object') | ||
4393 | 2032 | self.app.segment_size = 2 | ||
4394 | 2033 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4395 | 2034 | body='12345') | ||
4396 | 2035 | resp = controller.PUT(req) | ||
4397 | 2036 | self.assertEquals(resp.status_int, 201) | ||
4398 | 2037 | self.assertEquals(req.bytes_transferred, 5) | ||
4399 | 2038 | |||
4400 | 2039 | def test_PUT_segmented_object2(self): | ||
4401 | 2040 | """ Same as 1, just with a chunky data source. """ | ||
4402 | 2041 | with save_globals(): | ||
4403 | 2042 | proxy_server.http_connect = \ | ||
4404 | 2043 | fake_http_connect(200, 200, # account, container checks | ||
4405 | 2044 | 201, 201, 201, # segment 0 | ||
4406 | 2045 | 201, 201, 201, # segment 1 | ||
4407 | 2046 | 201, 201, 201, # segment 2 | ||
4408 | 2047 | 201, 201, 201) # manifest | ||
4409 | 2048 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4410 | 2049 | 'container', 'object') | ||
4411 | 2050 | self.app.segment_size = 2 | ||
4412 | 2051 | |||
4413 | 2052 | class ChunkedReader(object): | ||
4414 | 2053 | |||
4415 | 2054 | def __init__(self): | ||
4416 | 2055 | self.chunk = 0 | ||
4417 | 2056 | |||
4418 | 2057 | def read(self, size): | ||
4419 | 2058 | self.chunk += 1 | ||
4420 | 2059 | if self.chunk == 1: | ||
4421 | 2060 | return '123' | ||
4422 | 2061 | elif self.chunk == 2: | ||
4423 | 2062 | return '45' | ||
4424 | 2063 | else: | ||
4425 | 2064 | return '' | ||
4426 | 2065 | |||
4427 | 2066 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', | ||
4428 | 2067 | 'wsgi.input': ChunkedReader()}, headers={'Content-Length': '5'}) | ||
4429 | 2068 | resp = controller.PUT(req) | ||
4430 | 2069 | self.assertEquals(resp.status_int, 201) | ||
4431 | 2070 | self.assertEquals(req.bytes_transferred, 5) | ||
4432 | 2071 | |||
4433 | 2072 | def test_PUT_segmented_object3(self): | ||
4434 | 2073 | """ Failed segment PUT. """ | ||
4435 | 2074 | with save_globals(): | ||
4436 | 2075 | proxy_server.http_connect = \ | ||
4437 | 2076 | fake_http_connect(200, 200, # account, container checks | ||
4438 | 2077 | 201, 201, 201, # segment 0 | ||
4439 | 2078 | 201, 500, 500, # segment 1 | ||
4440 | 2079 | 201, 201, 201, # segment 2 | ||
4441 | 2080 | 201, 201, 201) # manifest | ||
4442 | 2081 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4443 | 2082 | 'container', 'object') | ||
4444 | 2083 | self.app.segment_size = 2 | ||
4445 | 2084 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4446 | 2085 | body='12345') | ||
4447 | 2086 | resp = controller.PUT(req) | ||
4448 | 2087 | self.assertEquals(resp.status_int, 503) | ||
4449 | 2088 | self.assertEquals(resp.body.strip(), | ||
4450 | 2089 | 'Unable to complete very large file operation.') | ||
4451 | 2090 | |||
4452 | 2091 | def test_PUT_segmented_object4(self): | ||
4453 | 2092 | """ Non-matching etag sent. """ | ||
4454 | 2093 | with save_globals(): | ||
4455 | 2094 | proxy_server.http_connect = \ | ||
4456 | 2095 | fake_http_connect(200, 200, # account, container checks | ||
4457 | 2096 | 201, 201, 201, # segment 0 | ||
4458 | 2097 | 201, 201, 201, # segment 1 | ||
4459 | 2098 | 201, 201, 201, # segment 2 | ||
4460 | 2099 | 201, 201, 201) # manifest | ||
4461 | 2100 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4462 | 2101 | 'container', 'object') | ||
4463 | 2102 | self.app.segment_size = 2 | ||
4464 | 2103 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4465 | 2104 | body='12345', headers={'ETag': 'abc'}) | ||
4466 | 2105 | resp = controller.PUT(req) | ||
4467 | 2106 | self.assertEquals(resp.status_int, 422) | ||
4468 | 2107 | |||
4469 | 2108 | def test_PUT_segmented_object5(self): | ||
4470 | 2109 | """ Failed manifest PUT. """ | ||
4471 | 2110 | with save_globals(): | ||
4472 | 2111 | proxy_server.http_connect = \ | ||
4473 | 2112 | fake_http_connect(200, 200, # account, container checks | ||
4474 | 2113 | 201, 201, 201, # segment 0 | ||
4475 | 2114 | 201, 201, 201, # segment 1 | ||
4476 | 2115 | 201, 201, 201, # segment 2 | ||
4477 | 2116 | 201, 500, 500) # manifest | ||
4478 | 2117 | controller = proxy_server.ObjectController(self.app, 'account', | ||
4479 | 2118 | 'container', 'object') | ||
4480 | 2119 | self.app.segment_size = 2 | ||
4481 | 2120 | req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, | ||
4482 | 2121 | body='12345') | ||
4483 | 2122 | resp = controller.PUT(req) | ||
4484 | 2123 | self.assertEquals(resp.status_int, 503) | ||
4485 | 2124 | self.assertEquals(resp.body.strip(), | ||
4486 | 2125 | 'Unable to complete very large file operation.') | ||
4487 | 2126 | |||
4488 | 1922 | def test_COPY_calls_authorize(self): | 2127 | def test_COPY_calls_authorize(self): |
4489 | 1923 | called = [False] | 2128 | called = [False] |
4490 | 1924 | 2129 | ||
4491 | @@ -2080,7 +2285,9 @@ | |||
4492 | 2080 | self.assertEquals(resp.status_int, 404) | 2285 | self.assertEquals(resp.status_int, 404) |
4493 | 2081 | 2286 | ||
4494 | 2082 | def test_put_locking(self): | 2287 | def test_put_locking(self): |
4495 | 2288 | |||
4496 | 2083 | class MockMemcache(FakeMemcache): | 2289 | class MockMemcache(FakeMemcache): |
4497 | 2290 | |||
4498 | 2084 | def __init__(self, allow_lock=None): | 2291 | def __init__(self, allow_lock=None): |
4499 | 2085 | self.allow_lock = allow_lock | 2292 | self.allow_lock = allow_lock |
4500 | 2086 | super(MockMemcache, self).__init__() | 2293 | super(MockMemcache, self).__init__() |
4501 | @@ -2669,5 +2876,142 @@ | |||
4502 | 2669 | self.assertEquals(resp.status_int, 400) | 2876 | self.assertEquals(resp.status_int, 400) |
4503 | 2670 | 2877 | ||
4504 | 2671 | 2878 | ||
4505 | 2879 | class TestSegmentedIterable(unittest.TestCase): | ||
4506 | 2880 | |||
4507 | 2881 | def setUp(self): | ||
4508 | 2882 | self.app = proxy_server.Application(None, FakeMemcache(), | ||
4509 | 2883 | account_ring=FakeRing(), container_ring=FakeRing(), | ||
4510 | 2884 | object_ring=FakeRing()) | ||
4511 | 2885 | self.controller = proxy_server.ObjectController(self.app, 'account', | ||
4512 | 2886 | 'container', 'object') | ||
4513 | 2887 | |||
4514 | 2888 | def test_zero_bytes(self): | ||
4515 | 2889 | si = proxy_server.SegmentedIterable(self.controller, 0, 2, | ||
4516 | 2890 | normalize_timestamp(1)) | ||
4517 | 2891 | self.assertEquals(''.join(iter(si)), '') | ||
4518 | 2892 | |||
4519 | 2893 | def test_happy_path(self): | ||
4520 | 2894 | segment = [0] | ||
4521 | 2895 | |||
4522 | 2896 | def give_connect(*args, **kwargs): | ||
4523 | 2897 | self.assertEquals(int(kwargs['headers']['X-Object-Segment']), | ||
4524 | 2898 | segment[0]) | ||
4525 | 2899 | segment[0] += 1 | ||
4526 | 2900 | |||
4527 | 2901 | with save_globals(): | ||
4528 | 2902 | proxy_server.http_connect = fake_http_connect(200, 200, 200, | ||
4529 | 2903 | body='12', give_connect=give_connect) | ||
4530 | 2904 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4531 | 2905 | normalize_timestamp(1)) | ||
4532 | 2906 | self.assertEquals(''.join(iter(si)), '12121') | ||
4533 | 2907 | self.assertEquals(segment[0], 3) | ||
4534 | 2908 | |||
4535 | 2909 | def test_not_found_start(self): | ||
4536 | 2910 | with save_globals(): | ||
4537 | 2911 | proxy_server.http_connect = \ | ||
4538 | 2912 | fake_http_connect(404, 404, 404, 200, 200, 200, body='12') | ||
4539 | 2913 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4540 | 2914 | normalize_timestamp(1)) | ||
4541 | 2915 | exc = None | ||
4542 | 2916 | try: | ||
4543 | 2917 | for chunk in si: | ||
4544 | 2918 | raise Exception('Got data when we should not have.') | ||
4545 | 2919 | except Exception, err: | ||
4546 | 2920 | exc = err | ||
4547 | 2921 | self.assertEquals(str(exc), | ||
4548 | 2922 | 'Could not load segment 0 of /account/container/object') | ||
4549 | 2923 | |||
4550 | 2924 | def test_not_found_after_start(self): | ||
4551 | 2925 | with save_globals(): | ||
4552 | 2926 | proxy_server.http_connect = \ | ||
4553 | 2927 | fake_http_connect(200, 404, 404, 404, 200, 200, body='12') | ||
4554 | 2928 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4555 | 2929 | normalize_timestamp(1)) | ||
4556 | 2930 | exc = None | ||
4557 | 2931 | try: | ||
4558 | 2932 | for chunk in si: | ||
4559 | 2933 | self.assertEquals(chunk, '12') | ||
4560 | 2934 | except Exception, err: | ||
4561 | 2935 | exc = err | ||
4562 | 2936 | self.assertEquals(str(exc), | ||
4563 | 2937 | 'Could not load segment 1 of /account/container/object') | ||
4564 | 2938 | |||
4565 | 2939 | def test_partial_not_found(self): | ||
4566 | 2940 | with save_globals(): | ||
4567 | 2941 | proxy_server.http_connect = \ | ||
4568 | 2942 | fake_http_connect(404, 200, 404, 404, 200, 200, body='12') | ||
4569 | 2943 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4570 | 2944 | normalize_timestamp(1)) | ||
4571 | 2945 | self.assertEquals(''.join(iter(si)), '12121') | ||
4572 | 2946 | |||
4573 | 2947 | def test_bytes_transferred(self): | ||
4574 | 2948 | with save_globals(): | ||
4575 | 2949 | proxy_server.http_connect = \ | ||
4576 | 2950 | fake_http_connect(200, 200, 200, body='12') | ||
4577 | 2951 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4578 | 2952 | normalize_timestamp(1)) | ||
4579 | 2953 | |||
4580 | 2954 | class Stub(object): | ||
4581 | 2955 | pass | ||
4582 | 2956 | |||
4583 | 2957 | si.response = Stub() | ||
4584 | 2958 | self.assertEquals(''.join(iter(si)), '12121') | ||
4585 | 2959 | self.assertEquals(si.response.bytes_transferred, 5) | ||
4586 | 2960 | |||
4587 | 2961 | def test_bytes_transferred_app_iter_range(self): | ||
4588 | 2962 | with save_globals(): | ||
4589 | 2963 | proxy_server.http_connect = \ | ||
4590 | 2964 | fake_http_connect(200, 200, 200, body='12') | ||
4591 | 2965 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4592 | 2966 | normalize_timestamp(1)) | ||
4593 | 2967 | |||
4594 | 2968 | class Stub(object): | ||
4595 | 2969 | pass | ||
4596 | 2970 | |||
4597 | 2971 | si.response = Stub() | ||
4598 | 2972 | self.assertEquals(''.join(si.app_iter_range(1, 3)), '212') | ||
4599 | 2973 | self.assertEquals(si.response.bytes_transferred, 3) | ||
4600 | 2974 | |||
4601 | 2975 | def test_app_iter_range_past_end(self): | ||
4602 | 2976 | with save_globals(): | ||
4603 | 2977 | proxy_server.http_connect = \ | ||
4604 | 2978 | fake_http_connect(200, 200, 200, body='12') | ||
4605 | 2979 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4606 | 2980 | normalize_timestamp(1)) | ||
4607 | 2981 | self.assertEquals(''.join(si.app_iter_range(1, 30)), '2121') | ||
4608 | 2982 | |||
4609 | 2983 | def test_app_iter_range_start_past_end(self): | ||
4610 | 2984 | with save_globals(): | ||
4611 | 2985 | proxy_server.http_connect = \ | ||
4612 | 2986 | fake_http_connect(200, 200, 200, body='12') | ||
4613 | 2987 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4614 | 2988 | normalize_timestamp(1)) | ||
4615 | 2989 | self.assertEquals(''.join(si.app_iter_range(30, 31)), '') | ||
4616 | 2990 | |||
4617 | 2991 | def test_app_iter_range_to_end(self): | ||
4618 | 2992 | with save_globals(): | ||
4619 | 2993 | proxy_server.http_connect = \ | ||
4620 | 2994 | fake_http_connect(200, 200, 200, body='12') | ||
4621 | 2995 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4622 | 2996 | normalize_timestamp(1)) | ||
4623 | 2997 | self.assertEquals(''.join(si.app_iter_range(3, None)), '21') | ||
4624 | 2998 | |||
4625 | 2999 | def test_app_iter_range_to_an_end(self): | ||
4626 | 3000 | with save_globals(): | ||
4627 | 3001 | proxy_server.http_connect = \ | ||
4628 | 3002 | fake_http_connect(200, 200, 200, body='12') | ||
4629 | 3003 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4630 | 3004 | normalize_timestamp(1)) | ||
4631 | 3005 | self.assertEquals(''.join(si.app_iter_range(None, 3)), '121') | ||
4632 | 3006 | |||
4633 | 3007 | def test_app_iter_range_full(self): | ||
4634 | 3008 | with save_globals(): | ||
4635 | 3009 | proxy_server.http_connect = \ | ||
4636 | 3010 | fake_http_connect(200, 200, 200, body='12') | ||
4637 | 3011 | si = proxy_server.SegmentedIterable(self.controller, 5, 2, | ||
4638 | 3012 | normalize_timestamp(1)) | ||
4639 | 3013 | self.assertEquals(''.join(si.app_iter_range(None, None)), '12121') | ||
4640 | 3014 | |||
4641 | 3015 | |||
4642 | 2672 | if __name__ == '__main__': | 3016 | if __name__ == '__main__': |
4643 | 2673 | unittest.main() | 3017 | unittest.main() |
After a thorough discussion, this idea of transparent large object support is going to be dropped in favor of a combination client and cluster side solution.