Merge lp:~notmyname/swift/renamest into lp:~hudson-openstack/swift/trunk
- renamest
- Merge into trunk
Proposed by
John Dickinson
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | gholt | ||||
Approved revision: | 311 | ||||
Merged at revision: | 311 | ||||
Proposed branch: | lp:~notmyname/swift/renamest | ||||
Merge into: | lp:~hudson-openstack/swift/trunk | ||||
Diff against target: |
3837 lines (+1860/-1860) 7 files modified
bin/st (+0/-1812) bin/swift (+1812/-0) doc/source/development_saio.rst (+1/-1) doc/source/howto_installmultinode.rst (+28/-28) doc/source/overview_large_objects.rst (+12/-12) setup.py (+1/-1) swift/common/middleware/staticweb.py (+6/-6) |
||||
To merge this branch: | bzr merge lp:~notmyname/swift/renamest | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Swift Core security contacts | Pending | ||
Review via email: mp+64566@code.launchpad.net |
Commit message
Description of the change
rename st to swift
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === removed file 'bin/st' | |||
2 | --- bin/st 2011-05-19 14:48:15 +0000 | |||
3 | +++ bin/st 1970-01-01 00:00:00 +0000 | |||
4 | @@ -1,1812 +0,0 @@ | |||
5 | 1 | #!/usr/bin/python -u | ||
6 | 2 | # Copyright (c) 2010-2011 OpenStack, LLC. | ||
7 | 3 | # | ||
8 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
9 | 5 | # you may not use this file except in compliance with the License. | ||
10 | 6 | # You may obtain a copy of the License at | ||
11 | 7 | # | ||
12 | 8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
13 | 9 | # | ||
14 | 10 | # Unless required by applicable law or agreed to in writing, software | ||
15 | 11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
16 | 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
17 | 13 | # implied. | ||
18 | 14 | # See the License for the specific language governing permissions and | ||
19 | 15 | # limitations under the License. | ||
20 | 16 | |||
21 | 17 | from errno import EEXIST, ENOENT | ||
22 | 18 | from hashlib import md5 | ||
23 | 19 | from optparse import OptionParser | ||
24 | 20 | from os import environ, listdir, makedirs, utime | ||
25 | 21 | from os.path import basename, dirname, getmtime, getsize, isdir, join | ||
26 | 22 | from Queue import Empty, Queue | ||
27 | 23 | from sys import argv, exc_info, exit, stderr, stdout | ||
28 | 24 | from threading import enumerate as threading_enumerate, Thread | ||
29 | 25 | from time import sleep | ||
30 | 26 | from traceback import format_exception | ||
31 | 27 | |||
32 | 28 | |||
33 | 29 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | ||
34 | 30 | # Inclusion of swift.common.client for convenience of single file distribution | ||
35 | 31 | |||
36 | 32 | import socket | ||
37 | 33 | from cStringIO import StringIO | ||
38 | 34 | from re import compile, DOTALL | ||
39 | 35 | from tokenize import generate_tokens, STRING, NAME, OP | ||
40 | 36 | from urllib import quote as _quote, unquote | ||
41 | 37 | from urlparse import urlparse, urlunparse | ||
42 | 38 | |||
43 | 39 | try: | ||
44 | 40 | from eventlet.green.httplib import HTTPException, HTTPSConnection | ||
45 | 41 | except ImportError: | ||
46 | 42 | from httplib import HTTPException, HTTPSConnection | ||
47 | 43 | |||
48 | 44 | try: | ||
49 | 45 | from eventlet import sleep | ||
50 | 46 | except ImportError: | ||
51 | 47 | from time import sleep | ||
52 | 48 | |||
53 | 49 | try: | ||
54 | 50 | from swift.common.bufferedhttp \ | ||
55 | 51 | import BufferedHTTPConnection as HTTPConnection | ||
56 | 52 | except ImportError: | ||
57 | 53 | try: | ||
58 | 54 | from eventlet.green.httplib import HTTPConnection | ||
59 | 55 | except ImportError: | ||
60 | 56 | from httplib import HTTPConnection | ||
61 | 57 | |||
62 | 58 | |||
63 | 59 | def quote(value, safe='/'): | ||
64 | 60 | """ | ||
65 | 61 | Patched version of urllib.quote that encodes utf8 strings before quoting | ||
66 | 62 | """ | ||
67 | 63 | if isinstance(value, unicode): | ||
68 | 64 | value = value.encode('utf8') | ||
69 | 65 | return _quote(value, safe) | ||
70 | 66 | |||
71 | 67 | |||
72 | 68 | # look for a real json parser first | ||
73 | 69 | try: | ||
74 | 70 | # simplejson is popular and pretty good | ||
75 | 71 | from simplejson import loads as json_loads | ||
76 | 72 | except ImportError: | ||
77 | 73 | try: | ||
78 | 74 | # 2.6 will have a json module in the stdlib | ||
79 | 75 | from json import loads as json_loads | ||
80 | 76 | except ImportError: | ||
81 | 77 | # fall back on local parser otherwise | ||
82 | 78 | comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) | ||
83 | 79 | |||
84 | 80 | def json_loads(string): | ||
85 | 81 | ''' | ||
86 | 82 | Fairly competent json parser exploiting the python tokenizer and | ||
87 | 83 | eval(). -- From python-cloudfiles | ||
88 | 84 | |||
89 | 85 | _loads(serialized_json) -> object | ||
90 | 86 | ''' | ||
91 | 87 | try: | ||
92 | 88 | res = [] | ||
93 | 89 | consts = {'true': True, 'false': False, 'null': None} | ||
94 | 90 | string = '(' + comments.sub('', string) + ')' | ||
95 | 91 | for type, val, _junk, _junk, _junk in \ | ||
96 | 92 | generate_tokens(StringIO(string).readline): | ||
97 | 93 | if (type == OP and val not in '[]{}:,()-') or \ | ||
98 | 94 | (type == NAME and val not in consts): | ||
99 | 95 | raise AttributeError() | ||
100 | 96 | elif type == STRING: | ||
101 | 97 | res.append('u') | ||
102 | 98 | res.append(val.replace('\\/', '/')) | ||
103 | 99 | else: | ||
104 | 100 | res.append(val) | ||
105 | 101 | return eval(''.join(res), {}, consts) | ||
106 | 102 | except Exception: | ||
107 | 103 | raise AttributeError() | ||
108 | 104 | |||
109 | 105 | |||
110 | 106 | class ClientException(Exception): | ||
111 | 107 | |||
112 | 108 | def __init__(self, msg, http_scheme='', http_host='', http_port='', | ||
113 | 109 | http_path='', http_query='', http_status=0, http_reason='', | ||
114 | 110 | http_device=''): | ||
115 | 111 | Exception.__init__(self, msg) | ||
116 | 112 | self.msg = msg | ||
117 | 113 | self.http_scheme = http_scheme | ||
118 | 114 | self.http_host = http_host | ||
119 | 115 | self.http_port = http_port | ||
120 | 116 | self.http_path = http_path | ||
121 | 117 | self.http_query = http_query | ||
122 | 118 | self.http_status = http_status | ||
123 | 119 | self.http_reason = http_reason | ||
124 | 120 | self.http_device = http_device | ||
125 | 121 | |||
126 | 122 | def __str__(self): | ||
127 | 123 | a = self.msg | ||
128 | 124 | b = '' | ||
129 | 125 | if self.http_scheme: | ||
130 | 126 | b += '%s://' % self.http_scheme | ||
131 | 127 | if self.http_host: | ||
132 | 128 | b += self.http_host | ||
133 | 129 | if self.http_port: | ||
134 | 130 | b += ':%s' % self.http_port | ||
135 | 131 | if self.http_path: | ||
136 | 132 | b += self.http_path | ||
137 | 133 | if self.http_query: | ||
138 | 134 | b += '?%s' % self.http_query | ||
139 | 135 | if self.http_status: | ||
140 | 136 | if b: | ||
141 | 137 | b = '%s %s' % (b, self.http_status) | ||
142 | 138 | else: | ||
143 | 139 | b = str(self.http_status) | ||
144 | 140 | if self.http_reason: | ||
145 | 141 | if b: | ||
146 | 142 | b = '%s %s' % (b, self.http_reason) | ||
147 | 143 | else: | ||
148 | 144 | b = '- %s' % self.http_reason | ||
149 | 145 | if self.http_device: | ||
150 | 146 | if b: | ||
151 | 147 | b = '%s: device %s' % (b, self.http_device) | ||
152 | 148 | else: | ||
153 | 149 | b = 'device %s' % self.http_device | ||
154 | 150 | return b and '%s: %s' % (a, b) or a | ||
155 | 151 | |||
156 | 152 | |||
157 | 153 | def http_connection(url): | ||
158 | 154 | """ | ||
159 | 155 | Make an HTTPConnection or HTTPSConnection | ||
160 | 156 | |||
161 | 157 | :param url: url to connect to | ||
162 | 158 | :returns: tuple of (parsed url, connection object) | ||
163 | 159 | :raises ClientException: Unable to handle protocol scheme | ||
164 | 160 | """ | ||
165 | 161 | parsed = urlparse(url) | ||
166 | 162 | if parsed.scheme == 'http': | ||
167 | 163 | conn = HTTPConnection(parsed.netloc) | ||
168 | 164 | elif parsed.scheme == 'https': | ||
169 | 165 | conn = HTTPSConnection(parsed.netloc) | ||
170 | 166 | else: | ||
171 | 167 | raise ClientException('Cannot handle protocol scheme %s for url %s' % | ||
172 | 168 | (parsed.scheme, repr(url))) | ||
173 | 169 | return parsed, conn | ||
174 | 170 | |||
175 | 171 | |||
176 | 172 | def get_auth(url, user, key, snet=False): | ||
177 | 173 | """ | ||
178 | 174 | Get authentication/authorization credentials. | ||
179 | 175 | |||
180 | 176 | The snet parameter is used for Rackspace's ServiceNet internal network | ||
181 | 177 | implementation. In this function, it simply adds *snet-* to the beginning | ||
182 | 178 | of the host name for the returned storage URL. With Rackspace Cloud Files, | ||
183 | 179 | use of this network path causes no bandwidth charges but requires the | ||
184 | 180 | client to be running on Rackspace's ServiceNet network. | ||
185 | 181 | |||
186 | 182 | :param url: authentication/authorization URL | ||
187 | 183 | :param user: user to authenticate as | ||
188 | 184 | :param key: key or password for authorization | ||
189 | 185 | :param snet: use SERVICENET internal network (see above), default is False | ||
190 | 186 | :returns: tuple of (storage URL, auth token) | ||
191 | 187 | :raises ClientException: HTTP GET request to auth URL failed | ||
192 | 188 | """ | ||
193 | 189 | parsed, conn = http_connection(url) | ||
194 | 190 | conn.request('GET', parsed.path, '', | ||
195 | 191 | {'X-Auth-User': user, 'X-Auth-Key': key}) | ||
196 | 192 | resp = conn.getresponse() | ||
197 | 193 | resp.read() | ||
198 | 194 | if resp.status < 200 or resp.status >= 300: | ||
199 | 195 | raise ClientException('Auth GET failed', http_scheme=parsed.scheme, | ||
200 | 196 | http_host=conn.host, http_port=conn.port, | ||
201 | 197 | http_path=parsed.path, http_status=resp.status, | ||
202 | 198 | http_reason=resp.reason) | ||
203 | 199 | url = resp.getheader('x-storage-url') | ||
204 | 200 | if snet: | ||
205 | 201 | parsed = list(urlparse(url)) | ||
206 | 202 | # Second item in the list is the netloc | ||
207 | 203 | parsed[1] = 'snet-' + parsed[1] | ||
208 | 204 | url = urlunparse(parsed) | ||
209 | 205 | return url, resp.getheader('x-storage-token', | ||
210 | 206 | resp.getheader('x-auth-token')) | ||
211 | 207 | |||
212 | 208 | |||
213 | 209 | def get_account(url, token, marker=None, limit=None, prefix=None, | ||
214 | 210 | http_conn=None, full_listing=False): | ||
215 | 211 | """ | ||
216 | 212 | Get a listing of containers for the account. | ||
217 | 213 | |||
218 | 214 | :param url: storage URL | ||
219 | 215 | :param token: auth token | ||
220 | 216 | :param marker: marker query | ||
221 | 217 | :param limit: limit query | ||
222 | 218 | :param prefix: prefix query | ||
223 | 219 | :param http_conn: HTTP connection object (If None, it will create the | ||
224 | 220 | conn object) | ||
225 | 221 | :param full_listing: if True, return a full listing, else returns a max | ||
226 | 222 | of 10000 listings | ||
227 | 223 | :returns: a tuple of (response headers, a list of containers) The response | ||
228 | 224 | headers will be a dict and all header names will be lowercase. | ||
229 | 225 | :raises ClientException: HTTP GET request failed | ||
230 | 226 | """ | ||
231 | 227 | if not http_conn: | ||
232 | 228 | http_conn = http_connection(url) | ||
233 | 229 | if full_listing: | ||
234 | 230 | rv = get_account(url, token, marker, limit, prefix, http_conn) | ||
235 | 231 | listing = rv[1] | ||
236 | 232 | while listing: | ||
237 | 233 | marker = listing[-1]['name'] | ||
238 | 234 | listing = \ | ||
239 | 235 | get_account(url, token, marker, limit, prefix, http_conn)[1] | ||
240 | 236 | if listing: | ||
241 | 237 | rv[1].extend(listing) | ||
242 | 238 | return rv | ||
243 | 239 | parsed, conn = http_conn | ||
244 | 240 | qs = 'format=json' | ||
245 | 241 | if marker: | ||
246 | 242 | qs += '&marker=%s' % quote(marker) | ||
247 | 243 | if limit: | ||
248 | 244 | qs += '&limit=%d' % limit | ||
249 | 245 | if prefix: | ||
250 | 246 | qs += '&prefix=%s' % quote(prefix) | ||
251 | 247 | conn.request('GET', '%s?%s' % (parsed.path, qs), '', | ||
252 | 248 | {'X-Auth-Token': token}) | ||
253 | 249 | resp = conn.getresponse() | ||
254 | 250 | resp_headers = {} | ||
255 | 251 | for header, value in resp.getheaders(): | ||
256 | 252 | resp_headers[header.lower()] = value | ||
257 | 253 | if resp.status < 200 or resp.status >= 300: | ||
258 | 254 | resp.read() | ||
259 | 255 | raise ClientException('Account GET failed', http_scheme=parsed.scheme, | ||
260 | 256 | http_host=conn.host, http_port=conn.port, | ||
261 | 257 | http_path=parsed.path, http_query=qs, http_status=resp.status, | ||
262 | 258 | http_reason=resp.reason) | ||
263 | 259 | if resp.status == 204: | ||
264 | 260 | resp.read() | ||
265 | 261 | return resp_headers, [] | ||
266 | 262 | return resp_headers, json_loads(resp.read()) | ||
267 | 263 | |||
268 | 264 | |||
269 | 265 | def head_account(url, token, http_conn=None): | ||
270 | 266 | """ | ||
271 | 267 | Get account stats. | ||
272 | 268 | |||
273 | 269 | :param url: storage URL | ||
274 | 270 | :param token: auth token | ||
275 | 271 | :param http_conn: HTTP connection object (If None, it will create the | ||
276 | 272 | conn object) | ||
277 | 273 | :returns: a dict containing the response's headers (all header names will | ||
278 | 274 | be lowercase) | ||
279 | 275 | :raises ClientException: HTTP HEAD request failed | ||
280 | 276 | """ | ||
281 | 277 | if http_conn: | ||
282 | 278 | parsed, conn = http_conn | ||
283 | 279 | else: | ||
284 | 280 | parsed, conn = http_connection(url) | ||
285 | 281 | conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) | ||
286 | 282 | resp = conn.getresponse() | ||
287 | 283 | resp.read() | ||
288 | 284 | if resp.status < 200 or resp.status >= 300: | ||
289 | 285 | raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, | ||
290 | 286 | http_host=conn.host, http_port=conn.port, | ||
291 | 287 | http_path=parsed.path, http_status=resp.status, | ||
292 | 288 | http_reason=resp.reason) | ||
293 | 289 | resp_headers = {} | ||
294 | 290 | for header, value in resp.getheaders(): | ||
295 | 291 | resp_headers[header.lower()] = value | ||
296 | 292 | return resp_headers | ||
297 | 293 | |||
298 | 294 | |||
299 | 295 | def post_account(url, token, headers, http_conn=None): | ||
300 | 296 | """ | ||
301 | 297 | Update an account's metadata. | ||
302 | 298 | |||
303 | 299 | :param url: storage URL | ||
304 | 300 | :param token: auth token | ||
305 | 301 | :param headers: additional headers to include in the request | ||
306 | 302 | :param http_conn: HTTP connection object (If None, it will create the | ||
307 | 303 | conn object) | ||
308 | 304 | :raises ClientException: HTTP POST request failed | ||
309 | 305 | """ | ||
310 | 306 | if http_conn: | ||
311 | 307 | parsed, conn = http_conn | ||
312 | 308 | else: | ||
313 | 309 | parsed, conn = http_connection(url) | ||
314 | 310 | headers['X-Auth-Token'] = token | ||
315 | 311 | conn.request('POST', parsed.path, '', headers) | ||
316 | 312 | resp = conn.getresponse() | ||
317 | 313 | resp.read() | ||
318 | 314 | if resp.status < 200 or resp.status >= 300: | ||
319 | 315 | raise ClientException('Account POST failed', | ||
320 | 316 | http_scheme=parsed.scheme, http_host=conn.host, | ||
321 | 317 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
322 | 318 | http_reason=resp.reason) | ||
323 | 319 | |||
324 | 320 | |||
325 | 321 | def get_container(url, token, container, marker=None, limit=None, | ||
326 | 322 | prefix=None, delimiter=None, http_conn=None, | ||
327 | 323 | full_listing=False): | ||
328 | 324 | """ | ||
329 | 325 | Get a listing of objects for the container. | ||
330 | 326 | |||
331 | 327 | :param url: storage URL | ||
332 | 328 | :param token: auth token | ||
333 | 329 | :param container: container name to get a listing for | ||
334 | 330 | :param marker: marker query | ||
335 | 331 | :param limit: limit query | ||
336 | 332 | :param prefix: prefix query | ||
337 | 333 | :param delimeter: string to delimit the queries on | ||
338 | 334 | :param http_conn: HTTP connection object (If None, it will create the | ||
339 | 335 | conn object) | ||
340 | 336 | :param full_listing: if True, return a full listing, else returns a max | ||
341 | 337 | of 10000 listings | ||
342 | 338 | :returns: a tuple of (response headers, a list of objects) The response | ||
343 | 339 | headers will be a dict and all header names will be lowercase. | ||
344 | 340 | :raises ClientException: HTTP GET request failed | ||
345 | 341 | """ | ||
346 | 342 | if not http_conn: | ||
347 | 343 | http_conn = http_connection(url) | ||
348 | 344 | if full_listing: | ||
349 | 345 | rv = get_container(url, token, container, marker, limit, prefix, | ||
350 | 346 | delimiter, http_conn) | ||
351 | 347 | listing = rv[1] | ||
352 | 348 | while listing: | ||
353 | 349 | if not delimiter: | ||
354 | 350 | marker = listing[-1]['name'] | ||
355 | 351 | else: | ||
356 | 352 | marker = listing[-1].get('name', listing[-1].get('subdir')) | ||
357 | 353 | listing = get_container(url, token, container, marker, limit, | ||
358 | 354 | prefix, delimiter, http_conn)[1] | ||
359 | 355 | if listing: | ||
360 | 356 | rv[1].extend(listing) | ||
361 | 357 | return rv | ||
362 | 358 | parsed, conn = http_conn | ||
363 | 359 | path = '%s/%s' % (parsed.path, quote(container)) | ||
364 | 360 | qs = 'format=json' | ||
365 | 361 | if marker: | ||
366 | 362 | qs += '&marker=%s' % quote(marker) | ||
367 | 363 | if limit: | ||
368 | 364 | qs += '&limit=%d' % limit | ||
369 | 365 | if prefix: | ||
370 | 366 | qs += '&prefix=%s' % quote(prefix) | ||
371 | 367 | if delimiter: | ||
372 | 368 | qs += '&delimiter=%s' % quote(delimiter) | ||
373 | 369 | conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) | ||
374 | 370 | resp = conn.getresponse() | ||
375 | 371 | if resp.status < 200 or resp.status >= 300: | ||
376 | 372 | resp.read() | ||
377 | 373 | raise ClientException('Container GET failed', | ||
378 | 374 | http_scheme=parsed.scheme, http_host=conn.host, | ||
379 | 375 | http_port=conn.port, http_path=path, http_query=qs, | ||
380 | 376 | http_status=resp.status, http_reason=resp.reason) | ||
381 | 377 | resp_headers = {} | ||
382 | 378 | for header, value in resp.getheaders(): | ||
383 | 379 | resp_headers[header.lower()] = value | ||
384 | 380 | if resp.status == 204: | ||
385 | 381 | resp.read() | ||
386 | 382 | return resp_headers, [] | ||
387 | 383 | return resp_headers, json_loads(resp.read()) | ||
388 | 384 | |||
389 | 385 | |||
390 | 386 | def head_container(url, token, container, http_conn=None): | ||
391 | 387 | """ | ||
392 | 388 | Get container stats. | ||
393 | 389 | |||
394 | 390 | :param url: storage URL | ||
395 | 391 | :param token: auth token | ||
396 | 392 | :param container: container name to get stats for | ||
397 | 393 | :param http_conn: HTTP connection object (If None, it will create the | ||
398 | 394 | conn object) | ||
399 | 395 | :returns: a dict containing the response's headers (all header names will | ||
400 | 396 | be lowercase) | ||
401 | 397 | :raises ClientException: HTTP HEAD request failed | ||
402 | 398 | """ | ||
403 | 399 | if http_conn: | ||
404 | 400 | parsed, conn = http_conn | ||
405 | 401 | else: | ||
406 | 402 | parsed, conn = http_connection(url) | ||
407 | 403 | path = '%s/%s' % (parsed.path, quote(container)) | ||
408 | 404 | conn.request('HEAD', path, '', {'X-Auth-Token': token}) | ||
409 | 405 | resp = conn.getresponse() | ||
410 | 406 | resp.read() | ||
411 | 407 | if resp.status < 200 or resp.status >= 300: | ||
412 | 408 | raise ClientException('Container HEAD failed', | ||
413 | 409 | http_scheme=parsed.scheme, http_host=conn.host, | ||
414 | 410 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
415 | 411 | http_reason=resp.reason) | ||
416 | 412 | resp_headers = {} | ||
417 | 413 | for header, value in resp.getheaders(): | ||
418 | 414 | resp_headers[header.lower()] = value | ||
419 | 415 | return resp_headers | ||
420 | 416 | |||
421 | 417 | |||
422 | 418 | def put_container(url, token, container, headers=None, http_conn=None): | ||
423 | 419 | """ | ||
424 | 420 | Create a container | ||
425 | 421 | |||
426 | 422 | :param url: storage URL | ||
427 | 423 | :param token: auth token | ||
428 | 424 | :param container: container name to create | ||
429 | 425 | :param headers: additional headers to include in the request | ||
430 | 426 | :param http_conn: HTTP connection object (If None, it will create the | ||
431 | 427 | conn object) | ||
432 | 428 | :raises ClientException: HTTP PUT request failed | ||
433 | 429 | """ | ||
434 | 430 | if http_conn: | ||
435 | 431 | parsed, conn = http_conn | ||
436 | 432 | else: | ||
437 | 433 | parsed, conn = http_connection(url) | ||
438 | 434 | path = '%s/%s' % (parsed.path, quote(container)) | ||
439 | 435 | if not headers: | ||
440 | 436 | headers = {} | ||
441 | 437 | headers['X-Auth-Token'] = token | ||
442 | 438 | conn.request('PUT', path, '', headers) | ||
443 | 439 | resp = conn.getresponse() | ||
444 | 440 | resp.read() | ||
445 | 441 | if resp.status < 200 or resp.status >= 300: | ||
446 | 442 | raise ClientException('Container PUT failed', | ||
447 | 443 | http_scheme=parsed.scheme, http_host=conn.host, | ||
448 | 444 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
449 | 445 | http_reason=resp.reason) | ||
450 | 446 | |||
451 | 447 | |||
452 | 448 | def post_container(url, token, container, headers, http_conn=None): | ||
453 | 449 | """ | ||
454 | 450 | Update a container's metadata. | ||
455 | 451 | |||
456 | 452 | :param url: storage URL | ||
457 | 453 | :param token: auth token | ||
458 | 454 | :param container: container name to update | ||
459 | 455 | :param headers: additional headers to include in the request | ||
460 | 456 | :param http_conn: HTTP connection object (If None, it will create the | ||
461 | 457 | conn object) | ||
462 | 458 | :raises ClientException: HTTP POST request failed | ||
463 | 459 | """ | ||
464 | 460 | if http_conn: | ||
465 | 461 | parsed, conn = http_conn | ||
466 | 462 | else: | ||
467 | 463 | parsed, conn = http_connection(url) | ||
468 | 464 | path = '%s/%s' % (parsed.path, quote(container)) | ||
469 | 465 | headers['X-Auth-Token'] = token | ||
470 | 466 | conn.request('POST', path, '', headers) | ||
471 | 467 | resp = conn.getresponse() | ||
472 | 468 | resp.read() | ||
473 | 469 | if resp.status < 200 or resp.status >= 300: | ||
474 | 470 | raise ClientException('Container POST failed', | ||
475 | 471 | http_scheme=parsed.scheme, http_host=conn.host, | ||
476 | 472 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
477 | 473 | http_reason=resp.reason) | ||
478 | 474 | |||
479 | 475 | |||
480 | 476 | def delete_container(url, token, container, http_conn=None): | ||
481 | 477 | """ | ||
482 | 478 | Delete a container | ||
483 | 479 | |||
484 | 480 | :param url: storage URL | ||
485 | 481 | :param token: auth token | ||
486 | 482 | :param container: container name to delete | ||
487 | 483 | :param http_conn: HTTP connection object (If None, it will create the | ||
488 | 484 | conn object) | ||
489 | 485 | :raises ClientException: HTTP DELETE request failed | ||
490 | 486 | """ | ||
491 | 487 | if http_conn: | ||
492 | 488 | parsed, conn = http_conn | ||
493 | 489 | else: | ||
494 | 490 | parsed, conn = http_connection(url) | ||
495 | 491 | path = '%s/%s' % (parsed.path, quote(container)) | ||
496 | 492 | conn.request('DELETE', path, '', {'X-Auth-Token': token}) | ||
497 | 493 | resp = conn.getresponse() | ||
498 | 494 | resp.read() | ||
499 | 495 | if resp.status < 200 or resp.status >= 300: | ||
500 | 496 | raise ClientException('Container DELETE failed', | ||
501 | 497 | http_scheme=parsed.scheme, http_host=conn.host, | ||
502 | 498 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
503 | 499 | http_reason=resp.reason) | ||
504 | 500 | |||
505 | 501 | |||
506 | 502 | def get_object(url, token, container, name, http_conn=None, | ||
507 | 503 | resp_chunk_size=None): | ||
508 | 504 | """ | ||
509 | 505 | Get an object | ||
510 | 506 | |||
511 | 507 | :param url: storage URL | ||
512 | 508 | :param token: auth token | ||
513 | 509 | :param container: container name that the object is in | ||
514 | 510 | :param name: object name to get | ||
515 | 511 | :param http_conn: HTTP connection object (If None, it will create the | ||
516 | 512 | conn object) | ||
517 | 513 | :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If | ||
518 | 514 | you specify a resp_chunk_size you must fully read | ||
519 | 515 | the object's contents before making another | ||
520 | 516 | request. | ||
521 | 517 | :returns: a tuple of (response headers, the object's contents) The response | ||
522 | 518 | headers will be a dict and all header names will be lowercase. | ||
523 | 519 | :raises ClientException: HTTP GET request failed | ||
524 | 520 | """ | ||
525 | 521 | if http_conn: | ||
526 | 522 | parsed, conn = http_conn | ||
527 | 523 | else: | ||
528 | 524 | parsed, conn = http_connection(url) | ||
529 | 525 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
530 | 526 | conn.request('GET', path, '', {'X-Auth-Token': token}) | ||
531 | 527 | resp = conn.getresponse() | ||
532 | 528 | if resp.status < 200 or resp.status >= 300: | ||
533 | 529 | resp.read() | ||
534 | 530 | raise ClientException('Object GET failed', http_scheme=parsed.scheme, | ||
535 | 531 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
536 | 532 | http_status=resp.status, http_reason=resp.reason) | ||
537 | 533 | if resp_chunk_size: | ||
538 | 534 | |||
539 | 535 | def _object_body(): | ||
540 | 536 | buf = resp.read(resp_chunk_size) | ||
541 | 537 | while buf: | ||
542 | 538 | yield buf | ||
543 | 539 | buf = resp.read(resp_chunk_size) | ||
544 | 540 | object_body = _object_body() | ||
545 | 541 | else: | ||
546 | 542 | object_body = resp.read() | ||
547 | 543 | resp_headers = {} | ||
548 | 544 | for header, value in resp.getheaders(): | ||
549 | 545 | resp_headers[header.lower()] = value | ||
550 | 546 | return resp_headers, object_body | ||
551 | 547 | |||
552 | 548 | |||
553 | 549 | def head_object(url, token, container, name, http_conn=None): | ||
554 | 550 | """ | ||
555 | 551 | Get object info | ||
556 | 552 | |||
557 | 553 | :param url: storage URL | ||
558 | 554 | :param token: auth token | ||
559 | 555 | :param container: container name that the object is in | ||
560 | 556 | :param name: object name to get info for | ||
561 | 557 | :param http_conn: HTTP connection object (If None, it will create the | ||
562 | 558 | conn object) | ||
563 | 559 | :returns: a dict containing the response's headers (all header names will | ||
564 | 560 | be lowercase) | ||
565 | 561 | :raises ClientException: HTTP HEAD request failed | ||
566 | 562 | """ | ||
567 | 563 | if http_conn: | ||
568 | 564 | parsed, conn = http_conn | ||
569 | 565 | else: | ||
570 | 566 | parsed, conn = http_connection(url) | ||
571 | 567 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
572 | 568 | conn.request('HEAD', path, '', {'X-Auth-Token': token}) | ||
573 | 569 | resp = conn.getresponse() | ||
574 | 570 | resp.read() | ||
575 | 571 | if resp.status < 200 or resp.status >= 300: | ||
576 | 572 | raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, | ||
577 | 573 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
578 | 574 | http_status=resp.status, http_reason=resp.reason) | ||
579 | 575 | resp_headers = {} | ||
580 | 576 | for header, value in resp.getheaders(): | ||
581 | 577 | resp_headers[header.lower()] = value | ||
582 | 578 | return resp_headers | ||
583 | 579 | |||
584 | 580 | |||
585 | 581 | def put_object(url, token, container, name, contents, content_length=None, | ||
586 | 582 | etag=None, chunk_size=65536, content_type=None, headers=None, | ||
587 | 583 | http_conn=None): | ||
588 | 584 | """ | ||
589 | 585 | Put an object | ||
590 | 586 | |||
591 | 587 | :param url: storage URL | ||
592 | 588 | :param token: auth token | ||
593 | 589 | :param container: container name that the object is in | ||
594 | 590 | :param name: object name to put | ||
595 | 591 | :param contents: a string or a file like object to read object data from | ||
596 | 592 | :param content_length: value to send as content-length header; also limits | ||
597 | 593 | the amount read from contents | ||
598 | 594 | :param etag: etag of contents | ||
599 | 595 | :param chunk_size: chunk size of data to write | ||
600 | 596 | :param content_type: value to send as content-type header | ||
601 | 597 | :param headers: additional headers to include in the request | ||
602 | 598 | :param http_conn: HTTP connection object (If None, it will create the | ||
603 | 599 | conn object) | ||
604 | 600 | :returns: etag from server response | ||
605 | 601 | :raises ClientException: HTTP PUT request failed | ||
606 | 602 | """ | ||
607 | 603 | if http_conn: | ||
608 | 604 | parsed, conn = http_conn | ||
609 | 605 | else: | ||
610 | 606 | parsed, conn = http_connection(url) | ||
611 | 607 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
612 | 608 | if not headers: | ||
613 | 609 | headers = {} | ||
614 | 610 | headers['X-Auth-Token'] = token | ||
615 | 611 | if etag: | ||
616 | 612 | headers['ETag'] = etag.strip('"') | ||
617 | 613 | if content_length is not None: | ||
618 | 614 | headers['Content-Length'] = str(content_length) | ||
619 | 615 | if content_type is not None: | ||
620 | 616 | headers['Content-Type'] = content_type | ||
621 | 617 | if not contents: | ||
622 | 618 | headers['Content-Length'] = '0' | ||
623 | 619 | if hasattr(contents, 'read'): | ||
624 | 620 | conn.putrequest('PUT', path) | ||
625 | 621 | for header, value in headers.iteritems(): | ||
626 | 622 | conn.putheader(header, value) | ||
627 | 623 | if content_length is None: | ||
628 | 624 | conn.putheader('Transfer-Encoding', 'chunked') | ||
629 | 625 | conn.endheaders() | ||
630 | 626 | chunk = contents.read(chunk_size) | ||
631 | 627 | while chunk: | ||
632 | 628 | conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) | ||
633 | 629 | chunk = contents.read(chunk_size) | ||
634 | 630 | conn.send('0\r\n\r\n') | ||
635 | 631 | else: | ||
636 | 632 | conn.endheaders() | ||
637 | 633 | left = content_length | ||
638 | 634 | while left > 0: | ||
639 | 635 | size = chunk_size | ||
640 | 636 | if size > left: | ||
641 | 637 | size = left | ||
642 | 638 | chunk = contents.read(size) | ||
643 | 639 | conn.send(chunk) | ||
644 | 640 | left -= len(chunk) | ||
645 | 641 | else: | ||
646 | 642 | conn.request('PUT', path, contents, headers) | ||
647 | 643 | resp = conn.getresponse() | ||
648 | 644 | resp.read() | ||
649 | 645 | if resp.status < 200 or resp.status >= 300: | ||
650 | 646 | raise ClientException('Object PUT failed', http_scheme=parsed.scheme, | ||
651 | 647 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
652 | 648 | http_status=resp.status, http_reason=resp.reason) | ||
653 | 649 | return resp.getheader('etag').strip('"') | ||
654 | 650 | |||
655 | 651 | |||
656 | 652 | def post_object(url, token, container, name, headers, http_conn=None): | ||
657 | 653 | """ | ||
658 | 654 | Update object metadata | ||
659 | 655 | |||
660 | 656 | :param url: storage URL | ||
661 | 657 | :param token: auth token | ||
662 | 658 | :param container: container name that the object is in | ||
663 | 659 | :param name: name of the object to update | ||
664 | 660 | :param headers: additional headers to include in the request | ||
665 | 661 | :param http_conn: HTTP connection object (If None, it will create the | ||
666 | 662 | conn object) | ||
667 | 663 | :raises ClientException: HTTP POST request failed | ||
668 | 664 | """ | ||
669 | 665 | if http_conn: | ||
670 | 666 | parsed, conn = http_conn | ||
671 | 667 | else: | ||
672 | 668 | parsed, conn = http_connection(url) | ||
673 | 669 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
674 | 670 | headers['X-Auth-Token'] = token | ||
675 | 671 | conn.request('POST', path, '', headers) | ||
676 | 672 | resp = conn.getresponse() | ||
677 | 673 | resp.read() | ||
678 | 674 | if resp.status < 200 or resp.status >= 300: | ||
679 | 675 | raise ClientException('Object POST failed', http_scheme=parsed.scheme, | ||
680 | 676 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
681 | 677 | http_status=resp.status, http_reason=resp.reason) | ||
682 | 678 | |||
683 | 679 | |||
684 | 680 | def delete_object(url, token, container, name, http_conn=None): | ||
685 | 681 | """ | ||
686 | 682 | Delete object | ||
687 | 683 | |||
688 | 684 | :param url: storage URL | ||
689 | 685 | :param token: auth token | ||
690 | 686 | :param container: container name that the object is in | ||
691 | 687 | :param name: object name to delete | ||
692 | 688 | :param http_conn: HTTP connection object (If None, it will create the | ||
693 | 689 | conn object) | ||
694 | 690 | :raises ClientException: HTTP DELETE request failed | ||
695 | 691 | """ | ||
696 | 692 | if http_conn: | ||
697 | 693 | parsed, conn = http_conn | ||
698 | 694 | else: | ||
699 | 695 | parsed, conn = http_connection(url) | ||
700 | 696 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
701 | 697 | conn.request('DELETE', path, '', {'X-Auth-Token': token}) | ||
702 | 698 | resp = conn.getresponse() | ||
703 | 699 | resp.read() | ||
704 | 700 | if resp.status < 200 or resp.status >= 300: | ||
705 | 701 | raise ClientException('Object DELETE failed', | ||
706 | 702 | http_scheme=parsed.scheme, http_host=conn.host, | ||
707 | 703 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
708 | 704 | http_reason=resp.reason) | ||
709 | 705 | |||
710 | 706 | |||
711 | 707 | class Connection(object): | ||
712 | 708 | """Convenience class to make requests that will also retry the request""" | ||
713 | 709 | |||
714 | 710 | def __init__(self, authurl, user, key, retries=5, preauthurl=None, | ||
715 | 711 | preauthtoken=None, snet=False, starting_backoff=1): | ||
716 | 712 | """ | ||
717 | 713 | :param authurl: authenitcation URL | ||
718 | 714 | :param user: user name to authenticate as | ||
719 | 715 | :param key: key/password to authenticate with | ||
720 | 716 | :param retries: Number of times to retry the request before failing | ||
721 | 717 | :param preauthurl: storage URL (if you have already authenticated) | ||
722 | 718 | :param preauthtoken: authentication token (if you have already | ||
723 | 719 | authenticated) | ||
724 | 720 | :param snet: use SERVICENET internal network default is False | ||
725 | 721 | """ | ||
726 | 722 | self.authurl = authurl | ||
727 | 723 | self.user = user | ||
728 | 724 | self.key = key | ||
729 | 725 | self.retries = retries | ||
730 | 726 | self.http_conn = None | ||
731 | 727 | self.url = preauthurl | ||
732 | 728 | self.token = preauthtoken | ||
733 | 729 | self.attempts = 0 | ||
734 | 730 | self.snet = snet | ||
735 | 731 | self.starting_backoff = starting_backoff | ||
736 | 732 | |||
737 | 733 | def get_auth(self): | ||
738 | 734 | return get_auth(self.authurl, self.user, self.key, snet=self.snet) | ||
739 | 735 | |||
740 | 736 | def http_connection(self): | ||
741 | 737 | return http_connection(self.url) | ||
742 | 738 | |||
743 | 739 | def _retry(self, reset_func, func, *args, **kwargs): | ||
744 | 740 | self.attempts = 0 | ||
745 | 741 | backoff = self.starting_backoff | ||
746 | 742 | while self.attempts <= self.retries: | ||
747 | 743 | self.attempts += 1 | ||
748 | 744 | try: | ||
749 | 745 | if not self.url or not self.token: | ||
750 | 746 | self.url, self.token = self.get_auth() | ||
751 | 747 | self.http_conn = None | ||
752 | 748 | if not self.http_conn: | ||
753 | 749 | self.http_conn = self.http_connection() | ||
754 | 750 | kwargs['http_conn'] = self.http_conn | ||
755 | 751 | rv = func(self.url, self.token, *args, **kwargs) | ||
756 | 752 | return rv | ||
757 | 753 | except (socket.error, HTTPException): | ||
758 | 754 | if self.attempts > self.retries: | ||
759 | 755 | raise | ||
760 | 756 | self.http_conn = None | ||
761 | 757 | except ClientException, err: | ||
762 | 758 | if self.attempts > self.retries: | ||
763 | 759 | raise | ||
764 | 760 | if err.http_status == 401: | ||
765 | 761 | self.url = self.token = None | ||
766 | 762 | if self.attempts > 1: | ||
767 | 763 | raise | ||
768 | 764 | elif err.http_status == 408: | ||
769 | 765 | self.http_conn = None | ||
770 | 766 | elif 500 <= err.http_status <= 599: | ||
771 | 767 | pass | ||
772 | 768 | else: | ||
773 | 769 | raise | ||
774 | 770 | sleep(backoff) | ||
775 | 771 | backoff *= 2 | ||
776 | 772 | if reset_func: | ||
777 | 773 | reset_func(func, *args, **kwargs) | ||
778 | 774 | |||
779 | 775 | def head_account(self): | ||
780 | 776 | """Wrapper for :func:`head_account`""" | ||
781 | 777 | return self._retry(None, head_account) | ||
782 | 778 | |||
783 | 779 | def get_account(self, marker=None, limit=None, prefix=None, | ||
784 | 780 | full_listing=False): | ||
785 | 781 | """Wrapper for :func:`get_account`""" | ||
786 | 782 | # TODO(unknown): With full_listing=True this will restart the entire | ||
787 | 783 | # listing with each retry. Need to make a better version that just | ||
788 | 784 | # retries where it left off. | ||
789 | 785 | return self._retry(None, get_account, marker=marker, limit=limit, | ||
790 | 786 | prefix=prefix, full_listing=full_listing) | ||
791 | 787 | |||
792 | 788 | def post_account(self, headers): | ||
793 | 789 | """Wrapper for :func:`post_account`""" | ||
794 | 790 | return self._retry(None, post_account, headers) | ||
795 | 791 | |||
796 | 792 | def head_container(self, container): | ||
797 | 793 | """Wrapper for :func:`head_container`""" | ||
798 | 794 | return self._retry(None, head_container, container) | ||
799 | 795 | |||
800 | 796 | def get_container(self, container, marker=None, limit=None, prefix=None, | ||
801 | 797 | delimiter=None, full_listing=False): | ||
802 | 798 | """Wrapper for :func:`get_container`""" | ||
803 | 799 | # TODO(unknown): With full_listing=True this will restart the entire | ||
804 | 800 | # listing with each retry. Need to make a better version that just | ||
805 | 801 | # retries where it left off. | ||
806 | 802 | return self._retry(None, get_container, container, marker=marker, | ||
807 | 803 | limit=limit, prefix=prefix, delimiter=delimiter, | ||
808 | 804 | full_listing=full_listing) | ||
809 | 805 | |||
810 | 806 | def put_container(self, container, headers=None): | ||
811 | 807 | """Wrapper for :func:`put_container`""" | ||
812 | 808 | return self._retry(None, put_container, container, headers=headers) | ||
813 | 809 | |||
814 | 810 | def post_container(self, container, headers): | ||
815 | 811 | """Wrapper for :func:`post_container`""" | ||
816 | 812 | return self._retry(None, post_container, container, headers) | ||
817 | 813 | |||
818 | 814 | def delete_container(self, container): | ||
819 | 815 | """Wrapper for :func:`delete_container`""" | ||
820 | 816 | return self._retry(None, delete_container, container) | ||
821 | 817 | |||
822 | 818 | def head_object(self, container, obj): | ||
823 | 819 | """Wrapper for :func:`head_object`""" | ||
824 | 820 | return self._retry(None, head_object, container, obj) | ||
825 | 821 | |||
826 | 822 | def get_object(self, container, obj, resp_chunk_size=None): | ||
827 | 823 | """Wrapper for :func:`get_object`""" | ||
828 | 824 | return self._retry(None, get_object, container, obj, | ||
829 | 825 | resp_chunk_size=resp_chunk_size) | ||
830 | 826 | |||
831 | 827 | def put_object(self, container, obj, contents, content_length=None, | ||
832 | 828 | etag=None, chunk_size=65536, content_type=None, | ||
833 | 829 | headers=None): | ||
834 | 830 | """Wrapper for :func:`put_object`""" | ||
835 | 831 | |||
836 | 832 | def _default_reset(*args, **kwargs): | ||
837 | 833 | raise ClientException('put_object(%r, %r, ...) failure and no ' | ||
838 | 834 | 'ability to reset contents for reupload.' % (container, obj)) | ||
839 | 835 | |||
840 | 836 | reset_func = _default_reset | ||
841 | 837 | tell = getattr(contents, 'tell', None) | ||
842 | 838 | seek = getattr(contents, 'seek', None) | ||
843 | 839 | if tell and seek: | ||
844 | 840 | orig_pos = tell() | ||
845 | 841 | reset_func = lambda *a, **k: seek(orig_pos) | ||
846 | 842 | elif not contents: | ||
847 | 843 | reset_func = lambda *a, **k: None | ||
848 | 844 | |||
849 | 845 | return self._retry(reset_func, put_object, container, obj, contents, | ||
850 | 846 | content_length=content_length, etag=etag, chunk_size=chunk_size, | ||
851 | 847 | content_type=content_type, headers=headers) | ||
852 | 848 | |||
853 | 849 | def post_object(self, container, obj, headers): | ||
854 | 850 | """Wrapper for :func:`post_object`""" | ||
855 | 851 | return self._retry(None, post_object, container, obj, headers) | ||
856 | 852 | |||
857 | 853 | def delete_object(self, container, obj): | ||
858 | 854 | """Wrapper for :func:`delete_object`""" | ||
859 | 855 | return self._retry(None, delete_object, container, obj) | ||
860 | 856 | |||
861 | 857 | # End inclusion of swift.common.client | ||
862 | 858 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | ||
863 | 859 | |||
864 | 860 | |||
865 | 861 | def mkdirs(path): | ||
866 | 862 | try: | ||
867 | 863 | makedirs(path) | ||
868 | 864 | except OSError, err: | ||
869 | 865 | if err.errno != EEXIST: | ||
870 | 866 | raise | ||
871 | 867 | |||
872 | 868 | |||
873 | 869 | def put_errors_from_threads(threads, error_queue): | ||
874 | 870 | """ | ||
875 | 871 | Places any errors from the threads into error_queue. | ||
876 | 872 | :param threads: A list of QueueFunctionThread instances. | ||
877 | 873 | :param error_queue: A queue to put error strings into. | ||
878 | 874 | :returns: True if any errors were found. | ||
879 | 875 | """ | ||
880 | 876 | was_error = False | ||
881 | 877 | for thread in threads: | ||
882 | 878 | for info in thread.exc_infos: | ||
883 | 879 | was_error = True | ||
884 | 880 | if isinstance(info[1], ClientException): | ||
885 | 881 | error_queue.put(str(info[1])) | ||
886 | 882 | else: | ||
887 | 883 | error_queue.put(''.join(format_exception(*info))) | ||
888 | 884 | return was_error | ||
889 | 885 | |||
890 | 886 | |||
891 | 887 | class QueueFunctionThread(Thread): | ||
892 | 888 | |||
893 | 889 | def __init__(self, queue, func, *args, **kwargs): | ||
894 | 890 | """ Calls func for each item in queue; func is called with a queued | ||
895 | 891 | item as the first arg followed by *args and **kwargs. Use the abort | ||
896 | 892 | attribute to have the thread empty the queue (without processing) | ||
897 | 893 | and exit. """ | ||
898 | 894 | Thread.__init__(self) | ||
899 | 895 | self.abort = False | ||
900 | 896 | self.queue = queue | ||
901 | 897 | self.func = func | ||
902 | 898 | self.args = args | ||
903 | 899 | self.kwargs = kwargs | ||
904 | 900 | self.exc_infos = [] | ||
905 | 901 | |||
906 | 902 | def run(self): | ||
907 | 903 | try: | ||
908 | 904 | while True: | ||
909 | 905 | try: | ||
910 | 906 | item = self.queue.get_nowait() | ||
911 | 907 | if not self.abort: | ||
912 | 908 | self.func(item, *self.args, **self.kwargs) | ||
913 | 909 | self.queue.task_done() | ||
914 | 910 | except Empty: | ||
915 | 911 | if self.abort: | ||
916 | 912 | break | ||
917 | 913 | sleep(0.01) | ||
918 | 914 | except Exception: | ||
919 | 915 | self.exc_infos.append(exc_info()) | ||
920 | 916 | |||
921 | 917 | |||
922 | 918 | st_delete_help = ''' | ||
923 | 919 | delete --all OR delete container [--leave-segments] [object] [object] ... | ||
924 | 920 | Deletes everything in the account (with --all), or everything in a | ||
925 | 921 | container, or a list of objects depending on the args given. Segments of | ||
926 | 922 | manifest objects will be deleted as well, unless you specify the | ||
927 | 923 | --leave-segments option.'''.strip('\n') | ||
928 | 924 | |||
929 | 925 | |||
930 | 926 | def st_delete(parser, args, print_queue, error_queue): | ||
931 | 927 | parser.add_option('-a', '--all', action='store_true', dest='yes_all', | ||
932 | 928 | default=False, help='Indicates that you really want to delete ' | ||
933 | 929 | 'everything in the account') | ||
934 | 930 | parser.add_option('', '--leave-segments', action='store_true', | ||
935 | 931 | dest='leave_segments', default=False, help='Indicates that you want ' | ||
936 | 932 | 'the segments of manifest objects left alone') | ||
937 | 933 | (options, args) = parse_args(parser, args) | ||
938 | 934 | args = args[1:] | ||
939 | 935 | if (not args and not options.yes_all) or (args and options.yes_all): | ||
940 | 936 | error_queue.put('Usage: %s [options] %s' % | ||
941 | 937 | (basename(argv[0]), st_delete_help)) | ||
942 | 938 | return | ||
943 | 939 | |||
944 | 940 | def _delete_segment((container, obj), conn): | ||
945 | 941 | conn.delete_object(container, obj) | ||
946 | 942 | if options.verbose: | ||
947 | 943 | if conn.attempts > 2: | ||
948 | 944 | print_queue.put('%s/%s [after %d attempts]' % | ||
949 | 945 | (container, obj, conn.attempts)) | ||
950 | 946 | else: | ||
951 | 947 | print_queue.put('%s/%s' % (container, obj)) | ||
952 | 948 | |||
953 | 949 | object_queue = Queue(10000) | ||
954 | 950 | |||
955 | 951 | def _delete_object((container, obj), conn): | ||
956 | 952 | try: | ||
957 | 953 | old_manifest = None | ||
958 | 954 | if not options.leave_segments: | ||
959 | 955 | try: | ||
960 | 956 | old_manifest = conn.head_object(container, obj).get( | ||
961 | 957 | 'x-object-manifest') | ||
962 | 958 | except ClientException, err: | ||
963 | 959 | if err.http_status != 404: | ||
964 | 960 | raise | ||
965 | 961 | conn.delete_object(container, obj) | ||
966 | 962 | if old_manifest: | ||
967 | 963 | segment_queue = Queue(10000) | ||
968 | 964 | scontainer, sprefix = old_manifest.split('/', 1) | ||
969 | 965 | for delobj in conn.get_container(scontainer, | ||
970 | 966 | prefix=sprefix)[1]: | ||
971 | 967 | segment_queue.put((scontainer, delobj['name'])) | ||
972 | 968 | if not segment_queue.empty(): | ||
973 | 969 | segment_threads = [QueueFunctionThread(segment_queue, | ||
974 | 970 | _delete_segment, create_connection()) for _junk in | ||
975 | 971 | xrange(10)] | ||
976 | 972 | for thread in segment_threads: | ||
977 | 973 | thread.start() | ||
978 | 974 | while not segment_queue.empty(): | ||
979 | 975 | sleep(0.01) | ||
980 | 976 | for thread in segment_threads: | ||
981 | 977 | thread.abort = True | ||
982 | 978 | while thread.isAlive(): | ||
983 | 979 | thread.join(0.01) | ||
984 | 980 | put_errors_from_threads(segment_threads, error_queue) | ||
985 | 981 | if options.verbose: | ||
986 | 982 | path = options.yes_all and join(container, obj) or obj | ||
987 | 983 | if path[:1] in ('/', '\\'): | ||
988 | 984 | path = path[1:] | ||
989 | 985 | if conn.attempts > 1: | ||
990 | 986 | print_queue.put('%s [after %d attempts]' % | ||
991 | 987 | (path, conn.attempts)) | ||
992 | 988 | else: | ||
993 | 989 | print_queue.put(path) | ||
994 | 990 | except ClientException, err: | ||
995 | 991 | if err.http_status != 404: | ||
996 | 992 | raise | ||
997 | 993 | error_queue.put('Object %s not found' % | ||
998 | 994 | repr('%s/%s' % (container, obj))) | ||
999 | 995 | |||
1000 | 996 | container_queue = Queue(10000) | ||
1001 | 997 | |||
1002 | 998 | def _delete_container(container, conn): | ||
1003 | 999 | try: | ||
1004 | 1000 | marker = '' | ||
1005 | 1001 | while True: | ||
1006 | 1002 | objects = [o['name'] for o in | ||
1007 | 1003 | conn.get_container(container, marker=marker)[1]] | ||
1008 | 1004 | if not objects: | ||
1009 | 1005 | break | ||
1010 | 1006 | for obj in objects: | ||
1011 | 1007 | object_queue.put((container, obj)) | ||
1012 | 1008 | marker = objects[-1] | ||
1013 | 1009 | while not object_queue.empty(): | ||
1014 | 1010 | sleep(0.01) | ||
1015 | 1011 | attempts = 1 | ||
1016 | 1012 | while True: | ||
1017 | 1013 | try: | ||
1018 | 1014 | conn.delete_container(container) | ||
1019 | 1015 | break | ||
1020 | 1016 | except ClientException, err: | ||
1021 | 1017 | if err.http_status != 409: | ||
1022 | 1018 | raise | ||
1023 | 1019 | if attempts > 10: | ||
1024 | 1020 | raise | ||
1025 | 1021 | attempts += 1 | ||
1026 | 1022 | sleep(1) | ||
1027 | 1023 | except ClientException, err: | ||
1028 | 1024 | if err.http_status != 404: | ||
1029 | 1025 | raise | ||
1030 | 1026 | error_queue.put('Container %s not found' % repr(container)) | ||
1031 | 1027 | |||
1032 | 1028 | url, token = get_auth(options.auth, options.user, options.key, | ||
1033 | 1029 | snet=options.snet) | ||
1034 | 1030 | create_connection = lambda: Connection(options.auth, options.user, | ||
1035 | 1031 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
1036 | 1032 | object_threads = [QueueFunctionThread(object_queue, _delete_object, | ||
1037 | 1033 | create_connection()) for _junk in xrange(10)] | ||
1038 | 1034 | for thread in object_threads: | ||
1039 | 1035 | thread.start() | ||
1040 | 1036 | container_threads = [QueueFunctionThread(container_queue, | ||
1041 | 1037 | _delete_container, create_connection()) for _junk in xrange(10)] | ||
1042 | 1038 | for thread in container_threads: | ||
1043 | 1039 | thread.start() | ||
1044 | 1040 | if not args: | ||
1045 | 1041 | conn = create_connection() | ||
1046 | 1042 | try: | ||
1047 | 1043 | marker = '' | ||
1048 | 1044 | while True: | ||
1049 | 1045 | containers = \ | ||
1050 | 1046 | [c['name'] for c in conn.get_account(marker=marker)[1]] | ||
1051 | 1047 | if not containers: | ||
1052 | 1048 | break | ||
1053 | 1049 | for container in containers: | ||
1054 | 1050 | container_queue.put(container) | ||
1055 | 1051 | marker = containers[-1] | ||
1056 | 1052 | while not container_queue.empty(): | ||
1057 | 1053 | sleep(0.01) | ||
1058 | 1054 | while not object_queue.empty(): | ||
1059 | 1055 | sleep(0.01) | ||
1060 | 1056 | except ClientException, err: | ||
1061 | 1057 | if err.http_status != 404: | ||
1062 | 1058 | raise | ||
1063 | 1059 | error_queue.put('Account not found') | ||
1064 | 1060 | elif len(args) == 1: | ||
1065 | 1061 | if '/' in args[0]: | ||
1066 | 1062 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
1067 | 1063 | 'meant %r instead of %r.' % \ | ||
1068 | 1064 | (args[0].replace('/', ' ', 1), args[0]) | ||
1069 | 1065 | conn = create_connection() | ||
1070 | 1066 | _delete_container(args[0], conn) | ||
1071 | 1067 | else: | ||
1072 | 1068 | for obj in args[1:]: | ||
1073 | 1069 | object_queue.put((args[0], obj)) | ||
1074 | 1070 | while not container_queue.empty(): | ||
1075 | 1071 | sleep(0.01) | ||
1076 | 1072 | for thread in container_threads: | ||
1077 | 1073 | thread.abort = True | ||
1078 | 1074 | while thread.isAlive(): | ||
1079 | 1075 | thread.join(0.01) | ||
1080 | 1076 | put_errors_from_threads(container_threads, error_queue) | ||
1081 | 1077 | while not object_queue.empty(): | ||
1082 | 1078 | sleep(0.01) | ||
1083 | 1079 | for thread in object_threads: | ||
1084 | 1080 | thread.abort = True | ||
1085 | 1081 | while thread.isAlive(): | ||
1086 | 1082 | thread.join(0.01) | ||
1087 | 1083 | put_errors_from_threads(object_threads, error_queue) | ||
1088 | 1084 | |||
1089 | 1085 | |||
1090 | 1086 | st_download_help = ''' | ||
1091 | 1087 | download --all OR download container [options] [object] [object] ... | ||
1092 | 1088 | Downloads everything in the account (with --all), or everything in a | ||
1093 | 1089 | container, or a list of objects depending on the args given. For a single | ||
1094 | 1090 | object download, you may use the -o [--output] <filename> option to | ||
1095 | 1091 | redirect the output to a specific file or if "-" then just redirect to | ||
1096 | 1092 | stdout.'''.strip('\n') | ||
1097 | 1093 | |||
1098 | 1094 | |||
1099 | 1095 | def st_download(options, args, print_queue, error_queue): | ||
1100 | 1096 | parser.add_option('-a', '--all', action='store_true', dest='yes_all', | ||
1101 | 1097 | default=False, help='Indicates that you really want to download ' | ||
1102 | 1098 | 'everything in the account') | ||
1103 | 1099 | parser.add_option('-o', '--output', dest='out_file', help='For a single ' | ||
1104 | 1100 | 'file download, stream the output to an alternate location ') | ||
1105 | 1101 | (options, args) = parse_args(parser, args) | ||
1106 | 1102 | args = args[1:] | ||
1107 | 1103 | if options.out_file == '-': | ||
1108 | 1104 | options.verbose = 0 | ||
1109 | 1105 | if options.out_file and len(args) != 2: | ||
1110 | 1106 | exit('-o option only allowed for single file downloads') | ||
1111 | 1107 | if (not args and not options.yes_all) or (args and options.yes_all): | ||
1112 | 1108 | error_queue.put('Usage: %s [options] %s' % | ||
1113 | 1109 | (basename(argv[0]), st_download_help)) | ||
1114 | 1110 | return | ||
1115 | 1111 | |||
1116 | 1112 | object_queue = Queue(10000) | ||
1117 | 1113 | |||
1118 | 1114 | def _download_object(queue_arg, conn): | ||
1119 | 1115 | if len(queue_arg) == 2: | ||
1120 | 1116 | container, obj = queue_arg | ||
1121 | 1117 | out_file = None | ||
1122 | 1118 | elif len(queue_arg) == 3: | ||
1123 | 1119 | container, obj, out_file = queue_arg | ||
1124 | 1120 | else: | ||
1125 | 1121 | raise Exception("Invalid queue_arg length of %s" % len(queue_arg)) | ||
1126 | 1122 | try: | ||
1127 | 1123 | headers, body = \ | ||
1128 | 1124 | conn.get_object(container, obj, resp_chunk_size=65536) | ||
1129 | 1125 | content_type = headers.get('content-type') | ||
1130 | 1126 | if 'content-length' in headers: | ||
1131 | 1127 | content_length = int(headers.get('content-length')) | ||
1132 | 1128 | else: | ||
1133 | 1129 | content_length = None | ||
1134 | 1130 | etag = headers.get('etag') | ||
1135 | 1131 | path = options.yes_all and join(container, obj) or obj | ||
1136 | 1132 | if path[:1] in ('/', '\\'): | ||
1137 | 1133 | path = path[1:] | ||
1138 | 1134 | md5sum = None | ||
1139 | 1135 | make_dir = out_file != "-" | ||
1140 | 1136 | if content_type.split(';', 1)[0] == 'text/directory': | ||
1141 | 1137 | if make_dir and not isdir(path): | ||
1142 | 1138 | mkdirs(path) | ||
1143 | 1139 | read_length = 0 | ||
1144 | 1140 | if 'x-object-manifest' not in headers: | ||
1145 | 1141 | md5sum = md5() | ||
1146 | 1142 | for chunk in body: | ||
1147 | 1143 | read_length += len(chunk) | ||
1148 | 1144 | if md5sum: | ||
1149 | 1145 | md5sum.update(chunk) | ||
1150 | 1146 | else: | ||
1151 | 1147 | dirpath = dirname(path) | ||
1152 | 1148 | if make_dir and dirpath and not isdir(dirpath): | ||
1153 | 1149 | mkdirs(dirpath) | ||
1154 | 1150 | if out_file == "-": | ||
1155 | 1151 | fp = stdout | ||
1156 | 1152 | elif out_file: | ||
1157 | 1153 | fp = open(out_file, 'wb') | ||
1158 | 1154 | else: | ||
1159 | 1155 | fp = open(path, 'wb') | ||
1160 | 1156 | read_length = 0 | ||
1161 | 1157 | if 'x-object-manifest' not in headers: | ||
1162 | 1158 | md5sum = md5() | ||
1163 | 1159 | for chunk in body: | ||
1164 | 1160 | fp.write(chunk) | ||
1165 | 1161 | read_length += len(chunk) | ||
1166 | 1162 | if md5sum: | ||
1167 | 1163 | md5sum.update(chunk) | ||
1168 | 1164 | fp.close() | ||
1169 | 1165 | if md5sum and md5sum.hexdigest() != etag: | ||
1170 | 1166 | error_queue.put('%s: md5sum != etag, %s != %s' % | ||
1171 | 1167 | (path, md5sum.hexdigest(), etag)) | ||
1172 | 1168 | if content_length is not None and read_length != content_length: | ||
1173 | 1169 | error_queue.put('%s: read_length != content_length, %d != %d' % | ||
1174 | 1170 | (path, read_length, content_length)) | ||
1175 | 1171 | if 'x-object-meta-mtime' in headers and not options.out_file: | ||
1176 | 1172 | mtime = float(headers['x-object-meta-mtime']) | ||
1177 | 1173 | utime(path, (mtime, mtime)) | ||
1178 | 1174 | if options.verbose: | ||
1179 | 1175 | if conn.attempts > 1: | ||
1180 | 1176 | print_queue.put('%s [after %d attempts' % | ||
1181 | 1177 | (path, conn.attempts)) | ||
1182 | 1178 | else: | ||
1183 | 1179 | print_queue.put(path) | ||
1184 | 1180 | except ClientException, err: | ||
1185 | 1181 | if err.http_status != 404: | ||
1186 | 1182 | raise | ||
1187 | 1183 | error_queue.put('Object %s not found' % | ||
1188 | 1184 | repr('%s/%s' % (container, obj))) | ||
1189 | 1185 | |||
1190 | 1186 | container_queue = Queue(10000) | ||
1191 | 1187 | |||
1192 | 1188 | def _download_container(container, conn): | ||
1193 | 1189 | try: | ||
1194 | 1190 | marker = '' | ||
1195 | 1191 | while True: | ||
1196 | 1192 | objects = [o['name'] for o in | ||
1197 | 1193 | conn.get_container(container, marker=marker)[1]] | ||
1198 | 1194 | if not objects: | ||
1199 | 1195 | break | ||
1200 | 1196 | for obj in objects: | ||
1201 | 1197 | object_queue.put((container, obj)) | ||
1202 | 1198 | marker = objects[-1] | ||
1203 | 1199 | except ClientException, err: | ||
1204 | 1200 | if err.http_status != 404: | ||
1205 | 1201 | raise | ||
1206 | 1202 | error_queue.put('Container %s not found' % repr(container)) | ||
1207 | 1203 | |||
1208 | 1204 | url, token = get_auth(options.auth, options.user, options.key, | ||
1209 | 1205 | snet=options.snet) | ||
1210 | 1206 | create_connection = lambda: Connection(options.auth, options.user, | ||
1211 | 1207 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
1212 | 1208 | object_threads = [QueueFunctionThread(object_queue, _download_object, | ||
1213 | 1209 | create_connection()) for _junk in xrange(10)] | ||
1214 | 1210 | for thread in object_threads: | ||
1215 | 1211 | thread.start() | ||
1216 | 1212 | container_threads = [QueueFunctionThread(container_queue, | ||
1217 | 1213 | _download_container, create_connection()) for _junk in xrange(10)] | ||
1218 | 1214 | for thread in container_threads: | ||
1219 | 1215 | thread.start() | ||
1220 | 1216 | if not args: | ||
1221 | 1217 | conn = create_connection() | ||
1222 | 1218 | try: | ||
1223 | 1219 | marker = '' | ||
1224 | 1220 | while True: | ||
1225 | 1221 | containers = [c['name'] | ||
1226 | 1222 | for c in conn.get_account(marker=marker)[1]] | ||
1227 | 1223 | if not containers: | ||
1228 | 1224 | break | ||
1229 | 1225 | for container in containers: | ||
1230 | 1226 | container_queue.put(container) | ||
1231 | 1227 | marker = containers[-1] | ||
1232 | 1228 | except ClientException, err: | ||
1233 | 1229 | if err.http_status != 404: | ||
1234 | 1230 | raise | ||
1235 | 1231 | error_queue.put('Account not found') | ||
1236 | 1232 | elif len(args) == 1: | ||
1237 | 1233 | if '/' in args[0]: | ||
1238 | 1234 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
1239 | 1235 | 'meant %r instead of %r.' % \ | ||
1240 | 1236 | (args[0].replace('/', ' ', 1), args[0]) | ||
1241 | 1237 | _download_container(args[0], create_connection()) | ||
1242 | 1238 | else: | ||
1243 | 1239 | if len(args) == 2: | ||
1244 | 1240 | obj = args[1] | ||
1245 | 1241 | object_queue.put((args[0], obj, options.out_file)) | ||
1246 | 1242 | else: | ||
1247 | 1243 | for obj in args[1:]: | ||
1248 | 1244 | object_queue.put((args[0], obj)) | ||
1249 | 1245 | while not container_queue.empty(): | ||
1250 | 1246 | sleep(0.01) | ||
1251 | 1247 | for thread in container_threads: | ||
1252 | 1248 | thread.abort = True | ||
1253 | 1249 | while thread.isAlive(): | ||
1254 | 1250 | thread.join(0.01) | ||
1255 | 1251 | put_errors_from_threads(container_threads, error_queue) | ||
1256 | 1252 | while not object_queue.empty(): | ||
1257 | 1253 | sleep(0.01) | ||
1258 | 1254 | for thread in object_threads: | ||
1259 | 1255 | thread.abort = True | ||
1260 | 1256 | while thread.isAlive(): | ||
1261 | 1257 | thread.join(0.01) | ||
1262 | 1258 | put_errors_from_threads(object_threads, error_queue) | ||
1263 | 1259 | |||
1264 | 1260 | |||
1265 | 1261 | st_list_help = ''' | ||
1266 | 1262 | list [options] [container] | ||
1267 | 1263 | Lists the containers for the account or the objects for a container. -p or | ||
1268 | 1264 | --prefix is an option that will only list items beginning with that prefix. | ||
1269 | 1265 | -d or --delimiter is option (for container listings only) that will roll up | ||
1270 | 1266 | items with the given delimiter (see Cloud Files general documentation for | ||
1271 | 1267 | what this means). | ||
1272 | 1268 | '''.strip('\n') | ||
1273 | 1269 | |||
1274 | 1270 | |||
1275 | 1271 | def st_list(options, args, print_queue, error_queue): | ||
1276 | 1272 | parser.add_option('-p', '--prefix', dest='prefix', help='Will only list ' | ||
1277 | 1273 | 'items beginning with the prefix') | ||
1278 | 1274 | parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll ' | ||
1279 | 1275 | 'up items with the given delimiter (see Cloud Files general ' | ||
1280 | 1276 | 'documentation for what this means)') | ||
1281 | 1277 | (options, args) = parse_args(parser, args) | ||
1282 | 1278 | args = args[1:] | ||
1283 | 1279 | if options.delimiter and not args: | ||
1284 | 1280 | exit('-d option only allowed for container listings') | ||
1285 | 1281 | if len(args) > 1: | ||
1286 | 1282 | error_queue.put('Usage: %s [options] %s' % | ||
1287 | 1283 | (basename(argv[0]), st_list_help)) | ||
1288 | 1284 | return | ||
1289 | 1285 | conn = Connection(options.auth, options.user, options.key, | ||
1290 | 1286 | snet=options.snet) | ||
1291 | 1287 | try: | ||
1292 | 1288 | marker = '' | ||
1293 | 1289 | while True: | ||
1294 | 1290 | if not args: | ||
1295 | 1291 | items = \ | ||
1296 | 1292 | conn.get_account(marker=marker, prefix=options.prefix)[1] | ||
1297 | 1293 | else: | ||
1298 | 1294 | items = conn.get_container(args[0], marker=marker, | ||
1299 | 1295 | prefix=options.prefix, delimiter=options.delimiter)[1] | ||
1300 | 1296 | if not items: | ||
1301 | 1297 | break | ||
1302 | 1298 | for item in items: | ||
1303 | 1299 | print_queue.put(item.get('name', item.get('subdir'))) | ||
1304 | 1300 | marker = items[-1].get('name', items[-1].get('subdir')) | ||
1305 | 1301 | except ClientException, err: | ||
1306 | 1302 | if err.http_status != 404: | ||
1307 | 1303 | raise | ||
1308 | 1304 | if not args: | ||
1309 | 1305 | error_queue.put('Account not found') | ||
1310 | 1306 | else: | ||
1311 | 1307 | error_queue.put('Container %s not found' % repr(args[0])) | ||
1312 | 1308 | |||
1313 | 1309 | |||
1314 | 1310 | st_stat_help = ''' | ||
1315 | 1311 | stat [container] [object] | ||
1316 | 1312 | Displays information for the account, container, or object depending on the | ||
1317 | 1313 | args given (if any).'''.strip('\n') | ||
1318 | 1314 | |||
1319 | 1315 | |||
1320 | 1316 | def st_stat(options, args, print_queue, error_queue): | ||
1321 | 1317 | (options, args) = parse_args(parser, args) | ||
1322 | 1318 | args = args[1:] | ||
1323 | 1319 | conn = Connection(options.auth, options.user, options.key) | ||
1324 | 1320 | if not args: | ||
1325 | 1321 | try: | ||
1326 | 1322 | headers = conn.head_account() | ||
1327 | 1323 | if options.verbose > 1: | ||
1328 | 1324 | print_queue.put(''' | ||
1329 | 1325 | StorageURL: %s | ||
1330 | 1326 | Auth Token: %s | ||
1331 | 1327 | '''.strip('\n') % (conn.url, conn.token)) | ||
1332 | 1328 | container_count = int(headers.get('x-account-container-count', 0)) | ||
1333 | 1329 | object_count = int(headers.get('x-account-object-count', 0)) | ||
1334 | 1330 | bytes_used = int(headers.get('x-account-bytes-used', 0)) | ||
1335 | 1331 | print_queue.put(''' | ||
1336 | 1332 | Account: %s | ||
1337 | 1333 | Containers: %d | ||
1338 | 1334 | Objects: %d | ||
1339 | 1335 | Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count, | ||
1340 | 1336 | object_count, bytes_used)) | ||
1341 | 1337 | for key, value in headers.items(): | ||
1342 | 1338 | if key.startswith('x-account-meta-'): | ||
1343 | 1339 | print_queue.put('%10s: %s' % ('Meta %s' % | ||
1344 | 1340 | key[len('x-account-meta-'):].title(), value)) | ||
1345 | 1341 | for key, value in headers.items(): | ||
1346 | 1342 | if not key.startswith('x-account-meta-') and key not in ( | ||
1347 | 1343 | 'content-length', 'date', 'x-account-container-count', | ||
1348 | 1344 | 'x-account-object-count', 'x-account-bytes-used'): | ||
1349 | 1345 | print_queue.put( | ||
1350 | 1346 | '%10s: %s' % (key.title(), value)) | ||
1351 | 1347 | except ClientException, err: | ||
1352 | 1348 | if err.http_status != 404: | ||
1353 | 1349 | raise | ||
1354 | 1350 | error_queue.put('Account not found') | ||
1355 | 1351 | elif len(args) == 1: | ||
1356 | 1352 | if '/' in args[0]: | ||
1357 | 1353 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
1358 | 1354 | 'meant %r instead of %r.' % \ | ||
1359 | 1355 | (args[0].replace('/', ' ', 1), args[0]) | ||
1360 | 1356 | try: | ||
1361 | 1357 | headers = conn.head_container(args[0]) | ||
1362 | 1358 | object_count = int(headers.get('x-container-object-count', 0)) | ||
1363 | 1359 | bytes_used = int(headers.get('x-container-bytes-used', 0)) | ||
1364 | 1360 | print_queue.put(''' | ||
1365 | 1361 | Account: %s | ||
1366 | 1362 | Container: %s | ||
1367 | 1363 | Objects: %d | ||
1368 | 1364 | Bytes: %d | ||
1369 | 1365 | Read ACL: %s | ||
1370 | 1366 | Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], | ||
1371 | 1367 | object_count, bytes_used, | ||
1372 | 1368 | headers.get('x-container-read', ''), | ||
1373 | 1369 | headers.get('x-container-write', ''))) | ||
1374 | 1370 | for key, value in headers.items(): | ||
1375 | 1371 | if key.startswith('x-container-meta-'): | ||
1376 | 1372 | print_queue.put('%9s: %s' % ('Meta %s' % | ||
1377 | 1373 | key[len('x-container-meta-'):].title(), value)) | ||
1378 | 1374 | for key, value in headers.items(): | ||
1379 | 1375 | if not key.startswith('x-container-meta-') and key not in ( | ||
1380 | 1376 | 'content-length', 'date', 'x-container-object-count', | ||
1381 | 1377 | 'x-container-bytes-used', 'x-container-read', | ||
1382 | 1378 | 'x-container-write'): | ||
1383 | 1379 | print_queue.put( | ||
1384 | 1380 | '%9s: %s' % (key.title(), value)) | ||
1385 | 1381 | except ClientException, err: | ||
1386 | 1382 | if err.http_status != 404: | ||
1387 | 1383 | raise | ||
1388 | 1384 | error_queue.put('Container %s not found' % repr(args[0])) | ||
1389 | 1385 | elif len(args) == 2: | ||
1390 | 1386 | try: | ||
1391 | 1387 | headers = conn.head_object(args[0], args[1]) | ||
1392 | 1388 | print_queue.put(''' | ||
1393 | 1389 | Account: %s | ||
1394 | 1390 | Container: %s | ||
1395 | 1391 | Object: %s | ||
1396 | 1392 | Content Type: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], | ||
1397 | 1393 | args[1], headers.get('content-type'))) | ||
1398 | 1394 | if 'content-length' in headers: | ||
1399 | 1395 | print_queue.put('Content Length: %s' % | ||
1400 | 1396 | headers['content-length']) | ||
1401 | 1397 | if 'last-modified' in headers: | ||
1402 | 1398 | print_queue.put(' Last Modified: %s' % | ||
1403 | 1399 | headers['last-modified']) | ||
1404 | 1400 | if 'etag' in headers: | ||
1405 | 1401 | print_queue.put(' ETag: %s' % headers['etag']) | ||
1406 | 1402 | if 'x-object-manifest' in headers: | ||
1407 | 1403 | print_queue.put(' Manifest: %s' % | ||
1408 | 1404 | headers['x-object-manifest']) | ||
1409 | 1405 | for key, value in headers.items(): | ||
1410 | 1406 | if key.startswith('x-object-meta-'): | ||
1411 | 1407 | print_queue.put('%14s: %s' % ('Meta %s' % | ||
1412 | 1408 | key[len('x-object-meta-'):].title(), value)) | ||
1413 | 1409 | for key, value in headers.items(): | ||
1414 | 1410 | if not key.startswith('x-object-meta-') and key not in ( | ||
1415 | 1411 | 'content-type', 'content-length', 'last-modified', | ||
1416 | 1412 | 'etag', 'date', 'x-object-manifest'): | ||
1417 | 1413 | print_queue.put( | ||
1418 | 1414 | '%14s: %s' % (key.title(), value)) | ||
1419 | 1415 | except ClientException, err: | ||
1420 | 1416 | if err.http_status != 404: | ||
1421 | 1417 | raise | ||
1422 | 1418 | error_queue.put('Object %s not found' % | ||
1423 | 1419 | repr('%s/%s' % (args[0], args[1]))) | ||
1424 | 1420 | else: | ||
1425 | 1421 | error_queue.put('Usage: %s [options] %s' % | ||
1426 | 1422 | (basename(argv[0]), st_stat_help)) | ||
1427 | 1423 | |||
1428 | 1424 | |||
1429 | 1425 | st_post_help = ''' | ||
1430 | 1426 | post [options] [container] [object] | ||
1431 | 1427 | Updates meta information for the account, container, or object depending on | ||
1432 | 1428 | the args given. If the container is not found, it will be created | ||
1433 | 1429 | automatically; but this is not true for accounts and objects. Containers | ||
1434 | 1430 | also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m | ||
1435 | 1431 | or --meta option is allowed on all and used to define the user meta data | ||
1436 | 1432 | items to set in the form Name:Value. This option can be repeated. Example: | ||
1437 | 1433 | post -m Color:Blue -m Size:Large'''.strip('\n') | ||
1438 | 1434 | |||
1439 | 1435 | |||
1440 | 1436 | def st_post(options, args, print_queue, error_queue): | ||
1441 | 1437 | parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the ' | ||
1442 | 1438 | 'Read ACL for containers. Quick summary of ACL syntax: .r:*, ' | ||
1443 | 1439 | '.r:-.example.com, .r:www.example.com, account1, account2:user2') | ||
1444 | 1440 | parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the ' | ||
1445 | 1441 | 'Write ACL for containers. Quick summary of ACL syntax: account1, ' | ||
1446 | 1442 | 'account2:user2') | ||
1447 | 1443 | parser.add_option('-m', '--meta', action='append', dest='meta', default=[], | ||
1448 | 1444 | help='Sets a meta data item with the syntax name:value. This option ' | ||
1449 | 1445 | 'may be repeated. Example: -m Color:Blue -m Size:Large') | ||
1450 | 1446 | (options, args) = parse_args(parser, args) | ||
1451 | 1447 | args = args[1:] | ||
1452 | 1448 | if (options.read_acl or options.write_acl) and not args: | ||
1453 | 1449 | exit('-r and -w options only allowed for containers') | ||
1454 | 1450 | conn = Connection(options.auth, options.user, options.key) | ||
1455 | 1451 | if not args: | ||
1456 | 1452 | headers = {} | ||
1457 | 1453 | for item in options.meta: | ||
1458 | 1454 | split_item = item.split(':') | ||
1459 | 1455 | headers['X-Account-Meta-' + split_item[0]] = \ | ||
1460 | 1456 | len(split_item) > 1 and split_item[1] | ||
1461 | 1457 | try: | ||
1462 | 1458 | conn.post_account(headers=headers) | ||
1463 | 1459 | except ClientException, err: | ||
1464 | 1460 | if err.http_status != 404: | ||
1465 | 1461 | raise | ||
1466 | 1462 | error_queue.put('Account not found') | ||
1467 | 1463 | elif len(args) == 1: | ||
1468 | 1464 | if '/' in args[0]: | ||
1469 | 1465 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
1470 | 1466 | 'meant %r instead of %r.' % \ | ||
1471 | 1467 | (args[0].replace('/', ' ', 1), args[0]) | ||
1472 | 1468 | headers = {} | ||
1473 | 1469 | for item in options.meta: | ||
1474 | 1470 | split_item = item.split(':') | ||
1475 | 1471 | headers['X-Container-Meta-' + split_item[0]] = \ | ||
1476 | 1472 | len(split_item) > 1 and split_item[1] | ||
1477 | 1473 | if options.read_acl is not None: | ||
1478 | 1474 | headers['X-Container-Read'] = options.read_acl | ||
1479 | 1475 | if options.write_acl is not None: | ||
1480 | 1476 | headers['X-Container-Write'] = options.write_acl | ||
1481 | 1477 | try: | ||
1482 | 1478 | conn.post_container(args[0], headers=headers) | ||
1483 | 1479 | except ClientException, err: | ||
1484 | 1480 | if err.http_status != 404: | ||
1485 | 1481 | raise | ||
1486 | 1482 | conn.put_container(args[0], headers=headers) | ||
1487 | 1483 | elif len(args) == 2: | ||
1488 | 1484 | headers = {} | ||
1489 | 1485 | for item in options.meta: | ||
1490 | 1486 | split_item = item.split(':') | ||
1491 | 1487 | headers['X-Object-Meta-' + split_item[0]] = \ | ||
1492 | 1488 | len(split_item) > 1 and split_item[1] | ||
1493 | 1489 | try: | ||
1494 | 1490 | conn.post_object(args[0], args[1], headers=headers) | ||
1495 | 1491 | except ClientException, err: | ||
1496 | 1492 | if err.http_status != 404: | ||
1497 | 1493 | raise | ||
1498 | 1494 | error_queue.put('Object %s not found' % | ||
1499 | 1495 | repr('%s/%s' % (args[0], args[1]))) | ||
1500 | 1496 | else: | ||
1501 | 1497 | error_queue.put('Usage: %s [options] %s' % | ||
1502 | 1498 | (basename(argv[0]), st_post_help)) | ||
1503 | 1499 | |||
1504 | 1500 | |||
1505 | 1501 | st_upload_help = ''' | ||
1506 | 1502 | upload [options] container file_or_directory [file_or_directory] [...] | ||
1507 | 1503 | Uploads to the given container the files and directories specified by the | ||
1508 | 1504 | remaining args. -c or --changed is an option that will only upload files | ||
1509 | 1505 | that have changed since the last upload. -S <size> or --segment-size <size> | ||
1510 | 1506 | and --leave-segments are options as well (see --help for more). | ||
1511 | 1507 | '''.strip('\n') | ||
1512 | 1508 | |||
1513 | 1509 | |||
1514 | 1510 | def st_upload(options, args, print_queue, error_queue): | ||
1515 | 1511 | parser.add_option('-c', '--changed', action='store_true', dest='changed', | ||
1516 | 1512 | default=False, help='Will only upload files that have changed since ' | ||
1517 | 1513 | 'the last upload') | ||
1518 | 1514 | parser.add_option('-S', '--segment-size', dest='segment_size', help='Will ' | ||
1519 | 1515 | 'upload files in segments no larger than <size> and then create a ' | ||
1520 | 1516 | '"manifest" file that will download all the segments as if it were ' | ||
1521 | 1517 | 'the original file. The segments will be uploaded to a ' | ||
1522 | 1518 | '<container>_segments container so as to not pollute the main ' | ||
1523 | 1519 | '<container> listings.') | ||
1524 | 1520 | parser.add_option('', '--leave-segments', action='store_true', | ||
1525 | 1521 | dest='leave_segments', default=False, help='Indicates that you want ' | ||
1526 | 1522 | 'the older segments of manifest objects left alone (in the case of ' | ||
1527 | 1523 | 'overwrites)') | ||
1528 | 1524 | (options, args) = parse_args(parser, args) | ||
1529 | 1525 | args = args[1:] | ||
1530 | 1526 | if len(args) < 2: | ||
1531 | 1527 | error_queue.put('Usage: %s [options] %s' % | ||
1532 | 1528 | (basename(argv[0]), st_upload_help)) | ||
1533 | 1529 | return | ||
1534 | 1530 | object_queue = Queue(10000) | ||
1535 | 1531 | |||
1536 | 1532 | def _segment_job(job, conn): | ||
1537 | 1533 | if job.get('delete', False): | ||
1538 | 1534 | conn.delete_object(job['container'], job['obj']) | ||
1539 | 1535 | else: | ||
1540 | 1536 | fp = open(job['path'], 'rb') | ||
1541 | 1537 | fp.seek(job['segment_start']) | ||
1542 | 1538 | conn.put_object(job.get('container', args[0] + '_segments'), | ||
1543 | 1539 | job['obj'], fp, content_length=job['segment_size']) | ||
1544 | 1540 | if options.verbose and 'log_line' in job: | ||
1545 | 1541 | if conn.attempts > 1: | ||
1546 | 1542 | print_queue.put('%s [after %d attempts]' % | ||
1547 | 1543 | (job['log_line'], conn.attempts)) | ||
1548 | 1544 | else: | ||
1549 | 1545 | print_queue.put(job['log_line']) | ||
1550 | 1546 | |||
1551 | 1547 | def _object_job(job, conn): | ||
1552 | 1548 | path = job['path'] | ||
1553 | 1549 | container = job.get('container', args[0]) | ||
1554 | 1550 | dir_marker = job.get('dir_marker', False) | ||
1555 | 1551 | try: | ||
1556 | 1552 | obj = path | ||
1557 | 1553 | if obj.startswith('./') or obj.startswith('.\\'): | ||
1558 | 1554 | obj = obj[2:] | ||
1559 | 1555 | put_headers = {'x-object-meta-mtime': str(getmtime(path))} | ||
1560 | 1556 | if dir_marker: | ||
1561 | 1557 | if options.changed: | ||
1562 | 1558 | try: | ||
1563 | 1559 | headers = conn.head_object(container, obj) | ||
1564 | 1560 | ct = headers.get('content-type') | ||
1565 | 1561 | cl = int(headers.get('content-length')) | ||
1566 | 1562 | et = headers.get('etag') | ||
1567 | 1563 | mt = headers.get('x-object-meta-mtime') | ||
1568 | 1564 | if ct.split(';', 1)[0] == 'text/directory' and \ | ||
1569 | 1565 | cl == 0 and \ | ||
1570 | 1566 | et == 'd41d8cd98f00b204e9800998ecf8427e' and \ | ||
1571 | 1567 | mt == put_headers['x-object-meta-mtime']: | ||
1572 | 1568 | return | ||
1573 | 1569 | except ClientException, err: | ||
1574 | 1570 | if err.http_status != 404: | ||
1575 | 1571 | raise | ||
1576 | 1572 | conn.put_object(container, obj, '', content_length=0, | ||
1577 | 1573 | content_type='text/directory', | ||
1578 | 1574 | headers=put_headers) | ||
1579 | 1575 | else: | ||
1580 | 1576 | # We need to HEAD all objects now in case we're overwriting a | ||
1581 | 1577 | # manifest object and need to delete the old segments | ||
1582 | 1578 | # ourselves. | ||
1583 | 1579 | old_manifest = None | ||
1584 | 1580 | if options.changed or not options.leave_segments: | ||
1585 | 1581 | try: | ||
1586 | 1582 | headers = conn.head_object(container, obj) | ||
1587 | 1583 | cl = int(headers.get('content-length')) | ||
1588 | 1584 | mt = headers.get('x-object-meta-mtime') | ||
1589 | 1585 | if options.changed and cl == getsize(path) and \ | ||
1590 | 1586 | mt == put_headers['x-object-meta-mtime']: | ||
1591 | 1587 | return | ||
1592 | 1588 | if not options.leave_segments: | ||
1593 | 1589 | old_manifest = headers.get('x-object-manifest') | ||
1594 | 1590 | except ClientException, err: | ||
1595 | 1591 | if err.http_status != 404: | ||
1596 | 1592 | raise | ||
1597 | 1593 | if options.segment_size and \ | ||
1598 | 1594 | getsize(path) < options.segment_size: | ||
1599 | 1595 | full_size = getsize(path) | ||
1600 | 1596 | segment_queue = Queue(10000) | ||
1601 | 1597 | segment_threads = [QueueFunctionThread(segment_queue, | ||
1602 | 1598 | _segment_job, create_connection()) for _junk in | ||
1603 | 1599 | xrange(10)] | ||
1604 | 1600 | for thread in segment_threads: | ||
1605 | 1601 | thread.start() | ||
1606 | 1602 | segment = 0 | ||
1607 | 1603 | segment_start = 0 | ||
1608 | 1604 | while segment_start < full_size: | ||
1609 | 1605 | segment_size = int(options.segment_size) | ||
1610 | 1606 | if segment_start + segment_size > full_size: | ||
1611 | 1607 | segment_size = full_size - segment_start | ||
1612 | 1608 | segment_queue.put({'path': path, | ||
1613 | 1609 | 'obj': '%s/%s/%s/%08d' % (obj, | ||
1614 | 1610 | put_headers['x-object-meta-mtime'], full_size, | ||
1615 | 1611 | segment), | ||
1616 | 1612 | 'segment_start': segment_start, | ||
1617 | 1613 | 'segment_size': segment_size, | ||
1618 | 1614 | 'log_line': '%s segment %s' % (obj, segment)}) | ||
1619 | 1615 | segment += 1 | ||
1620 | 1616 | segment_start += segment_size | ||
1621 | 1617 | while not segment_queue.empty(): | ||
1622 | 1618 | sleep(0.01) | ||
1623 | 1619 | for thread in segment_threads: | ||
1624 | 1620 | thread.abort = True | ||
1625 | 1621 | while thread.isAlive(): | ||
1626 | 1622 | thread.join(0.01) | ||
1627 | 1623 | if put_errors_from_threads(segment_threads, error_queue): | ||
1628 | 1624 | raise ClientException('Aborting manifest creation ' | ||
1629 | 1625 | 'because not all segments could be uploaded. %s/%s' | ||
1630 | 1626 | % (container, obj)) | ||
1631 | 1627 | new_object_manifest = '%s_segments/%s/%s/%s/' % ( | ||
1632 | 1628 | container, obj, put_headers['x-object-meta-mtime'], | ||
1633 | 1629 | full_size) | ||
1634 | 1630 | if old_manifest == new_object_manifest: | ||
1635 | 1631 | old_manifest = None | ||
1636 | 1632 | put_headers['x-object-manifest'] = new_object_manifest | ||
1637 | 1633 | conn.put_object(container, obj, '', content_length=0, | ||
1638 | 1634 | headers=put_headers) | ||
1639 | 1635 | else: | ||
1640 | 1636 | conn.put_object(container, obj, open(path, 'rb'), | ||
1641 | 1637 | content_length=getsize(path), headers=put_headers) | ||
1642 | 1638 | if old_manifest: | ||
1643 | 1639 | segment_queue = Queue(10000) | ||
1644 | 1640 | scontainer, sprefix = old_manifest.split('/', 1) | ||
1645 | 1641 | for delobj in conn.get_container(scontainer, | ||
1646 | 1642 | prefix=sprefix)[1]: | ||
1647 | 1643 | segment_queue.put({'delete': True, | ||
1648 | 1644 | 'container': scontainer, 'obj': delobj['name']}) | ||
1649 | 1645 | if not segment_queue.empty(): | ||
1650 | 1646 | segment_threads = [QueueFunctionThread(segment_queue, | ||
1651 | 1647 | _segment_job, create_connection()) for _junk in | ||
1652 | 1648 | xrange(10)] | ||
1653 | 1649 | for thread in segment_threads: | ||
1654 | 1650 | thread.start() | ||
1655 | 1651 | while not segment_queue.empty(): | ||
1656 | 1652 | sleep(0.01) | ||
1657 | 1653 | for thread in segment_threads: | ||
1658 | 1654 | thread.abort = True | ||
1659 | 1655 | while thread.isAlive(): | ||
1660 | 1656 | thread.join(0.01) | ||
1661 | 1657 | put_errors_from_threads(segment_threads, error_queue) | ||
1662 | 1658 | if options.verbose: | ||
1663 | 1659 | if conn.attempts > 1: | ||
1664 | 1660 | print_queue.put( | ||
1665 | 1661 | '%s [after %d attempts]' % (obj, conn.attempts)) | ||
1666 | 1662 | else: | ||
1667 | 1663 | print_queue.put(obj) | ||
1668 | 1664 | except OSError, err: | ||
1669 | 1665 | if err.errno != ENOENT: | ||
1670 | 1666 | raise | ||
1671 | 1667 | error_queue.put('Local file %s not found' % repr(path)) | ||
1672 | 1668 | |||
1673 | 1669 | def _upload_dir(path): | ||
1674 | 1670 | names = listdir(path) | ||
1675 | 1671 | if not names: | ||
1676 | 1672 | object_queue.put({'path': path, 'dir_marker': True}) | ||
1677 | 1673 | else: | ||
1678 | 1674 | for name in listdir(path): | ||
1679 | 1675 | subpath = join(path, name) | ||
1680 | 1676 | if isdir(subpath): | ||
1681 | 1677 | _upload_dir(subpath) | ||
1682 | 1678 | else: | ||
1683 | 1679 | object_queue.put({'path': subpath}) | ||
1684 | 1680 | |||
1685 | 1681 | url, token = get_auth(options.auth, options.user, options.key, | ||
1686 | 1682 | snet=options.snet) | ||
1687 | 1683 | create_connection = lambda: Connection(options.auth, options.user, | ||
1688 | 1684 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
1689 | 1685 | object_threads = [QueueFunctionThread(object_queue, _object_job, | ||
1690 | 1686 | create_connection()) for _junk in xrange(10)] | ||
1691 | 1687 | for thread in object_threads: | ||
1692 | 1688 | thread.start() | ||
1693 | 1689 | conn = create_connection() | ||
1694 | 1690 | # Try to create the container, just in case it doesn't exist. If this | ||
1695 | 1691 | # fails, it might just be because the user doesn't have container PUT | ||
1696 | 1692 | # permissions, so we'll ignore any error. If there's really a problem, | ||
1697 | 1693 | # it'll surface on the first object PUT. | ||
1698 | 1694 | try: | ||
1699 | 1695 | conn.put_container(args[0]) | ||
1700 | 1696 | if options.segment_size is not None: | ||
1701 | 1697 | conn.put_container(args[0] + '_segments') | ||
1702 | 1698 | except Exception: | ||
1703 | 1699 | pass | ||
1704 | 1700 | try: | ||
1705 | 1701 | for arg in args[1:]: | ||
1706 | 1702 | if isdir(arg): | ||
1707 | 1703 | _upload_dir(arg) | ||
1708 | 1704 | else: | ||
1709 | 1705 | object_queue.put({'path': arg}) | ||
1710 | 1706 | while not object_queue.empty(): | ||
1711 | 1707 | sleep(0.01) | ||
1712 | 1708 | for thread in object_threads: | ||
1713 | 1709 | thread.abort = True | ||
1714 | 1710 | while thread.isAlive(): | ||
1715 | 1711 | thread.join(0.01) | ||
1716 | 1712 | put_errors_from_threads(object_threads, error_queue) | ||
1717 | 1713 | except ClientException, err: | ||
1718 | 1714 | if err.http_status != 404: | ||
1719 | 1715 | raise | ||
1720 | 1716 | error_queue.put('Account not found') | ||
1721 | 1717 | |||
1722 | 1718 | |||
1723 | 1719 | def parse_args(parser, args, enforce_requires=True): | ||
1724 | 1720 | if not args: | ||
1725 | 1721 | args = ['-h'] | ||
1726 | 1722 | (options, args) = parser.parse_args(args) | ||
1727 | 1723 | if enforce_requires and \ | ||
1728 | 1724 | not (options.auth and options.user and options.key): | ||
1729 | 1725 | exit(''' | ||
1730 | 1726 | Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or | ||
1731 | 1727 | overridden with -A, -U, or -K.'''.strip('\n')) | ||
1732 | 1728 | return options, args | ||
1733 | 1729 | |||
1734 | 1730 | |||
1735 | 1731 | if __name__ == '__main__': | ||
1736 | 1732 | parser = OptionParser(version='%prog 1.0', usage=''' | ||
1737 | 1733 | Usage: %%prog <command> [options] [args] | ||
1738 | 1734 | |||
1739 | 1735 | Commands: | ||
1740 | 1736 | %(st_stat_help)s | ||
1741 | 1737 | %(st_list_help)s | ||
1742 | 1738 | %(st_upload_help)s | ||
1743 | 1739 | %(st_post_help)s | ||
1744 | 1740 | %(st_download_help)s | ||
1745 | 1741 | %(st_delete_help)s | ||
1746 | 1742 | |||
1747 | 1743 | Example: | ||
1748 | 1744 | %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K key stat | ||
1749 | 1745 | '''.strip('\n') % globals()) | ||
1750 | 1746 | parser.add_option('-s', '--snet', action='store_true', dest='snet', | ||
1751 | 1747 | default=False, help='Use SERVICENET internal network') | ||
1752 | 1748 | parser.add_option('-v', '--verbose', action='count', dest='verbose', | ||
1753 | 1749 | default=1, help='Print more info') | ||
1754 | 1750 | parser.add_option('-q', '--quiet', action='store_const', dest='verbose', | ||
1755 | 1751 | const=0, default=1, help='Suppress status output') | ||
1756 | 1752 | parser.add_option('-A', '--auth', dest='auth', | ||
1757 | 1753 | default=environ.get('ST_AUTH'), | ||
1758 | 1754 | help='URL for obtaining an auth token') | ||
1759 | 1755 | parser.add_option('-U', '--user', dest='user', | ||
1760 | 1756 | default=environ.get('ST_USER'), | ||
1761 | 1757 | help='User name for obtaining an auth token') | ||
1762 | 1758 | parser.add_option('-K', '--key', dest='key', | ||
1763 | 1759 | default=environ.get('ST_KEY'), | ||
1764 | 1760 | help='Key for obtaining an auth token') | ||
1765 | 1761 | parser.disable_interspersed_args() | ||
1766 | 1762 | (options, args) = parse_args(parser, argv[1:], enforce_requires=False) | ||
1767 | 1763 | parser.enable_interspersed_args() | ||
1768 | 1764 | |||
1769 | 1765 | commands = ('delete', 'download', 'list', 'post', 'stat', 'upload') | ||
1770 | 1766 | if not args or args[0] not in commands: | ||
1771 | 1767 | parser.print_usage() | ||
1772 | 1768 | if args: | ||
1773 | 1769 | exit('no such command: %s' % args[0]) | ||
1774 | 1770 | exit() | ||
1775 | 1771 | |||
1776 | 1772 | print_queue = Queue(10000) | ||
1777 | 1773 | |||
1778 | 1774 | def _print(item): | ||
1779 | 1775 | if isinstance(item, unicode): | ||
1780 | 1776 | item = item.encode('utf8') | ||
1781 | 1777 | print item | ||
1782 | 1778 | |||
1783 | 1779 | print_thread = QueueFunctionThread(print_queue, _print) | ||
1784 | 1780 | print_thread.start() | ||
1785 | 1781 | |||
1786 | 1782 | error_queue = Queue(10000) | ||
1787 | 1783 | |||
1788 | 1784 | def _error(item): | ||
1789 | 1785 | if isinstance(item, unicode): | ||
1790 | 1786 | item = item.encode('utf8') | ||
1791 | 1787 | print >> stderr, item | ||
1792 | 1788 | |||
1793 | 1789 | error_thread = QueueFunctionThread(error_queue, _error) | ||
1794 | 1790 | error_thread.start() | ||
1795 | 1791 | |||
1796 | 1792 | try: | ||
1797 | 1793 | parser.usage = globals()['st_%s_help' % args[0]] | ||
1798 | 1794 | try: | ||
1799 | 1795 | globals()['st_%s' % args[0]](parser, argv[1:], print_queue, | ||
1800 | 1796 | error_queue) | ||
1801 | 1797 | except (ClientException, HTTPException, socket.error), err: | ||
1802 | 1798 | error_queue.put(str(err)) | ||
1803 | 1799 | while not print_queue.empty(): | ||
1804 | 1800 | sleep(0.01) | ||
1805 | 1801 | print_thread.abort = True | ||
1806 | 1802 | while print_thread.isAlive(): | ||
1807 | 1803 | print_thread.join(0.01) | ||
1808 | 1804 | while not error_queue.empty(): | ||
1809 | 1805 | sleep(0.01) | ||
1810 | 1806 | error_thread.abort = True | ||
1811 | 1807 | while error_thread.isAlive(): | ||
1812 | 1808 | error_thread.join(0.01) | ||
1813 | 1809 | except (SystemExit, Exception): | ||
1814 | 1810 | for thread in threading_enumerate(): | ||
1815 | 1811 | thread.abort = True | ||
1816 | 1812 | raise | ||
1817 | 1813 | 0 | ||
1818 | === added file 'bin/swift' | |||
1819 | --- bin/swift 1970-01-01 00:00:00 +0000 | |||
1820 | +++ bin/swift 2011-06-14 16:07:28 +0000 | |||
1821 | @@ -0,0 +1,1812 @@ | |||
1822 | 1 | #!/usr/bin/python -u | ||
1823 | 2 | # Copyright (c) 2010-2011 OpenStack, LLC. | ||
1824 | 3 | # | ||
1825 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
1826 | 5 | # you may not use this file except in compliance with the License. | ||
1827 | 6 | # You may obtain a copy of the License at | ||
1828 | 7 | # | ||
1829 | 8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
1830 | 9 | # | ||
1831 | 10 | # Unless required by applicable law or agreed to in writing, software | ||
1832 | 11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
1833 | 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
1834 | 13 | # implied. | ||
1835 | 14 | # See the License for the specific language governing permissions and | ||
1836 | 15 | # limitations under the License. | ||
1837 | 16 | |||
1838 | 17 | from errno import EEXIST, ENOENT | ||
1839 | 18 | from hashlib import md5 | ||
1840 | 19 | from optparse import OptionParser | ||
1841 | 20 | from os import environ, listdir, makedirs, utime | ||
1842 | 21 | from os.path import basename, dirname, getmtime, getsize, isdir, join | ||
1843 | 22 | from Queue import Empty, Queue | ||
1844 | 23 | from sys import argv, exc_info, exit, stderr, stdout | ||
1845 | 24 | from threading import enumerate as threading_enumerate, Thread | ||
1846 | 25 | from time import sleep | ||
1847 | 26 | from traceback import format_exception | ||
1848 | 27 | |||
1849 | 28 | |||
1850 | 29 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | ||
1851 | 30 | # Inclusion of swift.common.client for convenience of single file distribution | ||
1852 | 31 | |||
1853 | 32 | import socket | ||
1854 | 33 | from cStringIO import StringIO | ||
1855 | 34 | from re import compile, DOTALL | ||
1856 | 35 | from tokenize import generate_tokens, STRING, NAME, OP | ||
1857 | 36 | from urllib import quote as _quote, unquote | ||
1858 | 37 | from urlparse import urlparse, urlunparse | ||
1859 | 38 | |||
1860 | 39 | try: | ||
1861 | 40 | from eventlet.green.httplib import HTTPException, HTTPSConnection | ||
1862 | 41 | except ImportError: | ||
1863 | 42 | from httplib import HTTPException, HTTPSConnection | ||
1864 | 43 | |||
1865 | 44 | try: | ||
1866 | 45 | from eventlet import sleep | ||
1867 | 46 | except ImportError: | ||
1868 | 47 | from time import sleep | ||
1869 | 48 | |||
1870 | 49 | try: | ||
1871 | 50 | from swift.common.bufferedhttp \ | ||
1872 | 51 | import BufferedHTTPConnection as HTTPConnection | ||
1873 | 52 | except ImportError: | ||
1874 | 53 | try: | ||
1875 | 54 | from eventlet.green.httplib import HTTPConnection | ||
1876 | 55 | except ImportError: | ||
1877 | 56 | from httplib import HTTPConnection | ||
1878 | 57 | |||
1879 | 58 | |||
1880 | 59 | def quote(value, safe='/'): | ||
1881 | 60 | """ | ||
1882 | 61 | Patched version of urllib.quote that encodes utf8 strings before quoting | ||
1883 | 62 | """ | ||
1884 | 63 | if isinstance(value, unicode): | ||
1885 | 64 | value = value.encode('utf8') | ||
1886 | 65 | return _quote(value, safe) | ||
1887 | 66 | |||
1888 | 67 | |||
1889 | 68 | # look for a real json parser first | ||
1890 | 69 | try: | ||
1891 | 70 | # simplejson is popular and pretty good | ||
1892 | 71 | from simplejson import loads as json_loads | ||
1893 | 72 | except ImportError: | ||
1894 | 73 | try: | ||
1895 | 74 | # 2.6 will have a json module in the stdlib | ||
1896 | 75 | from json import loads as json_loads | ||
1897 | 76 | except ImportError: | ||
1898 | 77 | # fall back on local parser otherwise | ||
1899 | 78 | comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) | ||
1900 | 79 | |||
1901 | 80 | def json_loads(string): | ||
1902 | 81 | ''' | ||
1903 | 82 | Fairly competent json parser exploiting the python tokenizer and | ||
1904 | 83 | eval(). -- From python-cloudfiles | ||
1905 | 84 | |||
1906 | 85 | _loads(serialized_json) -> object | ||
1907 | 86 | ''' | ||
1908 | 87 | try: | ||
1909 | 88 | res = [] | ||
1910 | 89 | consts = {'true': True, 'false': False, 'null': None} | ||
1911 | 90 | string = '(' + comments.sub('', string) + ')' | ||
1912 | 91 | for type, val, _junk, _junk, _junk in \ | ||
1913 | 92 | generate_tokens(StringIO(string).readline): | ||
1914 | 93 | if (type == OP and val not in '[]{}:,()-') or \ | ||
1915 | 94 | (type == NAME and val not in consts): | ||
1916 | 95 | raise AttributeError() | ||
1917 | 96 | elif type == STRING: | ||
1918 | 97 | res.append('u') | ||
1919 | 98 | res.append(val.replace('\\/', '/')) | ||
1920 | 99 | else: | ||
1921 | 100 | res.append(val) | ||
1922 | 101 | return eval(''.join(res), {}, consts) | ||
1923 | 102 | except Exception: | ||
1924 | 103 | raise AttributeError() | ||
1925 | 104 | |||
1926 | 105 | |||
1927 | 106 | class ClientException(Exception): | ||
1928 | 107 | |||
1929 | 108 | def __init__(self, msg, http_scheme='', http_host='', http_port='', | ||
1930 | 109 | http_path='', http_query='', http_status=0, http_reason='', | ||
1931 | 110 | http_device=''): | ||
1932 | 111 | Exception.__init__(self, msg) | ||
1933 | 112 | self.msg = msg | ||
1934 | 113 | self.http_scheme = http_scheme | ||
1935 | 114 | self.http_host = http_host | ||
1936 | 115 | self.http_port = http_port | ||
1937 | 116 | self.http_path = http_path | ||
1938 | 117 | self.http_query = http_query | ||
1939 | 118 | self.http_status = http_status | ||
1940 | 119 | self.http_reason = http_reason | ||
1941 | 120 | self.http_device = http_device | ||
1942 | 121 | |||
1943 | 122 | def __str__(self): | ||
1944 | 123 | a = self.msg | ||
1945 | 124 | b = '' | ||
1946 | 125 | if self.http_scheme: | ||
1947 | 126 | b += '%s://' % self.http_scheme | ||
1948 | 127 | if self.http_host: | ||
1949 | 128 | b += self.http_host | ||
1950 | 129 | if self.http_port: | ||
1951 | 130 | b += ':%s' % self.http_port | ||
1952 | 131 | if self.http_path: | ||
1953 | 132 | b += self.http_path | ||
1954 | 133 | if self.http_query: | ||
1955 | 134 | b += '?%s' % self.http_query | ||
1956 | 135 | if self.http_status: | ||
1957 | 136 | if b: | ||
1958 | 137 | b = '%s %s' % (b, self.http_status) | ||
1959 | 138 | else: | ||
1960 | 139 | b = str(self.http_status) | ||
1961 | 140 | if self.http_reason: | ||
1962 | 141 | if b: | ||
1963 | 142 | b = '%s %s' % (b, self.http_reason) | ||
1964 | 143 | else: | ||
1965 | 144 | b = '- %s' % self.http_reason | ||
1966 | 145 | if self.http_device: | ||
1967 | 146 | if b: | ||
1968 | 147 | b = '%s: device %s' % (b, self.http_device) | ||
1969 | 148 | else: | ||
1970 | 149 | b = 'device %s' % self.http_device | ||
1971 | 150 | return b and '%s: %s' % (a, b) or a | ||
1972 | 151 | |||
1973 | 152 | |||
1974 | 153 | def http_connection(url): | ||
1975 | 154 | """ | ||
1976 | 155 | Make an HTTPConnection or HTTPSConnection | ||
1977 | 156 | |||
1978 | 157 | :param url: url to connect to | ||
1979 | 158 | :returns: tuple of (parsed url, connection object) | ||
1980 | 159 | :raises ClientException: Unable to handle protocol scheme | ||
1981 | 160 | """ | ||
1982 | 161 | parsed = urlparse(url) | ||
1983 | 162 | if parsed.scheme == 'http': | ||
1984 | 163 | conn = HTTPConnection(parsed.netloc) | ||
1985 | 164 | elif parsed.scheme == 'https': | ||
1986 | 165 | conn = HTTPSConnection(parsed.netloc) | ||
1987 | 166 | else: | ||
1988 | 167 | raise ClientException('Cannot handle protocol scheme %s for url %s' % | ||
1989 | 168 | (parsed.scheme, repr(url))) | ||
1990 | 169 | return parsed, conn | ||
1991 | 170 | |||
1992 | 171 | |||
1993 | 172 | def get_auth(url, user, key, snet=False): | ||
1994 | 173 | """ | ||
1995 | 174 | Get authentication/authorization credentials. | ||
1996 | 175 | |||
1997 | 176 | The snet parameter is used for Rackspace's ServiceNet internal network | ||
1998 | 177 | implementation. In this function, it simply adds *snet-* to the beginning | ||
1999 | 178 | of the host name for the returned storage URL. With Rackspace Cloud Files, | ||
2000 | 179 | use of this network path causes no bandwidth charges but requires the | ||
2001 | 180 | client to be running on Rackspace's ServiceNet network. | ||
2002 | 181 | |||
2003 | 182 | :param url: authentication/authorization URL | ||
2004 | 183 | :param user: user to authenticate as | ||
2005 | 184 | :param key: key or password for authorization | ||
2006 | 185 | :param snet: use SERVICENET internal network (see above), default is False | ||
2007 | 186 | :returns: tuple of (storage URL, auth token) | ||
2008 | 187 | :raises ClientException: HTTP GET request to auth URL failed | ||
2009 | 188 | """ | ||
2010 | 189 | parsed, conn = http_connection(url) | ||
2011 | 190 | conn.request('GET', parsed.path, '', | ||
2012 | 191 | {'X-Auth-User': user, 'X-Auth-Key': key}) | ||
2013 | 192 | resp = conn.getresponse() | ||
2014 | 193 | resp.read() | ||
2015 | 194 | if resp.status < 200 or resp.status >= 300: | ||
2016 | 195 | raise ClientException('Auth GET failed', http_scheme=parsed.scheme, | ||
2017 | 196 | http_host=conn.host, http_port=conn.port, | ||
2018 | 197 | http_path=parsed.path, http_status=resp.status, | ||
2019 | 198 | http_reason=resp.reason) | ||
2020 | 199 | url = resp.getheader('x-storage-url') | ||
2021 | 200 | if snet: | ||
2022 | 201 | parsed = list(urlparse(url)) | ||
2023 | 202 | # Second item in the list is the netloc | ||
2024 | 203 | parsed[1] = 'snet-' + parsed[1] | ||
2025 | 204 | url = urlunparse(parsed) | ||
2026 | 205 | return url, resp.getheader('x-storage-token', | ||
2027 | 206 | resp.getheader('x-auth-token')) | ||
2028 | 207 | |||
2029 | 208 | |||
2030 | 209 | def get_account(url, token, marker=None, limit=None, prefix=None, | ||
2031 | 210 | http_conn=None, full_listing=False): | ||
2032 | 211 | """ | ||
2033 | 212 | Get a listing of containers for the account. | ||
2034 | 213 | |||
2035 | 214 | :param url: storage URL | ||
2036 | 215 | :param token: auth token | ||
2037 | 216 | :param marker: marker query | ||
2038 | 217 | :param limit: limit query | ||
2039 | 218 | :param prefix: prefix query | ||
2040 | 219 | :param http_conn: HTTP connection object (If None, it will create the | ||
2041 | 220 | conn object) | ||
2042 | 221 | :param full_listing: if True, return a full listing, else returns a max | ||
2043 | 222 | of 10000 listings | ||
2044 | 223 | :returns: a tuple of (response headers, a list of containers) The response | ||
2045 | 224 | headers will be a dict and all header names will be lowercase. | ||
2046 | 225 | :raises ClientException: HTTP GET request failed | ||
2047 | 226 | """ | ||
2048 | 227 | if not http_conn: | ||
2049 | 228 | http_conn = http_connection(url) | ||
2050 | 229 | if full_listing: | ||
2051 | 230 | rv = get_account(url, token, marker, limit, prefix, http_conn) | ||
2052 | 231 | listing = rv[1] | ||
2053 | 232 | while listing: | ||
2054 | 233 | marker = listing[-1]['name'] | ||
2055 | 234 | listing = \ | ||
2056 | 235 | get_account(url, token, marker, limit, prefix, http_conn)[1] | ||
2057 | 236 | if listing: | ||
2058 | 237 | rv[1].extend(listing) | ||
2059 | 238 | return rv | ||
2060 | 239 | parsed, conn = http_conn | ||
2061 | 240 | qs = 'format=json' | ||
2062 | 241 | if marker: | ||
2063 | 242 | qs += '&marker=%s' % quote(marker) | ||
2064 | 243 | if limit: | ||
2065 | 244 | qs += '&limit=%d' % limit | ||
2066 | 245 | if prefix: | ||
2067 | 246 | qs += '&prefix=%s' % quote(prefix) | ||
2068 | 247 | conn.request('GET', '%s?%s' % (parsed.path, qs), '', | ||
2069 | 248 | {'X-Auth-Token': token}) | ||
2070 | 249 | resp = conn.getresponse() | ||
2071 | 250 | resp_headers = {} | ||
2072 | 251 | for header, value in resp.getheaders(): | ||
2073 | 252 | resp_headers[header.lower()] = value | ||
2074 | 253 | if resp.status < 200 or resp.status >= 300: | ||
2075 | 254 | resp.read() | ||
2076 | 255 | raise ClientException('Account GET failed', http_scheme=parsed.scheme, | ||
2077 | 256 | http_host=conn.host, http_port=conn.port, | ||
2078 | 257 | http_path=parsed.path, http_query=qs, http_status=resp.status, | ||
2079 | 258 | http_reason=resp.reason) | ||
2080 | 259 | if resp.status == 204: | ||
2081 | 260 | resp.read() | ||
2082 | 261 | return resp_headers, [] | ||
2083 | 262 | return resp_headers, json_loads(resp.read()) | ||
2084 | 263 | |||
2085 | 264 | |||
2086 | 265 | def head_account(url, token, http_conn=None): | ||
2087 | 266 | """ | ||
2088 | 267 | Get account stats. | ||
2089 | 268 | |||
2090 | 269 | :param url: storage URL | ||
2091 | 270 | :param token: auth token | ||
2092 | 271 | :param http_conn: HTTP connection object (If None, it will create the | ||
2093 | 272 | conn object) | ||
2094 | 273 | :returns: a dict containing the response's headers (all header names will | ||
2095 | 274 | be lowercase) | ||
2096 | 275 | :raises ClientException: HTTP HEAD request failed | ||
2097 | 276 | """ | ||
2098 | 277 | if http_conn: | ||
2099 | 278 | parsed, conn = http_conn | ||
2100 | 279 | else: | ||
2101 | 280 | parsed, conn = http_connection(url) | ||
2102 | 281 | conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) | ||
2103 | 282 | resp = conn.getresponse() | ||
2104 | 283 | resp.read() | ||
2105 | 284 | if resp.status < 200 or resp.status >= 300: | ||
2106 | 285 | raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, | ||
2107 | 286 | http_host=conn.host, http_port=conn.port, | ||
2108 | 287 | http_path=parsed.path, http_status=resp.status, | ||
2109 | 288 | http_reason=resp.reason) | ||
2110 | 289 | resp_headers = {} | ||
2111 | 290 | for header, value in resp.getheaders(): | ||
2112 | 291 | resp_headers[header.lower()] = value | ||
2113 | 292 | return resp_headers | ||
2114 | 293 | |||
2115 | 294 | |||
2116 | 295 | def post_account(url, token, headers, http_conn=None): | ||
2117 | 296 | """ | ||
2118 | 297 | Update an account's metadata. | ||
2119 | 298 | |||
2120 | 299 | :param url: storage URL | ||
2121 | 300 | :param token: auth token | ||
2122 | 301 | :param headers: additional headers to include in the request | ||
2123 | 302 | :param http_conn: HTTP connection object (If None, it will create the | ||
2124 | 303 | conn object) | ||
2125 | 304 | :raises ClientException: HTTP POST request failed | ||
2126 | 305 | """ | ||
2127 | 306 | if http_conn: | ||
2128 | 307 | parsed, conn = http_conn | ||
2129 | 308 | else: | ||
2130 | 309 | parsed, conn = http_connection(url) | ||
2131 | 310 | headers['X-Auth-Token'] = token | ||
2132 | 311 | conn.request('POST', parsed.path, '', headers) | ||
2133 | 312 | resp = conn.getresponse() | ||
2134 | 313 | resp.read() | ||
2135 | 314 | if resp.status < 200 or resp.status >= 300: | ||
2136 | 315 | raise ClientException('Account POST failed', | ||
2137 | 316 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2138 | 317 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2139 | 318 | http_reason=resp.reason) | ||
2140 | 319 | |||
2141 | 320 | |||
2142 | 321 | def get_container(url, token, container, marker=None, limit=None, | ||
2143 | 322 | prefix=None, delimiter=None, http_conn=None, | ||
2144 | 323 | full_listing=False): | ||
2145 | 324 | """ | ||
2146 | 325 | Get a listing of objects for the container. | ||
2147 | 326 | |||
2148 | 327 | :param url: storage URL | ||
2149 | 328 | :param token: auth token | ||
2150 | 329 | :param container: container name to get a listing for | ||
2151 | 330 | :param marker: marker query | ||
2152 | 331 | :param limit: limit query | ||
2153 | 332 | :param prefix: prefix query | ||
2154 | 333 | :param delimeter: string to delimit the queries on | ||
2155 | 334 | :param http_conn: HTTP connection object (If None, it will create the | ||
2156 | 335 | conn object) | ||
2157 | 336 | :param full_listing: if True, return a full listing, else returns a max | ||
2158 | 337 | of 10000 listings | ||
2159 | 338 | :returns: a tuple of (response headers, a list of objects) The response | ||
2160 | 339 | headers will be a dict and all header names will be lowercase. | ||
2161 | 340 | :raises ClientException: HTTP GET request failed | ||
2162 | 341 | """ | ||
2163 | 342 | if not http_conn: | ||
2164 | 343 | http_conn = http_connection(url) | ||
2165 | 344 | if full_listing: | ||
2166 | 345 | rv = get_container(url, token, container, marker, limit, prefix, | ||
2167 | 346 | delimiter, http_conn) | ||
2168 | 347 | listing = rv[1] | ||
2169 | 348 | while listing: | ||
2170 | 349 | if not delimiter: | ||
2171 | 350 | marker = listing[-1]['name'] | ||
2172 | 351 | else: | ||
2173 | 352 | marker = listing[-1].get('name', listing[-1].get('subdir')) | ||
2174 | 353 | listing = get_container(url, token, container, marker, limit, | ||
2175 | 354 | prefix, delimiter, http_conn)[1] | ||
2176 | 355 | if listing: | ||
2177 | 356 | rv[1].extend(listing) | ||
2178 | 357 | return rv | ||
2179 | 358 | parsed, conn = http_conn | ||
2180 | 359 | path = '%s/%s' % (parsed.path, quote(container)) | ||
2181 | 360 | qs = 'format=json' | ||
2182 | 361 | if marker: | ||
2183 | 362 | qs += '&marker=%s' % quote(marker) | ||
2184 | 363 | if limit: | ||
2185 | 364 | qs += '&limit=%d' % limit | ||
2186 | 365 | if prefix: | ||
2187 | 366 | qs += '&prefix=%s' % quote(prefix) | ||
2188 | 367 | if delimiter: | ||
2189 | 368 | qs += '&delimiter=%s' % quote(delimiter) | ||
2190 | 369 | conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) | ||
2191 | 370 | resp = conn.getresponse() | ||
2192 | 371 | if resp.status < 200 or resp.status >= 300: | ||
2193 | 372 | resp.read() | ||
2194 | 373 | raise ClientException('Container GET failed', | ||
2195 | 374 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2196 | 375 | http_port=conn.port, http_path=path, http_query=qs, | ||
2197 | 376 | http_status=resp.status, http_reason=resp.reason) | ||
2198 | 377 | resp_headers = {} | ||
2199 | 378 | for header, value in resp.getheaders(): | ||
2200 | 379 | resp_headers[header.lower()] = value | ||
2201 | 380 | if resp.status == 204: | ||
2202 | 381 | resp.read() | ||
2203 | 382 | return resp_headers, [] | ||
2204 | 383 | return resp_headers, json_loads(resp.read()) | ||
2205 | 384 | |||
2206 | 385 | |||
2207 | 386 | def head_container(url, token, container, http_conn=None): | ||
2208 | 387 | """ | ||
2209 | 388 | Get container stats. | ||
2210 | 389 | |||
2211 | 390 | :param url: storage URL | ||
2212 | 391 | :param token: auth token | ||
2213 | 392 | :param container: container name to get stats for | ||
2214 | 393 | :param http_conn: HTTP connection object (If None, it will create the | ||
2215 | 394 | conn object) | ||
2216 | 395 | :returns: a dict containing the response's headers (all header names will | ||
2217 | 396 | be lowercase) | ||
2218 | 397 | :raises ClientException: HTTP HEAD request failed | ||
2219 | 398 | """ | ||
2220 | 399 | if http_conn: | ||
2221 | 400 | parsed, conn = http_conn | ||
2222 | 401 | else: | ||
2223 | 402 | parsed, conn = http_connection(url) | ||
2224 | 403 | path = '%s/%s' % (parsed.path, quote(container)) | ||
2225 | 404 | conn.request('HEAD', path, '', {'X-Auth-Token': token}) | ||
2226 | 405 | resp = conn.getresponse() | ||
2227 | 406 | resp.read() | ||
2228 | 407 | if resp.status < 200 or resp.status >= 300: | ||
2229 | 408 | raise ClientException('Container HEAD failed', | ||
2230 | 409 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2231 | 410 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2232 | 411 | http_reason=resp.reason) | ||
2233 | 412 | resp_headers = {} | ||
2234 | 413 | for header, value in resp.getheaders(): | ||
2235 | 414 | resp_headers[header.lower()] = value | ||
2236 | 415 | return resp_headers | ||
2237 | 416 | |||
2238 | 417 | |||
2239 | 418 | def put_container(url, token, container, headers=None, http_conn=None): | ||
2240 | 419 | """ | ||
2241 | 420 | Create a container | ||
2242 | 421 | |||
2243 | 422 | :param url: storage URL | ||
2244 | 423 | :param token: auth token | ||
2245 | 424 | :param container: container name to create | ||
2246 | 425 | :param headers: additional headers to include in the request | ||
2247 | 426 | :param http_conn: HTTP connection object (If None, it will create the | ||
2248 | 427 | conn object) | ||
2249 | 428 | :raises ClientException: HTTP PUT request failed | ||
2250 | 429 | """ | ||
2251 | 430 | if http_conn: | ||
2252 | 431 | parsed, conn = http_conn | ||
2253 | 432 | else: | ||
2254 | 433 | parsed, conn = http_connection(url) | ||
2255 | 434 | path = '%s/%s' % (parsed.path, quote(container)) | ||
2256 | 435 | if not headers: | ||
2257 | 436 | headers = {} | ||
2258 | 437 | headers['X-Auth-Token'] = token | ||
2259 | 438 | conn.request('PUT', path, '', headers) | ||
2260 | 439 | resp = conn.getresponse() | ||
2261 | 440 | resp.read() | ||
2262 | 441 | if resp.status < 200 or resp.status >= 300: | ||
2263 | 442 | raise ClientException('Container PUT failed', | ||
2264 | 443 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2265 | 444 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2266 | 445 | http_reason=resp.reason) | ||
2267 | 446 | |||
2268 | 447 | |||
2269 | 448 | def post_container(url, token, container, headers, http_conn=None): | ||
2270 | 449 | """ | ||
2271 | 450 | Update a container's metadata. | ||
2272 | 451 | |||
2273 | 452 | :param url: storage URL | ||
2274 | 453 | :param token: auth token | ||
2275 | 454 | :param container: container name to update | ||
2276 | 455 | :param headers: additional headers to include in the request | ||
2277 | 456 | :param http_conn: HTTP connection object (If None, it will create the | ||
2278 | 457 | conn object) | ||
2279 | 458 | :raises ClientException: HTTP POST request failed | ||
2280 | 459 | """ | ||
2281 | 460 | if http_conn: | ||
2282 | 461 | parsed, conn = http_conn | ||
2283 | 462 | else: | ||
2284 | 463 | parsed, conn = http_connection(url) | ||
2285 | 464 | path = '%s/%s' % (parsed.path, quote(container)) | ||
2286 | 465 | headers['X-Auth-Token'] = token | ||
2287 | 466 | conn.request('POST', path, '', headers) | ||
2288 | 467 | resp = conn.getresponse() | ||
2289 | 468 | resp.read() | ||
2290 | 469 | if resp.status < 200 or resp.status >= 300: | ||
2291 | 470 | raise ClientException('Container POST failed', | ||
2292 | 471 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2293 | 472 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2294 | 473 | http_reason=resp.reason) | ||
2295 | 474 | |||
2296 | 475 | |||
2297 | 476 | def delete_container(url, token, container, http_conn=None): | ||
2298 | 477 | """ | ||
2299 | 478 | Delete a container | ||
2300 | 479 | |||
2301 | 480 | :param url: storage URL | ||
2302 | 481 | :param token: auth token | ||
2303 | 482 | :param container: container name to delete | ||
2304 | 483 | :param http_conn: HTTP connection object (If None, it will create the | ||
2305 | 484 | conn object) | ||
2306 | 485 | :raises ClientException: HTTP DELETE request failed | ||
2307 | 486 | """ | ||
2308 | 487 | if http_conn: | ||
2309 | 488 | parsed, conn = http_conn | ||
2310 | 489 | else: | ||
2311 | 490 | parsed, conn = http_connection(url) | ||
2312 | 491 | path = '%s/%s' % (parsed.path, quote(container)) | ||
2313 | 492 | conn.request('DELETE', path, '', {'X-Auth-Token': token}) | ||
2314 | 493 | resp = conn.getresponse() | ||
2315 | 494 | resp.read() | ||
2316 | 495 | if resp.status < 200 or resp.status >= 300: | ||
2317 | 496 | raise ClientException('Container DELETE failed', | ||
2318 | 497 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2319 | 498 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2320 | 499 | http_reason=resp.reason) | ||
2321 | 500 | |||
2322 | 501 | |||
2323 | 502 | def get_object(url, token, container, name, http_conn=None, | ||
2324 | 503 | resp_chunk_size=None): | ||
2325 | 504 | """ | ||
2326 | 505 | Get an object | ||
2327 | 506 | |||
2328 | 507 | :param url: storage URL | ||
2329 | 508 | :param token: auth token | ||
2330 | 509 | :param container: container name that the object is in | ||
2331 | 510 | :param name: object name to get | ||
2332 | 511 | :param http_conn: HTTP connection object (If None, it will create the | ||
2333 | 512 | conn object) | ||
2334 | 513 | :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If | ||
2335 | 514 | you specify a resp_chunk_size you must fully read | ||
2336 | 515 | the object's contents before making another | ||
2337 | 516 | request. | ||
2338 | 517 | :returns: a tuple of (response headers, the object's contents) The response | ||
2339 | 518 | headers will be a dict and all header names will be lowercase. | ||
2340 | 519 | :raises ClientException: HTTP GET request failed | ||
2341 | 520 | """ | ||
2342 | 521 | if http_conn: | ||
2343 | 522 | parsed, conn = http_conn | ||
2344 | 523 | else: | ||
2345 | 524 | parsed, conn = http_connection(url) | ||
2346 | 525 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
2347 | 526 | conn.request('GET', path, '', {'X-Auth-Token': token}) | ||
2348 | 527 | resp = conn.getresponse() | ||
2349 | 528 | if resp.status < 200 or resp.status >= 300: | ||
2350 | 529 | resp.read() | ||
2351 | 530 | raise ClientException('Object GET failed', http_scheme=parsed.scheme, | ||
2352 | 531 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
2353 | 532 | http_status=resp.status, http_reason=resp.reason) | ||
2354 | 533 | if resp_chunk_size: | ||
2355 | 534 | |||
2356 | 535 | def _object_body(): | ||
2357 | 536 | buf = resp.read(resp_chunk_size) | ||
2358 | 537 | while buf: | ||
2359 | 538 | yield buf | ||
2360 | 539 | buf = resp.read(resp_chunk_size) | ||
2361 | 540 | object_body = _object_body() | ||
2362 | 541 | else: | ||
2363 | 542 | object_body = resp.read() | ||
2364 | 543 | resp_headers = {} | ||
2365 | 544 | for header, value in resp.getheaders(): | ||
2366 | 545 | resp_headers[header.lower()] = value | ||
2367 | 546 | return resp_headers, object_body | ||
2368 | 547 | |||
2369 | 548 | |||
2370 | 549 | def head_object(url, token, container, name, http_conn=None): | ||
2371 | 550 | """ | ||
2372 | 551 | Get object info | ||
2373 | 552 | |||
2374 | 553 | :param url: storage URL | ||
2375 | 554 | :param token: auth token | ||
2376 | 555 | :param container: container name that the object is in | ||
2377 | 556 | :param name: object name to get info for | ||
2378 | 557 | :param http_conn: HTTP connection object (If None, it will create the | ||
2379 | 558 | conn object) | ||
2380 | 559 | :returns: a dict containing the response's headers (all header names will | ||
2381 | 560 | be lowercase) | ||
2382 | 561 | :raises ClientException: HTTP HEAD request failed | ||
2383 | 562 | """ | ||
2384 | 563 | if http_conn: | ||
2385 | 564 | parsed, conn = http_conn | ||
2386 | 565 | else: | ||
2387 | 566 | parsed, conn = http_connection(url) | ||
2388 | 567 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
2389 | 568 | conn.request('HEAD', path, '', {'X-Auth-Token': token}) | ||
2390 | 569 | resp = conn.getresponse() | ||
2391 | 570 | resp.read() | ||
2392 | 571 | if resp.status < 200 or resp.status >= 300: | ||
2393 | 572 | raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, | ||
2394 | 573 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
2395 | 574 | http_status=resp.status, http_reason=resp.reason) | ||
2396 | 575 | resp_headers = {} | ||
2397 | 576 | for header, value in resp.getheaders(): | ||
2398 | 577 | resp_headers[header.lower()] = value | ||
2399 | 578 | return resp_headers | ||
2400 | 579 | |||
2401 | 580 | |||
2402 | 581 | def put_object(url, token, container, name, contents, content_length=None, | ||
2403 | 582 | etag=None, chunk_size=65536, content_type=None, headers=None, | ||
2404 | 583 | http_conn=None): | ||
2405 | 584 | """ | ||
2406 | 585 | Put an object | ||
2407 | 586 | |||
2408 | 587 | :param url: storage URL | ||
2409 | 588 | :param token: auth token | ||
2410 | 589 | :param container: container name that the object is in | ||
2411 | 590 | :param name: object name to put | ||
2412 | 591 | :param contents: a string or a file like object to read object data from | ||
2413 | 592 | :param content_length: value to send as content-length header; also limits | ||
2414 | 593 | the amount read from contents | ||
2415 | 594 | :param etag: etag of contents | ||
2416 | 595 | :param chunk_size: chunk size of data to write | ||
2417 | 596 | :param content_type: value to send as content-type header | ||
2418 | 597 | :param headers: additional headers to include in the request | ||
2419 | 598 | :param http_conn: HTTP connection object (If None, it will create the | ||
2420 | 599 | conn object) | ||
2421 | 600 | :returns: etag from server response | ||
2422 | 601 | :raises ClientException: HTTP PUT request failed | ||
2423 | 602 | """ | ||
2424 | 603 | if http_conn: | ||
2425 | 604 | parsed, conn = http_conn | ||
2426 | 605 | else: | ||
2427 | 606 | parsed, conn = http_connection(url) | ||
2428 | 607 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
2429 | 608 | if not headers: | ||
2430 | 609 | headers = {} | ||
2431 | 610 | headers['X-Auth-Token'] = token | ||
2432 | 611 | if etag: | ||
2433 | 612 | headers['ETag'] = etag.strip('"') | ||
2434 | 613 | if content_length is not None: | ||
2435 | 614 | headers['Content-Length'] = str(content_length) | ||
2436 | 615 | if content_type is not None: | ||
2437 | 616 | headers['Content-Type'] = content_type | ||
2438 | 617 | if not contents: | ||
2439 | 618 | headers['Content-Length'] = '0' | ||
2440 | 619 | if hasattr(contents, 'read'): | ||
2441 | 620 | conn.putrequest('PUT', path) | ||
2442 | 621 | for header, value in headers.iteritems(): | ||
2443 | 622 | conn.putheader(header, value) | ||
2444 | 623 | if content_length is None: | ||
2445 | 624 | conn.putheader('Transfer-Encoding', 'chunked') | ||
2446 | 625 | conn.endheaders() | ||
2447 | 626 | chunk = contents.read(chunk_size) | ||
2448 | 627 | while chunk: | ||
2449 | 628 | conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) | ||
2450 | 629 | chunk = contents.read(chunk_size) | ||
2451 | 630 | conn.send('0\r\n\r\n') | ||
2452 | 631 | else: | ||
2453 | 632 | conn.endheaders() | ||
2454 | 633 | left = content_length | ||
2455 | 634 | while left > 0: | ||
2456 | 635 | size = chunk_size | ||
2457 | 636 | if size > left: | ||
2458 | 637 | size = left | ||
2459 | 638 | chunk = contents.read(size) | ||
2460 | 639 | conn.send(chunk) | ||
2461 | 640 | left -= len(chunk) | ||
2462 | 641 | else: | ||
2463 | 642 | conn.request('PUT', path, contents, headers) | ||
2464 | 643 | resp = conn.getresponse() | ||
2465 | 644 | resp.read() | ||
2466 | 645 | if resp.status < 200 or resp.status >= 300: | ||
2467 | 646 | raise ClientException('Object PUT failed', http_scheme=parsed.scheme, | ||
2468 | 647 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
2469 | 648 | http_status=resp.status, http_reason=resp.reason) | ||
2470 | 649 | return resp.getheader('etag').strip('"') | ||
2471 | 650 | |||
2472 | 651 | |||
2473 | 652 | def post_object(url, token, container, name, headers, http_conn=None): | ||
2474 | 653 | """ | ||
2475 | 654 | Update object metadata | ||
2476 | 655 | |||
2477 | 656 | :param url: storage URL | ||
2478 | 657 | :param token: auth token | ||
2479 | 658 | :param container: container name that the object is in | ||
2480 | 659 | :param name: name of the object to update | ||
2481 | 660 | :param headers: additional headers to include in the request | ||
2482 | 661 | :param http_conn: HTTP connection object (If None, it will create the | ||
2483 | 662 | conn object) | ||
2484 | 663 | :raises ClientException: HTTP POST request failed | ||
2485 | 664 | """ | ||
2486 | 665 | if http_conn: | ||
2487 | 666 | parsed, conn = http_conn | ||
2488 | 667 | else: | ||
2489 | 668 | parsed, conn = http_connection(url) | ||
2490 | 669 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
2491 | 670 | headers['X-Auth-Token'] = token | ||
2492 | 671 | conn.request('POST', path, '', headers) | ||
2493 | 672 | resp = conn.getresponse() | ||
2494 | 673 | resp.read() | ||
2495 | 674 | if resp.status < 200 or resp.status >= 300: | ||
2496 | 675 | raise ClientException('Object POST failed', http_scheme=parsed.scheme, | ||
2497 | 676 | http_host=conn.host, http_port=conn.port, http_path=path, | ||
2498 | 677 | http_status=resp.status, http_reason=resp.reason) | ||
2499 | 678 | |||
2500 | 679 | |||
2501 | 680 | def delete_object(url, token, container, name, http_conn=None): | ||
2502 | 681 | """ | ||
2503 | 682 | Delete object | ||
2504 | 683 | |||
2505 | 684 | :param url: storage URL | ||
2506 | 685 | :param token: auth token | ||
2507 | 686 | :param container: container name that the object is in | ||
2508 | 687 | :param name: object name to delete | ||
2509 | 688 | :param http_conn: HTTP connection object (If None, it will create the | ||
2510 | 689 | conn object) | ||
2511 | 690 | :raises ClientException: HTTP DELETE request failed | ||
2512 | 691 | """ | ||
2513 | 692 | if http_conn: | ||
2514 | 693 | parsed, conn = http_conn | ||
2515 | 694 | else: | ||
2516 | 695 | parsed, conn = http_connection(url) | ||
2517 | 696 | path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) | ||
2518 | 697 | conn.request('DELETE', path, '', {'X-Auth-Token': token}) | ||
2519 | 698 | resp = conn.getresponse() | ||
2520 | 699 | resp.read() | ||
2521 | 700 | if resp.status < 200 or resp.status >= 300: | ||
2522 | 701 | raise ClientException('Object DELETE failed', | ||
2523 | 702 | http_scheme=parsed.scheme, http_host=conn.host, | ||
2524 | 703 | http_port=conn.port, http_path=path, http_status=resp.status, | ||
2525 | 704 | http_reason=resp.reason) | ||
2526 | 705 | |||
2527 | 706 | |||
2528 | 707 | class Connection(object): | ||
2529 | 708 | """Convenience class to make requests that will also retry the request""" | ||
2530 | 709 | |||
2531 | 710 | def __init__(self, authurl, user, key, retries=5, preauthurl=None, | ||
2532 | 711 | preauthtoken=None, snet=False, starting_backoff=1): | ||
2533 | 712 | """ | ||
2534 | 713 | :param authurl: authenitcation URL | ||
2535 | 714 | :param user: user name to authenticate as | ||
2536 | 715 | :param key: key/password to authenticate with | ||
2537 | 716 | :param retries: Number of times to retry the request before failing | ||
2538 | 717 | :param preauthurl: storage URL (if you have already authenticated) | ||
2539 | 718 | :param preauthtoken: authentication token (if you have already | ||
2540 | 719 | authenticated) | ||
2541 | 720 | :param snet: use SERVICENET internal network default is False | ||
2542 | 721 | """ | ||
2543 | 722 | self.authurl = authurl | ||
2544 | 723 | self.user = user | ||
2545 | 724 | self.key = key | ||
2546 | 725 | self.retries = retries | ||
2547 | 726 | self.http_conn = None | ||
2548 | 727 | self.url = preauthurl | ||
2549 | 728 | self.token = preauthtoken | ||
2550 | 729 | self.attempts = 0 | ||
2551 | 730 | self.snet = snet | ||
2552 | 731 | self.starting_backoff = starting_backoff | ||
2553 | 732 | |||
2554 | 733 | def get_auth(self): | ||
2555 | 734 | return get_auth(self.authurl, self.user, self.key, snet=self.snet) | ||
2556 | 735 | |||
2557 | 736 | def http_connection(self): | ||
2558 | 737 | return http_connection(self.url) | ||
2559 | 738 | |||
2560 | 739 | def _retry(self, reset_func, func, *args, **kwargs): | ||
2561 | 740 | self.attempts = 0 | ||
2562 | 741 | backoff = self.starting_backoff | ||
2563 | 742 | while self.attempts <= self.retries: | ||
2564 | 743 | self.attempts += 1 | ||
2565 | 744 | try: | ||
2566 | 745 | if not self.url or not self.token: | ||
2567 | 746 | self.url, self.token = self.get_auth() | ||
2568 | 747 | self.http_conn = None | ||
2569 | 748 | if not self.http_conn: | ||
2570 | 749 | self.http_conn = self.http_connection() | ||
2571 | 750 | kwargs['http_conn'] = self.http_conn | ||
2572 | 751 | rv = func(self.url, self.token, *args, **kwargs) | ||
2573 | 752 | return rv | ||
2574 | 753 | except (socket.error, HTTPException): | ||
2575 | 754 | if self.attempts > self.retries: | ||
2576 | 755 | raise | ||
2577 | 756 | self.http_conn = None | ||
2578 | 757 | except ClientException, err: | ||
2579 | 758 | if self.attempts > self.retries: | ||
2580 | 759 | raise | ||
2581 | 760 | if err.http_status == 401: | ||
2582 | 761 | self.url = self.token = None | ||
2583 | 762 | if self.attempts > 1: | ||
2584 | 763 | raise | ||
2585 | 764 | elif err.http_status == 408: | ||
2586 | 765 | self.http_conn = None | ||
2587 | 766 | elif 500 <= err.http_status <= 599: | ||
2588 | 767 | pass | ||
2589 | 768 | else: | ||
2590 | 769 | raise | ||
2591 | 770 | sleep(backoff) | ||
2592 | 771 | backoff *= 2 | ||
2593 | 772 | if reset_func: | ||
2594 | 773 | reset_func(func, *args, **kwargs) | ||
2595 | 774 | |||
2596 | 775 | def head_account(self): | ||
2597 | 776 | """Wrapper for :func:`head_account`""" | ||
2598 | 777 | return self._retry(None, head_account) | ||
2599 | 778 | |||
2600 | 779 | def get_account(self, marker=None, limit=None, prefix=None, | ||
2601 | 780 | full_listing=False): | ||
2602 | 781 | """Wrapper for :func:`get_account`""" | ||
2603 | 782 | # TODO(unknown): With full_listing=True this will restart the entire | ||
2604 | 783 | # listing with each retry. Need to make a better version that just | ||
2605 | 784 | # retries where it left off. | ||
2606 | 785 | return self._retry(None, get_account, marker=marker, limit=limit, | ||
2607 | 786 | prefix=prefix, full_listing=full_listing) | ||
2608 | 787 | |||
2609 | 788 | def post_account(self, headers): | ||
2610 | 789 | """Wrapper for :func:`post_account`""" | ||
2611 | 790 | return self._retry(None, post_account, headers) | ||
2612 | 791 | |||
2613 | 792 | def head_container(self, container): | ||
2614 | 793 | """Wrapper for :func:`head_container`""" | ||
2615 | 794 | return self._retry(None, head_container, container) | ||
2616 | 795 | |||
2617 | 796 | def get_container(self, container, marker=None, limit=None, prefix=None, | ||
2618 | 797 | delimiter=None, full_listing=False): | ||
2619 | 798 | """Wrapper for :func:`get_container`""" | ||
2620 | 799 | # TODO(unknown): With full_listing=True this will restart the entire | ||
2621 | 800 | # listing with each retry. Need to make a better version that just | ||
2622 | 801 | # retries where it left off. | ||
2623 | 802 | return self._retry(None, get_container, container, marker=marker, | ||
2624 | 803 | limit=limit, prefix=prefix, delimiter=delimiter, | ||
2625 | 804 | full_listing=full_listing) | ||
2626 | 805 | |||
2627 | 806 | def put_container(self, container, headers=None): | ||
2628 | 807 | """Wrapper for :func:`put_container`""" | ||
2629 | 808 | return self._retry(None, put_container, container, headers=headers) | ||
2630 | 809 | |||
2631 | 810 | def post_container(self, container, headers): | ||
2632 | 811 | """Wrapper for :func:`post_container`""" | ||
2633 | 812 | return self._retry(None, post_container, container, headers) | ||
2634 | 813 | |||
2635 | 814 | def delete_container(self, container): | ||
2636 | 815 | """Wrapper for :func:`delete_container`""" | ||
2637 | 816 | return self._retry(None, delete_container, container) | ||
2638 | 817 | |||
2639 | 818 | def head_object(self, container, obj): | ||
2640 | 819 | """Wrapper for :func:`head_object`""" | ||
2641 | 820 | return self._retry(None, head_object, container, obj) | ||
2642 | 821 | |||
2643 | 822 | def get_object(self, container, obj, resp_chunk_size=None): | ||
2644 | 823 | """Wrapper for :func:`get_object`""" | ||
2645 | 824 | return self._retry(None, get_object, container, obj, | ||
2646 | 825 | resp_chunk_size=resp_chunk_size) | ||
2647 | 826 | |||
2648 | 827 | def put_object(self, container, obj, contents, content_length=None, | ||
2649 | 828 | etag=None, chunk_size=65536, content_type=None, | ||
2650 | 829 | headers=None): | ||
2651 | 830 | """Wrapper for :func:`put_object`""" | ||
2652 | 831 | |||
2653 | 832 | def _default_reset(*args, **kwargs): | ||
2654 | 833 | raise ClientException('put_object(%r, %r, ...) failure and no ' | ||
2655 | 834 | 'ability to reset contents for reupload.' % (container, obj)) | ||
2656 | 835 | |||
2657 | 836 | reset_func = _default_reset | ||
2658 | 837 | tell = getattr(contents, 'tell', None) | ||
2659 | 838 | seek = getattr(contents, 'seek', None) | ||
2660 | 839 | if tell and seek: | ||
2661 | 840 | orig_pos = tell() | ||
2662 | 841 | reset_func = lambda *a, **k: seek(orig_pos) | ||
2663 | 842 | elif not contents: | ||
2664 | 843 | reset_func = lambda *a, **k: None | ||
2665 | 844 | |||
2666 | 845 | return self._retry(reset_func, put_object, container, obj, contents, | ||
2667 | 846 | content_length=content_length, etag=etag, chunk_size=chunk_size, | ||
2668 | 847 | content_type=content_type, headers=headers) | ||
2669 | 848 | |||
2670 | 849 | def post_object(self, container, obj, headers): | ||
2671 | 850 | """Wrapper for :func:`post_object`""" | ||
2672 | 851 | return self._retry(None, post_object, container, obj, headers) | ||
2673 | 852 | |||
2674 | 853 | def delete_object(self, container, obj): | ||
2675 | 854 | """Wrapper for :func:`delete_object`""" | ||
2676 | 855 | return self._retry(None, delete_object, container, obj) | ||
2677 | 856 | |||
2678 | 857 | # End inclusion of swift.common.client | ||
2679 | 858 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | ||
2680 | 859 | |||
2681 | 860 | |||
2682 | 861 | def mkdirs(path): | ||
2683 | 862 | try: | ||
2684 | 863 | makedirs(path) | ||
2685 | 864 | except OSError, err: | ||
2686 | 865 | if err.errno != EEXIST: | ||
2687 | 866 | raise | ||
2688 | 867 | |||
2689 | 868 | |||
2690 | 869 | def put_errors_from_threads(threads, error_queue): | ||
2691 | 870 | """ | ||
2692 | 871 | Places any errors from the threads into error_queue. | ||
2693 | 872 | :param threads: A list of QueueFunctionThread instances. | ||
2694 | 873 | :param error_queue: A queue to put error strings into. | ||
2695 | 874 | :returns: True if any errors were found. | ||
2696 | 875 | """ | ||
2697 | 876 | was_error = False | ||
2698 | 877 | for thread in threads: | ||
2699 | 878 | for info in thread.exc_infos: | ||
2700 | 879 | was_error = True | ||
2701 | 880 | if isinstance(info[1], ClientException): | ||
2702 | 881 | error_queue.put(str(info[1])) | ||
2703 | 882 | else: | ||
2704 | 883 | error_queue.put(''.join(format_exception(*info))) | ||
2705 | 884 | return was_error | ||
2706 | 885 | |||
2707 | 886 | |||
2708 | 887 | class QueueFunctionThread(Thread): | ||
2709 | 888 | |||
2710 | 889 | def __init__(self, queue, func, *args, **kwargs): | ||
2711 | 890 | """ Calls func for each item in queue; func is called with a queued | ||
2712 | 891 | item as the first arg followed by *args and **kwargs. Use the abort | ||
2713 | 892 | attribute to have the thread empty the queue (without processing) | ||
2714 | 893 | and exit. """ | ||
2715 | 894 | Thread.__init__(self) | ||
2716 | 895 | self.abort = False | ||
2717 | 896 | self.queue = queue | ||
2718 | 897 | self.func = func | ||
2719 | 898 | self.args = args | ||
2720 | 899 | self.kwargs = kwargs | ||
2721 | 900 | self.exc_infos = [] | ||
2722 | 901 | |||
2723 | 902 | def run(self): | ||
2724 | 903 | try: | ||
2725 | 904 | while True: | ||
2726 | 905 | try: | ||
2727 | 906 | item = self.queue.get_nowait() | ||
2728 | 907 | if not self.abort: | ||
2729 | 908 | self.func(item, *self.args, **self.kwargs) | ||
2730 | 909 | self.queue.task_done() | ||
2731 | 910 | except Empty: | ||
2732 | 911 | if self.abort: | ||
2733 | 912 | break | ||
2734 | 913 | sleep(0.01) | ||
2735 | 914 | except Exception: | ||
2736 | 915 | self.exc_infos.append(exc_info()) | ||
2737 | 916 | |||
2738 | 917 | |||
2739 | 918 | st_delete_help = ''' | ||
2740 | 919 | delete --all OR delete container [--leave-segments] [object] [object] ... | ||
2741 | 920 | Deletes everything in the account (with --all), or everything in a | ||
2742 | 921 | container, or a list of objects depending on the args given. Segments of | ||
2743 | 922 | manifest objects will be deleted as well, unless you specify the | ||
2744 | 923 | --leave-segments option.'''.strip('\n') | ||
2745 | 924 | |||
2746 | 925 | |||
2747 | 926 | def st_delete(parser, args, print_queue, error_queue): | ||
2748 | 927 | parser.add_option('-a', '--all', action='store_true', dest='yes_all', | ||
2749 | 928 | default=False, help='Indicates that you really want to delete ' | ||
2750 | 929 | 'everything in the account') | ||
2751 | 930 | parser.add_option('', '--leave-segments', action='store_true', | ||
2752 | 931 | dest='leave_segments', default=False, help='Indicates that you want ' | ||
2753 | 932 | 'the segments of manifest objects left alone') | ||
2754 | 933 | (options, args) = parse_args(parser, args) | ||
2755 | 934 | args = args[1:] | ||
2756 | 935 | if (not args and not options.yes_all) or (args and options.yes_all): | ||
2757 | 936 | error_queue.put('Usage: %s [options] %s' % | ||
2758 | 937 | (basename(argv[0]), st_delete_help)) | ||
2759 | 938 | return | ||
2760 | 939 | |||
2761 | 940 | def _delete_segment((container, obj), conn): | ||
2762 | 941 | conn.delete_object(container, obj) | ||
2763 | 942 | if options.verbose: | ||
2764 | 943 | if conn.attempts > 2: | ||
2765 | 944 | print_queue.put('%s/%s [after %d attempts]' % | ||
2766 | 945 | (container, obj, conn.attempts)) | ||
2767 | 946 | else: | ||
2768 | 947 | print_queue.put('%s/%s' % (container, obj)) | ||
2769 | 948 | |||
2770 | 949 | object_queue = Queue(10000) | ||
2771 | 950 | |||
2772 | 951 | def _delete_object((container, obj), conn): | ||
2773 | 952 | try: | ||
2774 | 953 | old_manifest = None | ||
2775 | 954 | if not options.leave_segments: | ||
2776 | 955 | try: | ||
2777 | 956 | old_manifest = conn.head_object(container, obj).get( | ||
2778 | 957 | 'x-object-manifest') | ||
2779 | 958 | except ClientException, err: | ||
2780 | 959 | if err.http_status != 404: | ||
2781 | 960 | raise | ||
2782 | 961 | conn.delete_object(container, obj) | ||
2783 | 962 | if old_manifest: | ||
2784 | 963 | segment_queue = Queue(10000) | ||
2785 | 964 | scontainer, sprefix = old_manifest.split('/', 1) | ||
2786 | 965 | for delobj in conn.get_container(scontainer, | ||
2787 | 966 | prefix=sprefix)[1]: | ||
2788 | 967 | segment_queue.put((scontainer, delobj['name'])) | ||
2789 | 968 | if not segment_queue.empty(): | ||
2790 | 969 | segment_threads = [QueueFunctionThread(segment_queue, | ||
2791 | 970 | _delete_segment, create_connection()) for _junk in | ||
2792 | 971 | xrange(10)] | ||
2793 | 972 | for thread in segment_threads: | ||
2794 | 973 | thread.start() | ||
2795 | 974 | while not segment_queue.empty(): | ||
2796 | 975 | sleep(0.01) | ||
2797 | 976 | for thread in segment_threads: | ||
2798 | 977 | thread.abort = True | ||
2799 | 978 | while thread.isAlive(): | ||
2800 | 979 | thread.join(0.01) | ||
2801 | 980 | put_errors_from_threads(segment_threads, error_queue) | ||
2802 | 981 | if options.verbose: | ||
2803 | 982 | path = options.yes_all and join(container, obj) or obj | ||
2804 | 983 | if path[:1] in ('/', '\\'): | ||
2805 | 984 | path = path[1:] | ||
2806 | 985 | if conn.attempts > 1: | ||
2807 | 986 | print_queue.put('%s [after %d attempts]' % | ||
2808 | 987 | (path, conn.attempts)) | ||
2809 | 988 | else: | ||
2810 | 989 | print_queue.put(path) | ||
2811 | 990 | except ClientException, err: | ||
2812 | 991 | if err.http_status != 404: | ||
2813 | 992 | raise | ||
2814 | 993 | error_queue.put('Object %s not found' % | ||
2815 | 994 | repr('%s/%s' % (container, obj))) | ||
2816 | 995 | |||
2817 | 996 | container_queue = Queue(10000) | ||
2818 | 997 | |||
2819 | 998 | def _delete_container(container, conn): | ||
2820 | 999 | try: | ||
2821 | 1000 | marker = '' | ||
2822 | 1001 | while True: | ||
2823 | 1002 | objects = [o['name'] for o in | ||
2824 | 1003 | conn.get_container(container, marker=marker)[1]] | ||
2825 | 1004 | if not objects: | ||
2826 | 1005 | break | ||
2827 | 1006 | for obj in objects: | ||
2828 | 1007 | object_queue.put((container, obj)) | ||
2829 | 1008 | marker = objects[-1] | ||
2830 | 1009 | while not object_queue.empty(): | ||
2831 | 1010 | sleep(0.01) | ||
2832 | 1011 | attempts = 1 | ||
2833 | 1012 | while True: | ||
2834 | 1013 | try: | ||
2835 | 1014 | conn.delete_container(container) | ||
2836 | 1015 | break | ||
2837 | 1016 | except ClientException, err: | ||
2838 | 1017 | if err.http_status != 409: | ||
2839 | 1018 | raise | ||
2840 | 1019 | if attempts > 10: | ||
2841 | 1020 | raise | ||
2842 | 1021 | attempts += 1 | ||
2843 | 1022 | sleep(1) | ||
2844 | 1023 | except ClientException, err: | ||
2845 | 1024 | if err.http_status != 404: | ||
2846 | 1025 | raise | ||
2847 | 1026 | error_queue.put('Container %s not found' % repr(container)) | ||
2848 | 1027 | |||
2849 | 1028 | url, token = get_auth(options.auth, options.user, options.key, | ||
2850 | 1029 | snet=options.snet) | ||
2851 | 1030 | create_connection = lambda: Connection(options.auth, options.user, | ||
2852 | 1031 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
2853 | 1032 | object_threads = [QueueFunctionThread(object_queue, _delete_object, | ||
2854 | 1033 | create_connection()) for _junk in xrange(10)] | ||
2855 | 1034 | for thread in object_threads: | ||
2856 | 1035 | thread.start() | ||
2857 | 1036 | container_threads = [QueueFunctionThread(container_queue, | ||
2858 | 1037 | _delete_container, create_connection()) for _junk in xrange(10)] | ||
2859 | 1038 | for thread in container_threads: | ||
2860 | 1039 | thread.start() | ||
2861 | 1040 | if not args: | ||
2862 | 1041 | conn = create_connection() | ||
2863 | 1042 | try: | ||
2864 | 1043 | marker = '' | ||
2865 | 1044 | while True: | ||
2866 | 1045 | containers = \ | ||
2867 | 1046 | [c['name'] for c in conn.get_account(marker=marker)[1]] | ||
2868 | 1047 | if not containers: | ||
2869 | 1048 | break | ||
2870 | 1049 | for container in containers: | ||
2871 | 1050 | container_queue.put(container) | ||
2872 | 1051 | marker = containers[-1] | ||
2873 | 1052 | while not container_queue.empty(): | ||
2874 | 1053 | sleep(0.01) | ||
2875 | 1054 | while not object_queue.empty(): | ||
2876 | 1055 | sleep(0.01) | ||
2877 | 1056 | except ClientException, err: | ||
2878 | 1057 | if err.http_status != 404: | ||
2879 | 1058 | raise | ||
2880 | 1059 | error_queue.put('Account not found') | ||
2881 | 1060 | elif len(args) == 1: | ||
2882 | 1061 | if '/' in args[0]: | ||
2883 | 1062 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
2884 | 1063 | 'meant %r instead of %r.' % \ | ||
2885 | 1064 | (args[0].replace('/', ' ', 1), args[0]) | ||
2886 | 1065 | conn = create_connection() | ||
2887 | 1066 | _delete_container(args[0], conn) | ||
2888 | 1067 | else: | ||
2889 | 1068 | for obj in args[1:]: | ||
2890 | 1069 | object_queue.put((args[0], obj)) | ||
2891 | 1070 | while not container_queue.empty(): | ||
2892 | 1071 | sleep(0.01) | ||
2893 | 1072 | for thread in container_threads: | ||
2894 | 1073 | thread.abort = True | ||
2895 | 1074 | while thread.isAlive(): | ||
2896 | 1075 | thread.join(0.01) | ||
2897 | 1076 | put_errors_from_threads(container_threads, error_queue) | ||
2898 | 1077 | while not object_queue.empty(): | ||
2899 | 1078 | sleep(0.01) | ||
2900 | 1079 | for thread in object_threads: | ||
2901 | 1080 | thread.abort = True | ||
2902 | 1081 | while thread.isAlive(): | ||
2903 | 1082 | thread.join(0.01) | ||
2904 | 1083 | put_errors_from_threads(object_threads, error_queue) | ||
2905 | 1084 | |||
2906 | 1085 | |||
2907 | 1086 | st_download_help = ''' | ||
2908 | 1087 | download --all OR download container [options] [object] [object] ... | ||
2909 | 1088 | Downloads everything in the account (with --all), or everything in a | ||
2910 | 1089 | container, or a list of objects depending on the args given. For a single | ||
2911 | 1090 | object download, you may use the -o [--output] <filename> option to | ||
2912 | 1091 | redirect the output to a specific file or if "-" then just redirect to | ||
2913 | 1092 | stdout.'''.strip('\n') | ||
2914 | 1093 | |||
2915 | 1094 | |||
2916 | 1095 | def st_download(options, args, print_queue, error_queue): | ||
2917 | 1096 | parser.add_option('-a', '--all', action='store_true', dest='yes_all', | ||
2918 | 1097 | default=False, help='Indicates that you really want to download ' | ||
2919 | 1098 | 'everything in the account') | ||
2920 | 1099 | parser.add_option('-o', '--output', dest='out_file', help='For a single ' | ||
2921 | 1100 | 'file download, stream the output to an alternate location ') | ||
2922 | 1101 | (options, args) = parse_args(parser, args) | ||
2923 | 1102 | args = args[1:] | ||
2924 | 1103 | if options.out_file == '-': | ||
2925 | 1104 | options.verbose = 0 | ||
2926 | 1105 | if options.out_file and len(args) != 2: | ||
2927 | 1106 | exit('-o option only allowed for single file downloads') | ||
2928 | 1107 | if (not args and not options.yes_all) or (args and options.yes_all): | ||
2929 | 1108 | error_queue.put('Usage: %s [options] %s' % | ||
2930 | 1109 | (basename(argv[0]), st_download_help)) | ||
2931 | 1110 | return | ||
2932 | 1111 | |||
2933 | 1112 | object_queue = Queue(10000) | ||
2934 | 1113 | |||
2935 | 1114 | def _download_object(queue_arg, conn): | ||
2936 | 1115 | if len(queue_arg) == 2: | ||
2937 | 1116 | container, obj = queue_arg | ||
2938 | 1117 | out_file = None | ||
2939 | 1118 | elif len(queue_arg) == 3: | ||
2940 | 1119 | container, obj, out_file = queue_arg | ||
2941 | 1120 | else: | ||
2942 | 1121 | raise Exception("Invalid queue_arg length of %s" % len(queue_arg)) | ||
2943 | 1122 | try: | ||
2944 | 1123 | headers, body = \ | ||
2945 | 1124 | conn.get_object(container, obj, resp_chunk_size=65536) | ||
2946 | 1125 | content_type = headers.get('content-type') | ||
2947 | 1126 | if 'content-length' in headers: | ||
2948 | 1127 | content_length = int(headers.get('content-length')) | ||
2949 | 1128 | else: | ||
2950 | 1129 | content_length = None | ||
2951 | 1130 | etag = headers.get('etag') | ||
2952 | 1131 | path = options.yes_all and join(container, obj) or obj | ||
2953 | 1132 | if path[:1] in ('/', '\\'): | ||
2954 | 1133 | path = path[1:] | ||
2955 | 1134 | md5sum = None | ||
2956 | 1135 | make_dir = out_file != "-" | ||
2957 | 1136 | if content_type.split(';', 1)[0] == 'text/directory': | ||
2958 | 1137 | if make_dir and not isdir(path): | ||
2959 | 1138 | mkdirs(path) | ||
2960 | 1139 | read_length = 0 | ||
2961 | 1140 | if 'x-object-manifest' not in headers: | ||
2962 | 1141 | md5sum = md5() | ||
2963 | 1142 | for chunk in body: | ||
2964 | 1143 | read_length += len(chunk) | ||
2965 | 1144 | if md5sum: | ||
2966 | 1145 | md5sum.update(chunk) | ||
2967 | 1146 | else: | ||
2968 | 1147 | dirpath = dirname(path) | ||
2969 | 1148 | if make_dir and dirpath and not isdir(dirpath): | ||
2970 | 1149 | mkdirs(dirpath) | ||
2971 | 1150 | if out_file == "-": | ||
2972 | 1151 | fp = stdout | ||
2973 | 1152 | elif out_file: | ||
2974 | 1153 | fp = open(out_file, 'wb') | ||
2975 | 1154 | else: | ||
2976 | 1155 | fp = open(path, 'wb') | ||
2977 | 1156 | read_length = 0 | ||
2978 | 1157 | if 'x-object-manifest' not in headers: | ||
2979 | 1158 | md5sum = md5() | ||
2980 | 1159 | for chunk in body: | ||
2981 | 1160 | fp.write(chunk) | ||
2982 | 1161 | read_length += len(chunk) | ||
2983 | 1162 | if md5sum: | ||
2984 | 1163 | md5sum.update(chunk) | ||
2985 | 1164 | fp.close() | ||
2986 | 1165 | if md5sum and md5sum.hexdigest() != etag: | ||
2987 | 1166 | error_queue.put('%s: md5sum != etag, %s != %s' % | ||
2988 | 1167 | (path, md5sum.hexdigest(), etag)) | ||
2989 | 1168 | if content_length is not None and read_length != content_length: | ||
2990 | 1169 | error_queue.put('%s: read_length != content_length, %d != %d' % | ||
2991 | 1170 | (path, read_length, content_length)) | ||
2992 | 1171 | if 'x-object-meta-mtime' in headers and not options.out_file: | ||
2993 | 1172 | mtime = float(headers['x-object-meta-mtime']) | ||
2994 | 1173 | utime(path, (mtime, mtime)) | ||
2995 | 1174 | if options.verbose: | ||
2996 | 1175 | if conn.attempts > 1: | ||
2997 | 1176 | print_queue.put('%s [after %d attempts' % | ||
2998 | 1177 | (path, conn.attempts)) | ||
2999 | 1178 | else: | ||
3000 | 1179 | print_queue.put(path) | ||
3001 | 1180 | except ClientException, err: | ||
3002 | 1181 | if err.http_status != 404: | ||
3003 | 1182 | raise | ||
3004 | 1183 | error_queue.put('Object %s not found' % | ||
3005 | 1184 | repr('%s/%s' % (container, obj))) | ||
3006 | 1185 | |||
3007 | 1186 | container_queue = Queue(10000) | ||
3008 | 1187 | |||
3009 | 1188 | def _download_container(container, conn): | ||
3010 | 1189 | try: | ||
3011 | 1190 | marker = '' | ||
3012 | 1191 | while True: | ||
3013 | 1192 | objects = [o['name'] for o in | ||
3014 | 1193 | conn.get_container(container, marker=marker)[1]] | ||
3015 | 1194 | if not objects: | ||
3016 | 1195 | break | ||
3017 | 1196 | for obj in objects: | ||
3018 | 1197 | object_queue.put((container, obj)) | ||
3019 | 1198 | marker = objects[-1] | ||
3020 | 1199 | except ClientException, err: | ||
3021 | 1200 | if err.http_status != 404: | ||
3022 | 1201 | raise | ||
3023 | 1202 | error_queue.put('Container %s not found' % repr(container)) | ||
3024 | 1203 | |||
3025 | 1204 | url, token = get_auth(options.auth, options.user, options.key, | ||
3026 | 1205 | snet=options.snet) | ||
3027 | 1206 | create_connection = lambda: Connection(options.auth, options.user, | ||
3028 | 1207 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
3029 | 1208 | object_threads = [QueueFunctionThread(object_queue, _download_object, | ||
3030 | 1209 | create_connection()) for _junk in xrange(10)] | ||
3031 | 1210 | for thread in object_threads: | ||
3032 | 1211 | thread.start() | ||
3033 | 1212 | container_threads = [QueueFunctionThread(container_queue, | ||
3034 | 1213 | _download_container, create_connection()) for _junk in xrange(10)] | ||
3035 | 1214 | for thread in container_threads: | ||
3036 | 1215 | thread.start() | ||
3037 | 1216 | if not args: | ||
3038 | 1217 | conn = create_connection() | ||
3039 | 1218 | try: | ||
3040 | 1219 | marker = '' | ||
3041 | 1220 | while True: | ||
3042 | 1221 | containers = [c['name'] | ||
3043 | 1222 | for c in conn.get_account(marker=marker)[1]] | ||
3044 | 1223 | if not containers: | ||
3045 | 1224 | break | ||
3046 | 1225 | for container in containers: | ||
3047 | 1226 | container_queue.put(container) | ||
3048 | 1227 | marker = containers[-1] | ||
3049 | 1228 | except ClientException, err: | ||
3050 | 1229 | if err.http_status != 404: | ||
3051 | 1230 | raise | ||
3052 | 1231 | error_queue.put('Account not found') | ||
3053 | 1232 | elif len(args) == 1: | ||
3054 | 1233 | if '/' in args[0]: | ||
3055 | 1234 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
3056 | 1235 | 'meant %r instead of %r.' % \ | ||
3057 | 1236 | (args[0].replace('/', ' ', 1), args[0]) | ||
3058 | 1237 | _download_container(args[0], create_connection()) | ||
3059 | 1238 | else: | ||
3060 | 1239 | if len(args) == 2: | ||
3061 | 1240 | obj = args[1] | ||
3062 | 1241 | object_queue.put((args[0], obj, options.out_file)) | ||
3063 | 1242 | else: | ||
3064 | 1243 | for obj in args[1:]: | ||
3065 | 1244 | object_queue.put((args[0], obj)) | ||
3066 | 1245 | while not container_queue.empty(): | ||
3067 | 1246 | sleep(0.01) | ||
3068 | 1247 | for thread in container_threads: | ||
3069 | 1248 | thread.abort = True | ||
3070 | 1249 | while thread.isAlive(): | ||
3071 | 1250 | thread.join(0.01) | ||
3072 | 1251 | put_errors_from_threads(container_threads, error_queue) | ||
3073 | 1252 | while not object_queue.empty(): | ||
3074 | 1253 | sleep(0.01) | ||
3075 | 1254 | for thread in object_threads: | ||
3076 | 1255 | thread.abort = True | ||
3077 | 1256 | while thread.isAlive(): | ||
3078 | 1257 | thread.join(0.01) | ||
3079 | 1258 | put_errors_from_threads(object_threads, error_queue) | ||
3080 | 1259 | |||
3081 | 1260 | |||
3082 | 1261 | st_list_help = ''' | ||
3083 | 1262 | list [options] [container] | ||
3084 | 1263 | Lists the containers for the account or the objects for a container. -p or | ||
3085 | 1264 | --prefix is an option that will only list items beginning with that prefix. | ||
3086 | 1265 | -d or --delimiter is option (for container listings only) that will roll up | ||
3087 | 1266 | items with the given delimiter (see Cloud Files general documentation for | ||
3088 | 1267 | what this means). | ||
3089 | 1268 | '''.strip('\n') | ||
3090 | 1269 | |||
3091 | 1270 | |||
3092 | 1271 | def st_list(options, args, print_queue, error_queue): | ||
3093 | 1272 | parser.add_option('-p', '--prefix', dest='prefix', help='Will only list ' | ||
3094 | 1273 | 'items beginning with the prefix') | ||
3095 | 1274 | parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll ' | ||
3096 | 1275 | 'up items with the given delimiter (see Cloud Files general ' | ||
3097 | 1276 | 'documentation for what this means)') | ||
3098 | 1277 | (options, args) = parse_args(parser, args) | ||
3099 | 1278 | args = args[1:] | ||
3100 | 1279 | if options.delimiter and not args: | ||
3101 | 1280 | exit('-d option only allowed for container listings') | ||
3102 | 1281 | if len(args) > 1: | ||
3103 | 1282 | error_queue.put('Usage: %s [options] %s' % | ||
3104 | 1283 | (basename(argv[0]), st_list_help)) | ||
3105 | 1284 | return | ||
3106 | 1285 | conn = Connection(options.auth, options.user, options.key, | ||
3107 | 1286 | snet=options.snet) | ||
3108 | 1287 | try: | ||
3109 | 1288 | marker = '' | ||
3110 | 1289 | while True: | ||
3111 | 1290 | if not args: | ||
3112 | 1291 | items = \ | ||
3113 | 1292 | conn.get_account(marker=marker, prefix=options.prefix)[1] | ||
3114 | 1293 | else: | ||
3115 | 1294 | items = conn.get_container(args[0], marker=marker, | ||
3116 | 1295 | prefix=options.prefix, delimiter=options.delimiter)[1] | ||
3117 | 1296 | if not items: | ||
3118 | 1297 | break | ||
3119 | 1298 | for item in items: | ||
3120 | 1299 | print_queue.put(item.get('name', item.get('subdir'))) | ||
3121 | 1300 | marker = items[-1].get('name', items[-1].get('subdir')) | ||
3122 | 1301 | except ClientException, err: | ||
3123 | 1302 | if err.http_status != 404: | ||
3124 | 1303 | raise | ||
3125 | 1304 | if not args: | ||
3126 | 1305 | error_queue.put('Account not found') | ||
3127 | 1306 | else: | ||
3128 | 1307 | error_queue.put('Container %s not found' % repr(args[0])) | ||
3129 | 1308 | |||
3130 | 1309 | |||
3131 | 1310 | st_stat_help = ''' | ||
3132 | 1311 | stat [container] [object] | ||
3133 | 1312 | Displays information for the account, container, or object depending on the | ||
3134 | 1313 | args given (if any).'''.strip('\n') | ||
3135 | 1314 | |||
3136 | 1315 | |||
3137 | 1316 | def st_stat(options, args, print_queue, error_queue): | ||
3138 | 1317 | (options, args) = parse_args(parser, args) | ||
3139 | 1318 | args = args[1:] | ||
3140 | 1319 | conn = Connection(options.auth, options.user, options.key) | ||
3141 | 1320 | if not args: | ||
3142 | 1321 | try: | ||
3143 | 1322 | headers = conn.head_account() | ||
3144 | 1323 | if options.verbose > 1: | ||
3145 | 1324 | print_queue.put(''' | ||
3146 | 1325 | StorageURL: %s | ||
3147 | 1326 | Auth Token: %s | ||
3148 | 1327 | '''.strip('\n') % (conn.url, conn.token)) | ||
3149 | 1328 | container_count = int(headers.get('x-account-container-count', 0)) | ||
3150 | 1329 | object_count = int(headers.get('x-account-object-count', 0)) | ||
3151 | 1330 | bytes_used = int(headers.get('x-account-bytes-used', 0)) | ||
3152 | 1331 | print_queue.put(''' | ||
3153 | 1332 | Account: %s | ||
3154 | 1333 | Containers: %d | ||
3155 | 1334 | Objects: %d | ||
3156 | 1335 | Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count, | ||
3157 | 1336 | object_count, bytes_used)) | ||
3158 | 1337 | for key, value in headers.items(): | ||
3159 | 1338 | if key.startswith('x-account-meta-'): | ||
3160 | 1339 | print_queue.put('%10s: %s' % ('Meta %s' % | ||
3161 | 1340 | key[len('x-account-meta-'):].title(), value)) | ||
3162 | 1341 | for key, value in headers.items(): | ||
3163 | 1342 | if not key.startswith('x-account-meta-') and key not in ( | ||
3164 | 1343 | 'content-length', 'date', 'x-account-container-count', | ||
3165 | 1344 | 'x-account-object-count', 'x-account-bytes-used'): | ||
3166 | 1345 | print_queue.put( | ||
3167 | 1346 | '%10s: %s' % (key.title(), value)) | ||
3168 | 1347 | except ClientException, err: | ||
3169 | 1348 | if err.http_status != 404: | ||
3170 | 1349 | raise | ||
3171 | 1350 | error_queue.put('Account not found') | ||
3172 | 1351 | elif len(args) == 1: | ||
3173 | 1352 | if '/' in args[0]: | ||
3174 | 1353 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
3175 | 1354 | 'meant %r instead of %r.' % \ | ||
3176 | 1355 | (args[0].replace('/', ' ', 1), args[0]) | ||
3177 | 1356 | try: | ||
3178 | 1357 | headers = conn.head_container(args[0]) | ||
3179 | 1358 | object_count = int(headers.get('x-container-object-count', 0)) | ||
3180 | 1359 | bytes_used = int(headers.get('x-container-bytes-used', 0)) | ||
3181 | 1360 | print_queue.put(''' | ||
3182 | 1361 | Account: %s | ||
3183 | 1362 | Container: %s | ||
3184 | 1363 | Objects: %d | ||
3185 | 1364 | Bytes: %d | ||
3186 | 1365 | Read ACL: %s | ||
3187 | 1366 | Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], | ||
3188 | 1367 | object_count, bytes_used, | ||
3189 | 1368 | headers.get('x-container-read', ''), | ||
3190 | 1369 | headers.get('x-container-write', ''))) | ||
3191 | 1370 | for key, value in headers.items(): | ||
3192 | 1371 | if key.startswith('x-container-meta-'): | ||
3193 | 1372 | print_queue.put('%9s: %s' % ('Meta %s' % | ||
3194 | 1373 | key[len('x-container-meta-'):].title(), value)) | ||
3195 | 1374 | for key, value in headers.items(): | ||
3196 | 1375 | if not key.startswith('x-container-meta-') and key not in ( | ||
3197 | 1376 | 'content-length', 'date', 'x-container-object-count', | ||
3198 | 1377 | 'x-container-bytes-used', 'x-container-read', | ||
3199 | 1378 | 'x-container-write'): | ||
3200 | 1379 | print_queue.put( | ||
3201 | 1380 | '%9s: %s' % (key.title(), value)) | ||
3202 | 1381 | except ClientException, err: | ||
3203 | 1382 | if err.http_status != 404: | ||
3204 | 1383 | raise | ||
3205 | 1384 | error_queue.put('Container %s not found' % repr(args[0])) | ||
3206 | 1385 | elif len(args) == 2: | ||
3207 | 1386 | try: | ||
3208 | 1387 | headers = conn.head_object(args[0], args[1]) | ||
3209 | 1388 | print_queue.put(''' | ||
3210 | 1389 | Account: %s | ||
3211 | 1390 | Container: %s | ||
3212 | 1391 | Object: %s | ||
3213 | 1392 | Content Type: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], | ||
3214 | 1393 | args[1], headers.get('content-type'))) | ||
3215 | 1394 | if 'content-length' in headers: | ||
3216 | 1395 | print_queue.put('Content Length: %s' % | ||
3217 | 1396 | headers['content-length']) | ||
3218 | 1397 | if 'last-modified' in headers: | ||
3219 | 1398 | print_queue.put(' Last Modified: %s' % | ||
3220 | 1399 | headers['last-modified']) | ||
3221 | 1400 | if 'etag' in headers: | ||
3222 | 1401 | print_queue.put(' ETag: %s' % headers['etag']) | ||
3223 | 1402 | if 'x-object-manifest' in headers: | ||
3224 | 1403 | print_queue.put(' Manifest: %s' % | ||
3225 | 1404 | headers['x-object-manifest']) | ||
3226 | 1405 | for key, value in headers.items(): | ||
3227 | 1406 | if key.startswith('x-object-meta-'): | ||
3228 | 1407 | print_queue.put('%14s: %s' % ('Meta %s' % | ||
3229 | 1408 | key[len('x-object-meta-'):].title(), value)) | ||
3230 | 1409 | for key, value in headers.items(): | ||
3231 | 1410 | if not key.startswith('x-object-meta-') and key not in ( | ||
3232 | 1411 | 'content-type', 'content-length', 'last-modified', | ||
3233 | 1412 | 'etag', 'date', 'x-object-manifest'): | ||
3234 | 1413 | print_queue.put( | ||
3235 | 1414 | '%14s: %s' % (key.title(), value)) | ||
3236 | 1415 | except ClientException, err: | ||
3237 | 1416 | if err.http_status != 404: | ||
3238 | 1417 | raise | ||
3239 | 1418 | error_queue.put('Object %s not found' % | ||
3240 | 1419 | repr('%s/%s' % (args[0], args[1]))) | ||
3241 | 1420 | else: | ||
3242 | 1421 | error_queue.put('Usage: %s [options] %s' % | ||
3243 | 1422 | (basename(argv[0]), st_stat_help)) | ||
3244 | 1423 | |||
3245 | 1424 | |||
3246 | 1425 | st_post_help = ''' | ||
3247 | 1426 | post [options] [container] [object] | ||
3248 | 1427 | Updates meta information for the account, container, or object depending on | ||
3249 | 1428 | the args given. If the container is not found, it will be created | ||
3250 | 1429 | automatically; but this is not true for accounts and objects. Containers | ||
3251 | 1430 | also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m | ||
3252 | 1431 | or --meta option is allowed on all and used to define the user meta data | ||
3253 | 1432 | items to set in the form Name:Value. This option can be repeated. Example: | ||
3254 | 1433 | post -m Color:Blue -m Size:Large'''.strip('\n') | ||
3255 | 1434 | |||
3256 | 1435 | |||
3257 | 1436 | def st_post(options, args, print_queue, error_queue): | ||
3258 | 1437 | parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the ' | ||
3259 | 1438 | 'Read ACL for containers. Quick summary of ACL syntax: .r:*, ' | ||
3260 | 1439 | '.r:-.example.com, .r:www.example.com, account1, account2:user2') | ||
3261 | 1440 | parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the ' | ||
3262 | 1441 | 'Write ACL for containers. Quick summary of ACL syntax: account1, ' | ||
3263 | 1442 | 'account2:user2') | ||
3264 | 1443 | parser.add_option('-m', '--meta', action='append', dest='meta', default=[], | ||
3265 | 1444 | help='Sets a meta data item with the syntax name:value. This option ' | ||
3266 | 1445 | 'may be repeated. Example: -m Color:Blue -m Size:Large') | ||
3267 | 1446 | (options, args) = parse_args(parser, args) | ||
3268 | 1447 | args = args[1:] | ||
3269 | 1448 | if (options.read_acl or options.write_acl) and not args: | ||
3270 | 1449 | exit('-r and -w options only allowed for containers') | ||
3271 | 1450 | conn = Connection(options.auth, options.user, options.key) | ||
3272 | 1451 | if not args: | ||
3273 | 1452 | headers = {} | ||
3274 | 1453 | for item in options.meta: | ||
3275 | 1454 | split_item = item.split(':') | ||
3276 | 1455 | headers['X-Account-Meta-' + split_item[0]] = \ | ||
3277 | 1456 | len(split_item) > 1 and split_item[1] | ||
3278 | 1457 | try: | ||
3279 | 1458 | conn.post_account(headers=headers) | ||
3280 | 1459 | except ClientException, err: | ||
3281 | 1460 | if err.http_status != 404: | ||
3282 | 1461 | raise | ||
3283 | 1462 | error_queue.put('Account not found') | ||
3284 | 1463 | elif len(args) == 1: | ||
3285 | 1464 | if '/' in args[0]: | ||
3286 | 1465 | print >> stderr, 'WARNING: / in container name; you might have ' \ | ||
3287 | 1466 | 'meant %r instead of %r.' % \ | ||
3288 | 1467 | (args[0].replace('/', ' ', 1), args[0]) | ||
3289 | 1468 | headers = {} | ||
3290 | 1469 | for item in options.meta: | ||
3291 | 1470 | split_item = item.split(':') | ||
3292 | 1471 | headers['X-Container-Meta-' + split_item[0]] = \ | ||
3293 | 1472 | len(split_item) > 1 and split_item[1] | ||
3294 | 1473 | if options.read_acl is not None: | ||
3295 | 1474 | headers['X-Container-Read'] = options.read_acl | ||
3296 | 1475 | if options.write_acl is not None: | ||
3297 | 1476 | headers['X-Container-Write'] = options.write_acl | ||
3298 | 1477 | try: | ||
3299 | 1478 | conn.post_container(args[0], headers=headers) | ||
3300 | 1479 | except ClientException, err: | ||
3301 | 1480 | if err.http_status != 404: | ||
3302 | 1481 | raise | ||
3303 | 1482 | conn.put_container(args[0], headers=headers) | ||
3304 | 1483 | elif len(args) == 2: | ||
3305 | 1484 | headers = {} | ||
3306 | 1485 | for item in options.meta: | ||
3307 | 1486 | split_item = item.split(':') | ||
3308 | 1487 | headers['X-Object-Meta-' + split_item[0]] = \ | ||
3309 | 1488 | len(split_item) > 1 and split_item[1] | ||
3310 | 1489 | try: | ||
3311 | 1490 | conn.post_object(args[0], args[1], headers=headers) | ||
3312 | 1491 | except ClientException, err: | ||
3313 | 1492 | if err.http_status != 404: | ||
3314 | 1493 | raise | ||
3315 | 1494 | error_queue.put('Object %s not found' % | ||
3316 | 1495 | repr('%s/%s' % (args[0], args[1]))) | ||
3317 | 1496 | else: | ||
3318 | 1497 | error_queue.put('Usage: %s [options] %s' % | ||
3319 | 1498 | (basename(argv[0]), st_post_help)) | ||
3320 | 1499 | |||
3321 | 1500 | |||
3322 | 1501 | st_upload_help = ''' | ||
3323 | 1502 | upload [options] container file_or_directory [file_or_directory] [...] | ||
3324 | 1503 | Uploads to the given container the files and directories specified by the | ||
3325 | 1504 | remaining args. -c or --changed is an option that will only upload files | ||
3326 | 1505 | that have changed since the last upload. -S <size> or --segment-size <size> | ||
3327 | 1506 | and --leave-segments are options as well (see --help for more). | ||
3328 | 1507 | '''.strip('\n') | ||
3329 | 1508 | |||
3330 | 1509 | |||
3331 | 1510 | def st_upload(options, args, print_queue, error_queue): | ||
3332 | 1511 | parser.add_option('-c', '--changed', action='store_true', dest='changed', | ||
3333 | 1512 | default=False, help='Will only upload files that have changed since ' | ||
3334 | 1513 | 'the last upload') | ||
3335 | 1514 | parser.add_option('-S', '--segment-size', dest='segment_size', help='Will ' | ||
3336 | 1515 | 'upload files in segments no larger than <size> and then create a ' | ||
3337 | 1516 | '"manifest" file that will download all the segments as if it were ' | ||
3338 | 1517 | 'the original file. The segments will be uploaded to a ' | ||
3339 | 1518 | '<container>_segments container so as to not pollute the main ' | ||
3340 | 1519 | '<container> listings.') | ||
3341 | 1520 | parser.add_option('', '--leave-segments', action='store_true', | ||
3342 | 1521 | dest='leave_segments', default=False, help='Indicates that you want ' | ||
3343 | 1522 | 'the older segments of manifest objects left alone (in the case of ' | ||
3344 | 1523 | 'overwrites)') | ||
3345 | 1524 | (options, args) = parse_args(parser, args) | ||
3346 | 1525 | args = args[1:] | ||
3347 | 1526 | if len(args) < 2: | ||
3348 | 1527 | error_queue.put('Usage: %s [options] %s' % | ||
3349 | 1528 | (basename(argv[0]), st_upload_help)) | ||
3350 | 1529 | return | ||
3351 | 1530 | object_queue = Queue(10000) | ||
3352 | 1531 | |||
3353 | 1532 | def _segment_job(job, conn): | ||
3354 | 1533 | if job.get('delete', False): | ||
3355 | 1534 | conn.delete_object(job['container'], job['obj']) | ||
3356 | 1535 | else: | ||
3357 | 1536 | fp = open(job['path'], 'rb') | ||
3358 | 1537 | fp.seek(job['segment_start']) | ||
3359 | 1538 | conn.put_object(job.get('container', args[0] + '_segments'), | ||
3360 | 1539 | job['obj'], fp, content_length=job['segment_size']) | ||
3361 | 1540 | if options.verbose and 'log_line' in job: | ||
3362 | 1541 | if conn.attempts > 1: | ||
3363 | 1542 | print_queue.put('%s [after %d attempts]' % | ||
3364 | 1543 | (job['log_line'], conn.attempts)) | ||
3365 | 1544 | else: | ||
3366 | 1545 | print_queue.put(job['log_line']) | ||
3367 | 1546 | |||
3368 | 1547 | def _object_job(job, conn): | ||
3369 | 1548 | path = job['path'] | ||
3370 | 1549 | container = job.get('container', args[0]) | ||
3371 | 1550 | dir_marker = job.get('dir_marker', False) | ||
3372 | 1551 | try: | ||
3373 | 1552 | obj = path | ||
3374 | 1553 | if obj.startswith('./') or obj.startswith('.\\'): | ||
3375 | 1554 | obj = obj[2:] | ||
3376 | 1555 | put_headers = {'x-object-meta-mtime': str(getmtime(path))} | ||
3377 | 1556 | if dir_marker: | ||
3378 | 1557 | if options.changed: | ||
3379 | 1558 | try: | ||
3380 | 1559 | headers = conn.head_object(container, obj) | ||
3381 | 1560 | ct = headers.get('content-type') | ||
3382 | 1561 | cl = int(headers.get('content-length')) | ||
3383 | 1562 | et = headers.get('etag') | ||
3384 | 1563 | mt = headers.get('x-object-meta-mtime') | ||
3385 | 1564 | if ct.split(';', 1)[0] == 'text/directory' and \ | ||
3386 | 1565 | cl == 0 and \ | ||
3387 | 1566 | et == 'd41d8cd98f00b204e9800998ecf8427e' and \ | ||
3388 | 1567 | mt == put_headers['x-object-meta-mtime']: | ||
3389 | 1568 | return | ||
3390 | 1569 | except ClientException, err: | ||
3391 | 1570 | if err.http_status != 404: | ||
3392 | 1571 | raise | ||
3393 | 1572 | conn.put_object(container, obj, '', content_length=0, | ||
3394 | 1573 | content_type='text/directory', | ||
3395 | 1574 | headers=put_headers) | ||
3396 | 1575 | else: | ||
3397 | 1576 | # We need to HEAD all objects now in case we're overwriting a | ||
3398 | 1577 | # manifest object and need to delete the old segments | ||
3399 | 1578 | # ourselves. | ||
3400 | 1579 | old_manifest = None | ||
3401 | 1580 | if options.changed or not options.leave_segments: | ||
3402 | 1581 | try: | ||
3403 | 1582 | headers = conn.head_object(container, obj) | ||
3404 | 1583 | cl = int(headers.get('content-length')) | ||
3405 | 1584 | mt = headers.get('x-object-meta-mtime') | ||
3406 | 1585 | if options.changed and cl == getsize(path) and \ | ||
3407 | 1586 | mt == put_headers['x-object-meta-mtime']: | ||
3408 | 1587 | return | ||
3409 | 1588 | if not options.leave_segments: | ||
3410 | 1589 | old_manifest = headers.get('x-object-manifest') | ||
3411 | 1590 | except ClientException, err: | ||
3412 | 1591 | if err.http_status != 404: | ||
3413 | 1592 | raise | ||
3414 | 1593 | if options.segment_size and \ | ||
3415 | 1594 | getsize(path) < options.segment_size: | ||
3416 | 1595 | full_size = getsize(path) | ||
3417 | 1596 | segment_queue = Queue(10000) | ||
3418 | 1597 | segment_threads = [QueueFunctionThread(segment_queue, | ||
3419 | 1598 | _segment_job, create_connection()) for _junk in | ||
3420 | 1599 | xrange(10)] | ||
3421 | 1600 | for thread in segment_threads: | ||
3422 | 1601 | thread.start() | ||
3423 | 1602 | segment = 0 | ||
3424 | 1603 | segment_start = 0 | ||
3425 | 1604 | while segment_start < full_size: | ||
3426 | 1605 | segment_size = int(options.segment_size) | ||
3427 | 1606 | if segment_start + segment_size > full_size: | ||
3428 | 1607 | segment_size = full_size - segment_start | ||
3429 | 1608 | segment_queue.put({'path': path, | ||
3430 | 1609 | 'obj': '%s/%s/%s/%08d' % (obj, | ||
3431 | 1610 | put_headers['x-object-meta-mtime'], full_size, | ||
3432 | 1611 | segment), | ||
3433 | 1612 | 'segment_start': segment_start, | ||
3434 | 1613 | 'segment_size': segment_size, | ||
3435 | 1614 | 'log_line': '%s segment %s' % (obj, segment)}) | ||
3436 | 1615 | segment += 1 | ||
3437 | 1616 | segment_start += segment_size | ||
3438 | 1617 | while not segment_queue.empty(): | ||
3439 | 1618 | sleep(0.01) | ||
3440 | 1619 | for thread in segment_threads: | ||
3441 | 1620 | thread.abort = True | ||
3442 | 1621 | while thread.isAlive(): | ||
3443 | 1622 | thread.join(0.01) | ||
3444 | 1623 | if put_errors_from_threads(segment_threads, error_queue): | ||
3445 | 1624 | raise ClientException('Aborting manifest creation ' | ||
3446 | 1625 | 'because not all segments could be uploaded. %s/%s' | ||
3447 | 1626 | % (container, obj)) | ||
3448 | 1627 | new_object_manifest = '%s_segments/%s/%s/%s/' % ( | ||
3449 | 1628 | container, obj, put_headers['x-object-meta-mtime'], | ||
3450 | 1629 | full_size) | ||
3451 | 1630 | if old_manifest == new_object_manifest: | ||
3452 | 1631 | old_manifest = None | ||
3453 | 1632 | put_headers['x-object-manifest'] = new_object_manifest | ||
3454 | 1633 | conn.put_object(container, obj, '', content_length=0, | ||
3455 | 1634 | headers=put_headers) | ||
3456 | 1635 | else: | ||
3457 | 1636 | conn.put_object(container, obj, open(path, 'rb'), | ||
3458 | 1637 | content_length=getsize(path), headers=put_headers) | ||
3459 | 1638 | if old_manifest: | ||
3460 | 1639 | segment_queue = Queue(10000) | ||
3461 | 1640 | scontainer, sprefix = old_manifest.split('/', 1) | ||
3462 | 1641 | for delobj in conn.get_container(scontainer, | ||
3463 | 1642 | prefix=sprefix)[1]: | ||
3464 | 1643 | segment_queue.put({'delete': True, | ||
3465 | 1644 | 'container': scontainer, 'obj': delobj['name']}) | ||
3466 | 1645 | if not segment_queue.empty(): | ||
3467 | 1646 | segment_threads = [QueueFunctionThread(segment_queue, | ||
3468 | 1647 | _segment_job, create_connection()) for _junk in | ||
3469 | 1648 | xrange(10)] | ||
3470 | 1649 | for thread in segment_threads: | ||
3471 | 1650 | thread.start() | ||
3472 | 1651 | while not segment_queue.empty(): | ||
3473 | 1652 | sleep(0.01) | ||
3474 | 1653 | for thread in segment_threads: | ||
3475 | 1654 | thread.abort = True | ||
3476 | 1655 | while thread.isAlive(): | ||
3477 | 1656 | thread.join(0.01) | ||
3478 | 1657 | put_errors_from_threads(segment_threads, error_queue) | ||
3479 | 1658 | if options.verbose: | ||
3480 | 1659 | if conn.attempts > 1: | ||
3481 | 1660 | print_queue.put( | ||
3482 | 1661 | '%s [after %d attempts]' % (obj, conn.attempts)) | ||
3483 | 1662 | else: | ||
3484 | 1663 | print_queue.put(obj) | ||
3485 | 1664 | except OSError, err: | ||
3486 | 1665 | if err.errno != ENOENT: | ||
3487 | 1666 | raise | ||
3488 | 1667 | error_queue.put('Local file %s not found' % repr(path)) | ||
3489 | 1668 | |||
3490 | 1669 | def _upload_dir(path): | ||
3491 | 1670 | names = listdir(path) | ||
3492 | 1671 | if not names: | ||
3493 | 1672 | object_queue.put({'path': path, 'dir_marker': True}) | ||
3494 | 1673 | else: | ||
3495 | 1674 | for name in listdir(path): | ||
3496 | 1675 | subpath = join(path, name) | ||
3497 | 1676 | if isdir(subpath): | ||
3498 | 1677 | _upload_dir(subpath) | ||
3499 | 1678 | else: | ||
3500 | 1679 | object_queue.put({'path': subpath}) | ||
3501 | 1680 | |||
3502 | 1681 | url, token = get_auth(options.auth, options.user, options.key, | ||
3503 | 1682 | snet=options.snet) | ||
3504 | 1683 | create_connection = lambda: Connection(options.auth, options.user, | ||
3505 | 1684 | options.key, preauthurl=url, preauthtoken=token, snet=options.snet) | ||
3506 | 1685 | object_threads = [QueueFunctionThread(object_queue, _object_job, | ||
3507 | 1686 | create_connection()) for _junk in xrange(10)] | ||
3508 | 1687 | for thread in object_threads: | ||
3509 | 1688 | thread.start() | ||
3510 | 1689 | conn = create_connection() | ||
3511 | 1690 | # Try to create the container, just in case it doesn't exist. If this | ||
3512 | 1691 | # fails, it might just be because the user doesn't have container PUT | ||
3513 | 1692 | # permissions, so we'll ignore any error. If there's really a problem, | ||
3514 | 1693 | # it'll surface on the first object PUT. | ||
3515 | 1694 | try: | ||
3516 | 1695 | conn.put_container(args[0]) | ||
3517 | 1696 | if options.segment_size is not None: | ||
3518 | 1697 | conn.put_container(args[0] + '_segments') | ||
3519 | 1698 | except Exception: | ||
3520 | 1699 | pass | ||
3521 | 1700 | try: | ||
3522 | 1701 | for arg in args[1:]: | ||
3523 | 1702 | if isdir(arg): | ||
3524 | 1703 | _upload_dir(arg) | ||
3525 | 1704 | else: | ||
3526 | 1705 | object_queue.put({'path': arg}) | ||
3527 | 1706 | while not object_queue.empty(): | ||
3528 | 1707 | sleep(0.01) | ||
3529 | 1708 | for thread in object_threads: | ||
3530 | 1709 | thread.abort = True | ||
3531 | 1710 | while thread.isAlive(): | ||
3532 | 1711 | thread.join(0.01) | ||
3533 | 1712 | put_errors_from_threads(object_threads, error_queue) | ||
3534 | 1713 | except ClientException, err: | ||
3535 | 1714 | if err.http_status != 404: | ||
3536 | 1715 | raise | ||
3537 | 1716 | error_queue.put('Account not found') | ||
3538 | 1717 | |||
3539 | 1718 | |||
3540 | 1719 | def parse_args(parser, args, enforce_requires=True): | ||
3541 | 1720 | if not args: | ||
3542 | 1721 | args = ['-h'] | ||
3543 | 1722 | (options, args) = parser.parse_args(args) | ||
3544 | 1723 | if enforce_requires and \ | ||
3545 | 1724 | not (options.auth and options.user and options.key): | ||
3546 | 1725 | exit(''' | ||
3547 | 1726 | Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or | ||
3548 | 1727 | overridden with -A, -U, or -K.'''.strip('\n')) | ||
3549 | 1728 | return options, args | ||
3550 | 1729 | |||
3551 | 1730 | |||
3552 | 1731 | if __name__ == '__main__': | ||
3553 | 1732 | parser = OptionParser(version='%prog 1.0', usage=''' | ||
3554 | 1733 | Usage: %%prog <command> [options] [args] | ||
3555 | 1734 | |||
3556 | 1735 | Commands: | ||
3557 | 1736 | %(st_stat_help)s | ||
3558 | 1737 | %(st_list_help)s | ||
3559 | 1738 | %(st_upload_help)s | ||
3560 | 1739 | %(st_post_help)s | ||
3561 | 1740 | %(st_download_help)s | ||
3562 | 1741 | %(st_delete_help)s | ||
3563 | 1742 | |||
3564 | 1743 | Example: | ||
3565 | 1744 | %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K key stat | ||
3566 | 1745 | '''.strip('\n') % globals()) | ||
3567 | 1746 | parser.add_option('-s', '--snet', action='store_true', dest='snet', | ||
3568 | 1747 | default=False, help='Use SERVICENET internal network') | ||
3569 | 1748 | parser.add_option('-v', '--verbose', action='count', dest='verbose', | ||
3570 | 1749 | default=1, help='Print more info') | ||
3571 | 1750 | parser.add_option('-q', '--quiet', action='store_const', dest='verbose', | ||
3572 | 1751 | const=0, default=1, help='Suppress status output') | ||
3573 | 1752 | parser.add_option('-A', '--auth', dest='auth', | ||
3574 | 1753 | default=environ.get('ST_AUTH'), | ||
3575 | 1754 | help='URL for obtaining an auth token') | ||
3576 | 1755 | parser.add_option('-U', '--user', dest='user', | ||
3577 | 1756 | default=environ.get('ST_USER'), | ||
3578 | 1757 | help='User name for obtaining an auth token') | ||
3579 | 1758 | parser.add_option('-K', '--key', dest='key', | ||
3580 | 1759 | default=environ.get('ST_KEY'), | ||
3581 | 1760 | help='Key for obtaining an auth token') | ||
3582 | 1761 | parser.disable_interspersed_args() | ||
3583 | 1762 | (options, args) = parse_args(parser, argv[1:], enforce_requires=False) | ||
3584 | 1763 | parser.enable_interspersed_args() | ||
3585 | 1764 | |||
3586 | 1765 | commands = ('delete', 'download', 'list', 'post', 'stat', 'upload') | ||
3587 | 1766 | if not args or args[0] not in commands: | ||
3588 | 1767 | parser.print_usage() | ||
3589 | 1768 | if args: | ||
3590 | 1769 | exit('no such command: %s' % args[0]) | ||
3591 | 1770 | exit() | ||
3592 | 1771 | |||
3593 | 1772 | print_queue = Queue(10000) | ||
3594 | 1773 | |||
3595 | 1774 | def _print(item): | ||
3596 | 1775 | if isinstance(item, unicode): | ||
3597 | 1776 | item = item.encode('utf8') | ||
3598 | 1777 | print item | ||
3599 | 1778 | |||
3600 | 1779 | print_thread = QueueFunctionThread(print_queue, _print) | ||
3601 | 1780 | print_thread.start() | ||
3602 | 1781 | |||
3603 | 1782 | error_queue = Queue(10000) | ||
3604 | 1783 | |||
3605 | 1784 | def _error(item): | ||
3606 | 1785 | if isinstance(item, unicode): | ||
3607 | 1786 | item = item.encode('utf8') | ||
3608 | 1787 | print >> stderr, item | ||
3609 | 1788 | |||
3610 | 1789 | error_thread = QueueFunctionThread(error_queue, _error) | ||
3611 | 1790 | error_thread.start() | ||
3612 | 1791 | |||
3613 | 1792 | try: | ||
3614 | 1793 | parser.usage = globals()['st_%s_help' % args[0]] | ||
3615 | 1794 | try: | ||
3616 | 1795 | globals()['st_%s' % args[0]](parser, argv[1:], print_queue, | ||
3617 | 1796 | error_queue) | ||
3618 | 1797 | except (ClientException, HTTPException, socket.error), err: | ||
3619 | 1798 | error_queue.put(str(err)) | ||
3620 | 1799 | while not print_queue.empty(): | ||
3621 | 1800 | sleep(0.01) | ||
3622 | 1801 | print_thread.abort = True | ||
3623 | 1802 | while print_thread.isAlive(): | ||
3624 | 1803 | print_thread.join(0.01) | ||
3625 | 1804 | while not error_queue.empty(): | ||
3626 | 1805 | sleep(0.01) | ||
3627 | 1806 | error_thread.abort = True | ||
3628 | 1807 | while error_thread.isAlive(): | ||
3629 | 1808 | error_thread.join(0.01) | ||
3630 | 1809 | except (SystemExit, Exception): | ||
3631 | 1810 | for thread in threading_enumerate(): | ||
3632 | 1811 | thread.abort = True | ||
3633 | 1812 | raise | ||
3634 | 0 | 1813 | ||
3635 | === modified file 'doc/source/development_saio.rst' | |||
3636 | --- doc/source/development_saio.rst 2011-05-26 02:24:12 +0000 | |||
3637 | +++ doc/source/development_saio.rst 2011-06-14 16:07:28 +0000 | |||
3638 | @@ -625,7 +625,7 @@ | |||
3639 | 625 | #. `recreateaccounts` | 625 | #. `recreateaccounts` |
3640 | 626 | #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0`` | 626 | #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0`` |
3641 | 627 | #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>`` | 627 | #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>`` |
3643 | 628 | #. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` | 628 | #. Check that `swift` works: `swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` |
3644 | 629 | #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` | 629 | #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` |
3645 | 630 | #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete | 630 | #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete |
3646 | 631 | everything in the configured accounts.) | 631 | everything in the configured accounts.) |
3647 | 632 | 632 | ||
3648 | === modified file 'doc/source/howto_installmultinode.rst' | |||
3649 | --- doc/source/howto_installmultinode.rst 2011-05-26 02:24:12 +0000 | |||
3650 | +++ doc/source/howto_installmultinode.rst 2011-06-14 16:07:28 +0000 | |||
3651 | @@ -372,34 +372,34 @@ | |||
3652 | 372 | 372 | ||
3653 | 373 | curl -k -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above> | 373 | curl -k -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above> |
3654 | 374 | 374 | ||
3683 | 375 | #. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes):: | 375 | #. Check that ``swift`` works (at this point, expect zero containers, zero objects, and zero bytes):: |
3684 | 376 | 376 | ||
3685 | 377 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat | 377 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat |
3686 | 378 | 378 | ||
3687 | 379 | #. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: | 379 | #. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: |
3688 | 380 | 380 | ||
3689 | 381 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz | 381 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz |
3690 | 382 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz | 382 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz |
3691 | 383 | 383 | ||
3692 | 384 | #. Use ``st`` to download all files from the 'myfiles' container:: | 384 | #. Use ``swift`` to download all files from the 'myfiles' container:: |
3693 | 385 | 385 | ||
3694 | 386 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles | 386 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles |
3695 | 387 | 387 | ||
3696 | 388 | #. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: | 388 | #. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: |
3697 | 389 | 389 | ||
3698 | 390 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder | 390 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder |
3699 | 391 | 391 | ||
3700 | 392 | #. Use ``st`` to list your containers:: | 392 | #. Use ``swift`` to list your containers:: |
3701 | 393 | 393 | ||
3702 | 394 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list | 394 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list |
3703 | 395 | 395 | ||
3704 | 396 | #. Use ``st`` to list the contents of your 'builders' container:: | 396 | #. Use ``swift`` to list the contents of your 'builders' container:: |
3705 | 397 | 397 | ||
3706 | 398 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders | 398 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders |
3707 | 399 | 399 | ||
3708 | 400 | #. Use ``st`` to download all files from the 'builders' container:: | 400 | #. Use ``swift`` to download all files from the 'builders' container:: |
3709 | 401 | 401 | ||
3710 | 402 | st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders | 402 | swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders |
3711 | 403 | 403 | ||
3712 | 404 | .. _add-proxy-server: | 404 | .. _add-proxy-server: |
3713 | 405 | 405 | ||
3714 | 406 | 406 | ||
3715 | === modified file 'doc/source/overview_large_objects.rst' | |||
3716 | --- doc/source/overview_large_objects.rst 2010-12-06 20:01:19 +0000 | |||
3717 | +++ doc/source/overview_large_objects.rst 2011-06-14 16:07:28 +0000 | |||
3718 | @@ -14,24 +14,24 @@ | |||
3719 | 14 | with the possibility of parallel uploads of the segments. | 14 | with the possibility of parallel uploads of the segments. |
3720 | 15 | 15 | ||
3721 | 16 | ---------------------------------- | 16 | ---------------------------------- |
3723 | 17 | Using ``st`` for Segmented Objects | 17 | Using ``swift`` for Segmented Objects |
3724 | 18 | ---------------------------------- | 18 | ---------------------------------- |
3725 | 19 | 19 | ||
3727 | 20 | The quickest way to try out this feature is use the included ``st`` Swift Tool. | 20 | The quickest way to try out this feature is use the included ``swift`` Swift Tool. |
3728 | 21 | You can use the ``-S`` option to specify the segment size to use when splitting | 21 | You can use the ``-S`` option to specify the segment size to use when splitting |
3729 | 22 | a large file. For example:: | 22 | a large file. For example:: |
3730 | 23 | 23 | ||
3732 | 24 | st upload test_container -S 1073741824 large_file | 24 | swift upload test_container -S 1073741824 large_file |
3733 | 25 | 25 | ||
3734 | 26 | This would split the large_file into 1G segments and begin uploading those | 26 | This would split the large_file into 1G segments and begin uploading those |
3736 | 27 | segments in parallel. Once all the segments have been uploaded, ``st`` will | 27 | segments in parallel. Once all the segments have been uploaded, ``swift`` will |
3737 | 28 | then create the manifest file so the segments can be downloaded as one. | 28 | then create the manifest file so the segments can be downloaded as one. |
3738 | 29 | 29 | ||
3744 | 30 | So now, the following ``st`` command would download the entire large object:: | 30 | So now, the following ``swift`` command would download the entire large object:: |
3745 | 31 | 31 | ||
3746 | 32 | st download test_container large_file | 32 | swift download test_container large_file |
3747 | 33 | 33 | ||
3748 | 34 | ``st`` uses a strict convention for its segmented object support. In the above | 34 | ``swift`` uses a strict convention for its segmented object support. In the above |
3749 | 35 | example it will upload all the segments into a second container named | 35 | example it will upload all the segments into a second container named |
3750 | 36 | test_container_segments. These segments will have names like | 36 | test_container_segments. These segments will have names like |
3751 | 37 | large_file/1290206778.25/21474836480/00000000, | 37 | large_file/1290206778.25/21474836480/00000000, |
3752 | @@ -43,7 +43,7 @@ | |||
3753 | 43 | upload of a new file with the same name won't overwrite the contents of the | 43 | upload of a new file with the same name won't overwrite the contents of the |
3754 | 44 | first until the last moment when the manifest file is updated. | 44 | first until the last moment when the manifest file is updated. |
3755 | 45 | 45 | ||
3757 | 46 | ``st`` will manage these segment files for you, deleting old segments on | 46 | ``swift`` will manage these segment files for you, deleting old segments on |
3758 | 47 | deletes and overwrites, etc. You can override this behavior with the | 47 | deletes and overwrites, etc. You can override this behavior with the |
3759 | 48 | ``--leave-segments`` option if desired; this is useful if you want to have | 48 | ``--leave-segments`` option if desired; this is useful if you want to have |
3760 | 49 | multiple versions of the same large object available. | 49 | multiple versions of the same large object available. |
3761 | @@ -53,14 +53,14 @@ | |||
3762 | 53 | ---------- | 53 | ---------- |
3763 | 54 | 54 | ||
3764 | 55 | You can also work with the segments and manifests directly with HTTP requests | 55 | You can also work with the segments and manifests directly with HTTP requests |
3766 | 56 | instead of having ``st`` do that for you. You can just upload the segments like | 56 | instead of having ``swift`` do that for you. You can just upload the segments like |
3767 | 57 | you would any other object and the manifest is just a zero-byte file with an | 57 | you would any other object and the manifest is just a zero-byte file with an |
3768 | 58 | extra ``X-Object-Manifest`` header. | 58 | extra ``X-Object-Manifest`` header. |
3769 | 59 | 59 | ||
3770 | 60 | All the object segments need to be in the same container, have a common object | 60 | All the object segments need to be in the same container, have a common object |
3771 | 61 | name prefix, and their names sort in the order they should be concatenated. | 61 | name prefix, and their names sort in the order they should be concatenated. |
3772 | 62 | They don't have to be in the same container as the manifest file will be, which | 62 | They don't have to be in the same container as the manifest file will be, which |
3774 | 63 | is useful to keep container listings clean as explained above with ``st``. | 63 | is useful to keep container listings clean as explained above with ``swift``. |
3775 | 64 | 64 | ||
3776 | 65 | The manifest file is simply a zero-byte file with the extra | 65 | The manifest file is simply a zero-byte file with the extra |
3777 | 66 | ``X-Object-Manifest: <container>/<prefix>`` header, where ``<container>`` is | 66 | ``X-Object-Manifest: <container>/<prefix>`` header, where ``<container>`` is |
3778 | 67 | 67 | ||
3779 | === modified file 'setup.py' | |||
3780 | --- setup.py 2011-05-31 18:09:53 +0000 | |||
3781 | +++ setup.py 2011-06-14 16:07:28 +0000 | |||
3782 | @@ -76,7 +76,7 @@ | |||
3783 | 76 | ], | 76 | ], |
3784 | 77 | install_requires=[], # removed for better compat | 77 | install_requires=[], # removed for better compat |
3785 | 78 | scripts=[ | 78 | scripts=[ |
3787 | 79 | 'bin/st', 'bin/swift-account-auditor', | 79 | 'bin/swift', 'bin/swift-account-auditor', |
3788 | 80 | 'bin/swift-account-audit', 'bin/swift-account-reaper', | 80 | 'bin/swift-account-audit', 'bin/swift-account-reaper', |
3789 | 81 | 'bin/swift-account-replicator', 'bin/swift-account-server', | 81 | 'bin/swift-account-replicator', 'bin/swift-account-server', |
3790 | 82 | 'bin/swift-container-auditor', | 82 | 'bin/swift-container-auditor', |
3791 | 83 | 83 | ||
3792 | === modified file 'swift/common/middleware/staticweb.py' | |||
3793 | --- swift/common/middleware/staticweb.py 2011-06-05 23:22:35 +0000 | |||
3794 | +++ swift/common/middleware/staticweb.py 2011-06-14 16:07:28 +0000 | |||
3795 | @@ -74,36 +74,36 @@ | |||
3796 | 74 | listing page, you will see the well defined document structure that can be | 74 | listing page, you will see the well defined document structure that can be |
3797 | 75 | styled. | 75 | styled. |
3798 | 76 | 76 | ||
3800 | 77 | Example usage of this middleware via ``st``: | 77 | Example usage of this middleware via ``swift``: |
3801 | 78 | 78 | ||
3802 | 79 | Make the container publicly readable:: | 79 | Make the container publicly readable:: |
3803 | 80 | 80 | ||
3805 | 81 | st post -r '.r:*' container | 81 | swift post -r '.r:*' container |
3806 | 82 | 82 | ||
3807 | 83 | You should be able to get objects directly, but no index.html resolution or | 83 | You should be able to get objects directly, but no index.html resolution or |
3808 | 84 | listings. | 84 | listings. |
3809 | 85 | 85 | ||
3810 | 86 | Set an index file directive:: | 86 | Set an index file directive:: |
3811 | 87 | 87 | ||
3813 | 88 | st post -m 'web-index:index.html' container | 88 | swift post -m 'web-index:index.html' container |
3814 | 89 | 89 | ||
3815 | 90 | You should be able to hit paths that have an index.html without needing to | 90 | You should be able to hit paths that have an index.html without needing to |
3816 | 91 | type the index.html part. | 91 | type the index.html part. |
3817 | 92 | 92 | ||
3818 | 93 | Turn on listings:: | 93 | Turn on listings:: |
3819 | 94 | 94 | ||
3821 | 95 | st post -m 'web-listings: true' container | 95 | swift post -m 'web-listings: true' container |
3822 | 96 | 96 | ||
3823 | 97 | Now you should see object listings for paths and pseudo paths that have no | 97 | Now you should see object listings for paths and pseudo paths that have no |
3824 | 98 | index.html. | 98 | index.html. |
3825 | 99 | 99 | ||
3826 | 100 | Enable a custom listings style sheet:: | 100 | Enable a custom listings style sheet:: |
3827 | 101 | 101 | ||
3829 | 102 | st post -m 'web-listings-css:listings.css' container | 102 | swift post -m 'web-listings-css:listings.css' container |
3830 | 103 | 103 | ||
3831 | 104 | Set an error file:: | 104 | Set an error file:: |
3832 | 105 | 105 | ||
3834 | 106 | st post -m 'web-error:error.html' container | 106 | swift post -m 'web-error:error.html' container |
3835 | 107 | 107 | ||
3836 | 108 | Now 401's should load 401error.html, 404's should load 404error.html, etc. | 108 | Now 401's should load 401error.html, 404's should load 404error.html, etc. |
3837 | 109 | """ | 109 | """ |