Merge lp:~notmyname/swift/renamest into lp:~hudson-openstack/swift/trunk

Proposed by John Dickinson
Status: Merged
Approved by: gholt
Approved revision: 311
Merged at revision: 311
Proposed branch: lp:~notmyname/swift/renamest
Merge into: lp:~hudson-openstack/swift/trunk
Diff against target: 3837 lines (+1860/-1860)
7 files modified
bin/st (+0/-1812)
bin/swift (+1812/-0)
doc/source/development_saio.rst (+1/-1)
doc/source/howto_installmultinode.rst (+28/-28)
doc/source/overview_large_objects.rst (+12/-12)
setup.py (+1/-1)
swift/common/middleware/staticweb.py (+6/-6)
To merge this branch: bzr merge lp:~notmyname/swift/renamest
Reviewer Review Type Date Requested Status
Swift Core security contacts Pending
Review via email: mp+64566@code.launchpad.net

Description of the change

rename st to swift

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== removed file 'bin/st'
2--- bin/st 2011-05-19 14:48:15 +0000
3+++ bin/st 1970-01-01 00:00:00 +0000
4@@ -1,1812 +0,0 @@
5-#!/usr/bin/python -u
6-# Copyright (c) 2010-2011 OpenStack, LLC.
7-#
8-# Licensed under the Apache License, Version 2.0 (the "License");
9-# you may not use this file except in compliance with the License.
10-# You may obtain a copy of the License at
11-#
12-# http://www.apache.org/licenses/LICENSE-2.0
13-#
14-# Unless required by applicable law or agreed to in writing, software
15-# distributed under the License is distributed on an "AS IS" BASIS,
16-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
17-# implied.
18-# See the License for the specific language governing permissions and
19-# limitations under the License.
20-
21-from errno import EEXIST, ENOENT
22-from hashlib import md5
23-from optparse import OptionParser
24-from os import environ, listdir, makedirs, utime
25-from os.path import basename, dirname, getmtime, getsize, isdir, join
26-from Queue import Empty, Queue
27-from sys import argv, exc_info, exit, stderr, stdout
28-from threading import enumerate as threading_enumerate, Thread
29-from time import sleep
30-from traceback import format_exception
31-
32-
33-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
34-# Inclusion of swift.common.client for convenience of single file distribution
35-
36-import socket
37-from cStringIO import StringIO
38-from re import compile, DOTALL
39-from tokenize import generate_tokens, STRING, NAME, OP
40-from urllib import quote as _quote, unquote
41-from urlparse import urlparse, urlunparse
42-
43-try:
44- from eventlet.green.httplib import HTTPException, HTTPSConnection
45-except ImportError:
46- from httplib import HTTPException, HTTPSConnection
47-
48-try:
49- from eventlet import sleep
50-except ImportError:
51- from time import sleep
52-
53-try:
54- from swift.common.bufferedhttp \
55- import BufferedHTTPConnection as HTTPConnection
56-except ImportError:
57- try:
58- from eventlet.green.httplib import HTTPConnection
59- except ImportError:
60- from httplib import HTTPConnection
61-
62-
63-def quote(value, safe='/'):
64- """
65- Patched version of urllib.quote that encodes utf8 strings before quoting
66- """
67- if isinstance(value, unicode):
68- value = value.encode('utf8')
69- return _quote(value, safe)
70-
71-
72-# look for a real json parser first
73-try:
74- # simplejson is popular and pretty good
75- from simplejson import loads as json_loads
76-except ImportError:
77- try:
78- # 2.6 will have a json module in the stdlib
79- from json import loads as json_loads
80- except ImportError:
81- # fall back on local parser otherwise
82- comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL)
83-
84- def json_loads(string):
85- '''
86- Fairly competent json parser exploiting the python tokenizer and
87- eval(). -- From python-cloudfiles
88-
89- _loads(serialized_json) -> object
90- '''
91- try:
92- res = []
93- consts = {'true': True, 'false': False, 'null': None}
94- string = '(' + comments.sub('', string) + ')'
95- for type, val, _junk, _junk, _junk in \
96- generate_tokens(StringIO(string).readline):
97- if (type == OP and val not in '[]{}:,()-') or \
98- (type == NAME and val not in consts):
99- raise AttributeError()
100- elif type == STRING:
101- res.append('u')
102- res.append(val.replace('\\/', '/'))
103- else:
104- res.append(val)
105- return eval(''.join(res), {}, consts)
106- except Exception:
107- raise AttributeError()
108-
109-
110-class ClientException(Exception):
111-
112- def __init__(self, msg, http_scheme='', http_host='', http_port='',
113- http_path='', http_query='', http_status=0, http_reason='',
114- http_device=''):
115- Exception.__init__(self, msg)
116- self.msg = msg
117- self.http_scheme = http_scheme
118- self.http_host = http_host
119- self.http_port = http_port
120- self.http_path = http_path
121- self.http_query = http_query
122- self.http_status = http_status
123- self.http_reason = http_reason
124- self.http_device = http_device
125-
126- def __str__(self):
127- a = self.msg
128- b = ''
129- if self.http_scheme:
130- b += '%s://' % self.http_scheme
131- if self.http_host:
132- b += self.http_host
133- if self.http_port:
134- b += ':%s' % self.http_port
135- if self.http_path:
136- b += self.http_path
137- if self.http_query:
138- b += '?%s' % self.http_query
139- if self.http_status:
140- if b:
141- b = '%s %s' % (b, self.http_status)
142- else:
143- b = str(self.http_status)
144- if self.http_reason:
145- if b:
146- b = '%s %s' % (b, self.http_reason)
147- else:
148- b = '- %s' % self.http_reason
149- if self.http_device:
150- if b:
151- b = '%s: device %s' % (b, self.http_device)
152- else:
153- b = 'device %s' % self.http_device
154- return b and '%s: %s' % (a, b) or a
155-
156-
157-def http_connection(url):
158- """
159- Make an HTTPConnection or HTTPSConnection
160-
161- :param url: url to connect to
162- :returns: tuple of (parsed url, connection object)
163- :raises ClientException: Unable to handle protocol scheme
164- """
165- parsed = urlparse(url)
166- if parsed.scheme == 'http':
167- conn = HTTPConnection(parsed.netloc)
168- elif parsed.scheme == 'https':
169- conn = HTTPSConnection(parsed.netloc)
170- else:
171- raise ClientException('Cannot handle protocol scheme %s for url %s' %
172- (parsed.scheme, repr(url)))
173- return parsed, conn
174-
175-
176-def get_auth(url, user, key, snet=False):
177- """
178- Get authentication/authorization credentials.
179-
180- The snet parameter is used for Rackspace's ServiceNet internal network
181- implementation. In this function, it simply adds *snet-* to the beginning
182- of the host name for the returned storage URL. With Rackspace Cloud Files,
183- use of this network path causes no bandwidth charges but requires the
184- client to be running on Rackspace's ServiceNet network.
185-
186- :param url: authentication/authorization URL
187- :param user: user to authenticate as
188- :param key: key or password for authorization
189- :param snet: use SERVICENET internal network (see above), default is False
190- :returns: tuple of (storage URL, auth token)
191- :raises ClientException: HTTP GET request to auth URL failed
192- """
193- parsed, conn = http_connection(url)
194- conn.request('GET', parsed.path, '',
195- {'X-Auth-User': user, 'X-Auth-Key': key})
196- resp = conn.getresponse()
197- resp.read()
198- if resp.status < 200 or resp.status >= 300:
199- raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
200- http_host=conn.host, http_port=conn.port,
201- http_path=parsed.path, http_status=resp.status,
202- http_reason=resp.reason)
203- url = resp.getheader('x-storage-url')
204- if snet:
205- parsed = list(urlparse(url))
206- # Second item in the list is the netloc
207- parsed[1] = 'snet-' + parsed[1]
208- url = urlunparse(parsed)
209- return url, resp.getheader('x-storage-token',
210- resp.getheader('x-auth-token'))
211-
212-
213-def get_account(url, token, marker=None, limit=None, prefix=None,
214- http_conn=None, full_listing=False):
215- """
216- Get a listing of containers for the account.
217-
218- :param url: storage URL
219- :param token: auth token
220- :param marker: marker query
221- :param limit: limit query
222- :param prefix: prefix query
223- :param http_conn: HTTP connection object (If None, it will create the
224- conn object)
225- :param full_listing: if True, return a full listing, else returns a max
226- of 10000 listings
227- :returns: a tuple of (response headers, a list of containers) The response
228- headers will be a dict and all header names will be lowercase.
229- :raises ClientException: HTTP GET request failed
230- """
231- if not http_conn:
232- http_conn = http_connection(url)
233- if full_listing:
234- rv = get_account(url, token, marker, limit, prefix, http_conn)
235- listing = rv[1]
236- while listing:
237- marker = listing[-1]['name']
238- listing = \
239- get_account(url, token, marker, limit, prefix, http_conn)[1]
240- if listing:
241- rv[1].extend(listing)
242- return rv
243- parsed, conn = http_conn
244- qs = 'format=json'
245- if marker:
246- qs += '&marker=%s' % quote(marker)
247- if limit:
248- qs += '&limit=%d' % limit
249- if prefix:
250- qs += '&prefix=%s' % quote(prefix)
251- conn.request('GET', '%s?%s' % (parsed.path, qs), '',
252- {'X-Auth-Token': token})
253- resp = conn.getresponse()
254- resp_headers = {}
255- for header, value in resp.getheaders():
256- resp_headers[header.lower()] = value
257- if resp.status < 200 or resp.status >= 300:
258- resp.read()
259- raise ClientException('Account GET failed', http_scheme=parsed.scheme,
260- http_host=conn.host, http_port=conn.port,
261- http_path=parsed.path, http_query=qs, http_status=resp.status,
262- http_reason=resp.reason)
263- if resp.status == 204:
264- resp.read()
265- return resp_headers, []
266- return resp_headers, json_loads(resp.read())
267-
268-
269-def head_account(url, token, http_conn=None):
270- """
271- Get account stats.
272-
273- :param url: storage URL
274- :param token: auth token
275- :param http_conn: HTTP connection object (If None, it will create the
276- conn object)
277- :returns: a dict containing the response's headers (all header names will
278- be lowercase)
279- :raises ClientException: HTTP HEAD request failed
280- """
281- if http_conn:
282- parsed, conn = http_conn
283- else:
284- parsed, conn = http_connection(url)
285- conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
286- resp = conn.getresponse()
287- resp.read()
288- if resp.status < 200 or resp.status >= 300:
289- raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
290- http_host=conn.host, http_port=conn.port,
291- http_path=parsed.path, http_status=resp.status,
292- http_reason=resp.reason)
293- resp_headers = {}
294- for header, value in resp.getheaders():
295- resp_headers[header.lower()] = value
296- return resp_headers
297-
298-
299-def post_account(url, token, headers, http_conn=None):
300- """
301- Update an account's metadata.
302-
303- :param url: storage URL
304- :param token: auth token
305- :param headers: additional headers to include in the request
306- :param http_conn: HTTP connection object (If None, it will create the
307- conn object)
308- :raises ClientException: HTTP POST request failed
309- """
310- if http_conn:
311- parsed, conn = http_conn
312- else:
313- parsed, conn = http_connection(url)
314- headers['X-Auth-Token'] = token
315- conn.request('POST', parsed.path, '', headers)
316- resp = conn.getresponse()
317- resp.read()
318- if resp.status < 200 or resp.status >= 300:
319- raise ClientException('Account POST failed',
320- http_scheme=parsed.scheme, http_host=conn.host,
321- http_port=conn.port, http_path=path, http_status=resp.status,
322- http_reason=resp.reason)
323-
324-
325-def get_container(url, token, container, marker=None, limit=None,
326- prefix=None, delimiter=None, http_conn=None,
327- full_listing=False):
328- """
329- Get a listing of objects for the container.
330-
331- :param url: storage URL
332- :param token: auth token
333- :param container: container name to get a listing for
334- :param marker: marker query
335- :param limit: limit query
336- :param prefix: prefix query
337- :param delimeter: string to delimit the queries on
338- :param http_conn: HTTP connection object (If None, it will create the
339- conn object)
340- :param full_listing: if True, return a full listing, else returns a max
341- of 10000 listings
342- :returns: a tuple of (response headers, a list of objects) The response
343- headers will be a dict and all header names will be lowercase.
344- :raises ClientException: HTTP GET request failed
345- """
346- if not http_conn:
347- http_conn = http_connection(url)
348- if full_listing:
349- rv = get_container(url, token, container, marker, limit, prefix,
350- delimiter, http_conn)
351- listing = rv[1]
352- while listing:
353- if not delimiter:
354- marker = listing[-1]['name']
355- else:
356- marker = listing[-1].get('name', listing[-1].get('subdir'))
357- listing = get_container(url, token, container, marker, limit,
358- prefix, delimiter, http_conn)[1]
359- if listing:
360- rv[1].extend(listing)
361- return rv
362- parsed, conn = http_conn
363- path = '%s/%s' % (parsed.path, quote(container))
364- qs = 'format=json'
365- if marker:
366- qs += '&marker=%s' % quote(marker)
367- if limit:
368- qs += '&limit=%d' % limit
369- if prefix:
370- qs += '&prefix=%s' % quote(prefix)
371- if delimiter:
372- qs += '&delimiter=%s' % quote(delimiter)
373- conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token})
374- resp = conn.getresponse()
375- if resp.status < 200 or resp.status >= 300:
376- resp.read()
377- raise ClientException('Container GET failed',
378- http_scheme=parsed.scheme, http_host=conn.host,
379- http_port=conn.port, http_path=path, http_query=qs,
380- http_status=resp.status, http_reason=resp.reason)
381- resp_headers = {}
382- for header, value in resp.getheaders():
383- resp_headers[header.lower()] = value
384- if resp.status == 204:
385- resp.read()
386- return resp_headers, []
387- return resp_headers, json_loads(resp.read())
388-
389-
390-def head_container(url, token, container, http_conn=None):
391- """
392- Get container stats.
393-
394- :param url: storage URL
395- :param token: auth token
396- :param container: container name to get stats for
397- :param http_conn: HTTP connection object (If None, it will create the
398- conn object)
399- :returns: a dict containing the response's headers (all header names will
400- be lowercase)
401- :raises ClientException: HTTP HEAD request failed
402- """
403- if http_conn:
404- parsed, conn = http_conn
405- else:
406- parsed, conn = http_connection(url)
407- path = '%s/%s' % (parsed.path, quote(container))
408- conn.request('HEAD', path, '', {'X-Auth-Token': token})
409- resp = conn.getresponse()
410- resp.read()
411- if resp.status < 200 or resp.status >= 300:
412- raise ClientException('Container HEAD failed',
413- http_scheme=parsed.scheme, http_host=conn.host,
414- http_port=conn.port, http_path=path, http_status=resp.status,
415- http_reason=resp.reason)
416- resp_headers = {}
417- for header, value in resp.getheaders():
418- resp_headers[header.lower()] = value
419- return resp_headers
420-
421-
422-def put_container(url, token, container, headers=None, http_conn=None):
423- """
424- Create a container
425-
426- :param url: storage URL
427- :param token: auth token
428- :param container: container name to create
429- :param headers: additional headers to include in the request
430- :param http_conn: HTTP connection object (If None, it will create the
431- conn object)
432- :raises ClientException: HTTP PUT request failed
433- """
434- if http_conn:
435- parsed, conn = http_conn
436- else:
437- parsed, conn = http_connection(url)
438- path = '%s/%s' % (parsed.path, quote(container))
439- if not headers:
440- headers = {}
441- headers['X-Auth-Token'] = token
442- conn.request('PUT', path, '', headers)
443- resp = conn.getresponse()
444- resp.read()
445- if resp.status < 200 or resp.status >= 300:
446- raise ClientException('Container PUT failed',
447- http_scheme=parsed.scheme, http_host=conn.host,
448- http_port=conn.port, http_path=path, http_status=resp.status,
449- http_reason=resp.reason)
450-
451-
452-def post_container(url, token, container, headers, http_conn=None):
453- """
454- Update a container's metadata.
455-
456- :param url: storage URL
457- :param token: auth token
458- :param container: container name to update
459- :param headers: additional headers to include in the request
460- :param http_conn: HTTP connection object (If None, it will create the
461- conn object)
462- :raises ClientException: HTTP POST request failed
463- """
464- if http_conn:
465- parsed, conn = http_conn
466- else:
467- parsed, conn = http_connection(url)
468- path = '%s/%s' % (parsed.path, quote(container))
469- headers['X-Auth-Token'] = token
470- conn.request('POST', path, '', headers)
471- resp = conn.getresponse()
472- resp.read()
473- if resp.status < 200 or resp.status >= 300:
474- raise ClientException('Container POST failed',
475- http_scheme=parsed.scheme, http_host=conn.host,
476- http_port=conn.port, http_path=path, http_status=resp.status,
477- http_reason=resp.reason)
478-
479-
480-def delete_container(url, token, container, http_conn=None):
481- """
482- Delete a container
483-
484- :param url: storage URL
485- :param token: auth token
486- :param container: container name to delete
487- :param http_conn: HTTP connection object (If None, it will create the
488- conn object)
489- :raises ClientException: HTTP DELETE request failed
490- """
491- if http_conn:
492- parsed, conn = http_conn
493- else:
494- parsed, conn = http_connection(url)
495- path = '%s/%s' % (parsed.path, quote(container))
496- conn.request('DELETE', path, '', {'X-Auth-Token': token})
497- resp = conn.getresponse()
498- resp.read()
499- if resp.status < 200 or resp.status >= 300:
500- raise ClientException('Container DELETE failed',
501- http_scheme=parsed.scheme, http_host=conn.host,
502- http_port=conn.port, http_path=path, http_status=resp.status,
503- http_reason=resp.reason)
504-
505-
506-def get_object(url, token, container, name, http_conn=None,
507- resp_chunk_size=None):
508- """
509- Get an object
510-
511- :param url: storage URL
512- :param token: auth token
513- :param container: container name that the object is in
514- :param name: object name to get
515- :param http_conn: HTTP connection object (If None, it will create the
516- conn object)
517- :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
518- you specify a resp_chunk_size you must fully read
519- the object's contents before making another
520- request.
521- :returns: a tuple of (response headers, the object's contents) The response
522- headers will be a dict and all header names will be lowercase.
523- :raises ClientException: HTTP GET request failed
524- """
525- if http_conn:
526- parsed, conn = http_conn
527- else:
528- parsed, conn = http_connection(url)
529- path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
530- conn.request('GET', path, '', {'X-Auth-Token': token})
531- resp = conn.getresponse()
532- if resp.status < 200 or resp.status >= 300:
533- resp.read()
534- raise ClientException('Object GET failed', http_scheme=parsed.scheme,
535- http_host=conn.host, http_port=conn.port, http_path=path,
536- http_status=resp.status, http_reason=resp.reason)
537- if resp_chunk_size:
538-
539- def _object_body():
540- buf = resp.read(resp_chunk_size)
541- while buf:
542- yield buf
543- buf = resp.read(resp_chunk_size)
544- object_body = _object_body()
545- else:
546- object_body = resp.read()
547- resp_headers = {}
548- for header, value in resp.getheaders():
549- resp_headers[header.lower()] = value
550- return resp_headers, object_body
551-
552-
553-def head_object(url, token, container, name, http_conn=None):
554- """
555- Get object info
556-
557- :param url: storage URL
558- :param token: auth token
559- :param container: container name that the object is in
560- :param name: object name to get info for
561- :param http_conn: HTTP connection object (If None, it will create the
562- conn object)
563- :returns: a dict containing the response's headers (all header names will
564- be lowercase)
565- :raises ClientException: HTTP HEAD request failed
566- """
567- if http_conn:
568- parsed, conn = http_conn
569- else:
570- parsed, conn = http_connection(url)
571- path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
572- conn.request('HEAD', path, '', {'X-Auth-Token': token})
573- resp = conn.getresponse()
574- resp.read()
575- if resp.status < 200 or resp.status >= 300:
576- raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
577- http_host=conn.host, http_port=conn.port, http_path=path,
578- http_status=resp.status, http_reason=resp.reason)
579- resp_headers = {}
580- for header, value in resp.getheaders():
581- resp_headers[header.lower()] = value
582- return resp_headers
583-
584-
585-def put_object(url, token, container, name, contents, content_length=None,
586- etag=None, chunk_size=65536, content_type=None, headers=None,
587- http_conn=None):
588- """
589- Put an object
590-
591- :param url: storage URL
592- :param token: auth token
593- :param container: container name that the object is in
594- :param name: object name to put
595- :param contents: a string or a file like object to read object data from
596- :param content_length: value to send as content-length header; also limits
597- the amount read from contents
598- :param etag: etag of contents
599- :param chunk_size: chunk size of data to write
600- :param content_type: value to send as content-type header
601- :param headers: additional headers to include in the request
602- :param http_conn: HTTP connection object (If None, it will create the
603- conn object)
604- :returns: etag from server response
605- :raises ClientException: HTTP PUT request failed
606- """
607- if http_conn:
608- parsed, conn = http_conn
609- else:
610- parsed, conn = http_connection(url)
611- path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
612- if not headers:
613- headers = {}
614- headers['X-Auth-Token'] = token
615- if etag:
616- headers['ETag'] = etag.strip('"')
617- if content_length is not None:
618- headers['Content-Length'] = str(content_length)
619- if content_type is not None:
620- headers['Content-Type'] = content_type
621- if not contents:
622- headers['Content-Length'] = '0'
623- if hasattr(contents, 'read'):
624- conn.putrequest('PUT', path)
625- for header, value in headers.iteritems():
626- conn.putheader(header, value)
627- if content_length is None:
628- conn.putheader('Transfer-Encoding', 'chunked')
629- conn.endheaders()
630- chunk = contents.read(chunk_size)
631- while chunk:
632- conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
633- chunk = contents.read(chunk_size)
634- conn.send('0\r\n\r\n')
635- else:
636- conn.endheaders()
637- left = content_length
638- while left > 0:
639- size = chunk_size
640- if size > left:
641- size = left
642- chunk = contents.read(size)
643- conn.send(chunk)
644- left -= len(chunk)
645- else:
646- conn.request('PUT', path, contents, headers)
647- resp = conn.getresponse()
648- resp.read()
649- if resp.status < 200 or resp.status >= 300:
650- raise ClientException('Object PUT failed', http_scheme=parsed.scheme,
651- http_host=conn.host, http_port=conn.port, http_path=path,
652- http_status=resp.status, http_reason=resp.reason)
653- return resp.getheader('etag').strip('"')
654-
655-
656-def post_object(url, token, container, name, headers, http_conn=None):
657- """
658- Update object metadata
659-
660- :param url: storage URL
661- :param token: auth token
662- :param container: container name that the object is in
663- :param name: name of the object to update
664- :param headers: additional headers to include in the request
665- :param http_conn: HTTP connection object (If None, it will create the
666- conn object)
667- :raises ClientException: HTTP POST request failed
668- """
669- if http_conn:
670- parsed, conn = http_conn
671- else:
672- parsed, conn = http_connection(url)
673- path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
674- headers['X-Auth-Token'] = token
675- conn.request('POST', path, '', headers)
676- resp = conn.getresponse()
677- resp.read()
678- if resp.status < 200 or resp.status >= 300:
679- raise ClientException('Object POST failed', http_scheme=parsed.scheme,
680- http_host=conn.host, http_port=conn.port, http_path=path,
681- http_status=resp.status, http_reason=resp.reason)
682-
683-
684-def delete_object(url, token, container, name, http_conn=None):
685- """
686- Delete object
687-
688- :param url: storage URL
689- :param token: auth token
690- :param container: container name that the object is in
691- :param name: object name to delete
692- :param http_conn: HTTP connection object (If None, it will create the
693- conn object)
694- :raises ClientException: HTTP DELETE request failed
695- """
696- if http_conn:
697- parsed, conn = http_conn
698- else:
699- parsed, conn = http_connection(url)
700- path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
701- conn.request('DELETE', path, '', {'X-Auth-Token': token})
702- resp = conn.getresponse()
703- resp.read()
704- if resp.status < 200 or resp.status >= 300:
705- raise ClientException('Object DELETE failed',
706- http_scheme=parsed.scheme, http_host=conn.host,
707- http_port=conn.port, http_path=path, http_status=resp.status,
708- http_reason=resp.reason)
709-
710-
711-class Connection(object):
712- """Convenience class to make requests that will also retry the request"""
713-
714- def __init__(self, authurl, user, key, retries=5, preauthurl=None,
715- preauthtoken=None, snet=False, starting_backoff=1):
716- """
717- :param authurl: authenitcation URL
718- :param user: user name to authenticate as
719- :param key: key/password to authenticate with
720- :param retries: Number of times to retry the request before failing
721- :param preauthurl: storage URL (if you have already authenticated)
722- :param preauthtoken: authentication token (if you have already
723- authenticated)
724- :param snet: use SERVICENET internal network default is False
725- """
726- self.authurl = authurl
727- self.user = user
728- self.key = key
729- self.retries = retries
730- self.http_conn = None
731- self.url = preauthurl
732- self.token = preauthtoken
733- self.attempts = 0
734- self.snet = snet
735- self.starting_backoff = starting_backoff
736-
737- def get_auth(self):
738- return get_auth(self.authurl, self.user, self.key, snet=self.snet)
739-
740- def http_connection(self):
741- return http_connection(self.url)
742-
743- def _retry(self, reset_func, func, *args, **kwargs):
744- self.attempts = 0
745- backoff = self.starting_backoff
746- while self.attempts <= self.retries:
747- self.attempts += 1
748- try:
749- if not self.url or not self.token:
750- self.url, self.token = self.get_auth()
751- self.http_conn = None
752- if not self.http_conn:
753- self.http_conn = self.http_connection()
754- kwargs['http_conn'] = self.http_conn
755- rv = func(self.url, self.token, *args, **kwargs)
756- return rv
757- except (socket.error, HTTPException):
758- if self.attempts > self.retries:
759- raise
760- self.http_conn = None
761- except ClientException, err:
762- if self.attempts > self.retries:
763- raise
764- if err.http_status == 401:
765- self.url = self.token = None
766- if self.attempts > 1:
767- raise
768- elif err.http_status == 408:
769- self.http_conn = None
770- elif 500 <= err.http_status <= 599:
771- pass
772- else:
773- raise
774- sleep(backoff)
775- backoff *= 2
776- if reset_func:
777- reset_func(func, *args, **kwargs)
778-
779- def head_account(self):
780- """Wrapper for :func:`head_account`"""
781- return self._retry(None, head_account)
782-
783- def get_account(self, marker=None, limit=None, prefix=None,
784- full_listing=False):
785- """Wrapper for :func:`get_account`"""
786- # TODO(unknown): With full_listing=True this will restart the entire
787- # listing with each retry. Need to make a better version that just
788- # retries where it left off.
789- return self._retry(None, get_account, marker=marker, limit=limit,
790- prefix=prefix, full_listing=full_listing)
791-
792- def post_account(self, headers):
793- """Wrapper for :func:`post_account`"""
794- return self._retry(None, post_account, headers)
795-
796- def head_container(self, container):
797- """Wrapper for :func:`head_container`"""
798- return self._retry(None, head_container, container)
799-
800- def get_container(self, container, marker=None, limit=None, prefix=None,
801- delimiter=None, full_listing=False):
802- """Wrapper for :func:`get_container`"""
803- # TODO(unknown): With full_listing=True this will restart the entire
804- # listing with each retry. Need to make a better version that just
805- # retries where it left off.
806- return self._retry(None, get_container, container, marker=marker,
807- limit=limit, prefix=prefix, delimiter=delimiter,
808- full_listing=full_listing)
809-
810- def put_container(self, container, headers=None):
811- """Wrapper for :func:`put_container`"""
812- return self._retry(None, put_container, container, headers=headers)
813-
814- def post_container(self, container, headers):
815- """Wrapper for :func:`post_container`"""
816- return self._retry(None, post_container, container, headers)
817-
818- def delete_container(self, container):
819- """Wrapper for :func:`delete_container`"""
820- return self._retry(None, delete_container, container)
821-
822- def head_object(self, container, obj):
823- """Wrapper for :func:`head_object`"""
824- return self._retry(None, head_object, container, obj)
825-
826- def get_object(self, container, obj, resp_chunk_size=None):
827- """Wrapper for :func:`get_object`"""
828- return self._retry(None, get_object, container, obj,
829- resp_chunk_size=resp_chunk_size)
830-
831- def put_object(self, container, obj, contents, content_length=None,
832- etag=None, chunk_size=65536, content_type=None,
833- headers=None):
834- """Wrapper for :func:`put_object`"""
835-
836- def _default_reset(*args, **kwargs):
837- raise ClientException('put_object(%r, %r, ...) failure and no '
838- 'ability to reset contents for reupload.' % (container, obj))
839-
840- reset_func = _default_reset
841- tell = getattr(contents, 'tell', None)
842- seek = getattr(contents, 'seek', None)
843- if tell and seek:
844- orig_pos = tell()
845- reset_func = lambda *a, **k: seek(orig_pos)
846- elif not contents:
847- reset_func = lambda *a, **k: None
848-
849- return self._retry(reset_func, put_object, container, obj, contents,
850- content_length=content_length, etag=etag, chunk_size=chunk_size,
851- content_type=content_type, headers=headers)
852-
853- def post_object(self, container, obj, headers):
854- """Wrapper for :func:`post_object`"""
855- return self._retry(None, post_object, container, obj, headers)
856-
857- def delete_object(self, container, obj):
858- """Wrapper for :func:`delete_object`"""
859- return self._retry(None, delete_object, container, obj)
860-
861-# End inclusion of swift.common.client
862-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
863-
864-
865-def mkdirs(path):
866- try:
867- makedirs(path)
868- except OSError, err:
869- if err.errno != EEXIST:
870- raise
871-
872-
873-def put_errors_from_threads(threads, error_queue):
874- """
875- Places any errors from the threads into error_queue.
876- :param threads: A list of QueueFunctionThread instances.
877- :param error_queue: A queue to put error strings into.
878- :returns: True if any errors were found.
879- """
880- was_error = False
881- for thread in threads:
882- for info in thread.exc_infos:
883- was_error = True
884- if isinstance(info[1], ClientException):
885- error_queue.put(str(info[1]))
886- else:
887- error_queue.put(''.join(format_exception(*info)))
888- return was_error
889-
890-
891-class QueueFunctionThread(Thread):
892-
893- def __init__(self, queue, func, *args, **kwargs):
894- """ Calls func for each item in queue; func is called with a queued
895- item as the first arg followed by *args and **kwargs. Use the abort
896- attribute to have the thread empty the queue (without processing)
897- and exit. """
898- Thread.__init__(self)
899- self.abort = False
900- self.queue = queue
901- self.func = func
902- self.args = args
903- self.kwargs = kwargs
904- self.exc_infos = []
905-
906- def run(self):
907- try:
908- while True:
909- try:
910- item = self.queue.get_nowait()
911- if not self.abort:
912- self.func(item, *self.args, **self.kwargs)
913- self.queue.task_done()
914- except Empty:
915- if self.abort:
916- break
917- sleep(0.01)
918- except Exception:
919- self.exc_infos.append(exc_info())
920-
921-
922-st_delete_help = '''
923-delete --all OR delete container [--leave-segments] [object] [object] ...
924- Deletes everything in the account (with --all), or everything in a
925- container, or a list of objects depending on the args given. Segments of
926- manifest objects will be deleted as well, unless you specify the
927- --leave-segments option.'''.strip('\n')
928-
929-
930-def st_delete(parser, args, print_queue, error_queue):
931- parser.add_option('-a', '--all', action='store_true', dest='yes_all',
932- default=False, help='Indicates that you really want to delete '
933- 'everything in the account')
934- parser.add_option('', '--leave-segments', action='store_true',
935- dest='leave_segments', default=False, help='Indicates that you want '
936- 'the segments of manifest objects left alone')
937- (options, args) = parse_args(parser, args)
938- args = args[1:]
939- if (not args and not options.yes_all) or (args and options.yes_all):
940- error_queue.put('Usage: %s [options] %s' %
941- (basename(argv[0]), st_delete_help))
942- return
943-
944- def _delete_segment((container, obj), conn):
945- conn.delete_object(container, obj)
946- if options.verbose:
947- if conn.attempts > 2:
948- print_queue.put('%s/%s [after %d attempts]' %
949- (container, obj, conn.attempts))
950- else:
951- print_queue.put('%s/%s' % (container, obj))
952-
953- object_queue = Queue(10000)
954-
955- def _delete_object((container, obj), conn):
956- try:
957- old_manifest = None
958- if not options.leave_segments:
959- try:
960- old_manifest = conn.head_object(container, obj).get(
961- 'x-object-manifest')
962- except ClientException, err:
963- if err.http_status != 404:
964- raise
965- conn.delete_object(container, obj)
966- if old_manifest:
967- segment_queue = Queue(10000)
968- scontainer, sprefix = old_manifest.split('/', 1)
969- for delobj in conn.get_container(scontainer,
970- prefix=sprefix)[1]:
971- segment_queue.put((scontainer, delobj['name']))
972- if not segment_queue.empty():
973- segment_threads = [QueueFunctionThread(segment_queue,
974- _delete_segment, create_connection()) for _junk in
975- xrange(10)]
976- for thread in segment_threads:
977- thread.start()
978- while not segment_queue.empty():
979- sleep(0.01)
980- for thread in segment_threads:
981- thread.abort = True
982- while thread.isAlive():
983- thread.join(0.01)
984- put_errors_from_threads(segment_threads, error_queue)
985- if options.verbose:
986- path = options.yes_all and join(container, obj) or obj
987- if path[:1] in ('/', '\\'):
988- path = path[1:]
989- if conn.attempts > 1:
990- print_queue.put('%s [after %d attempts]' %
991- (path, conn.attempts))
992- else:
993- print_queue.put(path)
994- except ClientException, err:
995- if err.http_status != 404:
996- raise
997- error_queue.put('Object %s not found' %
998- repr('%s/%s' % (container, obj)))
999-
1000- container_queue = Queue(10000)
1001-
1002- def _delete_container(container, conn):
1003- try:
1004- marker = ''
1005- while True:
1006- objects = [o['name'] for o in
1007- conn.get_container(container, marker=marker)[1]]
1008- if not objects:
1009- break
1010- for obj in objects:
1011- object_queue.put((container, obj))
1012- marker = objects[-1]
1013- while not object_queue.empty():
1014- sleep(0.01)
1015- attempts = 1
1016- while True:
1017- try:
1018- conn.delete_container(container)
1019- break
1020- except ClientException, err:
1021- if err.http_status != 409:
1022- raise
1023- if attempts > 10:
1024- raise
1025- attempts += 1
1026- sleep(1)
1027- except ClientException, err:
1028- if err.http_status != 404:
1029- raise
1030- error_queue.put('Container %s not found' % repr(container))
1031-
1032- url, token = get_auth(options.auth, options.user, options.key,
1033- snet=options.snet)
1034- create_connection = lambda: Connection(options.auth, options.user,
1035- options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
1036- object_threads = [QueueFunctionThread(object_queue, _delete_object,
1037- create_connection()) for _junk in xrange(10)]
1038- for thread in object_threads:
1039- thread.start()
1040- container_threads = [QueueFunctionThread(container_queue,
1041- _delete_container, create_connection()) for _junk in xrange(10)]
1042- for thread in container_threads:
1043- thread.start()
1044- if not args:
1045- conn = create_connection()
1046- try:
1047- marker = ''
1048- while True:
1049- containers = \
1050- [c['name'] for c in conn.get_account(marker=marker)[1]]
1051- if not containers:
1052- break
1053- for container in containers:
1054- container_queue.put(container)
1055- marker = containers[-1]
1056- while not container_queue.empty():
1057- sleep(0.01)
1058- while not object_queue.empty():
1059- sleep(0.01)
1060- except ClientException, err:
1061- if err.http_status != 404:
1062- raise
1063- error_queue.put('Account not found')
1064- elif len(args) == 1:
1065- if '/' in args[0]:
1066- print >> stderr, 'WARNING: / in container name; you might have ' \
1067- 'meant %r instead of %r.' % \
1068- (args[0].replace('/', ' ', 1), args[0])
1069- conn = create_connection()
1070- _delete_container(args[0], conn)
1071- else:
1072- for obj in args[1:]:
1073- object_queue.put((args[0], obj))
1074- while not container_queue.empty():
1075- sleep(0.01)
1076- for thread in container_threads:
1077- thread.abort = True
1078- while thread.isAlive():
1079- thread.join(0.01)
1080- put_errors_from_threads(container_threads, error_queue)
1081- while not object_queue.empty():
1082- sleep(0.01)
1083- for thread in object_threads:
1084- thread.abort = True
1085- while thread.isAlive():
1086- thread.join(0.01)
1087- put_errors_from_threads(object_threads, error_queue)
1088-
1089-
1090-st_download_help = '''
1091-download --all OR download container [options] [object] [object] ...
1092- Downloads everything in the account (with --all), or everything in a
1093- container, or a list of objects depending on the args given. For a single
1094- object download, you may use the -o [--output] <filename> option to
1095- redirect the output to a specific file or if "-" then just redirect to
1096- stdout.'''.strip('\n')
1097-
1098-
1099-def st_download(options, args, print_queue, error_queue):
1100- parser.add_option('-a', '--all', action='store_true', dest='yes_all',
1101- default=False, help='Indicates that you really want to download '
1102- 'everything in the account')
1103- parser.add_option('-o', '--output', dest='out_file', help='For a single '
1104- 'file download, stream the output to an alternate location ')
1105- (options, args) = parse_args(parser, args)
1106- args = args[1:]
1107- if options.out_file == '-':
1108- options.verbose = 0
1109- if options.out_file and len(args) != 2:
1110- exit('-o option only allowed for single file downloads')
1111- if (not args and not options.yes_all) or (args and options.yes_all):
1112- error_queue.put('Usage: %s [options] %s' %
1113- (basename(argv[0]), st_download_help))
1114- return
1115-
1116- object_queue = Queue(10000)
1117-
1118- def _download_object(queue_arg, conn):
1119- if len(queue_arg) == 2:
1120- container, obj = queue_arg
1121- out_file = None
1122- elif len(queue_arg) == 3:
1123- container, obj, out_file = queue_arg
1124- else:
1125- raise Exception("Invalid queue_arg length of %s" % len(queue_arg))
1126- try:
1127- headers, body = \
1128- conn.get_object(container, obj, resp_chunk_size=65536)
1129- content_type = headers.get('content-type')
1130- if 'content-length' in headers:
1131- content_length = int(headers.get('content-length'))
1132- else:
1133- content_length = None
1134- etag = headers.get('etag')
1135- path = options.yes_all and join(container, obj) or obj
1136- if path[:1] in ('/', '\\'):
1137- path = path[1:]
1138- md5sum = None
1139- make_dir = out_file != "-"
1140- if content_type.split(';', 1)[0] == 'text/directory':
1141- if make_dir and not isdir(path):
1142- mkdirs(path)
1143- read_length = 0
1144- if 'x-object-manifest' not in headers:
1145- md5sum = md5()
1146- for chunk in body:
1147- read_length += len(chunk)
1148- if md5sum:
1149- md5sum.update(chunk)
1150- else:
1151- dirpath = dirname(path)
1152- if make_dir and dirpath and not isdir(dirpath):
1153- mkdirs(dirpath)
1154- if out_file == "-":
1155- fp = stdout
1156- elif out_file:
1157- fp = open(out_file, 'wb')
1158- else:
1159- fp = open(path, 'wb')
1160- read_length = 0
1161- if 'x-object-manifest' not in headers:
1162- md5sum = md5()
1163- for chunk in body:
1164- fp.write(chunk)
1165- read_length += len(chunk)
1166- if md5sum:
1167- md5sum.update(chunk)
1168- fp.close()
1169- if md5sum and md5sum.hexdigest() != etag:
1170- error_queue.put('%s: md5sum != etag, %s != %s' %
1171- (path, md5sum.hexdigest(), etag))
1172- if content_length is not None and read_length != content_length:
1173- error_queue.put('%s: read_length != content_length, %d != %d' %
1174- (path, read_length, content_length))
1175- if 'x-object-meta-mtime' in headers and not options.out_file:
1176- mtime = float(headers['x-object-meta-mtime'])
1177- utime(path, (mtime, mtime))
1178- if options.verbose:
1179- if conn.attempts > 1:
1180- print_queue.put('%s [after %d attempts' %
1181- (path, conn.attempts))
1182- else:
1183- print_queue.put(path)
1184- except ClientException, err:
1185- if err.http_status != 404:
1186- raise
1187- error_queue.put('Object %s not found' %
1188- repr('%s/%s' % (container, obj)))
1189-
1190- container_queue = Queue(10000)
1191-
1192- def _download_container(container, conn):
1193- try:
1194- marker = ''
1195- while True:
1196- objects = [o['name'] for o in
1197- conn.get_container(container, marker=marker)[1]]
1198- if not objects:
1199- break
1200- for obj in objects:
1201- object_queue.put((container, obj))
1202- marker = objects[-1]
1203- except ClientException, err:
1204- if err.http_status != 404:
1205- raise
1206- error_queue.put('Container %s not found' % repr(container))
1207-
1208- url, token = get_auth(options.auth, options.user, options.key,
1209- snet=options.snet)
1210- create_connection = lambda: Connection(options.auth, options.user,
1211- options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
1212- object_threads = [QueueFunctionThread(object_queue, _download_object,
1213- create_connection()) for _junk in xrange(10)]
1214- for thread in object_threads:
1215- thread.start()
1216- container_threads = [QueueFunctionThread(container_queue,
1217- _download_container, create_connection()) for _junk in xrange(10)]
1218- for thread in container_threads:
1219- thread.start()
1220- if not args:
1221- conn = create_connection()
1222- try:
1223- marker = ''
1224- while True:
1225- containers = [c['name']
1226- for c in conn.get_account(marker=marker)[1]]
1227- if not containers:
1228- break
1229- for container in containers:
1230- container_queue.put(container)
1231- marker = containers[-1]
1232- except ClientException, err:
1233- if err.http_status != 404:
1234- raise
1235- error_queue.put('Account not found')
1236- elif len(args) == 1:
1237- if '/' in args[0]:
1238- print >> stderr, 'WARNING: / in container name; you might have ' \
1239- 'meant %r instead of %r.' % \
1240- (args[0].replace('/', ' ', 1), args[0])
1241- _download_container(args[0], create_connection())
1242- else:
1243- if len(args) == 2:
1244- obj = args[1]
1245- object_queue.put((args[0], obj, options.out_file))
1246- else:
1247- for obj in args[1:]:
1248- object_queue.put((args[0], obj))
1249- while not container_queue.empty():
1250- sleep(0.01)
1251- for thread in container_threads:
1252- thread.abort = True
1253- while thread.isAlive():
1254- thread.join(0.01)
1255- put_errors_from_threads(container_threads, error_queue)
1256- while not object_queue.empty():
1257- sleep(0.01)
1258- for thread in object_threads:
1259- thread.abort = True
1260- while thread.isAlive():
1261- thread.join(0.01)
1262- put_errors_from_threads(object_threads, error_queue)
1263-
1264-
1265-st_list_help = '''
1266-list [options] [container]
1267- Lists the containers for the account or the objects for a container. -p or
1268- --prefix is an option that will only list items beginning with that prefix.
1269- -d or --delimiter is option (for container listings only) that will roll up
1270- items with the given delimiter (see Cloud Files general documentation for
1271- what this means).
1272-'''.strip('\n')
1273-
1274-
1275-def st_list(options, args, print_queue, error_queue):
1276- parser.add_option('-p', '--prefix', dest='prefix', help='Will only list '
1277- 'items beginning with the prefix')
1278- parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll '
1279- 'up items with the given delimiter (see Cloud Files general '
1280- 'documentation for what this means)')
1281- (options, args) = parse_args(parser, args)
1282- args = args[1:]
1283- if options.delimiter and not args:
1284- exit('-d option only allowed for container listings')
1285- if len(args) > 1:
1286- error_queue.put('Usage: %s [options] %s' %
1287- (basename(argv[0]), st_list_help))
1288- return
1289- conn = Connection(options.auth, options.user, options.key,
1290- snet=options.snet)
1291- try:
1292- marker = ''
1293- while True:
1294- if not args:
1295- items = \
1296- conn.get_account(marker=marker, prefix=options.prefix)[1]
1297- else:
1298- items = conn.get_container(args[0], marker=marker,
1299- prefix=options.prefix, delimiter=options.delimiter)[1]
1300- if not items:
1301- break
1302- for item in items:
1303- print_queue.put(item.get('name', item.get('subdir')))
1304- marker = items[-1].get('name', items[-1].get('subdir'))
1305- except ClientException, err:
1306- if err.http_status != 404:
1307- raise
1308- if not args:
1309- error_queue.put('Account not found')
1310- else:
1311- error_queue.put('Container %s not found' % repr(args[0]))
1312-
1313-
1314-st_stat_help = '''
1315-stat [container] [object]
1316- Displays information for the account, container, or object depending on the
1317- args given (if any).'''.strip('\n')
1318-
1319-
1320-def st_stat(options, args, print_queue, error_queue):
1321- (options, args) = parse_args(parser, args)
1322- args = args[1:]
1323- conn = Connection(options.auth, options.user, options.key)
1324- if not args:
1325- try:
1326- headers = conn.head_account()
1327- if options.verbose > 1:
1328- print_queue.put('''
1329-StorageURL: %s
1330-Auth Token: %s
1331-'''.strip('\n') % (conn.url, conn.token))
1332- container_count = int(headers.get('x-account-container-count', 0))
1333- object_count = int(headers.get('x-account-object-count', 0))
1334- bytes_used = int(headers.get('x-account-bytes-used', 0))
1335- print_queue.put('''
1336- Account: %s
1337-Containers: %d
1338- Objects: %d
1339- Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count,
1340- object_count, bytes_used))
1341- for key, value in headers.items():
1342- if key.startswith('x-account-meta-'):
1343- print_queue.put('%10s: %s' % ('Meta %s' %
1344- key[len('x-account-meta-'):].title(), value))
1345- for key, value in headers.items():
1346- if not key.startswith('x-account-meta-') and key not in (
1347- 'content-length', 'date', 'x-account-container-count',
1348- 'x-account-object-count', 'x-account-bytes-used'):
1349- print_queue.put(
1350- '%10s: %s' % (key.title(), value))
1351- except ClientException, err:
1352- if err.http_status != 404:
1353- raise
1354- error_queue.put('Account not found')
1355- elif len(args) == 1:
1356- if '/' in args[0]:
1357- print >> stderr, 'WARNING: / in container name; you might have ' \
1358- 'meant %r instead of %r.' % \
1359- (args[0].replace('/', ' ', 1), args[0])
1360- try:
1361- headers = conn.head_container(args[0])
1362- object_count = int(headers.get('x-container-object-count', 0))
1363- bytes_used = int(headers.get('x-container-bytes-used', 0))
1364- print_queue.put('''
1365- Account: %s
1366-Container: %s
1367- Objects: %d
1368- Bytes: %d
1369- Read ACL: %s
1370-Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
1371- object_count, bytes_used,
1372- headers.get('x-container-read', ''),
1373- headers.get('x-container-write', '')))
1374- for key, value in headers.items():
1375- if key.startswith('x-container-meta-'):
1376- print_queue.put('%9s: %s' % ('Meta %s' %
1377- key[len('x-container-meta-'):].title(), value))
1378- for key, value in headers.items():
1379- if not key.startswith('x-container-meta-') and key not in (
1380- 'content-length', 'date', 'x-container-object-count',
1381- 'x-container-bytes-used', 'x-container-read',
1382- 'x-container-write'):
1383- print_queue.put(
1384- '%9s: %s' % (key.title(), value))
1385- except ClientException, err:
1386- if err.http_status != 404:
1387- raise
1388- error_queue.put('Container %s not found' % repr(args[0]))
1389- elif len(args) == 2:
1390- try:
1391- headers = conn.head_object(args[0], args[1])
1392- print_queue.put('''
1393- Account: %s
1394- Container: %s
1395- Object: %s
1396- Content Type: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
1397- args[1], headers.get('content-type')))
1398- if 'content-length' in headers:
1399- print_queue.put('Content Length: %s' %
1400- headers['content-length'])
1401- if 'last-modified' in headers:
1402- print_queue.put(' Last Modified: %s' %
1403- headers['last-modified'])
1404- if 'etag' in headers:
1405- print_queue.put(' ETag: %s' % headers['etag'])
1406- if 'x-object-manifest' in headers:
1407- print_queue.put(' Manifest: %s' %
1408- headers['x-object-manifest'])
1409- for key, value in headers.items():
1410- if key.startswith('x-object-meta-'):
1411- print_queue.put('%14s: %s' % ('Meta %s' %
1412- key[len('x-object-meta-'):].title(), value))
1413- for key, value in headers.items():
1414- if not key.startswith('x-object-meta-') and key not in (
1415- 'content-type', 'content-length', 'last-modified',
1416- 'etag', 'date', 'x-object-manifest'):
1417- print_queue.put(
1418- '%14s: %s' % (key.title(), value))
1419- except ClientException, err:
1420- if err.http_status != 404:
1421- raise
1422- error_queue.put('Object %s not found' %
1423- repr('%s/%s' % (args[0], args[1])))
1424- else:
1425- error_queue.put('Usage: %s [options] %s' %
1426- (basename(argv[0]), st_stat_help))
1427-
1428-
1429-st_post_help = '''
1430-post [options] [container] [object]
1431- Updates meta information for the account, container, or object depending on
1432- the args given. If the container is not found, it will be created
1433- automatically; but this is not true for accounts and objects. Containers
1434- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
1435- or --meta option is allowed on all and used to define the user meta data
1436- items to set in the form Name:Value. This option can be repeated. Example:
1437- post -m Color:Blue -m Size:Large'''.strip('\n')
1438-
1439-
1440-def st_post(options, args, print_queue, error_queue):
1441- parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the '
1442- 'Read ACL for containers. Quick summary of ACL syntax: .r:*, '
1443- '.r:-.example.com, .r:www.example.com, account1, account2:user2')
1444- parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the '
1445- 'Write ACL for containers. Quick summary of ACL syntax: account1, '
1446- 'account2:user2')
1447- parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
1448- help='Sets a meta data item with the syntax name:value. This option '
1449- 'may be repeated. Example: -m Color:Blue -m Size:Large')
1450- (options, args) = parse_args(parser, args)
1451- args = args[1:]
1452- if (options.read_acl or options.write_acl) and not args:
1453- exit('-r and -w options only allowed for containers')
1454- conn = Connection(options.auth, options.user, options.key)
1455- if not args:
1456- headers = {}
1457- for item in options.meta:
1458- split_item = item.split(':')
1459- headers['X-Account-Meta-' + split_item[0]] = \
1460- len(split_item) > 1 and split_item[1]
1461- try:
1462- conn.post_account(headers=headers)
1463- except ClientException, err:
1464- if err.http_status != 404:
1465- raise
1466- error_queue.put('Account not found')
1467- elif len(args) == 1:
1468- if '/' in args[0]:
1469- print >> stderr, 'WARNING: / in container name; you might have ' \
1470- 'meant %r instead of %r.' % \
1471- (args[0].replace('/', ' ', 1), args[0])
1472- headers = {}
1473- for item in options.meta:
1474- split_item = item.split(':')
1475- headers['X-Container-Meta-' + split_item[0]] = \
1476- len(split_item) > 1 and split_item[1]
1477- if options.read_acl is not None:
1478- headers['X-Container-Read'] = options.read_acl
1479- if options.write_acl is not None:
1480- headers['X-Container-Write'] = options.write_acl
1481- try:
1482- conn.post_container(args[0], headers=headers)
1483- except ClientException, err:
1484- if err.http_status != 404:
1485- raise
1486- conn.put_container(args[0], headers=headers)
1487- elif len(args) == 2:
1488- headers = {}
1489- for item in options.meta:
1490- split_item = item.split(':')
1491- headers['X-Object-Meta-' + split_item[0]] = \
1492- len(split_item) > 1 and split_item[1]
1493- try:
1494- conn.post_object(args[0], args[1], headers=headers)
1495- except ClientException, err:
1496- if err.http_status != 404:
1497- raise
1498- error_queue.put('Object %s not found' %
1499- repr('%s/%s' % (args[0], args[1])))
1500- else:
1501- error_queue.put('Usage: %s [options] %s' %
1502- (basename(argv[0]), st_post_help))
1503-
1504-
1505-st_upload_help = '''
1506-upload [options] container file_or_directory [file_or_directory] [...]
1507- Uploads to the given container the files and directories specified by the
1508- remaining args. -c or --changed is an option that will only upload files
1509- that have changed since the last upload. -S <size> or --segment-size <size>
1510- and --leave-segments are options as well (see --help for more).
1511-'''.strip('\n')
1512-
1513-
1514-def st_upload(options, args, print_queue, error_queue):
1515- parser.add_option('-c', '--changed', action='store_true', dest='changed',
1516- default=False, help='Will only upload files that have changed since '
1517- 'the last upload')
1518- parser.add_option('-S', '--segment-size', dest='segment_size', help='Will '
1519- 'upload files in segments no larger than <size> and then create a '
1520- '"manifest" file that will download all the segments as if it were '
1521- 'the original file. The segments will be uploaded to a '
1522- '<container>_segments container so as to not pollute the main '
1523- '<container> listings.')
1524- parser.add_option('', '--leave-segments', action='store_true',
1525- dest='leave_segments', default=False, help='Indicates that you want '
1526- 'the older segments of manifest objects left alone (in the case of '
1527- 'overwrites)')
1528- (options, args) = parse_args(parser, args)
1529- args = args[1:]
1530- if len(args) < 2:
1531- error_queue.put('Usage: %s [options] %s' %
1532- (basename(argv[0]), st_upload_help))
1533- return
1534- object_queue = Queue(10000)
1535-
1536- def _segment_job(job, conn):
1537- if job.get('delete', False):
1538- conn.delete_object(job['container'], job['obj'])
1539- else:
1540- fp = open(job['path'], 'rb')
1541- fp.seek(job['segment_start'])
1542- conn.put_object(job.get('container', args[0] + '_segments'),
1543- job['obj'], fp, content_length=job['segment_size'])
1544- if options.verbose and 'log_line' in job:
1545- if conn.attempts > 1:
1546- print_queue.put('%s [after %d attempts]' %
1547- (job['log_line'], conn.attempts))
1548- else:
1549- print_queue.put(job['log_line'])
1550-
1551- def _object_job(job, conn):
1552- path = job['path']
1553- container = job.get('container', args[0])
1554- dir_marker = job.get('dir_marker', False)
1555- try:
1556- obj = path
1557- if obj.startswith('./') or obj.startswith('.\\'):
1558- obj = obj[2:]
1559- put_headers = {'x-object-meta-mtime': str(getmtime(path))}
1560- if dir_marker:
1561- if options.changed:
1562- try:
1563- headers = conn.head_object(container, obj)
1564- ct = headers.get('content-type')
1565- cl = int(headers.get('content-length'))
1566- et = headers.get('etag')
1567- mt = headers.get('x-object-meta-mtime')
1568- if ct.split(';', 1)[0] == 'text/directory' and \
1569- cl == 0 and \
1570- et == 'd41d8cd98f00b204e9800998ecf8427e' and \
1571- mt == put_headers['x-object-meta-mtime']:
1572- return
1573- except ClientException, err:
1574- if err.http_status != 404:
1575- raise
1576- conn.put_object(container, obj, '', content_length=0,
1577- content_type='text/directory',
1578- headers=put_headers)
1579- else:
1580- # We need to HEAD all objects now in case we're overwriting a
1581- # manifest object and need to delete the old segments
1582- # ourselves.
1583- old_manifest = None
1584- if options.changed or not options.leave_segments:
1585- try:
1586- headers = conn.head_object(container, obj)
1587- cl = int(headers.get('content-length'))
1588- mt = headers.get('x-object-meta-mtime')
1589- if options.changed and cl == getsize(path) and \
1590- mt == put_headers['x-object-meta-mtime']:
1591- return
1592- if not options.leave_segments:
1593- old_manifest = headers.get('x-object-manifest')
1594- except ClientException, err:
1595- if err.http_status != 404:
1596- raise
1597- if options.segment_size and \
1598- getsize(path) < options.segment_size:
1599- full_size = getsize(path)
1600- segment_queue = Queue(10000)
1601- segment_threads = [QueueFunctionThread(segment_queue,
1602- _segment_job, create_connection()) for _junk in
1603- xrange(10)]
1604- for thread in segment_threads:
1605- thread.start()
1606- segment = 0
1607- segment_start = 0
1608- while segment_start < full_size:
1609- segment_size = int(options.segment_size)
1610- if segment_start + segment_size > full_size:
1611- segment_size = full_size - segment_start
1612- segment_queue.put({'path': path,
1613- 'obj': '%s/%s/%s/%08d' % (obj,
1614- put_headers['x-object-meta-mtime'], full_size,
1615- segment),
1616- 'segment_start': segment_start,
1617- 'segment_size': segment_size,
1618- 'log_line': '%s segment %s' % (obj, segment)})
1619- segment += 1
1620- segment_start += segment_size
1621- while not segment_queue.empty():
1622- sleep(0.01)
1623- for thread in segment_threads:
1624- thread.abort = True
1625- while thread.isAlive():
1626- thread.join(0.01)
1627- if put_errors_from_threads(segment_threads, error_queue):
1628- raise ClientException('Aborting manifest creation '
1629- 'because not all segments could be uploaded. %s/%s'
1630- % (container, obj))
1631- new_object_manifest = '%s_segments/%s/%s/%s/' % (
1632- container, obj, put_headers['x-object-meta-mtime'],
1633- full_size)
1634- if old_manifest == new_object_manifest:
1635- old_manifest = None
1636- put_headers['x-object-manifest'] = new_object_manifest
1637- conn.put_object(container, obj, '', content_length=0,
1638- headers=put_headers)
1639- else:
1640- conn.put_object(container, obj, open(path, 'rb'),
1641- content_length=getsize(path), headers=put_headers)
1642- if old_manifest:
1643- segment_queue = Queue(10000)
1644- scontainer, sprefix = old_manifest.split('/', 1)
1645- for delobj in conn.get_container(scontainer,
1646- prefix=sprefix)[1]:
1647- segment_queue.put({'delete': True,
1648- 'container': scontainer, 'obj': delobj['name']})
1649- if not segment_queue.empty():
1650- segment_threads = [QueueFunctionThread(segment_queue,
1651- _segment_job, create_connection()) for _junk in
1652- xrange(10)]
1653- for thread in segment_threads:
1654- thread.start()
1655- while not segment_queue.empty():
1656- sleep(0.01)
1657- for thread in segment_threads:
1658- thread.abort = True
1659- while thread.isAlive():
1660- thread.join(0.01)
1661- put_errors_from_threads(segment_threads, error_queue)
1662- if options.verbose:
1663- if conn.attempts > 1:
1664- print_queue.put(
1665- '%s [after %d attempts]' % (obj, conn.attempts))
1666- else:
1667- print_queue.put(obj)
1668- except OSError, err:
1669- if err.errno != ENOENT:
1670- raise
1671- error_queue.put('Local file %s not found' % repr(path))
1672-
1673- def _upload_dir(path):
1674- names = listdir(path)
1675- if not names:
1676- object_queue.put({'path': path, 'dir_marker': True})
1677- else:
1678- for name in listdir(path):
1679- subpath = join(path, name)
1680- if isdir(subpath):
1681- _upload_dir(subpath)
1682- else:
1683- object_queue.put({'path': subpath})
1684-
1685- url, token = get_auth(options.auth, options.user, options.key,
1686- snet=options.snet)
1687- create_connection = lambda: Connection(options.auth, options.user,
1688- options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
1689- object_threads = [QueueFunctionThread(object_queue, _object_job,
1690- create_connection()) for _junk in xrange(10)]
1691- for thread in object_threads:
1692- thread.start()
1693- conn = create_connection()
1694- # Try to create the container, just in case it doesn't exist. If this
1695- # fails, it might just be because the user doesn't have container PUT
1696- # permissions, so we'll ignore any error. If there's really a problem,
1697- # it'll surface on the first object PUT.
1698- try:
1699- conn.put_container(args[0])
1700- if options.segment_size is not None:
1701- conn.put_container(args[0] + '_segments')
1702- except Exception:
1703- pass
1704- try:
1705- for arg in args[1:]:
1706- if isdir(arg):
1707- _upload_dir(arg)
1708- else:
1709- object_queue.put({'path': arg})
1710- while not object_queue.empty():
1711- sleep(0.01)
1712- for thread in object_threads:
1713- thread.abort = True
1714- while thread.isAlive():
1715- thread.join(0.01)
1716- put_errors_from_threads(object_threads, error_queue)
1717- except ClientException, err:
1718- if err.http_status != 404:
1719- raise
1720- error_queue.put('Account not found')
1721-
1722-
1723-def parse_args(parser, args, enforce_requires=True):
1724- if not args:
1725- args = ['-h']
1726- (options, args) = parser.parse_args(args)
1727- if enforce_requires and \
1728- not (options.auth and options.user and options.key):
1729- exit('''
1730-Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or
1731-overridden with -A, -U, or -K.'''.strip('\n'))
1732- return options, args
1733-
1734-
1735-if __name__ == '__main__':
1736- parser = OptionParser(version='%prog 1.0', usage='''
1737-Usage: %%prog <command> [options] [args]
1738-
1739-Commands:
1740- %(st_stat_help)s
1741- %(st_list_help)s
1742- %(st_upload_help)s
1743- %(st_post_help)s
1744- %(st_download_help)s
1745- %(st_delete_help)s
1746-
1747-Example:
1748- %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K key stat
1749-'''.strip('\n') % globals())
1750- parser.add_option('-s', '--snet', action='store_true', dest='snet',
1751- default=False, help='Use SERVICENET internal network')
1752- parser.add_option('-v', '--verbose', action='count', dest='verbose',
1753- default=1, help='Print more info')
1754- parser.add_option('-q', '--quiet', action='store_const', dest='verbose',
1755- const=0, default=1, help='Suppress status output')
1756- parser.add_option('-A', '--auth', dest='auth',
1757- default=environ.get('ST_AUTH'),
1758- help='URL for obtaining an auth token')
1759- parser.add_option('-U', '--user', dest='user',
1760- default=environ.get('ST_USER'),
1761- help='User name for obtaining an auth token')
1762- parser.add_option('-K', '--key', dest='key',
1763- default=environ.get('ST_KEY'),
1764- help='Key for obtaining an auth token')
1765- parser.disable_interspersed_args()
1766- (options, args) = parse_args(parser, argv[1:], enforce_requires=False)
1767- parser.enable_interspersed_args()
1768-
1769- commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
1770- if not args or args[0] not in commands:
1771- parser.print_usage()
1772- if args:
1773- exit('no such command: %s' % args[0])
1774- exit()
1775-
1776- print_queue = Queue(10000)
1777-
1778- def _print(item):
1779- if isinstance(item, unicode):
1780- item = item.encode('utf8')
1781- print item
1782-
1783- print_thread = QueueFunctionThread(print_queue, _print)
1784- print_thread.start()
1785-
1786- error_queue = Queue(10000)
1787-
1788- def _error(item):
1789- if isinstance(item, unicode):
1790- item = item.encode('utf8')
1791- print >> stderr, item
1792-
1793- error_thread = QueueFunctionThread(error_queue, _error)
1794- error_thread.start()
1795-
1796- try:
1797- parser.usage = globals()['st_%s_help' % args[0]]
1798- try:
1799- globals()['st_%s' % args[0]](parser, argv[1:], print_queue,
1800- error_queue)
1801- except (ClientException, HTTPException, socket.error), err:
1802- error_queue.put(str(err))
1803- while not print_queue.empty():
1804- sleep(0.01)
1805- print_thread.abort = True
1806- while print_thread.isAlive():
1807- print_thread.join(0.01)
1808- while not error_queue.empty():
1809- sleep(0.01)
1810- error_thread.abort = True
1811- while error_thread.isAlive():
1812- error_thread.join(0.01)
1813- except (SystemExit, Exception):
1814- for thread in threading_enumerate():
1815- thread.abort = True
1816- raise
1817
1818=== added file 'bin/swift'
1819--- bin/swift 1970-01-01 00:00:00 +0000
1820+++ bin/swift 2011-06-14 16:07:28 +0000
1821@@ -0,0 +1,1812 @@
1822+#!/usr/bin/python -u
1823+# Copyright (c) 2010-2011 OpenStack, LLC.
1824+#
1825+# Licensed under the Apache License, Version 2.0 (the "License");
1826+# you may not use this file except in compliance with the License.
1827+# You may obtain a copy of the License at
1828+#
1829+# http://www.apache.org/licenses/LICENSE-2.0
1830+#
1831+# Unless required by applicable law or agreed to in writing, software
1832+# distributed under the License is distributed on an "AS IS" BASIS,
1833+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
1834+# implied.
1835+# See the License for the specific language governing permissions and
1836+# limitations under the License.
1837+
1838+from errno import EEXIST, ENOENT
1839+from hashlib import md5
1840+from optparse import OptionParser
1841+from os import environ, listdir, makedirs, utime
1842+from os.path import basename, dirname, getmtime, getsize, isdir, join
1843+from Queue import Empty, Queue
1844+from sys import argv, exc_info, exit, stderr, stdout
1845+from threading import enumerate as threading_enumerate, Thread
1846+from time import sleep
1847+from traceback import format_exception
1848+
1849+
1850+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
1851+# Inclusion of swift.common.client for convenience of single file distribution
1852+
1853+import socket
1854+from cStringIO import StringIO
1855+from re import compile, DOTALL
1856+from tokenize import generate_tokens, STRING, NAME, OP
1857+from urllib import quote as _quote, unquote
1858+from urlparse import urlparse, urlunparse
1859+
1860+try:
1861+ from eventlet.green.httplib import HTTPException, HTTPSConnection
1862+except ImportError:
1863+ from httplib import HTTPException, HTTPSConnection
1864+
1865+try:
1866+ from eventlet import sleep
1867+except ImportError:
1868+ from time import sleep
1869+
1870+try:
1871+ from swift.common.bufferedhttp \
1872+ import BufferedHTTPConnection as HTTPConnection
1873+except ImportError:
1874+ try:
1875+ from eventlet.green.httplib import HTTPConnection
1876+ except ImportError:
1877+ from httplib import HTTPConnection
1878+
1879+
1880+def quote(value, safe='/'):
1881+ """
1882+ Patched version of urllib.quote that encodes utf8 strings before quoting
1883+ """
1884+ if isinstance(value, unicode):
1885+ value = value.encode('utf8')
1886+ return _quote(value, safe)
1887+
1888+
1889+# look for a real json parser first
1890+try:
1891+ # simplejson is popular and pretty good
1892+ from simplejson import loads as json_loads
1893+except ImportError:
1894+ try:
1895+ # 2.6 will have a json module in the stdlib
1896+ from json import loads as json_loads
1897+ except ImportError:
1898+ # fall back on local parser otherwise
1899+ comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL)
1900+
1901+ def json_loads(string):
1902+ '''
1903+ Fairly competent json parser exploiting the python tokenizer and
1904+ eval(). -- From python-cloudfiles
1905+
1906+ _loads(serialized_json) -> object
1907+ '''
1908+ try:
1909+ res = []
1910+ consts = {'true': True, 'false': False, 'null': None}
1911+ string = '(' + comments.sub('', string) + ')'
1912+ for type, val, _junk, _junk, _junk in \
1913+ generate_tokens(StringIO(string).readline):
1914+ if (type == OP and val not in '[]{}:,()-') or \
1915+ (type == NAME and val not in consts):
1916+ raise AttributeError()
1917+ elif type == STRING:
1918+ res.append('u')
1919+ res.append(val.replace('\\/', '/'))
1920+ else:
1921+ res.append(val)
1922+ return eval(''.join(res), {}, consts)
1923+ except Exception:
1924+ raise AttributeError()
1925+
1926+
1927+class ClientException(Exception):
1928+
1929+ def __init__(self, msg, http_scheme='', http_host='', http_port='',
1930+ http_path='', http_query='', http_status=0, http_reason='',
1931+ http_device=''):
1932+ Exception.__init__(self, msg)
1933+ self.msg = msg
1934+ self.http_scheme = http_scheme
1935+ self.http_host = http_host
1936+ self.http_port = http_port
1937+ self.http_path = http_path
1938+ self.http_query = http_query
1939+ self.http_status = http_status
1940+ self.http_reason = http_reason
1941+ self.http_device = http_device
1942+
1943+ def __str__(self):
1944+ a = self.msg
1945+ b = ''
1946+ if self.http_scheme:
1947+ b += '%s://' % self.http_scheme
1948+ if self.http_host:
1949+ b += self.http_host
1950+ if self.http_port:
1951+ b += ':%s' % self.http_port
1952+ if self.http_path:
1953+ b += self.http_path
1954+ if self.http_query:
1955+ b += '?%s' % self.http_query
1956+ if self.http_status:
1957+ if b:
1958+ b = '%s %s' % (b, self.http_status)
1959+ else:
1960+ b = str(self.http_status)
1961+ if self.http_reason:
1962+ if b:
1963+ b = '%s %s' % (b, self.http_reason)
1964+ else:
1965+ b = '- %s' % self.http_reason
1966+ if self.http_device:
1967+ if b:
1968+ b = '%s: device %s' % (b, self.http_device)
1969+ else:
1970+ b = 'device %s' % self.http_device
1971+ return b and '%s: %s' % (a, b) or a
1972+
1973+
1974+def http_connection(url):
1975+ """
1976+ Make an HTTPConnection or HTTPSConnection
1977+
1978+ :param url: url to connect to
1979+ :returns: tuple of (parsed url, connection object)
1980+ :raises ClientException: Unable to handle protocol scheme
1981+ """
1982+ parsed = urlparse(url)
1983+ if parsed.scheme == 'http':
1984+ conn = HTTPConnection(parsed.netloc)
1985+ elif parsed.scheme == 'https':
1986+ conn = HTTPSConnection(parsed.netloc)
1987+ else:
1988+ raise ClientException('Cannot handle protocol scheme %s for url %s' %
1989+ (parsed.scheme, repr(url)))
1990+ return parsed, conn
1991+
1992+
1993+def get_auth(url, user, key, snet=False):
1994+ """
1995+ Get authentication/authorization credentials.
1996+
1997+ The snet parameter is used for Rackspace's ServiceNet internal network
1998+ implementation. In this function, it simply adds *snet-* to the beginning
1999+ of the host name for the returned storage URL. With Rackspace Cloud Files,
2000+ use of this network path causes no bandwidth charges but requires the
2001+ client to be running on Rackspace's ServiceNet network.
2002+
2003+ :param url: authentication/authorization URL
2004+ :param user: user to authenticate as
2005+ :param key: key or password for authorization
2006+ :param snet: use SERVICENET internal network (see above), default is False
2007+ :returns: tuple of (storage URL, auth token)
2008+ :raises ClientException: HTTP GET request to auth URL failed
2009+ """
2010+ parsed, conn = http_connection(url)
2011+ conn.request('GET', parsed.path, '',
2012+ {'X-Auth-User': user, 'X-Auth-Key': key})
2013+ resp = conn.getresponse()
2014+ resp.read()
2015+ if resp.status < 200 or resp.status >= 300:
2016+ raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
2017+ http_host=conn.host, http_port=conn.port,
2018+ http_path=parsed.path, http_status=resp.status,
2019+ http_reason=resp.reason)
2020+ url = resp.getheader('x-storage-url')
2021+ if snet:
2022+ parsed = list(urlparse(url))
2023+ # Second item in the list is the netloc
2024+ parsed[1] = 'snet-' + parsed[1]
2025+ url = urlunparse(parsed)
2026+ return url, resp.getheader('x-storage-token',
2027+ resp.getheader('x-auth-token'))
2028+
2029+
2030+def get_account(url, token, marker=None, limit=None, prefix=None,
2031+ http_conn=None, full_listing=False):
2032+ """
2033+ Get a listing of containers for the account.
2034+
2035+ :param url: storage URL
2036+ :param token: auth token
2037+ :param marker: marker query
2038+ :param limit: limit query
2039+ :param prefix: prefix query
2040+ :param http_conn: HTTP connection object (If None, it will create the
2041+ conn object)
2042+ :param full_listing: if True, return a full listing, else returns a max
2043+ of 10000 listings
2044+ :returns: a tuple of (response headers, a list of containers) The response
2045+ headers will be a dict and all header names will be lowercase.
2046+ :raises ClientException: HTTP GET request failed
2047+ """
2048+ if not http_conn:
2049+ http_conn = http_connection(url)
2050+ if full_listing:
2051+ rv = get_account(url, token, marker, limit, prefix, http_conn)
2052+ listing = rv[1]
2053+ while listing:
2054+ marker = listing[-1]['name']
2055+ listing = \
2056+ get_account(url, token, marker, limit, prefix, http_conn)[1]
2057+ if listing:
2058+ rv[1].extend(listing)
2059+ return rv
2060+ parsed, conn = http_conn
2061+ qs = 'format=json'
2062+ if marker:
2063+ qs += '&marker=%s' % quote(marker)
2064+ if limit:
2065+ qs += '&limit=%d' % limit
2066+ if prefix:
2067+ qs += '&prefix=%s' % quote(prefix)
2068+ conn.request('GET', '%s?%s' % (parsed.path, qs), '',
2069+ {'X-Auth-Token': token})
2070+ resp = conn.getresponse()
2071+ resp_headers = {}
2072+ for header, value in resp.getheaders():
2073+ resp_headers[header.lower()] = value
2074+ if resp.status < 200 or resp.status >= 300:
2075+ resp.read()
2076+ raise ClientException('Account GET failed', http_scheme=parsed.scheme,
2077+ http_host=conn.host, http_port=conn.port,
2078+ http_path=parsed.path, http_query=qs, http_status=resp.status,
2079+ http_reason=resp.reason)
2080+ if resp.status == 204:
2081+ resp.read()
2082+ return resp_headers, []
2083+ return resp_headers, json_loads(resp.read())
2084+
2085+
2086+def head_account(url, token, http_conn=None):
2087+ """
2088+ Get account stats.
2089+
2090+ :param url: storage URL
2091+ :param token: auth token
2092+ :param http_conn: HTTP connection object (If None, it will create the
2093+ conn object)
2094+ :returns: a dict containing the response's headers (all header names will
2095+ be lowercase)
2096+ :raises ClientException: HTTP HEAD request failed
2097+ """
2098+ if http_conn:
2099+ parsed, conn = http_conn
2100+ else:
2101+ parsed, conn = http_connection(url)
2102+ conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
2103+ resp = conn.getresponse()
2104+ resp.read()
2105+ if resp.status < 200 or resp.status >= 300:
2106+ raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
2107+ http_host=conn.host, http_port=conn.port,
2108+ http_path=parsed.path, http_status=resp.status,
2109+ http_reason=resp.reason)
2110+ resp_headers = {}
2111+ for header, value in resp.getheaders():
2112+ resp_headers[header.lower()] = value
2113+ return resp_headers
2114+
2115+
2116+def post_account(url, token, headers, http_conn=None):
2117+ """
2118+ Update an account's metadata.
2119+
2120+ :param url: storage URL
2121+ :param token: auth token
2122+ :param headers: additional headers to include in the request
2123+ :param http_conn: HTTP connection object (If None, it will create the
2124+ conn object)
2125+ :raises ClientException: HTTP POST request failed
2126+ """
2127+ if http_conn:
2128+ parsed, conn = http_conn
2129+ else:
2130+ parsed, conn = http_connection(url)
2131+ headers['X-Auth-Token'] = token
2132+ conn.request('POST', parsed.path, '', headers)
2133+ resp = conn.getresponse()
2134+ resp.read()
2135+ if resp.status < 200 or resp.status >= 300:
2136+ raise ClientException('Account POST failed',
2137+ http_scheme=parsed.scheme, http_host=conn.host,
2138+ http_port=conn.port, http_path=path, http_status=resp.status,
2139+ http_reason=resp.reason)
2140+
2141+
2142+def get_container(url, token, container, marker=None, limit=None,
2143+ prefix=None, delimiter=None, http_conn=None,
2144+ full_listing=False):
2145+ """
2146+ Get a listing of objects for the container.
2147+
2148+ :param url: storage URL
2149+ :param token: auth token
2150+ :param container: container name to get a listing for
2151+ :param marker: marker query
2152+ :param limit: limit query
2153+ :param prefix: prefix query
2154+ :param delimeter: string to delimit the queries on
2155+ :param http_conn: HTTP connection object (If None, it will create the
2156+ conn object)
2157+ :param full_listing: if True, return a full listing, else returns a max
2158+ of 10000 listings
2159+ :returns: a tuple of (response headers, a list of objects) The response
2160+ headers will be a dict and all header names will be lowercase.
2161+ :raises ClientException: HTTP GET request failed
2162+ """
2163+ if not http_conn:
2164+ http_conn = http_connection(url)
2165+ if full_listing:
2166+ rv = get_container(url, token, container, marker, limit, prefix,
2167+ delimiter, http_conn)
2168+ listing = rv[1]
2169+ while listing:
2170+ if not delimiter:
2171+ marker = listing[-1]['name']
2172+ else:
2173+ marker = listing[-1].get('name', listing[-1].get('subdir'))
2174+ listing = get_container(url, token, container, marker, limit,
2175+ prefix, delimiter, http_conn)[1]
2176+ if listing:
2177+ rv[1].extend(listing)
2178+ return rv
2179+ parsed, conn = http_conn
2180+ path = '%s/%s' % (parsed.path, quote(container))
2181+ qs = 'format=json'
2182+ if marker:
2183+ qs += '&marker=%s' % quote(marker)
2184+ if limit:
2185+ qs += '&limit=%d' % limit
2186+ if prefix:
2187+ qs += '&prefix=%s' % quote(prefix)
2188+ if delimiter:
2189+ qs += '&delimiter=%s' % quote(delimiter)
2190+ conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token})
2191+ resp = conn.getresponse()
2192+ if resp.status < 200 or resp.status >= 300:
2193+ resp.read()
2194+ raise ClientException('Container GET failed',
2195+ http_scheme=parsed.scheme, http_host=conn.host,
2196+ http_port=conn.port, http_path=path, http_query=qs,
2197+ http_status=resp.status, http_reason=resp.reason)
2198+ resp_headers = {}
2199+ for header, value in resp.getheaders():
2200+ resp_headers[header.lower()] = value
2201+ if resp.status == 204:
2202+ resp.read()
2203+ return resp_headers, []
2204+ return resp_headers, json_loads(resp.read())
2205+
2206+
2207+def head_container(url, token, container, http_conn=None):
2208+ """
2209+ Get container stats.
2210+
2211+ :param url: storage URL
2212+ :param token: auth token
2213+ :param container: container name to get stats for
2214+ :param http_conn: HTTP connection object (If None, it will create the
2215+ conn object)
2216+ :returns: a dict containing the response's headers (all header names will
2217+ be lowercase)
2218+ :raises ClientException: HTTP HEAD request failed
2219+ """
2220+ if http_conn:
2221+ parsed, conn = http_conn
2222+ else:
2223+ parsed, conn = http_connection(url)
2224+ path = '%s/%s' % (parsed.path, quote(container))
2225+ conn.request('HEAD', path, '', {'X-Auth-Token': token})
2226+ resp = conn.getresponse()
2227+ resp.read()
2228+ if resp.status < 200 or resp.status >= 300:
2229+ raise ClientException('Container HEAD failed',
2230+ http_scheme=parsed.scheme, http_host=conn.host,
2231+ http_port=conn.port, http_path=path, http_status=resp.status,
2232+ http_reason=resp.reason)
2233+ resp_headers = {}
2234+ for header, value in resp.getheaders():
2235+ resp_headers[header.lower()] = value
2236+ return resp_headers
2237+
2238+
2239+def put_container(url, token, container, headers=None, http_conn=None):
2240+ """
2241+ Create a container
2242+
2243+ :param url: storage URL
2244+ :param token: auth token
2245+ :param container: container name to create
2246+ :param headers: additional headers to include in the request
2247+ :param http_conn: HTTP connection object (If None, it will create the
2248+ conn object)
2249+ :raises ClientException: HTTP PUT request failed
2250+ """
2251+ if http_conn:
2252+ parsed, conn = http_conn
2253+ else:
2254+ parsed, conn = http_connection(url)
2255+ path = '%s/%s' % (parsed.path, quote(container))
2256+ if not headers:
2257+ headers = {}
2258+ headers['X-Auth-Token'] = token
2259+ conn.request('PUT', path, '', headers)
2260+ resp = conn.getresponse()
2261+ resp.read()
2262+ if resp.status < 200 or resp.status >= 300:
2263+ raise ClientException('Container PUT failed',
2264+ http_scheme=parsed.scheme, http_host=conn.host,
2265+ http_port=conn.port, http_path=path, http_status=resp.status,
2266+ http_reason=resp.reason)
2267+
2268+
2269+def post_container(url, token, container, headers, http_conn=None):
2270+ """
2271+ Update a container's metadata.
2272+
2273+ :param url: storage URL
2274+ :param token: auth token
2275+ :param container: container name to update
2276+ :param headers: additional headers to include in the request
2277+ :param http_conn: HTTP connection object (If None, it will create the
2278+ conn object)
2279+ :raises ClientException: HTTP POST request failed
2280+ """
2281+ if http_conn:
2282+ parsed, conn = http_conn
2283+ else:
2284+ parsed, conn = http_connection(url)
2285+ path = '%s/%s' % (parsed.path, quote(container))
2286+ headers['X-Auth-Token'] = token
2287+ conn.request('POST', path, '', headers)
2288+ resp = conn.getresponse()
2289+ resp.read()
2290+ if resp.status < 200 or resp.status >= 300:
2291+ raise ClientException('Container POST failed',
2292+ http_scheme=parsed.scheme, http_host=conn.host,
2293+ http_port=conn.port, http_path=path, http_status=resp.status,
2294+ http_reason=resp.reason)
2295+
2296+
2297+def delete_container(url, token, container, http_conn=None):
2298+ """
2299+ Delete a container
2300+
2301+ :param url: storage URL
2302+ :param token: auth token
2303+ :param container: container name to delete
2304+ :param http_conn: HTTP connection object (If None, it will create the
2305+ conn object)
2306+ :raises ClientException: HTTP DELETE request failed
2307+ """
2308+ if http_conn:
2309+ parsed, conn = http_conn
2310+ else:
2311+ parsed, conn = http_connection(url)
2312+ path = '%s/%s' % (parsed.path, quote(container))
2313+ conn.request('DELETE', path, '', {'X-Auth-Token': token})
2314+ resp = conn.getresponse()
2315+ resp.read()
2316+ if resp.status < 200 or resp.status >= 300:
2317+ raise ClientException('Container DELETE failed',
2318+ http_scheme=parsed.scheme, http_host=conn.host,
2319+ http_port=conn.port, http_path=path, http_status=resp.status,
2320+ http_reason=resp.reason)
2321+
2322+
2323+def get_object(url, token, container, name, http_conn=None,
2324+ resp_chunk_size=None):
2325+ """
2326+ Get an object
2327+
2328+ :param url: storage URL
2329+ :param token: auth token
2330+ :param container: container name that the object is in
2331+ :param name: object name to get
2332+ :param http_conn: HTTP connection object (If None, it will create the
2333+ conn object)
2334+ :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
2335+ you specify a resp_chunk_size you must fully read
2336+ the object's contents before making another
2337+ request.
2338+ :returns: a tuple of (response headers, the object's contents) The response
2339+ headers will be a dict and all header names will be lowercase.
2340+ :raises ClientException: HTTP GET request failed
2341+ """
2342+ if http_conn:
2343+ parsed, conn = http_conn
2344+ else:
2345+ parsed, conn = http_connection(url)
2346+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
2347+ conn.request('GET', path, '', {'X-Auth-Token': token})
2348+ resp = conn.getresponse()
2349+ if resp.status < 200 or resp.status >= 300:
2350+ resp.read()
2351+ raise ClientException('Object GET failed', http_scheme=parsed.scheme,
2352+ http_host=conn.host, http_port=conn.port, http_path=path,
2353+ http_status=resp.status, http_reason=resp.reason)
2354+ if resp_chunk_size:
2355+
2356+ def _object_body():
2357+ buf = resp.read(resp_chunk_size)
2358+ while buf:
2359+ yield buf
2360+ buf = resp.read(resp_chunk_size)
2361+ object_body = _object_body()
2362+ else:
2363+ object_body = resp.read()
2364+ resp_headers = {}
2365+ for header, value in resp.getheaders():
2366+ resp_headers[header.lower()] = value
2367+ return resp_headers, object_body
2368+
2369+
2370+def head_object(url, token, container, name, http_conn=None):
2371+ """
2372+ Get object info
2373+
2374+ :param url: storage URL
2375+ :param token: auth token
2376+ :param container: container name that the object is in
2377+ :param name: object name to get info for
2378+ :param http_conn: HTTP connection object (If None, it will create the
2379+ conn object)
2380+ :returns: a dict containing the response's headers (all header names will
2381+ be lowercase)
2382+ :raises ClientException: HTTP HEAD request failed
2383+ """
2384+ if http_conn:
2385+ parsed, conn = http_conn
2386+ else:
2387+ parsed, conn = http_connection(url)
2388+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
2389+ conn.request('HEAD', path, '', {'X-Auth-Token': token})
2390+ resp = conn.getresponse()
2391+ resp.read()
2392+ if resp.status < 200 or resp.status >= 300:
2393+ raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
2394+ http_host=conn.host, http_port=conn.port, http_path=path,
2395+ http_status=resp.status, http_reason=resp.reason)
2396+ resp_headers = {}
2397+ for header, value in resp.getheaders():
2398+ resp_headers[header.lower()] = value
2399+ return resp_headers
2400+
2401+
2402+def put_object(url, token, container, name, contents, content_length=None,
2403+ etag=None, chunk_size=65536, content_type=None, headers=None,
2404+ http_conn=None):
2405+ """
2406+ Put an object
2407+
2408+ :param url: storage URL
2409+ :param token: auth token
2410+ :param container: container name that the object is in
2411+ :param name: object name to put
2412+ :param contents: a string or a file like object to read object data from
2413+ :param content_length: value to send as content-length header; also limits
2414+ the amount read from contents
2415+ :param etag: etag of contents
2416+ :param chunk_size: chunk size of data to write
2417+ :param content_type: value to send as content-type header
2418+ :param headers: additional headers to include in the request
2419+ :param http_conn: HTTP connection object (If None, it will create the
2420+ conn object)
2421+ :returns: etag from server response
2422+ :raises ClientException: HTTP PUT request failed
2423+ """
2424+ if http_conn:
2425+ parsed, conn = http_conn
2426+ else:
2427+ parsed, conn = http_connection(url)
2428+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
2429+ if not headers:
2430+ headers = {}
2431+ headers['X-Auth-Token'] = token
2432+ if etag:
2433+ headers['ETag'] = etag.strip('"')
2434+ if content_length is not None:
2435+ headers['Content-Length'] = str(content_length)
2436+ if content_type is not None:
2437+ headers['Content-Type'] = content_type
2438+ if not contents:
2439+ headers['Content-Length'] = '0'
2440+ if hasattr(contents, 'read'):
2441+ conn.putrequest('PUT', path)
2442+ for header, value in headers.iteritems():
2443+ conn.putheader(header, value)
2444+ if content_length is None:
2445+ conn.putheader('Transfer-Encoding', 'chunked')
2446+ conn.endheaders()
2447+ chunk = contents.read(chunk_size)
2448+ while chunk:
2449+ conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
2450+ chunk = contents.read(chunk_size)
2451+ conn.send('0\r\n\r\n')
2452+ else:
2453+ conn.endheaders()
2454+ left = content_length
2455+ while left > 0:
2456+ size = chunk_size
2457+ if size > left:
2458+ size = left
2459+ chunk = contents.read(size)
2460+ conn.send(chunk)
2461+ left -= len(chunk)
2462+ else:
2463+ conn.request('PUT', path, contents, headers)
2464+ resp = conn.getresponse()
2465+ resp.read()
2466+ if resp.status < 200 or resp.status >= 300:
2467+ raise ClientException('Object PUT failed', http_scheme=parsed.scheme,
2468+ http_host=conn.host, http_port=conn.port, http_path=path,
2469+ http_status=resp.status, http_reason=resp.reason)
2470+ return resp.getheader('etag').strip('"')
2471+
2472+
2473+def post_object(url, token, container, name, headers, http_conn=None):
2474+ """
2475+ Update object metadata
2476+
2477+ :param url: storage URL
2478+ :param token: auth token
2479+ :param container: container name that the object is in
2480+ :param name: name of the object to update
2481+ :param headers: additional headers to include in the request
2482+ :param http_conn: HTTP connection object (If None, it will create the
2483+ conn object)
2484+ :raises ClientException: HTTP POST request failed
2485+ """
2486+ if http_conn:
2487+ parsed, conn = http_conn
2488+ else:
2489+ parsed, conn = http_connection(url)
2490+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
2491+ headers['X-Auth-Token'] = token
2492+ conn.request('POST', path, '', headers)
2493+ resp = conn.getresponse()
2494+ resp.read()
2495+ if resp.status < 200 or resp.status >= 300:
2496+ raise ClientException('Object POST failed', http_scheme=parsed.scheme,
2497+ http_host=conn.host, http_port=conn.port, http_path=path,
2498+ http_status=resp.status, http_reason=resp.reason)
2499+
2500+
2501+def delete_object(url, token, container, name, http_conn=None):
2502+ """
2503+ Delete object
2504+
2505+ :param url: storage URL
2506+ :param token: auth token
2507+ :param container: container name that the object is in
2508+ :param name: object name to delete
2509+ :param http_conn: HTTP connection object (If None, it will create the
2510+ conn object)
2511+ :raises ClientException: HTTP DELETE request failed
2512+ """
2513+ if http_conn:
2514+ parsed, conn = http_conn
2515+ else:
2516+ parsed, conn = http_connection(url)
2517+ path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
2518+ conn.request('DELETE', path, '', {'X-Auth-Token': token})
2519+ resp = conn.getresponse()
2520+ resp.read()
2521+ if resp.status < 200 or resp.status >= 300:
2522+ raise ClientException('Object DELETE failed',
2523+ http_scheme=parsed.scheme, http_host=conn.host,
2524+ http_port=conn.port, http_path=path, http_status=resp.status,
2525+ http_reason=resp.reason)
2526+
2527+
2528+class Connection(object):
2529+ """Convenience class to make requests that will also retry the request"""
2530+
2531+ def __init__(self, authurl, user, key, retries=5, preauthurl=None,
2532+ preauthtoken=None, snet=False, starting_backoff=1):
2533+ """
2534+ :param authurl: authenitcation URL
2535+ :param user: user name to authenticate as
2536+ :param key: key/password to authenticate with
2537+ :param retries: Number of times to retry the request before failing
2538+ :param preauthurl: storage URL (if you have already authenticated)
2539+ :param preauthtoken: authentication token (if you have already
2540+ authenticated)
2541+ :param snet: use SERVICENET internal network default is False
2542+ """
2543+ self.authurl = authurl
2544+ self.user = user
2545+ self.key = key
2546+ self.retries = retries
2547+ self.http_conn = None
2548+ self.url = preauthurl
2549+ self.token = preauthtoken
2550+ self.attempts = 0
2551+ self.snet = snet
2552+ self.starting_backoff = starting_backoff
2553+
2554+ def get_auth(self):
2555+ return get_auth(self.authurl, self.user, self.key, snet=self.snet)
2556+
2557+ def http_connection(self):
2558+ return http_connection(self.url)
2559+
2560+ def _retry(self, reset_func, func, *args, **kwargs):
2561+ self.attempts = 0
2562+ backoff = self.starting_backoff
2563+ while self.attempts <= self.retries:
2564+ self.attempts += 1
2565+ try:
2566+ if not self.url or not self.token:
2567+ self.url, self.token = self.get_auth()
2568+ self.http_conn = None
2569+ if not self.http_conn:
2570+ self.http_conn = self.http_connection()
2571+ kwargs['http_conn'] = self.http_conn
2572+ rv = func(self.url, self.token, *args, **kwargs)
2573+ return rv
2574+ except (socket.error, HTTPException):
2575+ if self.attempts > self.retries:
2576+ raise
2577+ self.http_conn = None
2578+ except ClientException, err:
2579+ if self.attempts > self.retries:
2580+ raise
2581+ if err.http_status == 401:
2582+ self.url = self.token = None
2583+ if self.attempts > 1:
2584+ raise
2585+ elif err.http_status == 408:
2586+ self.http_conn = None
2587+ elif 500 <= err.http_status <= 599:
2588+ pass
2589+ else:
2590+ raise
2591+ sleep(backoff)
2592+ backoff *= 2
2593+ if reset_func:
2594+ reset_func(func, *args, **kwargs)
2595+
2596+ def head_account(self):
2597+ """Wrapper for :func:`head_account`"""
2598+ return self._retry(None, head_account)
2599+
2600+ def get_account(self, marker=None, limit=None, prefix=None,
2601+ full_listing=False):
2602+ """Wrapper for :func:`get_account`"""
2603+ # TODO(unknown): With full_listing=True this will restart the entire
2604+ # listing with each retry. Need to make a better version that just
2605+ # retries where it left off.
2606+ return self._retry(None, get_account, marker=marker, limit=limit,
2607+ prefix=prefix, full_listing=full_listing)
2608+
2609+ def post_account(self, headers):
2610+ """Wrapper for :func:`post_account`"""
2611+ return self._retry(None, post_account, headers)
2612+
2613+ def head_container(self, container):
2614+ """Wrapper for :func:`head_container`"""
2615+ return self._retry(None, head_container, container)
2616+
2617+ def get_container(self, container, marker=None, limit=None, prefix=None,
2618+ delimiter=None, full_listing=False):
2619+ """Wrapper for :func:`get_container`"""
2620+ # TODO(unknown): With full_listing=True this will restart the entire
2621+ # listing with each retry. Need to make a better version that just
2622+ # retries where it left off.
2623+ return self._retry(None, get_container, container, marker=marker,
2624+ limit=limit, prefix=prefix, delimiter=delimiter,
2625+ full_listing=full_listing)
2626+
2627+ def put_container(self, container, headers=None):
2628+ """Wrapper for :func:`put_container`"""
2629+ return self._retry(None, put_container, container, headers=headers)
2630+
2631+ def post_container(self, container, headers):
2632+ """Wrapper for :func:`post_container`"""
2633+ return self._retry(None, post_container, container, headers)
2634+
2635+ def delete_container(self, container):
2636+ """Wrapper for :func:`delete_container`"""
2637+ return self._retry(None, delete_container, container)
2638+
2639+ def head_object(self, container, obj):
2640+ """Wrapper for :func:`head_object`"""
2641+ return self._retry(None, head_object, container, obj)
2642+
2643+ def get_object(self, container, obj, resp_chunk_size=None):
2644+ """Wrapper for :func:`get_object`"""
2645+ return self._retry(None, get_object, container, obj,
2646+ resp_chunk_size=resp_chunk_size)
2647+
2648+ def put_object(self, container, obj, contents, content_length=None,
2649+ etag=None, chunk_size=65536, content_type=None,
2650+ headers=None):
2651+ """Wrapper for :func:`put_object`"""
2652+
2653+ def _default_reset(*args, **kwargs):
2654+ raise ClientException('put_object(%r, %r, ...) failure and no '
2655+ 'ability to reset contents for reupload.' % (container, obj))
2656+
2657+ reset_func = _default_reset
2658+ tell = getattr(contents, 'tell', None)
2659+ seek = getattr(contents, 'seek', None)
2660+ if tell and seek:
2661+ orig_pos = tell()
2662+ reset_func = lambda *a, **k: seek(orig_pos)
2663+ elif not contents:
2664+ reset_func = lambda *a, **k: None
2665+
2666+ return self._retry(reset_func, put_object, container, obj, contents,
2667+ content_length=content_length, etag=etag, chunk_size=chunk_size,
2668+ content_type=content_type, headers=headers)
2669+
2670+ def post_object(self, container, obj, headers):
2671+ """Wrapper for :func:`post_object`"""
2672+ return self._retry(None, post_object, container, obj, headers)
2673+
2674+ def delete_object(self, container, obj):
2675+ """Wrapper for :func:`delete_object`"""
2676+ return self._retry(None, delete_object, container, obj)
2677+
2678+# End inclusion of swift.common.client
2679+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
2680+
2681+
2682+def mkdirs(path):
2683+ try:
2684+ makedirs(path)
2685+ except OSError, err:
2686+ if err.errno != EEXIST:
2687+ raise
2688+
2689+
2690+def put_errors_from_threads(threads, error_queue):
2691+ """
2692+ Places any errors from the threads into error_queue.
2693+ :param threads: A list of QueueFunctionThread instances.
2694+ :param error_queue: A queue to put error strings into.
2695+ :returns: True if any errors were found.
2696+ """
2697+ was_error = False
2698+ for thread in threads:
2699+ for info in thread.exc_infos:
2700+ was_error = True
2701+ if isinstance(info[1], ClientException):
2702+ error_queue.put(str(info[1]))
2703+ else:
2704+ error_queue.put(''.join(format_exception(*info)))
2705+ return was_error
2706+
2707+
2708+class QueueFunctionThread(Thread):
2709+
2710+ def __init__(self, queue, func, *args, **kwargs):
2711+ """ Calls func for each item in queue; func is called with a queued
2712+ item as the first arg followed by *args and **kwargs. Use the abort
2713+ attribute to have the thread empty the queue (without processing)
2714+ and exit. """
2715+ Thread.__init__(self)
2716+ self.abort = False
2717+ self.queue = queue
2718+ self.func = func
2719+ self.args = args
2720+ self.kwargs = kwargs
2721+ self.exc_infos = []
2722+
2723+ def run(self):
2724+ try:
2725+ while True:
2726+ try:
2727+ item = self.queue.get_nowait()
2728+ if not self.abort:
2729+ self.func(item, *self.args, **self.kwargs)
2730+ self.queue.task_done()
2731+ except Empty:
2732+ if self.abort:
2733+ break
2734+ sleep(0.01)
2735+ except Exception:
2736+ self.exc_infos.append(exc_info())
2737+
2738+
2739+st_delete_help = '''
2740+delete --all OR delete container [--leave-segments] [object] [object] ...
2741+ Deletes everything in the account (with --all), or everything in a
2742+ container, or a list of objects depending on the args given. Segments of
2743+ manifest objects will be deleted as well, unless you specify the
2744+ --leave-segments option.'''.strip('\n')
2745+
2746+
2747+def st_delete(parser, args, print_queue, error_queue):
2748+ parser.add_option('-a', '--all', action='store_true', dest='yes_all',
2749+ default=False, help='Indicates that you really want to delete '
2750+ 'everything in the account')
2751+ parser.add_option('', '--leave-segments', action='store_true',
2752+ dest='leave_segments', default=False, help='Indicates that you want '
2753+ 'the segments of manifest objects left alone')
2754+ (options, args) = parse_args(parser, args)
2755+ args = args[1:]
2756+ if (not args and not options.yes_all) or (args and options.yes_all):
2757+ error_queue.put('Usage: %s [options] %s' %
2758+ (basename(argv[0]), st_delete_help))
2759+ return
2760+
2761+ def _delete_segment((container, obj), conn):
2762+ conn.delete_object(container, obj)
2763+ if options.verbose:
2764+ if conn.attempts > 2:
2765+ print_queue.put('%s/%s [after %d attempts]' %
2766+ (container, obj, conn.attempts))
2767+ else:
2768+ print_queue.put('%s/%s' % (container, obj))
2769+
2770+ object_queue = Queue(10000)
2771+
2772+ def _delete_object((container, obj), conn):
2773+ try:
2774+ old_manifest = None
2775+ if not options.leave_segments:
2776+ try:
2777+ old_manifest = conn.head_object(container, obj).get(
2778+ 'x-object-manifest')
2779+ except ClientException, err:
2780+ if err.http_status != 404:
2781+ raise
2782+ conn.delete_object(container, obj)
2783+ if old_manifest:
2784+ segment_queue = Queue(10000)
2785+ scontainer, sprefix = old_manifest.split('/', 1)
2786+ for delobj in conn.get_container(scontainer,
2787+ prefix=sprefix)[1]:
2788+ segment_queue.put((scontainer, delobj['name']))
2789+ if not segment_queue.empty():
2790+ segment_threads = [QueueFunctionThread(segment_queue,
2791+ _delete_segment, create_connection()) for _junk in
2792+ xrange(10)]
2793+ for thread in segment_threads:
2794+ thread.start()
2795+ while not segment_queue.empty():
2796+ sleep(0.01)
2797+ for thread in segment_threads:
2798+ thread.abort = True
2799+ while thread.isAlive():
2800+ thread.join(0.01)
2801+ put_errors_from_threads(segment_threads, error_queue)
2802+ if options.verbose:
2803+ path = options.yes_all and join(container, obj) or obj
2804+ if path[:1] in ('/', '\\'):
2805+ path = path[1:]
2806+ if conn.attempts > 1:
2807+ print_queue.put('%s [after %d attempts]' %
2808+ (path, conn.attempts))
2809+ else:
2810+ print_queue.put(path)
2811+ except ClientException, err:
2812+ if err.http_status != 404:
2813+ raise
2814+ error_queue.put('Object %s not found' %
2815+ repr('%s/%s' % (container, obj)))
2816+
2817+ container_queue = Queue(10000)
2818+
2819+ def _delete_container(container, conn):
2820+ try:
2821+ marker = ''
2822+ while True:
2823+ objects = [o['name'] for o in
2824+ conn.get_container(container, marker=marker)[1]]
2825+ if not objects:
2826+ break
2827+ for obj in objects:
2828+ object_queue.put((container, obj))
2829+ marker = objects[-1]
2830+ while not object_queue.empty():
2831+ sleep(0.01)
2832+ attempts = 1
2833+ while True:
2834+ try:
2835+ conn.delete_container(container)
2836+ break
2837+ except ClientException, err:
2838+ if err.http_status != 409:
2839+ raise
2840+ if attempts > 10:
2841+ raise
2842+ attempts += 1
2843+ sleep(1)
2844+ except ClientException, err:
2845+ if err.http_status != 404:
2846+ raise
2847+ error_queue.put('Container %s not found' % repr(container))
2848+
2849+ url, token = get_auth(options.auth, options.user, options.key,
2850+ snet=options.snet)
2851+ create_connection = lambda: Connection(options.auth, options.user,
2852+ options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
2853+ object_threads = [QueueFunctionThread(object_queue, _delete_object,
2854+ create_connection()) for _junk in xrange(10)]
2855+ for thread in object_threads:
2856+ thread.start()
2857+ container_threads = [QueueFunctionThread(container_queue,
2858+ _delete_container, create_connection()) for _junk in xrange(10)]
2859+ for thread in container_threads:
2860+ thread.start()
2861+ if not args:
2862+ conn = create_connection()
2863+ try:
2864+ marker = ''
2865+ while True:
2866+ containers = \
2867+ [c['name'] for c in conn.get_account(marker=marker)[1]]
2868+ if not containers:
2869+ break
2870+ for container in containers:
2871+ container_queue.put(container)
2872+ marker = containers[-1]
2873+ while not container_queue.empty():
2874+ sleep(0.01)
2875+ while not object_queue.empty():
2876+ sleep(0.01)
2877+ except ClientException, err:
2878+ if err.http_status != 404:
2879+ raise
2880+ error_queue.put('Account not found')
2881+ elif len(args) == 1:
2882+ if '/' in args[0]:
2883+ print >> stderr, 'WARNING: / in container name; you might have ' \
2884+ 'meant %r instead of %r.' % \
2885+ (args[0].replace('/', ' ', 1), args[0])
2886+ conn = create_connection()
2887+ _delete_container(args[0], conn)
2888+ else:
2889+ for obj in args[1:]:
2890+ object_queue.put((args[0], obj))
2891+ while not container_queue.empty():
2892+ sleep(0.01)
2893+ for thread in container_threads:
2894+ thread.abort = True
2895+ while thread.isAlive():
2896+ thread.join(0.01)
2897+ put_errors_from_threads(container_threads, error_queue)
2898+ while not object_queue.empty():
2899+ sleep(0.01)
2900+ for thread in object_threads:
2901+ thread.abort = True
2902+ while thread.isAlive():
2903+ thread.join(0.01)
2904+ put_errors_from_threads(object_threads, error_queue)
2905+
2906+
2907+st_download_help = '''
2908+download --all OR download container [options] [object] [object] ...
2909+ Downloads everything in the account (with --all), or everything in a
2910+ container, or a list of objects depending on the args given. For a single
2911+ object download, you may use the -o [--output] <filename> option to
2912+ redirect the output to a specific file or if "-" then just redirect to
2913+ stdout.'''.strip('\n')
2914+
2915+
2916+def st_download(options, args, print_queue, error_queue):
2917+ parser.add_option('-a', '--all', action='store_true', dest='yes_all',
2918+ default=False, help='Indicates that you really want to download '
2919+ 'everything in the account')
2920+ parser.add_option('-o', '--output', dest='out_file', help='For a single '
2921+ 'file download, stream the output to an alternate location ')
2922+ (options, args) = parse_args(parser, args)
2923+ args = args[1:]
2924+ if options.out_file == '-':
2925+ options.verbose = 0
2926+ if options.out_file and len(args) != 2:
2927+ exit('-o option only allowed for single file downloads')
2928+ if (not args and not options.yes_all) or (args and options.yes_all):
2929+ error_queue.put('Usage: %s [options] %s' %
2930+ (basename(argv[0]), st_download_help))
2931+ return
2932+
2933+ object_queue = Queue(10000)
2934+
2935+ def _download_object(queue_arg, conn):
2936+ if len(queue_arg) == 2:
2937+ container, obj = queue_arg
2938+ out_file = None
2939+ elif len(queue_arg) == 3:
2940+ container, obj, out_file = queue_arg
2941+ else:
2942+ raise Exception("Invalid queue_arg length of %s" % len(queue_arg))
2943+ try:
2944+ headers, body = \
2945+ conn.get_object(container, obj, resp_chunk_size=65536)
2946+ content_type = headers.get('content-type')
2947+ if 'content-length' in headers:
2948+ content_length = int(headers.get('content-length'))
2949+ else:
2950+ content_length = None
2951+ etag = headers.get('etag')
2952+ path = options.yes_all and join(container, obj) or obj
2953+ if path[:1] in ('/', '\\'):
2954+ path = path[1:]
2955+ md5sum = None
2956+ make_dir = out_file != "-"
2957+ if content_type.split(';', 1)[0] == 'text/directory':
2958+ if make_dir and not isdir(path):
2959+ mkdirs(path)
2960+ read_length = 0
2961+ if 'x-object-manifest' not in headers:
2962+ md5sum = md5()
2963+ for chunk in body:
2964+ read_length += len(chunk)
2965+ if md5sum:
2966+ md5sum.update(chunk)
2967+ else:
2968+ dirpath = dirname(path)
2969+ if make_dir and dirpath and not isdir(dirpath):
2970+ mkdirs(dirpath)
2971+ if out_file == "-":
2972+ fp = stdout
2973+ elif out_file:
2974+ fp = open(out_file, 'wb')
2975+ else:
2976+ fp = open(path, 'wb')
2977+ read_length = 0
2978+ if 'x-object-manifest' not in headers:
2979+ md5sum = md5()
2980+ for chunk in body:
2981+ fp.write(chunk)
2982+ read_length += len(chunk)
2983+ if md5sum:
2984+ md5sum.update(chunk)
2985+ fp.close()
2986+ if md5sum and md5sum.hexdigest() != etag:
2987+ error_queue.put('%s: md5sum != etag, %s != %s' %
2988+ (path, md5sum.hexdigest(), etag))
2989+ if content_length is not None and read_length != content_length:
2990+ error_queue.put('%s: read_length != content_length, %d != %d' %
2991+ (path, read_length, content_length))
2992+ if 'x-object-meta-mtime' in headers and not options.out_file:
2993+ mtime = float(headers['x-object-meta-mtime'])
2994+ utime(path, (mtime, mtime))
2995+ if options.verbose:
2996+ if conn.attempts > 1:
2997+ print_queue.put('%s [after %d attempts' %
2998+ (path, conn.attempts))
2999+ else:
3000+ print_queue.put(path)
3001+ except ClientException, err:
3002+ if err.http_status != 404:
3003+ raise
3004+ error_queue.put('Object %s not found' %
3005+ repr('%s/%s' % (container, obj)))
3006+
3007+ container_queue = Queue(10000)
3008+
3009+ def _download_container(container, conn):
3010+ try:
3011+ marker = ''
3012+ while True:
3013+ objects = [o['name'] for o in
3014+ conn.get_container(container, marker=marker)[1]]
3015+ if not objects:
3016+ break
3017+ for obj in objects:
3018+ object_queue.put((container, obj))
3019+ marker = objects[-1]
3020+ except ClientException, err:
3021+ if err.http_status != 404:
3022+ raise
3023+ error_queue.put('Container %s not found' % repr(container))
3024+
3025+ url, token = get_auth(options.auth, options.user, options.key,
3026+ snet=options.snet)
3027+ create_connection = lambda: Connection(options.auth, options.user,
3028+ options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
3029+ object_threads = [QueueFunctionThread(object_queue, _download_object,
3030+ create_connection()) for _junk in xrange(10)]
3031+ for thread in object_threads:
3032+ thread.start()
3033+ container_threads = [QueueFunctionThread(container_queue,
3034+ _download_container, create_connection()) for _junk in xrange(10)]
3035+ for thread in container_threads:
3036+ thread.start()
3037+ if not args:
3038+ conn = create_connection()
3039+ try:
3040+ marker = ''
3041+ while True:
3042+ containers = [c['name']
3043+ for c in conn.get_account(marker=marker)[1]]
3044+ if not containers:
3045+ break
3046+ for container in containers:
3047+ container_queue.put(container)
3048+ marker = containers[-1]
3049+ except ClientException, err:
3050+ if err.http_status != 404:
3051+ raise
3052+ error_queue.put('Account not found')
3053+ elif len(args) == 1:
3054+ if '/' in args[0]:
3055+ print >> stderr, 'WARNING: / in container name; you might have ' \
3056+ 'meant %r instead of %r.' % \
3057+ (args[0].replace('/', ' ', 1), args[0])
3058+ _download_container(args[0], create_connection())
3059+ else:
3060+ if len(args) == 2:
3061+ obj = args[1]
3062+ object_queue.put((args[0], obj, options.out_file))
3063+ else:
3064+ for obj in args[1:]:
3065+ object_queue.put((args[0], obj))
3066+ while not container_queue.empty():
3067+ sleep(0.01)
3068+ for thread in container_threads:
3069+ thread.abort = True
3070+ while thread.isAlive():
3071+ thread.join(0.01)
3072+ put_errors_from_threads(container_threads, error_queue)
3073+ while not object_queue.empty():
3074+ sleep(0.01)
3075+ for thread in object_threads:
3076+ thread.abort = True
3077+ while thread.isAlive():
3078+ thread.join(0.01)
3079+ put_errors_from_threads(object_threads, error_queue)
3080+
3081+
3082+st_list_help = '''
3083+list [options] [container]
3084+ Lists the containers for the account or the objects for a container. -p or
3085+ --prefix is an option that will only list items beginning with that prefix.
3086+ -d or --delimiter is option (for container listings only) that will roll up
3087+ items with the given delimiter (see Cloud Files general documentation for
3088+ what this means).
3089+'''.strip('\n')
3090+
3091+
3092+def st_list(options, args, print_queue, error_queue):
3093+ parser.add_option('-p', '--prefix', dest='prefix', help='Will only list '
3094+ 'items beginning with the prefix')
3095+ parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll '
3096+ 'up items with the given delimiter (see Cloud Files general '
3097+ 'documentation for what this means)')
3098+ (options, args) = parse_args(parser, args)
3099+ args = args[1:]
3100+ if options.delimiter and not args:
3101+ exit('-d option only allowed for container listings')
3102+ if len(args) > 1:
3103+ error_queue.put('Usage: %s [options] %s' %
3104+ (basename(argv[0]), st_list_help))
3105+ return
3106+ conn = Connection(options.auth, options.user, options.key,
3107+ snet=options.snet)
3108+ try:
3109+ marker = ''
3110+ while True:
3111+ if not args:
3112+ items = \
3113+ conn.get_account(marker=marker, prefix=options.prefix)[1]
3114+ else:
3115+ items = conn.get_container(args[0], marker=marker,
3116+ prefix=options.prefix, delimiter=options.delimiter)[1]
3117+ if not items:
3118+ break
3119+ for item in items:
3120+ print_queue.put(item.get('name', item.get('subdir')))
3121+ marker = items[-1].get('name', items[-1].get('subdir'))
3122+ except ClientException, err:
3123+ if err.http_status != 404:
3124+ raise
3125+ if not args:
3126+ error_queue.put('Account not found')
3127+ else:
3128+ error_queue.put('Container %s not found' % repr(args[0]))
3129+
3130+
3131+st_stat_help = '''
3132+stat [container] [object]
3133+ Displays information for the account, container, or object depending on the
3134+ args given (if any).'''.strip('\n')
3135+
3136+
3137+def st_stat(options, args, print_queue, error_queue):
3138+ (options, args) = parse_args(parser, args)
3139+ args = args[1:]
3140+ conn = Connection(options.auth, options.user, options.key)
3141+ if not args:
3142+ try:
3143+ headers = conn.head_account()
3144+ if options.verbose > 1:
3145+ print_queue.put('''
3146+StorageURL: %s
3147+Auth Token: %s
3148+'''.strip('\n') % (conn.url, conn.token))
3149+ container_count = int(headers.get('x-account-container-count', 0))
3150+ object_count = int(headers.get('x-account-object-count', 0))
3151+ bytes_used = int(headers.get('x-account-bytes-used', 0))
3152+ print_queue.put('''
3153+ Account: %s
3154+Containers: %d
3155+ Objects: %d
3156+ Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count,
3157+ object_count, bytes_used))
3158+ for key, value in headers.items():
3159+ if key.startswith('x-account-meta-'):
3160+ print_queue.put('%10s: %s' % ('Meta %s' %
3161+ key[len('x-account-meta-'):].title(), value))
3162+ for key, value in headers.items():
3163+ if not key.startswith('x-account-meta-') and key not in (
3164+ 'content-length', 'date', 'x-account-container-count',
3165+ 'x-account-object-count', 'x-account-bytes-used'):
3166+ print_queue.put(
3167+ '%10s: %s' % (key.title(), value))
3168+ except ClientException, err:
3169+ if err.http_status != 404:
3170+ raise
3171+ error_queue.put('Account not found')
3172+ elif len(args) == 1:
3173+ if '/' in args[0]:
3174+ print >> stderr, 'WARNING: / in container name; you might have ' \
3175+ 'meant %r instead of %r.' % \
3176+ (args[0].replace('/', ' ', 1), args[0])
3177+ try:
3178+ headers = conn.head_container(args[0])
3179+ object_count = int(headers.get('x-container-object-count', 0))
3180+ bytes_used = int(headers.get('x-container-bytes-used', 0))
3181+ print_queue.put('''
3182+ Account: %s
3183+Container: %s
3184+ Objects: %d
3185+ Bytes: %d
3186+ Read ACL: %s
3187+Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
3188+ object_count, bytes_used,
3189+ headers.get('x-container-read', ''),
3190+ headers.get('x-container-write', '')))
3191+ for key, value in headers.items():
3192+ if key.startswith('x-container-meta-'):
3193+ print_queue.put('%9s: %s' % ('Meta %s' %
3194+ key[len('x-container-meta-'):].title(), value))
3195+ for key, value in headers.items():
3196+ if not key.startswith('x-container-meta-') and key not in (
3197+ 'content-length', 'date', 'x-container-object-count',
3198+ 'x-container-bytes-used', 'x-container-read',
3199+ 'x-container-write'):
3200+ print_queue.put(
3201+ '%9s: %s' % (key.title(), value))
3202+ except ClientException, err:
3203+ if err.http_status != 404:
3204+ raise
3205+ error_queue.put('Container %s not found' % repr(args[0]))
3206+ elif len(args) == 2:
3207+ try:
3208+ headers = conn.head_object(args[0], args[1])
3209+ print_queue.put('''
3210+ Account: %s
3211+ Container: %s
3212+ Object: %s
3213+ Content Type: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
3214+ args[1], headers.get('content-type')))
3215+ if 'content-length' in headers:
3216+ print_queue.put('Content Length: %s' %
3217+ headers['content-length'])
3218+ if 'last-modified' in headers:
3219+ print_queue.put(' Last Modified: %s' %
3220+ headers['last-modified'])
3221+ if 'etag' in headers:
3222+ print_queue.put(' ETag: %s' % headers['etag'])
3223+ if 'x-object-manifest' in headers:
3224+ print_queue.put(' Manifest: %s' %
3225+ headers['x-object-manifest'])
3226+ for key, value in headers.items():
3227+ if key.startswith('x-object-meta-'):
3228+ print_queue.put('%14s: %s' % ('Meta %s' %
3229+ key[len('x-object-meta-'):].title(), value))
3230+ for key, value in headers.items():
3231+ if not key.startswith('x-object-meta-') and key not in (
3232+ 'content-type', 'content-length', 'last-modified',
3233+ 'etag', 'date', 'x-object-manifest'):
3234+ print_queue.put(
3235+ '%14s: %s' % (key.title(), value))
3236+ except ClientException, err:
3237+ if err.http_status != 404:
3238+ raise
3239+ error_queue.put('Object %s not found' %
3240+ repr('%s/%s' % (args[0], args[1])))
3241+ else:
3242+ error_queue.put('Usage: %s [options] %s' %
3243+ (basename(argv[0]), st_stat_help))
3244+
3245+
3246+st_post_help = '''
3247+post [options] [container] [object]
3248+ Updates meta information for the account, container, or object depending on
3249+ the args given. If the container is not found, it will be created
3250+ automatically; but this is not true for accounts and objects. Containers
3251+ also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
3252+ or --meta option is allowed on all and used to define the user meta data
3253+ items to set in the form Name:Value. This option can be repeated. Example:
3254+ post -m Color:Blue -m Size:Large'''.strip('\n')
3255+
3256+
3257+def st_post(options, args, print_queue, error_queue):
3258+ parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the '
3259+ 'Read ACL for containers. Quick summary of ACL syntax: .r:*, '
3260+ '.r:-.example.com, .r:www.example.com, account1, account2:user2')
3261+ parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the '
3262+ 'Write ACL for containers. Quick summary of ACL syntax: account1, '
3263+ 'account2:user2')
3264+ parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
3265+ help='Sets a meta data item with the syntax name:value. This option '
3266+ 'may be repeated. Example: -m Color:Blue -m Size:Large')
3267+ (options, args) = parse_args(parser, args)
3268+ args = args[1:]
3269+ if (options.read_acl or options.write_acl) and not args:
3270+ exit('-r and -w options only allowed for containers')
3271+ conn = Connection(options.auth, options.user, options.key)
3272+ if not args:
3273+ headers = {}
3274+ for item in options.meta:
3275+ split_item = item.split(':')
3276+ headers['X-Account-Meta-' + split_item[0]] = \
3277+ len(split_item) > 1 and split_item[1]
3278+ try:
3279+ conn.post_account(headers=headers)
3280+ except ClientException, err:
3281+ if err.http_status != 404:
3282+ raise
3283+ error_queue.put('Account not found')
3284+ elif len(args) == 1:
3285+ if '/' in args[0]:
3286+ print >> stderr, 'WARNING: / in container name; you might have ' \
3287+ 'meant %r instead of %r.' % \
3288+ (args[0].replace('/', ' ', 1), args[0])
3289+ headers = {}
3290+ for item in options.meta:
3291+ split_item = item.split(':')
3292+ headers['X-Container-Meta-' + split_item[0]] = \
3293+ len(split_item) > 1 and split_item[1]
3294+ if options.read_acl is not None:
3295+ headers['X-Container-Read'] = options.read_acl
3296+ if options.write_acl is not None:
3297+ headers['X-Container-Write'] = options.write_acl
3298+ try:
3299+ conn.post_container(args[0], headers=headers)
3300+ except ClientException, err:
3301+ if err.http_status != 404:
3302+ raise
3303+ conn.put_container(args[0], headers=headers)
3304+ elif len(args) == 2:
3305+ headers = {}
3306+ for item in options.meta:
3307+ split_item = item.split(':')
3308+ headers['X-Object-Meta-' + split_item[0]] = \
3309+ len(split_item) > 1 and split_item[1]
3310+ try:
3311+ conn.post_object(args[0], args[1], headers=headers)
3312+ except ClientException, err:
3313+ if err.http_status != 404:
3314+ raise
3315+ error_queue.put('Object %s not found' %
3316+ repr('%s/%s' % (args[0], args[1])))
3317+ else:
3318+ error_queue.put('Usage: %s [options] %s' %
3319+ (basename(argv[0]), st_post_help))
3320+
3321+
3322+st_upload_help = '''
3323+upload [options] container file_or_directory [file_or_directory] [...]
3324+ Uploads to the given container the files and directories specified by the
3325+ remaining args. -c or --changed is an option that will only upload files
3326+ that have changed since the last upload. -S <size> or --segment-size <size>
3327+ and --leave-segments are options as well (see --help for more).
3328+'''.strip('\n')
3329+
3330+
3331+def st_upload(options, args, print_queue, error_queue):
3332+ parser.add_option('-c', '--changed', action='store_true', dest='changed',
3333+ default=False, help='Will only upload files that have changed since '
3334+ 'the last upload')
3335+ parser.add_option('-S', '--segment-size', dest='segment_size', help='Will '
3336+ 'upload files in segments no larger than <size> and then create a '
3337+ '"manifest" file that will download all the segments as if it were '
3338+ 'the original file. The segments will be uploaded to a '
3339+ '<container>_segments container so as to not pollute the main '
3340+ '<container> listings.')
3341+ parser.add_option('', '--leave-segments', action='store_true',
3342+ dest='leave_segments', default=False, help='Indicates that you want '
3343+ 'the older segments of manifest objects left alone (in the case of '
3344+ 'overwrites)')
3345+ (options, args) = parse_args(parser, args)
3346+ args = args[1:]
3347+ if len(args) < 2:
3348+ error_queue.put('Usage: %s [options] %s' %
3349+ (basename(argv[0]), st_upload_help))
3350+ return
3351+ object_queue = Queue(10000)
3352+
3353+ def _segment_job(job, conn):
3354+ if job.get('delete', False):
3355+ conn.delete_object(job['container'], job['obj'])
3356+ else:
3357+ fp = open(job['path'], 'rb')
3358+ fp.seek(job['segment_start'])
3359+ conn.put_object(job.get('container', args[0] + '_segments'),
3360+ job['obj'], fp, content_length=job['segment_size'])
3361+ if options.verbose and 'log_line' in job:
3362+ if conn.attempts > 1:
3363+ print_queue.put('%s [after %d attempts]' %
3364+ (job['log_line'], conn.attempts))
3365+ else:
3366+ print_queue.put(job['log_line'])
3367+
3368+ def _object_job(job, conn):
3369+ path = job['path']
3370+ container = job.get('container', args[0])
3371+ dir_marker = job.get('dir_marker', False)
3372+ try:
3373+ obj = path
3374+ if obj.startswith('./') or obj.startswith('.\\'):
3375+ obj = obj[2:]
3376+ put_headers = {'x-object-meta-mtime': str(getmtime(path))}
3377+ if dir_marker:
3378+ if options.changed:
3379+ try:
3380+ headers = conn.head_object(container, obj)
3381+ ct = headers.get('content-type')
3382+ cl = int(headers.get('content-length'))
3383+ et = headers.get('etag')
3384+ mt = headers.get('x-object-meta-mtime')
3385+ if ct.split(';', 1)[0] == 'text/directory' and \
3386+ cl == 0 and \
3387+ et == 'd41d8cd98f00b204e9800998ecf8427e' and \
3388+ mt == put_headers['x-object-meta-mtime']:
3389+ return
3390+ except ClientException, err:
3391+ if err.http_status != 404:
3392+ raise
3393+ conn.put_object(container, obj, '', content_length=0,
3394+ content_type='text/directory',
3395+ headers=put_headers)
3396+ else:
3397+ # We need to HEAD all objects now in case we're overwriting a
3398+ # manifest object and need to delete the old segments
3399+ # ourselves.
3400+ old_manifest = None
3401+ if options.changed or not options.leave_segments:
3402+ try:
3403+ headers = conn.head_object(container, obj)
3404+ cl = int(headers.get('content-length'))
3405+ mt = headers.get('x-object-meta-mtime')
3406+ if options.changed and cl == getsize(path) and \
3407+ mt == put_headers['x-object-meta-mtime']:
3408+ return
3409+ if not options.leave_segments:
3410+ old_manifest = headers.get('x-object-manifest')
3411+ except ClientException, err:
3412+ if err.http_status != 404:
3413+ raise
3414+ if options.segment_size and \
3415+ getsize(path) < options.segment_size:
3416+ full_size = getsize(path)
3417+ segment_queue = Queue(10000)
3418+ segment_threads = [QueueFunctionThread(segment_queue,
3419+ _segment_job, create_connection()) for _junk in
3420+ xrange(10)]
3421+ for thread in segment_threads:
3422+ thread.start()
3423+ segment = 0
3424+ segment_start = 0
3425+ while segment_start < full_size:
3426+ segment_size = int(options.segment_size)
3427+ if segment_start + segment_size > full_size:
3428+ segment_size = full_size - segment_start
3429+ segment_queue.put({'path': path,
3430+ 'obj': '%s/%s/%s/%08d' % (obj,
3431+ put_headers['x-object-meta-mtime'], full_size,
3432+ segment),
3433+ 'segment_start': segment_start,
3434+ 'segment_size': segment_size,
3435+ 'log_line': '%s segment %s' % (obj, segment)})
3436+ segment += 1
3437+ segment_start += segment_size
3438+ while not segment_queue.empty():
3439+ sleep(0.01)
3440+ for thread in segment_threads:
3441+ thread.abort = True
3442+ while thread.isAlive():
3443+ thread.join(0.01)
3444+ if put_errors_from_threads(segment_threads, error_queue):
3445+ raise ClientException('Aborting manifest creation '
3446+ 'because not all segments could be uploaded. %s/%s'
3447+ % (container, obj))
3448+ new_object_manifest = '%s_segments/%s/%s/%s/' % (
3449+ container, obj, put_headers['x-object-meta-mtime'],
3450+ full_size)
3451+ if old_manifest == new_object_manifest:
3452+ old_manifest = None
3453+ put_headers['x-object-manifest'] = new_object_manifest
3454+ conn.put_object(container, obj, '', content_length=0,
3455+ headers=put_headers)
3456+ else:
3457+ conn.put_object(container, obj, open(path, 'rb'),
3458+ content_length=getsize(path), headers=put_headers)
3459+ if old_manifest:
3460+ segment_queue = Queue(10000)
3461+ scontainer, sprefix = old_manifest.split('/', 1)
3462+ for delobj in conn.get_container(scontainer,
3463+ prefix=sprefix)[1]:
3464+ segment_queue.put({'delete': True,
3465+ 'container': scontainer, 'obj': delobj['name']})
3466+ if not segment_queue.empty():
3467+ segment_threads = [QueueFunctionThread(segment_queue,
3468+ _segment_job, create_connection()) for _junk in
3469+ xrange(10)]
3470+ for thread in segment_threads:
3471+ thread.start()
3472+ while not segment_queue.empty():
3473+ sleep(0.01)
3474+ for thread in segment_threads:
3475+ thread.abort = True
3476+ while thread.isAlive():
3477+ thread.join(0.01)
3478+ put_errors_from_threads(segment_threads, error_queue)
3479+ if options.verbose:
3480+ if conn.attempts > 1:
3481+ print_queue.put(
3482+ '%s [after %d attempts]' % (obj, conn.attempts))
3483+ else:
3484+ print_queue.put(obj)
3485+ except OSError, err:
3486+ if err.errno != ENOENT:
3487+ raise
3488+ error_queue.put('Local file %s not found' % repr(path))
3489+
3490+ def _upload_dir(path):
3491+ names = listdir(path)
3492+ if not names:
3493+ object_queue.put({'path': path, 'dir_marker': True})
3494+ else:
3495+ for name in listdir(path):
3496+ subpath = join(path, name)
3497+ if isdir(subpath):
3498+ _upload_dir(subpath)
3499+ else:
3500+ object_queue.put({'path': subpath})
3501+
3502+ url, token = get_auth(options.auth, options.user, options.key,
3503+ snet=options.snet)
3504+ create_connection = lambda: Connection(options.auth, options.user,
3505+ options.key, preauthurl=url, preauthtoken=token, snet=options.snet)
3506+ object_threads = [QueueFunctionThread(object_queue, _object_job,
3507+ create_connection()) for _junk in xrange(10)]
3508+ for thread in object_threads:
3509+ thread.start()
3510+ conn = create_connection()
3511+ # Try to create the container, just in case it doesn't exist. If this
3512+ # fails, it might just be because the user doesn't have container PUT
3513+ # permissions, so we'll ignore any error. If there's really a problem,
3514+ # it'll surface on the first object PUT.
3515+ try:
3516+ conn.put_container(args[0])
3517+ if options.segment_size is not None:
3518+ conn.put_container(args[0] + '_segments')
3519+ except Exception:
3520+ pass
3521+ try:
3522+ for arg in args[1:]:
3523+ if isdir(arg):
3524+ _upload_dir(arg)
3525+ else:
3526+ object_queue.put({'path': arg})
3527+ while not object_queue.empty():
3528+ sleep(0.01)
3529+ for thread in object_threads:
3530+ thread.abort = True
3531+ while thread.isAlive():
3532+ thread.join(0.01)
3533+ put_errors_from_threads(object_threads, error_queue)
3534+ except ClientException, err:
3535+ if err.http_status != 404:
3536+ raise
3537+ error_queue.put('Account not found')
3538+
3539+
3540+def parse_args(parser, args, enforce_requires=True):
3541+ if not args:
3542+ args = ['-h']
3543+ (options, args) = parser.parse_args(args)
3544+ if enforce_requires and \
3545+ not (options.auth and options.user and options.key):
3546+ exit('''
3547+Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or
3548+overridden with -A, -U, or -K.'''.strip('\n'))
3549+ return options, args
3550+
3551+
3552+if __name__ == '__main__':
3553+ parser = OptionParser(version='%prog 1.0', usage='''
3554+Usage: %%prog <command> [options] [args]
3555+
3556+Commands:
3557+ %(st_stat_help)s
3558+ %(st_list_help)s
3559+ %(st_upload_help)s
3560+ %(st_post_help)s
3561+ %(st_download_help)s
3562+ %(st_delete_help)s
3563+
3564+Example:
3565+ %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K key stat
3566+'''.strip('\n') % globals())
3567+ parser.add_option('-s', '--snet', action='store_true', dest='snet',
3568+ default=False, help='Use SERVICENET internal network')
3569+ parser.add_option('-v', '--verbose', action='count', dest='verbose',
3570+ default=1, help='Print more info')
3571+ parser.add_option('-q', '--quiet', action='store_const', dest='verbose',
3572+ const=0, default=1, help='Suppress status output')
3573+ parser.add_option('-A', '--auth', dest='auth',
3574+ default=environ.get('ST_AUTH'),
3575+ help='URL for obtaining an auth token')
3576+ parser.add_option('-U', '--user', dest='user',
3577+ default=environ.get('ST_USER'),
3578+ help='User name for obtaining an auth token')
3579+ parser.add_option('-K', '--key', dest='key',
3580+ default=environ.get('ST_KEY'),
3581+ help='Key for obtaining an auth token')
3582+ parser.disable_interspersed_args()
3583+ (options, args) = parse_args(parser, argv[1:], enforce_requires=False)
3584+ parser.enable_interspersed_args()
3585+
3586+ commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
3587+ if not args or args[0] not in commands:
3588+ parser.print_usage()
3589+ if args:
3590+ exit('no such command: %s' % args[0])
3591+ exit()
3592+
3593+ print_queue = Queue(10000)
3594+
3595+ def _print(item):
3596+ if isinstance(item, unicode):
3597+ item = item.encode('utf8')
3598+ print item
3599+
3600+ print_thread = QueueFunctionThread(print_queue, _print)
3601+ print_thread.start()
3602+
3603+ error_queue = Queue(10000)
3604+
3605+ def _error(item):
3606+ if isinstance(item, unicode):
3607+ item = item.encode('utf8')
3608+ print >> stderr, item
3609+
3610+ error_thread = QueueFunctionThread(error_queue, _error)
3611+ error_thread.start()
3612+
3613+ try:
3614+ parser.usage = globals()['st_%s_help' % args[0]]
3615+ try:
3616+ globals()['st_%s' % args[0]](parser, argv[1:], print_queue,
3617+ error_queue)
3618+ except (ClientException, HTTPException, socket.error), err:
3619+ error_queue.put(str(err))
3620+ while not print_queue.empty():
3621+ sleep(0.01)
3622+ print_thread.abort = True
3623+ while print_thread.isAlive():
3624+ print_thread.join(0.01)
3625+ while not error_queue.empty():
3626+ sleep(0.01)
3627+ error_thread.abort = True
3628+ while error_thread.isAlive():
3629+ error_thread.join(0.01)
3630+ except (SystemExit, Exception):
3631+ for thread in threading_enumerate():
3632+ thread.abort = True
3633+ raise
3634
3635=== modified file 'doc/source/development_saio.rst'
3636--- doc/source/development_saio.rst 2011-05-26 02:24:12 +0000
3637+++ doc/source/development_saio.rst 2011-06-14 16:07:28 +0000
3638@@ -625,7 +625,7 @@
3639 #. `recreateaccounts`
3640 #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0``
3641 #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
3642- #. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat`
3643+ #. Check that `swift` works: `swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat`
3644 #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf`
3645 #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete
3646 everything in the configured accounts.)
3647
3648=== modified file 'doc/source/howto_installmultinode.rst'
3649--- doc/source/howto_installmultinode.rst 2011-05-26 02:24:12 +0000
3650+++ doc/source/howto_installmultinode.rst 2011-06-14 16:07:28 +0000
3651@@ -372,34 +372,34 @@
3652
3653 curl -k -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
3654
3655-#. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes)::
3656-
3657- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat
3658-
3659-#. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles'::
3660-
3661- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz
3662- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz
3663-
3664-#. Use ``st`` to download all files from the 'myfiles' container::
3665-
3666- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles
3667-
3668-#. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!::
3669-
3670- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder
3671-
3672-#. Use ``st`` to list your containers::
3673-
3674- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list
3675-
3676-#. Use ``st`` to list the contents of your 'builders' container::
3677-
3678- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders
3679-
3680-#. Use ``st`` to download all files from the 'builders' container::
3681-
3682- st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders
3683+#. Check that ``swift`` works (at this point, expect zero containers, zero objects, and zero bytes)::
3684+
3685+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat
3686+
3687+#. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles'::
3688+
3689+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz
3690+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz
3691+
3692+#. Use ``swift`` to download all files from the 'myfiles' container::
3693+
3694+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles
3695+
3696+#. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!::
3697+
3698+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder
3699+
3700+#. Use ``swift`` to list your containers::
3701+
3702+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list
3703+
3704+#. Use ``swift`` to list the contents of your 'builders' container::
3705+
3706+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders
3707+
3708+#. Use ``swift`` to download all files from the 'builders' container::
3709+
3710+ swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders
3711
3712 .. _add-proxy-server:
3713
3714
3715=== modified file 'doc/source/overview_large_objects.rst'
3716--- doc/source/overview_large_objects.rst 2010-12-06 20:01:19 +0000
3717+++ doc/source/overview_large_objects.rst 2011-06-14 16:07:28 +0000
3718@@ -14,24 +14,24 @@
3719 with the possibility of parallel uploads of the segments.
3720
3721 ----------------------------------
3722-Using ``st`` for Segmented Objects
3723+Using ``swift`` for Segmented Objects
3724 ----------------------------------
3725
3726-The quickest way to try out this feature is use the included ``st`` Swift Tool.
3727+The quickest way to try out this feature is use the included ``swift`` Swift Tool.
3728 You can use the ``-S`` option to specify the segment size to use when splitting
3729 a large file. For example::
3730
3731- st upload test_container -S 1073741824 large_file
3732+ swift upload test_container -S 1073741824 large_file
3733
3734 This would split the large_file into 1G segments and begin uploading those
3735-segments in parallel. Once all the segments have been uploaded, ``st`` will
3736+segments in parallel. Once all the segments have been uploaded, ``swift`` will
3737 then create the manifest file so the segments can be downloaded as one.
3738
3739-So now, the following ``st`` command would download the entire large object::
3740-
3741- st download test_container large_file
3742-
3743-``st`` uses a strict convention for its segmented object support. In the above
3744+So now, the following ``swift`` command would download the entire large object::
3745+
3746+ swift download test_container large_file
3747+
3748+``swift`` uses a strict convention for its segmented object support. In the above
3749 example it will upload all the segments into a second container named
3750 test_container_segments. These segments will have names like
3751 large_file/1290206778.25/21474836480/00000000,
3752@@ -43,7 +43,7 @@
3753 upload of a new file with the same name won't overwrite the contents of the
3754 first until the last moment when the manifest file is updated.
3755
3756-``st`` will manage these segment files for you, deleting old segments on
3757+``swift`` will manage these segment files for you, deleting old segments on
3758 deletes and overwrites, etc. You can override this behavior with the
3759 ``--leave-segments`` option if desired; this is useful if you want to have
3760 multiple versions of the same large object available.
3761@@ -53,14 +53,14 @@
3762 ----------
3763
3764 You can also work with the segments and manifests directly with HTTP requests
3765-instead of having ``st`` do that for you. You can just upload the segments like
3766+instead of having ``swift`` do that for you. You can just upload the segments like
3767 you would any other object and the manifest is just a zero-byte file with an
3768 extra ``X-Object-Manifest`` header.
3769
3770 All the object segments need to be in the same container, have a common object
3771 name prefix, and their names sort in the order they should be concatenated.
3772 They don't have to be in the same container as the manifest file will be, which
3773-is useful to keep container listings clean as explained above with ``st``.
3774+is useful to keep container listings clean as explained above with ``swift``.
3775
3776 The manifest file is simply a zero-byte file with the extra
3777 ``X-Object-Manifest: <container>/<prefix>`` header, where ``<container>`` is
3778
3779=== modified file 'setup.py'
3780--- setup.py 2011-05-31 18:09:53 +0000
3781+++ setup.py 2011-06-14 16:07:28 +0000
3782@@ -76,7 +76,7 @@
3783 ],
3784 install_requires=[], # removed for better compat
3785 scripts=[
3786- 'bin/st', 'bin/swift-account-auditor',
3787+ 'bin/swift', 'bin/swift-account-auditor',
3788 'bin/swift-account-audit', 'bin/swift-account-reaper',
3789 'bin/swift-account-replicator', 'bin/swift-account-server',
3790 'bin/swift-container-auditor',
3791
3792=== modified file 'swift/common/middleware/staticweb.py'
3793--- swift/common/middleware/staticweb.py 2011-06-05 23:22:35 +0000
3794+++ swift/common/middleware/staticweb.py 2011-06-14 16:07:28 +0000
3795@@ -74,36 +74,36 @@
3796 listing page, you will see the well defined document structure that can be
3797 styled.
3798
3799-Example usage of this middleware via ``st``:
3800+Example usage of this middleware via ``swift``:
3801
3802 Make the container publicly readable::
3803
3804- st post -r '.r:*' container
3805+ swift post -r '.r:*' container
3806
3807 You should be able to get objects directly, but no index.html resolution or
3808 listings.
3809
3810 Set an index file directive::
3811
3812- st post -m 'web-index:index.html' container
3813+ swift post -m 'web-index:index.html' container
3814
3815 You should be able to hit paths that have an index.html without needing to
3816 type the index.html part.
3817
3818 Turn on listings::
3819
3820- st post -m 'web-listings: true' container
3821+ swift post -m 'web-listings: true' container
3822
3823 Now you should see object listings for paths and pseudo paths that have no
3824 index.html.
3825
3826 Enable a custom listings style sheet::
3827
3828- st post -m 'web-listings-css:listings.css' container
3829+ swift post -m 'web-listings-css:listings.css' container
3830
3831 Set an error file::
3832
3833- st post -m 'web-error:error.html' container
3834+ swift post -m 'web-error:error.html' container
3835
3836 Now 401's should load 401error.html, 404's should load 404error.html, etc.
3837 """