Merge lp:~notmyname/swift/stats_fix into lp:swift/1.1

[John Dickinson]

Fixed lazy-load of internal_proxy attribute in log_processor

Unmerged revisions

120. By John Dickinson on 2010-11-05

merged with clayg's branch

119. By John Dickinson on 2010-11-04

fixed bug in internal_proxy

118. By John Dickinson on 2010-11-04

fixed typo on the getting started doc

117. By John Dickinson on 2010-11-03

Middleware that catches and logs errors that would otherwise be sent to the client

116. By John Dickinson on 2010-11-03

Middleware that translates account and container info from the host header to the path.

115. By Mike Barton on 2010-11-02

shore up accept header parsing

114. By David Goetz <email address hidden> on 2010-11-02

catching invalid urls and adding tests

113. By Anne Gentle on 2010-11-02

Okay, mtaylor worked out a way to have conf.py contain a build environment variable so that the build can insert the GA code but the local builds wouldn't have it.

112. By Anne Gentle on 2010-11-01

Add link to RHEL instructions in the SAIO doc

111. By John Dickinson on 2010-10-28

PUT with X-Copy-From header and a non-zero Content-Length will now error as a 400

=== added directory 'doc/source/_ga'
=== added file 'doc/source/_ga/layout.html'
--- doc/source/_ga/layout.html	1970-01-01 00:00:00 +0000
+++ doc/source/_ga/layout.html	2010-12-15 16:48:57 +0000
@@ -0,0 +1,17 @@
+{% extends "!layout.html" %}
+
+{% block footer %}
+{{ super() }}
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+try {
+var pageTracker = _gat._getTracker("UA-17511903-1");
+pageTracker._setDomainName("none");
+pageTracker._setAllowLinker(true);
+pageTracker._trackPageview();
+} catch(err) {}</script>
+{% endblock %}
+

=== added directory 'doc/source/_templates'
=== modified file 'doc/source/conf.py'
--- doc/source/conf.py	2010-10-18 15:34:21 +0000
+++ doc/source/conf.py	2010-12-15 16:48:57 +0000
@@ -41,7 +41,13 @@
 todo_include_todos = True
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+# Changing the path so that the Hudson build output contains GA code and the source
+# docs do not contain the code so local, offline sphinx builds are "clean."
+templates_path = []
+if os.getenv('HUDSON_PUBLISH_DOCS'):
+  templates_path = ['_ga', '_templates']
+else:
+  templates_path = ['_templates']
 
 # The suffix of source filenames.
 source_suffix = '.rst'

=== modified file 'doc/source/development_saio.rst'
--- doc/source/development_saio.rst	2010-10-20 20:24:59 +0000
+++ doc/source/development_saio.rst	2010-12-15 16:48:57 +0000
@@ -17,6 +17,9 @@
 
 * Create guest virtual machine from the Ubuntu image. 
 
+Additional information about setting up a Swift development snapshot on other distributions is 
+available on the wiki at http://wiki.openstack.org/SAIOInstructions.
+
 -----------------------------------------
 Installing dependencies and the core code
 -----------------------------------------

=== modified file 'doc/source/getting_started.rst'
--- doc/source/getting_started.rst	2010-10-14 18:11:21 +0000
+++ doc/source/getting_started.rst	2010-12-15 16:48:57 +0000
@@ -6,7 +6,7 @@
 System Requirements
 -------------------
 
-Swift development currently targets Unbuntu Server 10.04, but should work on 
+Swift development currently targets Ubuntu Server 10.04, but should work on 
 most Linux platforms with the following software:
 
 * Python 2.6
@@ -37,5 +37,4 @@
 ----------
 
 We do not have documentation yet on how to set up and configure Swift for a 
-production cluster, but hope to begin work on those soon.
-
+production cluster, but hope to begin work on those soon.
\ No newline at end of file

=== modified file 'doc/source/index.rst'
--- doc/source/index.rst	2010-10-13 20:51:11 +0000
+++ doc/source/index.rst	2010-12-15 16:48:57 +0000
@@ -40,7 +40,7 @@
 
 .. toctree::
     :maxdepth: 1
-
+    
     deployment_guide
     admin_guide
     debian_package_guide

=== modified file 'doc/source/ratelimit.rst'
--- doc/source/ratelimit.rst	2010-10-13 21:30:00 +0000
+++ doc/source/ratelimit.rst	2010-12-15 16:48:57 +0000
@@ -4,7 +4,7 @@
 
 Rate limiting in swift is implemented as a pluggable middleware.  Rate
 limiting is performed on requests that result in database writes to the
-account and container sqlite dbs.  It uses memcached and is dependant on
+account and container sqlite dbs.  It uses memcached and is dependent on
 the proxy servers having highly synchronized time.  The rate limits are
 limited by the accuracy of the proxy server clocks.
 
@@ -27,6 +27,9 @@
 max_sleep_time_seconds   60         App will immediately return a 498 response
                                     if the necessary sleep time ever exceeds
                                     the given max_sleep_time_seconds.
+log_sleep_time_seconds   0          To allow visibility into rate limiting set
+                                    this value > 0 and all sleeps greater than
+                                    the number will be logged.
 account_ratelimit        0          If set, will limit all requests to
                                     /account_name and PUTs to
                                     /account_name/container_name. Number is in

=== modified file 'etc/proxy-server.conf-sample'
--- etc/proxy-server.conf-sample	2010-10-14 23:47:14 +0000
+++ etc/proxy-server.conf-sample	2010-12-15 16:48:57 +0000
@@ -9,7 +9,7 @@
 # key_file = /etc/swift/proxy.key
 
 [pipeline:main]
-pipeline = healthcheck cache ratelimit auth proxy-server
+pipeline = catch_errors healthcheck cache ratelimit auth proxy-server
 
 [app:proxy-server]
 use = egg:swift#proxy
@@ -60,7 +60,8 @@
 # clock accuracy.
 # clock_accuracy = 1000
 # max_sleep_time_seconds = 60
-
+# log_sleep_time_seconds of 0 means disabled
+# log_sleep_time_seconds = 0
 # account_ratelimit of 0 means disabled
 # account_ratelimit = 0
 
@@ -75,3 +76,12 @@
 # container_ratelimit_0 = 100
 # container_ratelimit_10 = 50
 # container_ratelimit_50 = 20
+
+[filter:domain_remap]
+use = egg:swift#domain_remap
+# storage_domain = example.com
+# path_root = v1
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+

=== modified file 'setup.py'
--- setup.py	2010-10-18 15:34:21 +0000
+++ setup.py	2010-12-15 16:48:57 +0000
@@ -93,6 +93,8 @@
             'healthcheck=swift.common.middleware.healthcheck:filter_factory',
             'memcache=swift.common.middleware.memcache:filter_factory',
             'ratelimit=swift.common.middleware.ratelimit:filter_factory',
+            'catch_errors=swift.common.middleware.catch_errors:filter_factory',
+            'domain_remap=swift.common.middleware.domain_remap:filter_factory',
             ],
         },
     )

=== modified file 'swift/account/server.py'
--- swift/account/server.py	2010-11-02 19:41:24 +0000
+++ swift/account/server.py	2010-12-15 16:48:57 +0000
@@ -204,16 +204,15 @@
         except UnicodeDecodeError, err:
             return HTTPBadRequest(body='parameters not utf8',
                                   content_type='text/plain', request=req)
-        header_format = req.accept.first_match(['text/plain',
-                                                'application/json',
-                                                'application/xml'])
-        format = query_format if query_format else header_format
-        if format.startswith('application/'):
-            format = format[12:]
+        if query_format:
+            req.accept = 'application/%s' % query_format.lower()
+        out_content_type = req.accept.best_match(
+                                ['text/plain', 'application/json',
+                                 'application/xml', 'text/xml'],
+                                default_match='text/plain')
         account_list = broker.list_containers_iter(limit, marker, prefix,
                                                   delimiter)
-        if format == 'json':
-            out_content_type = 'application/json'
+        if out_content_type == 'application/json':
             json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
             json_pattern = '{' + ','.join(json_pattern) + '}'
             json_out = []
@@ -225,8 +224,7 @@
                     json_out.append(json_pattern %
                         (name, object_count, bytes_used))
             account_list = '[' + ','.join(json_out) + ']'
-        elif format == 'xml':
-            out_content_type = 'application/xml'
+        elif out_content_type.endswith('/xml'):
             output_list = ['<?xml version="1.0" encoding="UTF-8"?>',
                            '<account name="%s">' % account]
             for (name, object_count, bytes_used, is_subdir) in account_list:
@@ -243,7 +241,6 @@
         else:
             if not account_list:
                 return HTTPNoContent(request=req, headers=resp_headers)
-            out_content_type = 'text/plain'
             account_list = '\n'.join(r[0] for r in account_list) + '\n'
         ret = Response(body=account_list, request=req, headers=resp_headers)
         ret.content_type = out_content_type

=== modified file 'swift/common/constraints.py'
--- swift/common/constraints.py	2010-09-22 19:53:38 +0000
+++ swift/common/constraints.py	2010-12-15 16:48:57 +0000
@@ -100,6 +100,9 @@
     if req.content_length is None and \
             req.headers.get('transfer-encoding') != 'chunked':
         return HTTPLengthRequired(request=req)
+    if 'X-Copy-From' in req.headers and req.content_length:
+        return HTTPBadRequest(body='Copy requests require a zero byte body',
+                              request=req, content_type='text/plain')
     if len(object_name) > MAX_OBJECT_NAME_LENGTH:
         return HTTPBadRequest(body='Object name length of %d longer than %d' %
                 (len(object_name), MAX_OBJECT_NAME_LENGTH), request=req,

=== modified file 'swift/common/memcached.py'
--- swift/common/memcached.py	2010-10-20 17:31:50 +0000
+++ swift/common/memcached.py	2010-12-15 16:48:57 +0000
@@ -168,23 +168,42 @@
     def incr(self, key, delta=1, timeout=0):
         """
         Increments a key which has a numeric value by delta.
-        If the key can't be found, it's added as delta.
+        If the key can't be found, it's added as delta or 0 if delta < 0.
+        If passed a negative number, will use memcached's decr. Returns
+        the int stored in memcached
+        Note: The data memcached stores as the result of incr/decr is
+        an unsigned int.  decr's that result in a number below 0 are
+        stored as 0.
 
         :param key: key
         :param delta: amount to add to the value of key (or set as the value
-                      if the key is not found)
+                      if the key is not found) will be cast to an int
         :param timeout: ttl in memcache
         """
         key = md5hash(key)
+        command = 'incr'
+        if delta < 0:
+            command = 'decr'
+        delta = str(abs(int(delta)))
         for (server, fp, sock) in self._get_conns(key):
             try:
-                sock.sendall('incr %s %s\r\n' % (key, delta))
+                sock.sendall('%s %s %s\r\n' % (command, key, delta))
                 line = fp.readline().strip().split()
                 if line[0].upper() == 'NOT_FOUND':
-                    line[0] = str(delta)
-                    sock.sendall('add %s %d %d %s noreply\r\n%s\r\n' % \
-                                  (key, 0, timeout, len(line[0]), line[0]))
-                ret = int(line[0].strip())
+                    add_val = delta
+                    if command == 'decr':
+                        add_val = '0'
+                    sock.sendall('add %s %d %d %s\r\n%s\r\n' % \
+                                  (key, 0, timeout, len(add_val), add_val))
+                    line = fp.readline().strip().split()
+                    if line[0].upper() == 'NOT_STORED':
+                        sock.sendall('%s %s %s\r\n' % (command, key, delta))
+                        line = fp.readline().strip().split()
+                        ret = int(line[0].strip())
+                    else:
+                        ret = int(add_val)
+                else:
+                    ret = int(line[0].strip())
                 self._return_conn(server, fp, sock)
                 return ret
             except Exception, e:
@@ -192,29 +211,16 @@
 
     def decr(self, key, delta=1, timeout=0):
         """
-        Decrements a key which has a numeric value by delta.
-        If the key can't be found, it's added as 0.  Memcached
-        will treat data values below 0 as 0 with incr/decr.
+        Decrements a key which has a numeric value by delta. Calls incr with
+        -delta.
 
         :param key: key
-        :param delta: amount to subtract to the value of key (or set
-                      as the value if the key is not found)
+        :param delta: amount to subtract to the value of key (or set the
+                      value to 0 if the key is not found) will be cast to
+                      an int
         :param timeout: ttl in memcache
         """
-        key = md5hash(key)
-        for (server, fp, sock) in self._get_conns(key):
-            try:
-                sock.sendall('decr %s %s\r\n' % (key, delta))
-                line = fp.readline().strip().split()
-                if line[0].upper() == 'NOT_FOUND':
-                    line[0] = '0'
-                    sock.sendall('add %s %d %d %s noreply\r\n%s\r\n' %
-                                 (key, 0, timeout, len(line[0]), line[0]))
-                ret = int(line[0].strip())
-                self._return_conn(server, fp, sock)
-                return ret
-            except Exception, e:
-                self._exception_occurred(server, e)
+        self.incr(key, delta=-delta, timeout=timeout)
 
     def delete(self, key):
         """

=== added file 'swift/common/middleware/catch_errors.py'
--- swift/common/middleware/catch_errors.py	1970-01-01 00:00:00 +0000
+++ swift/common/middleware/catch_errors.py	2010-12-15 16:48:57 +0000
@@ -0,0 +1,48 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from webob import Request
+from webob.exc import HTTPServerError
+
+from swift.common.utils import get_logger
+
+
+class CatchErrorMiddleware(object):
+    """
+    Middleware that provides high-level error handling.
+    """
+
+    def __init__(self, app, conf):
+        self.app = app
+        self.logger = get_logger(conf)
+
+    def __call__(self, env, start_response):
+        try:
+            return self.app(env, start_response)
+        except Exception, err:
+            self.logger.exception('Error: %s' % err)
+            resp = HTTPServerError(request=Request(env),
+                                   body='An error occurred',
+                                   content_type='text/plain')
+            return resp(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def except_filter(app):
+        return CatchErrorMiddleware(app, conf)
+    return except_filter

=== added file 'swift/common/middleware/domain_remap.py'
--- swift/common/middleware/domain_remap.py	1970-01-01 00:00:00 +0000
+++ swift/common/middleware/domain_remap.py	2010-12-15 16:48:57 +0000
@@ -0,0 +1,80 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from webob import Request
+from webob.exc import HTTPBadRequest
+
+
+class DomainRemapMiddleware(object):
+    """
+    Middleware that translates container and account parts of a domain to
+    path parameters that the proxy server understands.
+
+    container.account.storageurl/object gets translated to
+    container.account.storageurl/path_root/account/container/object
+
+    account.storageurl/path_root/container/object gets translated to
+    account.storageurl/path_root/account/container/object
+    """
+
+    def __init__(self, app, conf):
+        self.app = app
+        self.storage_domain = conf.get('storage_domain', 'example.com')
+        if self.storage_domain and self.storage_domain[0] != '.':
+            self.storage_domain = '.' + self.storage_domain
+        self.path_root = conf.get('path_root', 'v1').strip('/')
+
+    def __call__(self, env, start_response):
+        if not self.storage_domain:
+            return self.app(env, start_response)
+        given_domain = env['HTTP_HOST']
+        port = ''
+        if ':' in given_domain:
+            given_domain, port = given_domain.rsplit(':', 1)
+        if given_domain.endswith(self.storage_domain):
+            parts_to_parse = given_domain[:-len(self.storage_domain)]
+            parts_to_parse = parts_to_parse.strip('.').split('.')
+            len_parts_to_parse = len(parts_to_parse)
+            if len_parts_to_parse == 2:
+                container, account = parts_to_parse
+            elif len_parts_to_parse == 1:
+                container, account = None, parts_to_parse[0]
+            else:
+                resp = HTTPBadRequest(request=Request(env),
+                                      body='Bad domain in host header',
+                                      content_type='text/plain')
+                return resp(env, start_response)
+            if '_' not in account and '-' in account:
+                account = account.replace('-', '_', 1)
+            path = env['PATH_INFO'].strip('/')
+            new_path_parts = ['', self.path_root, account]
+            if container:
+                new_path_parts.append(container)
+            if path.startswith(self.path_root):
+                path = path[len(self.path_root):].lstrip('/')
+            if path:
+                new_path_parts.append(path)
+            new_path = '/'.join(new_path_parts)
+            env['PATH_INFO'] = new_path
+        return self.app(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def domain_filter(app):
+        return DomainRemapMiddleware(app, conf)
+    return domain_filter

=== modified file 'swift/common/middleware/memcache.py'
--- swift/common/middleware/memcache.py	2010-09-16 03:52:54 +0000
+++ swift/common/middleware/memcache.py	2010-12-15 16:48:57 +0000
@@ -1,3 +1,4 @@
+# Copyright (c) 2010 OpenStack, LLC.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

=== modified file 'swift/common/middleware/ratelimit.py'
--- swift/common/middleware/ratelimit.py	2010-11-02 19:41:24 +0000
+++ swift/common/middleware/ratelimit.py	2010-12-15 16:48:57 +0000
@@ -41,6 +41,8 @@
         self.account_ratelimit = float(conf.get('account_ratelimit', 0))
         self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds',
                                                    60))
+        self.log_sleep_time_seconds = float(conf.get('log_sleep_time_seconds',
+                                                   0))
         self.clock_accuracy = int(conf.get('clock_accuracy', 1000))
         self.ratelimit_whitelist = [acc.strip() for acc in
             conf.get('account_whitelist', '').split(',')
@@ -177,6 +179,11 @@
             obj_name=obj_name):
             try:
                 need_to_sleep = self._get_sleep_time(key, max_rate)
+                if self.log_sleep_time_seconds and \
+                        need_to_sleep > self.log_sleep_time_seconds:
+                    self.logger.info("Ratelimit sleep log: %s for %s/%s/%s" % (
+                            need_to_sleep, account_name,
+                            container_name, obj_name))
                 if need_to_sleep > 0:
                     eventlet.sleep(need_to_sleep)
             except MaxSleepTimeHit, e:

=== modified file 'swift/container/server.py'
--- swift/container/server.py	2010-09-22 19:34:52 +0000
+++ swift/container/server.py	2010-12-15 16:48:57 +0000
@@ -278,16 +278,15 @@
         except UnicodeDecodeError, err:
             return HTTPBadRequest(body='parameters not utf8',
                                   content_type='text/plain', request=req)
-        header_format = req.accept.first_match(['text/plain',
-                                                'application/json',
-                                                'application/xml'])
-        format = query_format if query_format else header_format
-        if format.startswith('application/'):
-            format = format[12:]
+        if query_format:
+            req.accept = 'application/%s' % query_format.lower()
+        out_content_type = req.accept.best_match(
+                                ['text/plain', 'application/json',
+                                 'application/xml', 'text/xml'],
+                                default_match='text/plain')
         container_list = broker.list_objects_iter(limit, marker, prefix,
                                                   delimiter, path)
-        if format == 'json':
-            out_content_type = 'application/json'
+        if out_content_type == 'application/json':
             json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
                             '"content_type":%s, "last_modified":"%s"']
             json_pattern = '{' + ','.join(json_pattern) + '}'
@@ -307,8 +306,7 @@
                                                     content_type,
                                                     created_at))
             container_list = '[' + ','.join(json_out) + ']'
-        elif format == 'xml':
-            out_content_type = 'application/xml'
+        elif out_content_type.endswith('/xml'):
             xml_output = []
             for (name, created_at, size, content_type, etag) in container_list:
                 # escape name and format date here
@@ -330,7 +328,6 @@
         else:
             if not container_list:
                 return HTTPNoContent(request=req, headers=resp_headers)
-            out_content_type = 'text/plain'
             container_list = '\n'.join(r[0] for r in container_list) + '\n'
         ret = Response(body=container_list, request=req, headers=resp_headers)
         ret.content_type = out_content_type

=== modified file 'swift/proxy/server.py'
--- swift/proxy/server.py	2010-11-02 19:41:24 +0000
+++ swift/proxy/server.py	2010-12-15 16:48:57 +0000
@@ -644,7 +644,8 @@
                         environ=req.environ, headers=req.headers)
             new_req.content_length = source_resp.content_length
             new_req.etag = source_resp.etag
-            new_req.headers['X-Copy-From'] = source_header.split('/', 2)[2]
+            # we no longer need the X-Copy-From header
+            del new_req.headers['X-Copy-From']
             for k, v in source_resp.headers.items():
                 if k.lower().startswith('x-object-meta-'):
                     new_req.headers[k] = v
@@ -767,8 +768,9 @@
             bodies.append('')
         resp = self.best_response(req, statuses, reasons, bodies, 'Object PUT',
                                   etag=etag)
-        if 'x-copy-from' in req.headers:
-            resp.headers['X-Copied-From'] = req.headers['x-copy-from']
+        if source_header:
+            resp.headers['X-Copied-From'] = quote(
+                                                source_header.split('/', 2)[2])
             for k, v in req.headers.items():
                 if k.lower().startswith('x-object-meta-'):
                     resp.headers[k] = v

=== modified file 'swift/stats/log_processor.py'
--- swift/stats/log_processor.py	2010-10-08 22:20:43 +0000
+++ swift/stats/log_processor.py	2010-12-15 16:48:57 +0000
@@ -73,8 +73,7 @@
             self._internal_proxy = InternalProxy(proxy_server_conf,
                                                  self.logger,
                                                  retries=3)
-        else:
-            return self._internal_proxy
+        return self._internal_proxy
 
     def process_one_file(self, plugin_name, account, container, object_name):
         self.logger.info('Processing %s/%s/%s with plugin "%s"' % (account,

=== added file 'test/unit/common/middleware/test_domain_remap.py'
--- test/unit/common/middleware/test_domain_remap.py	1970-01-01 00:00:00 +0000
+++ test/unit/common/middleware/test_domain_remap.py	2010-12-15 16:48:57 +0000
@@ -0,0 +1,110 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from webob import Request
+
+from swift.common.middleware import domain_remap
+
+
+class FakeApp(object):
+
+    def __call__(self, env, start_response):
+        return env['PATH_INFO']
+
+
+def start_response(*args):
+    pass
+
+
+class TestDomainRemap(unittest.TestCase):
+
+    def setUp(self):
+        self.app = domain_remap.DomainRemapMiddleware(FakeApp(), {})
+
+    def test_domain_remap_passthrough(self):
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/')
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'example.com:8080'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/')
+
+    def test_domain_remap_account(self):
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a')
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'a-uuid.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a_uuid')
+
+    def test_domain_remap_account_container(self):
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a/c')
+
+    def test_domain_remap_extra_subdomains(self):
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'x.y.c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, ['Bad domain in host header'])
+
+    def test_domain_remap_account_with_path_root(self):
+        req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a')
+
+    def test_domain_remap_account_container_with_path_root(self):
+        req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a/c')
+
+    def test_domain_remap_account_container_with_path(self):
+        req = Request.blank('/obj', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a/c/obj')
+
+    def test_domain_remap_account_container_with_path_root_and_path(self):
+        req = Request.blank('/v1/obj', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/v1/a/c/obj')
+
+    def test_domain_remap_account_matching_ending_not_domain(self):
+        req = Request.blank('/dontchange', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.aexample.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/dontchange')
+
+    def test_domain_remap_configured_with_empty_storage_domain(self):
+        self.app = domain_remap.DomainRemapMiddleware(FakeApp(),
+                                                      {'storage_domain': ''})
+        req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'},
+                            headers={'Host': 'c.a.example.com'})
+        resp = self.app(req.environ, start_response)
+        self.assertEquals(resp, '/test')
+
+
+if __name__ == '__main__':
+    unittest.main()

=== added file 'test/unit/common/middleware/test_except.py'
--- test/unit/common/middleware/test_except.py	1970-01-01 00:00:00 +0000
+++ test/unit/common/middleware/test_except.py	2010-12-15 16:48:57 +0000
@@ -0,0 +1,49 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from webob import Request
+
+from swift.common.middleware import catch_errors
+
+class FakeApp(object):
+    def __init__(self, error=False):
+        self.error = error
+
+    def __call__(self, env, start_response):
+        if self.error:
+            raise Exception('augh!')
+        return "FAKE APP"
+
+def start_response(*args):
+    pass
+
+class TestCatchErrors(unittest.TestCase):
+
+    def test_catcherrors_passthrough(self):
+        app = catch_errors.CatchErrorMiddleware(FakeApp(), {})
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
+        resp = app(req.environ, start_response)
+        self.assertEquals(resp, 'FAKE APP')
+
+    def test_catcherrors(self):
+        app = catch_errors.CatchErrorMiddleware(FakeApp(True), {})
+        req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'})
+        resp = app(req.environ, start_response)
+        self.assertEquals(resp, ['An error occurred'])
+
+if __name__ == '__main__':
+    unittest.main()

=== modified file 'test/unit/common/middleware/test_ratelimit.py'
--- test/unit/common/middleware/test_ratelimit.py	2010-11-02 19:41:24 +0000
+++ test/unit/common/middleware/test_ratelimit.py	2010-12-15 16:48:57 +0000
@@ -36,19 +36,14 @@
         return True
 
     def incr(self, key, delta=1, timeout=0):
-        if delta < 0:
-            raise "Cannot incr by a negative number"
-        self.store[key] = int(self.store.setdefault(key, 0)) + delta
-        return int(self.store[key])
-
-    def decr(self, key, delta=1, timeout=0):
-        if delta < 0:
-            raise "Cannot decr by a negative number"
-        self.store[key] = int(self.store.setdefault(key, 0)) - delta
+        self.store[key] = int(self.store.setdefault(key, 0)) + int(delta)
         if self.store[key] < 0:
             self.store[key] = 0
         return int(self.store[key])
 
+    def decr(self, key, delta=1, timeout=0):
+        return self.incr(key, delta=-delta, timeout=timeout)
+
     @contextmanager
     def soft_lock(self, key, timeout=0, retries=5):
         yield True
@@ -98,9 +93,12 @@
 
 
 class FakeLogger(object):
+    # a thread safe logger
 
     def error(self, msg):
-        # a thread safe logger
+        pass
+
+    def info(self, msg):
         pass
 
 
@@ -289,7 +287,7 @@
         the_498s = [t for t in all_results if t.startswith('Slow down')]
         self.assertEquals(len(the_498s), 2)
         time_took = time.time() - begin
-        self.assert_(1.5 <= round(time_took,1) < 1.7, time_took)
+        self.assert_(1.5 <= round(time_took, 1) < 1.7, time_took)
 
     def test_ratelimit_max_rate_multiple_acc(self):
         num_calls = 4
@@ -326,7 +324,7 @@
             thread.join()
         time_took = time.time() - begin
         # the all 15 threads still take 1.5 secs
-        self.assert_(1.5 <= round(time_took,1) < 1.7)
+        self.assert_(1.5 <= round(time_took, 1) < 1.7)
 
     def test_ratelimit_acc_vrs_container(self):
         conf_dict = {'clock_accuracy': 1000,

=== modified file 'test/unit/common/test_memcached.py'
--- test/unit/common/test_memcached.py	2010-07-19 16:25:18 +0000
+++ test/unit/common/test_memcached.py	2010-12-15 16:48:57 +0000
@@ -98,6 +98,17 @@
                     self.outbuf += str(val[2]) + '\r\n'
                 else:
                     self.outbuf += 'NOT_FOUND\r\n'
+            elif parts[0].lower() == 'decr':
+                if parts[1] in self.cache:
+                    val = list(self.cache[parts[1]])
+                    if int(val[2]) - int(parts[2]) > 0:
+                        val[2] = str(int(val[2]) - int(parts[2]))
+                    else:
+                        val[2] = '0'
+                    self.cache[parts[1]] = val
+                    self.outbuf += str(val[2]) + '\r\n'
+                else:
+                    self.outbuf += 'NOT_FOUND\r\n'
     def readline(self):
         if self.down:
             raise Exception('mock is down')
@@ -151,6 +162,23 @@
         self.assertEquals(memcache_client.get('some_key'), '10')
         memcache_client.incr('some_key', delta=1)
         self.assertEquals(memcache_client.get('some_key'), '11')
+        memcache_client.incr('some_key', delta=-5)
+        self.assertEquals(memcache_client.get('some_key'), '6')
+        memcache_client.incr('some_key', delta=-15)
+        self.assertEquals(memcache_client.get('some_key'), '0')
+
+    def test_decr(self):
+        memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'])
+        mock = MockMemcached()
+        memcache_client._client_cache['1.2.3.4:11211'] = [(mock, mock)] * 2
+        memcache_client.decr('some_key', delta=5)
+        self.assertEquals(memcache_client.get('some_key'), '0')
+        memcache_client.incr('some_key', delta=15)
+        self.assertEquals(memcache_client.get('some_key'), '15')
+        memcache_client.decr('some_key', delta=4)
+        self.assertEquals(memcache_client.get('some_key'), '11')
+        memcache_client.decr('some_key', delta=15)
+        self.assertEquals(memcache_client.get('some_key'), '0')
 
     def test_retry(self):
         logging.getLogger().addHandler(NullLoggingHandler())

=== modified file 'test/unit/container/test_server.py'
--- test/unit/container/test_server.py	2010-09-09 05:37:27 +0000
+++ test/unit/container/test_server.py	2010-12-15 16:48:57 +0000
@@ -479,18 +479,20 @@
         resp = self.controller.GET(req)
         self.assertEquals(resp.status_int, 412)
 
-    def test_GET_format(self):
+    def test_GET_json(self):
         # make a container
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
+        req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT',
             'HTTP_X_TIMESTAMP': '0'})
         resp = self.controller.PUT(req)
         # test an empty container
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
+        req = Request.blank('/sda1/p/a/jsonc?format=json',
+            environ={'REQUEST_METHOD': 'GET'})
         resp = self.controller.GET(req)
-        self.assertEquals(resp.status_int, 204)
+        self.assertEquals(resp.status_int, 200)
+        self.assertEquals(eval(resp.body), [])
         # fill the container
         for i in range(3):
-            req = Request.blank('/sda1/p/a/c/%s'%i, environ=
+            req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ=
                     {'REQUEST_METHOD': 'PUT',
                     'HTTP_X_TIMESTAMP': '1',
                     'HTTP_X_CONTENT_TYPE': 'text/plain',
@@ -514,8 +516,88 @@
                     "bytes":0,
                     "content_type":"text/plain",
                     "last_modified":"1970-01-01T00:00:01"}]
+
+        req = Request.blank('/sda1/p/a/jsonc?format=json',
+                environ={'REQUEST_METHOD': 'GET'})
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.content_type, 'application/json')
+        self.assertEquals(eval(resp.body), json_body)
+
+        for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9',
+                 '*/*;q=0.9,application/json;q=1.0', 'application/*'):
+            req = Request.blank('/sda1/p/a/jsonc',
+                    environ={'REQUEST_METHOD': 'GET'})
+            req.accept = accept
+            resp = self.controller.GET(req)
+            self.assertEquals(eval(resp.body), json_body,
+                'Invalid body for Accept: %s' % accept)
+            self.assertEquals(resp.content_type, 'application/json',
+                'Invalid content_type for Accept: %s' % accept)
+
+    def test_GET_plain(self):
+        # make a container
+        req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT',
+            'HTTP_X_TIMESTAMP': '0'})
+        resp = self.controller.PUT(req)
+        # test an empty container
+        req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'})
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.status_int, 204)
+        # fill the container
+        for i in range(3):
+            req = Request.blank('/sda1/p/a/plainc/%s'%i, environ=
+                    {'REQUEST_METHOD': 'PUT',
+                    'HTTP_X_TIMESTAMP': '1',
+                    'HTTP_X_CONTENT_TYPE': 'text/plain',
+                    'HTTP_X_ETAG': 'x',
+                    'HTTP_X_SIZE': 0})
+            resp = self.controller.PUT(req)
+            self.assertEquals(resp.status_int, 201)
+        plain_body = '0\n1\n2\n'
+
+        req = Request.blank('/sda1/p/a/plainc',
+                environ={'REQUEST_METHOD': 'GET'})
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.content_type, 'text/plain')
+        self.assertEquals(resp.body, plain_body)
+
+        for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9',
+                '*/*;q=0.9,application/xml;q=0.8', '*/*',
+                'text/plain,application/xml'):
+            req = Request.blank('/sda1/p/a/plainc',
+                    environ={'REQUEST_METHOD': 'GET'})
+            req.accept = accept
+            resp = self.controller.GET(req)
+            self.assertEquals(resp.body, plain_body,
+                'Invalid body for Accept: %s' % accept)
+            self.assertEquals(resp.content_type, 'text/plain',
+                'Invalid content_type for Accept: %s' % accept)
+
+        # test conflicting formats
+        req = Request.blank('/sda1/p/a/plainc?format=plain',
+                environ={'REQUEST_METHOD': 'GET'})
+        req.accept = 'application/json'
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.content_type, 'text/plain')
+        self.assertEquals(resp.body, plain_body)
+
+    def test_GET_xml(self):
+        # make a container
+        req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'PUT',
+            'HTTP_X_TIMESTAMP': '0'})
+        resp = self.controller.PUT(req)
+        # fill the container
+        for i in range(3):
+            req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ=
+                    {'REQUEST_METHOD': 'PUT',
+                    'HTTP_X_TIMESTAMP': '1',
+                    'HTTP_X_CONTENT_TYPE': 'text/plain',
+                    'HTTP_X_ETAG': 'x',
+                    'HTTP_X_SIZE': 0})
+            resp = self.controller.PUT(req)
+            self.assertEquals(resp.status_int, 201)
         xml_body = '<?xml version="1.0" encoding="UTF-8"?>\n' \
-            '<container name="c">' \
+            '<container name="xmlc">' \
                 '<object><name>0</name><hash>x</hash><bytes>0</bytes>' \
                     '<content_type>text/plain</content_type>' \
                     '<last_modified>1970-01-01T00:00:01' \
@@ -529,46 +611,30 @@
                     '<last_modified>1970-01-01T00:00:01' \
                     '</last_modified></object>' \
             '</container>'
-        plain_body = '0\n1\n2\n'
-        req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'})
-        resp = self.controller.GET(req)
-        self.assertEquals(resp.content_type, 'application/json')
-        result = eval(resp.body)
-        self.assertEquals(result, json_body)
-        req = Request.blank('/sda1/p/a/c?format=xml', environ={'REQUEST_METHOD': 'GET'})
+        # tests
+        req = Request.blank('/sda1/p/a/xmlc?format=xml',
+                environ={'REQUEST_METHOD': 'GET'})
         resp = self.controller.GET(req)
         self.assertEquals(resp.content_type, 'application/xml')
-        result = resp.body
-        self.assertEquals(result, xml_body)
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
-        req.accept = 'application/json'
-        resp = self.controller.GET(req)
-        self.assertEquals(resp.content_type, 'application/json')
-        result = eval(resp.body)
-        self.assertEquals(result, json_body)
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
-        req.accept = '*/*'
-        resp = self.controller.GET(req)
-        self.assertEquals(resp.content_type, 'text/plain')
-        result = resp.body
-        self.assertEquals(result, plain_body)
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
-        req.accept = 'application/*'
-        resp = self.controller.GET(req)
-        result = eval(resp.body)
-        self.assertEquals(result, json_body)
-        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
-        req.accept = 'application/xml'
-        resp = self.controller.GET(req)
-        result = resp.body
-        self.assertEquals(result, xml_body)
-        # test conflicting formats
-        req = Request.blank('/sda1/p/a/c?format=plain', environ={'REQUEST_METHOD': 'GET'})
-        req.accept = 'application/json'
-        resp = self.controller.GET(req)
-        self.assertEquals(resp.content_type, 'text/plain')
-        result = resp.body
-        self.assertEquals(result, plain_body)
+        self.assertEquals(resp.body, xml_body)
+
+        for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9',
+                 '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'):
+            req = Request.blank('/sda1/p/a/xmlc',
+                    environ={'REQUEST_METHOD': 'GET'})
+            req.accept = xml_accept
+            resp = self.controller.GET(req)
+            self.assertEquals(resp.body, xml_body,
+                'Invalid body for Accept: %s' % xml_accept)
+            self.assertEquals(resp.content_type, 'application/xml',
+                'Invalid content_type for Accept: %s' % xml_accept)
+
+        req = Request.blank('/sda1/p/a/xmlc',
+                environ={'REQUEST_METHOD': 'GET'})
+        req.accept = 'text/xml'
+        resp = self.controller.GET(req)
+        self.assertEquals(resp.content_type, 'text/xml')
+        self.assertEquals(resp.body, xml_body)
 
     def test_GET_marker(self):
         # make a container

=== modified file 'test/unit/proxy/test_server.py'
--- test/unit/proxy/test_server.py	2010-10-15 15:07:19 +0000
+++ test/unit/proxy/test_server.py	2010-12-15 16:48:57 +0000
@@ -1090,6 +1090,17 @@
             self.assertEquals(resp.headers['x-copied-from'], 'c/o')
 
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                                headers={'Content-Length': '5',
+                                          'X-Copy-From': 'c/o'})
+            self.app.update_request(req)
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200)
+                #                 acct cont acct cont objc
+            self.app.memcache.store = {}
+            resp = controller.PUT(req)
+            self.assertEquals(resp.status_int, 400)
+
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': 'c/o/o2'})
             req.account = 'a'
@@ -1101,6 +1112,18 @@
             self.assertEquals(resp.status_int, 201)
             self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2')
 
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                                headers={'Content-Length': '0',
+                                          'X-Copy-From': 'c/o%20o2'})
+            req.account = 'a'
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
+                #                 acct cont acct cont objc obj  obj  obj
+            self.app.memcache.store = {}
+            resp = controller.PUT(req)
+            self.assertEquals(resp.status_int, 201)
+            self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2')
+
             # repeat tests with leading /
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',

=== modified file 'test/unit/stats/test_log_processor.py'
--- test/unit/stats/test_log_processor.py	2010-10-08 22:20:43 +0000
+++ test/unit/stats/test_log_processor.py	2010-12-15 16:48:57 +0000
@@ -14,9 +14,30 @@
 # limitations under the License.
 
 import unittest
+import os
+from contextlib import contextmanager
+from tempfile import NamedTemporaryFile
 
+from swift.common import internal_proxy
 from swift.stats import log_processor
 
+
+@contextmanager
+def tmpfile(content):
+    with NamedTemporaryFile('w', delete=False) as f:
+        file_name = f.name
+        f.write(str(content))
+    try:
+        yield file_name
+    finally:
+        os.unlink(file_name)
+
+
+class FakeUploadApp(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+
 class DumbLogger(object):
     def __getattr__(self, n):
         return self.foo
@@ -63,6 +84,27 @@
                     }
                    }
 
+    def test_lazy_load_internal_proxy(self):
+        # stub out internal_proxy's upload_app
+        internal_proxy.BaseApplication = FakeUploadApp
+        dummy_proxy_config = """[app:proxy-server]
+use = egg:swift#proxy
+"""
+        with tmpfile(dummy_proxy_config) as proxy_config_file:
+            conf = {'log-processor': {
+                    'proxy_server_conf': proxy_config_file,
+                }
+            }
+            p = log_processor.LogProcessor(conf, DumbLogger())
+            self.assert_(isinstance(p._internal_proxy,
+                                    None.__class__))
+            self.assert_(isinstance(p.internal_proxy,
+                                    log_processor.InternalProxy))
+            self.assertEquals(p.internal_proxy, p._internal_proxy)
+
+        # reset FakeUploadApp
+        reload(internal_proxy)
+
     def test_access_log_line_parser(self):
         access_proxy_config = self.proxy_config.copy()
         access_proxy_config.update({