Merge lp:~gholt/swift/postcopy into lp:~hudson-openstack/swift/trunk

[gholt]

You can specify X-Newest: true on GETs and HEADs to indicate you want Swift to query all backend copies and return the newest version retrieved.
Object COPY requests now always copy the newest object they can find.
Object POSTs are implemented as COPYs now by default (you can revert to previous implementation with conf object_post_as_copy = false)
Account and container GETs and HEADs now shuffle the nodes they use to balance load.

You can specify X-Newest: true on GETs and HEADs to indicate you want Swift to query all backend copies and return the newest version retrieved.
Object COPY requests now always copy the newest object they can find.
Object POSTs are implemented as COPYs now by default (you can revert to previous implementation with conf object_post_as_copy = false)
Account and container GETs and HEADs now shuffle the nodes they use to balance load.

That last part I added because at first I thought it'd be quick and easy; but it ended up having to improve quite a few tests. I can separate it out to another merge if desired; but since it's already here, I left it for the moment.

[lp:~gholt/swift/postcopy]

310. By gholt on 2011-06-09

Merged from trunk

[John Dickinson (notmyname)]

looks go to me

review: Approve

[lp:~gholt/swift/postcopy]

311. By gholt on 2011-06-14

Merged from trunk

[David Goetz (david-goetz)]

looks good

review: Approve

=== modified file 'doc/source/deployment_guide.rst'
--- doc/source/deployment_guide.rst	2011-06-05 23:22:35 +0000
+++ doc/source/deployment_guide.rst	2011-06-14 22:21:02 +0000
@@ -547,6 +547,16 @@
                                                node error limited
 allow_account_management      false            Whether account PUTs and DELETEs
                                                are even callable
+object_post_as_copy           true             Set object_post_as_copy = false
+                                               to turn on fast posts where only
+                                               the metadata changes are stored
+                                               anew and the original data file
+                                               is kept in place. This makes for
+                                               quicker posts; but since the
+                                               container metadata isn't updated
+                                               in this mode, features like
+                                               container sync won't be able to
+                                               sync posts.
 account_autocreate            false            If set to 'true' authorized
                                                accounts that do not yet exist
                                                within the Swift cluster will
 
=== modified file 'etc/proxy-server.conf-sample'
--- etc/proxy-server.conf-sample	2011-06-05 23:22:35 +0000
+++ etc/proxy-server.conf-sample	2011-06-14 22:21:02 +0000
@@ -40,6 +40,11 @@
 # If set to 'true' any authorized user may create and delete accounts; if
 # 'false' no one, even authorized, can.
 # allow_account_management = false
+# Set object_post_as_copy = false to turn on fast posts where only the metadata
+# changes are stored anew and the original data file is kept in place. This
+# makes for quicker posts; but since the container metadata isn't updated in
+# this mode, features like container sync won't be able to sync posts.
+# object_post_as_copy = true
 # If set to 'true' authorized accounts that do not yet exist within the Swift
 # cluster will be automatically created.
 # account_autocreate = false
 
=== modified file 'swift/common/direct_client.py'
--- swift/common/direct_client.py	2011-03-29 02:09:24 +0000
+++ swift/common/direct_client.py	2011-06-14 22:21:02 +0000
@@ -36,6 +36,57 @@
     return _quote(value, safe)
 
 
+def direct_get_account(node, part, account, marker=None, limit=None,
+                       prefix=None, delimiter=None, conn_timeout=5,
+                       response_timeout=15):
+    """
+    Get listings directly from the account server.
+
+    :param node: node dictionary from the ring
+    :param part: partition the account is on
+    :param account: account name
+    :param marker: marker query
+    :param limit: query limit
+    :param prefix: prefix query
+    :param delimeter: delimeter for the query
+    :param conn_timeout: timeout in seconds for establishing the connection
+    :param response_timeout: timeout in seconds for getting the response
+    :returns: a tuple of (response headers, a list of containers) The response
+              headers will be a dict and all header names will be lowercase.
+    """
+    path = '/' + account
+    qs = 'format=json'
+    if marker:
+        qs += '&marker=%s' % quote(marker)
+    if limit:
+        qs += '&limit=%d' % limit
+    if prefix:
+        qs += '&prefix=%s' % quote(prefix)
+    if delimiter:
+        qs += '&delimiter=%s' % quote(delimiter)
+    with Timeout(conn_timeout):
+        conn = http_connect(node['ip'], node['port'], node['device'], part,
+                            'GET', path, query_string='format=json')
+    with Timeout(response_timeout):
+        resp = conn.getresponse()
+    if resp.status < 200 or resp.status >= 300:
+        resp.read()
+        raise ClientException(
+            'Account server %s:%s direct GET %s gave status %s' % (node['ip'],
+                node['port'], repr('/%s/%s%s' % (node['device'], part, path)),
+                resp.status),
+            http_host=node['ip'], http_port=node['port'],
+            http_device=node['device'], http_status=resp.status,
+            http_reason=resp.reason)
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
+    if resp.status == 204:
+        resp.read()
+        return resp_headers, []
+    return resp_headers, json_loads(resp.read())
+
+
 def direct_head_container(node, part, account, container, conn_timeout=5,
                           response_timeout=15):
     """
 
=== modified file 'swift/obj/server.py'
--- swift/obj/server.py	2011-05-09 20:21:34 +0000
+++ swift/obj/server.py	2011-06-14 22:21:02 +0000
@@ -623,6 +623,7 @@
             file.keep_cache = True
         if 'Content-Encoding' in file.metadata:
             response.content_encoding = file.metadata['Content-Encoding']
+        response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
         return request.get_response(response)
 
     def HEAD(self, request):
@@ -657,6 +658,7 @@
         response.content_length = file_size
         if 'Content-Encoding' in file.metadata:
             response.content_encoding = file.metadata['Content-Encoding']
+        response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
         return response
 
     def DELETE(self, request):
 
=== modified file 'swift/proxy/server.py'
--- swift/proxy/server.py	2011-06-11 04:57:04 +0000
+++ swift/proxy/server.py	2011-06-14 22:21:02 +0000
@@ -162,6 +162,7 @@
             if self.segment > 10:
                 sleep(max(self.next_get_time - time.time(), 0))
                 self.next_get_time = time.time() + 1
+            shuffle(nodes)
             resp = self.controller.GETorHEAD_base(req, _('Object'), partition,
                 self.controller.iter_nodes(partition, nodes,
                 self.controller.app.object_ring), path,
@@ -605,6 +606,8 @@
         statuses = []
         reasons = []
         bodies = []
+        source = None
+        newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
         for node in nodes:
             if len(statuses) >= attempts:
                 break
@@ -617,23 +620,48 @@
                         headers=req.headers,
                         query_string=req.query_string)
                 with Timeout(self.app.node_timeout):
-                    source = conn.getresponse()
+                    possible_source = conn.getresponse()
             except (Exception, TimeoutError):
                 self.exception_occurred(node, server_type,
                     _('Trying to %(method)s %(path)s') %
                     {'method': req.method, 'path': req.path})
                 continue
-            if source.status == 507:
+            if possible_source.status == 507:
                 self.error_limit(node)
                 continue
-            if 200 <= source.status <= 399:
+            if 200 <= possible_source.status <= 399:
                 # 404 if we know we don't have a synced copy
-                if not float(source.getheader('X-PUT-Timestamp', '1')):
+                if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
                     statuses.append(404)
                     reasons.append('')
                     bodies.append('')
-                    source.read()
-                    continue
+                    possible_source.read()
+                    continue
+            if (req.method == 'GET' and
+                possible_source.status in (200, 206)) or \
+                    200 <= possible_source.status <= 399:
+                if newest:
+                    ts = 0
+                    if source:
+                        ts = float(source.getheader('x-put-timestamp') or
+                                   source.getheader('x-timestamp') or 0)
+                    pts = float(possible_source.getheader('x-put-timestamp') or
+                                possible_source.getheader('x-timestamp') or 0)
+                    if pts > ts:
+                        source = possible_source
+                    continue
+                else:
+                    source = possible_source
+                    break
+            statuses.append(possible_source.status)
+            reasons.append(possible_source.reason)
+            bodies.append(possible_source.read())
+            if possible_source.status >= 500:
+                self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
+                    'From %(type)s Server') %
+                    {'status': possible_source.status,
+                    'body': bodies[-1][:1024], 'type': server_type})
+        if source:
             if req.method == 'GET' and source.status in (200, 206):
                 res = Response(request=req, conditional_response=True)
                 res.bytes_transferred = 0
@@ -673,13 +701,6 @@
                         res.charset = None
                         res.content_type = source.getheader('Content-Type')
                 return res
-            statuses.append(source.status)
-            reasons.append(source.reason)
-            bodies.append(source.read())
-            if source.status >= 500:
-                self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
-                    'From %(type)s Server') % {'status': source.status,
-                    'body': bodies[-1][:1024], 'type': server_type})
         return self.best_response(req, statuses, reasons, bodies,
                                   '%s %s' % (server_type, req.method))
 
@@ -734,6 +755,7 @@
                 lreq = Request.blank('/%s/%s?prefix=%s&format=json&marker=%s' %
                     (quote(self.account_name), quote(lcontainer),
                      quote(lprefix), quote(marker)))
+                shuffle(lnodes)
                 lresp = self.GETorHEAD_base(lreq, _('Container'), lpartition,
                     lnodes, lreq.path_info,
                     self.app.container_ring.replica_count)
@@ -861,30 +883,40 @@
     @delay_denial
     def POST(self, req):
         """HTTP POST request handler."""
-        error_response = check_metadata(req, 'object')
-        if error_response:
-            return error_response
-        container_partition, containers, _junk, req.acl = \
-            self.container_info(self.account_name, self.container_name,
-                account_autocreate=self.app.account_autocreate)
-        if 'swift.authorize' in req.environ:
-            aresp = req.environ['swift.authorize'](req)
-            if aresp:
-                return aresp
-        if not containers:
-            return HTTPNotFound(request=req)
-        partition, nodes = self.app.object_ring.get_nodes(
-            self.account_name, self.container_name, self.object_name)
-        req.headers['X-Timestamp'] = normalize_timestamp(time.time())
-        headers = []
-        for container in containers:
-            nheaders = dict(req.headers.iteritems())
-            nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
-            nheaders['X-Container-Partition'] = container_partition
-            nheaders['X-Container-Device'] = container['device']
-            headers.append(nheaders)
-        return self.make_requests(req, self.app.object_ring,
-                partition, 'POST', req.path_info, headers)
+        if self.app.object_post_as_copy:
+            req.method = 'PUT'
+            req.path_info = '/%s/%s/%s' % (self.account_name,
+                self.container_name, self.object_name)
+            req.headers['Content-Length'] = 0
+            req.headers['X-Copy-From'] = '/%s/%s' % (self.container_name,
+                self.object_name)
+            req.headers['X-Fresh-Metadata'] = 'true'
+            return self.PUT(req)
+        else:
+            error_response = check_metadata(req, 'object')
+            if error_response:
+                return error_response
+            container_partition, containers, _junk, req.acl = \
+                self.container_info(self.account_name, self.container_name,
+                    account_autocreate=self.app.account_autocreate)
+            if 'swift.authorize' in req.environ:
+                aresp = req.environ['swift.authorize'](req)
+                if aresp:
+                    return aresp
+            if not containers:
+                return HTTPNotFound(request=req)
+            partition, nodes = self.app.object_ring.get_nodes(
+                self.account_name, self.container_name, self.object_name)
+            req.headers['X-Timestamp'] = normalize_timestamp(time.time())
+            headers = []
+            for container in containers:
+                nheaders = dict(req.headers.iteritems())
+                nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
+                nheaders['X-Container-Partition'] = container_partition
+                nheaders['X-Container-Device'] = container['device']
+                headers.append(nheaders)
+            return self.make_requests(req, self.app.object_ring,
+                    partition, 'POST', req.path_info, headers)
 
     def _send_file(self, conn, path):
         """Method for a file PUT coro"""
@@ -947,6 +979,7 @@
         reader = req.environ['wsgi.input'].read
         data_source = iter(lambda: reader(self.app.client_chunk_size), '')
         source_header = req.headers.get('X-Copy-From')
+        source_resp = None
         if source_header:
             source_header = unquote(source_header)
             acct = req.path_info.split('/', 2)[1]
@@ -962,6 +995,7 @@
                     '<container name>/<object name>')
             source_req = req.copy_get()
             source_req.path_info = source_header
+            source_req.headers['X-Newest'] = 'true'
             orig_obj_name = self.object_name
             orig_container_name = self.container_name
             self.object_name = src_obj_name
@@ -987,12 +1021,14 @@
             if not content_type_manually_set:
                 new_req.headers['Content-Type'] = \
                     source_resp.headers['Content-Type']
-            for k, v in source_resp.headers.items():
-                if k.lower().startswith('x-object-meta-'):
-                    new_req.headers[k] = v
-            for k, v in req.headers.items():
-                if k.lower().startswith('x-object-meta-'):
-                    new_req.headers[k] = v
+            if new_req.headers.get('x-fresh-metadata', 'false').lower() \
+                    not in TRUE_VALUES:
+                for k, v in source_resp.headers.items():
+                    if k.lower().startswith('x-object-meta-'):
+                        new_req.headers[k] = v
+                for k, v in req.headers.items():
+                    if k.lower().startswith('x-object-meta-'):
+                        new_req.headers[k] = v
             req = new_req
         node_iter = self.iter_nodes(partition, nodes, self.app.object_ring)
         pile = GreenPile(len(nodes))
@@ -1094,6 +1130,9 @@
         if source_header:
             resp.headers['X-Copied-From'] = quote(
                                                 source_header.split('/', 2)[2])
+            if 'last-modified' in source_resp.headers:
+                resp.headers['X-Copied-From-Last-Modified'] = \
+                    source_resp.headers['last-modified']
             for k, v in req.headers.items():
                 if k.lower().startswith('x-object-meta-'):
                     resp.headers[k] = v
@@ -1187,6 +1226,7 @@
             return HTTPNotFound(request=req)
         part, nodes = self.app.container_ring.get_nodes(
                         self.account_name, self.container_name)
+        shuffle(nodes)
         resp = self.GETorHEAD_base(req, _('Container'), part, nodes,
                 req.path_info, self.app.container_ring.replica_count)
 
@@ -1319,6 +1359,7 @@
     def GETorHEAD(self, req):
         """Handler for HTTP GET/HEAD requests."""
         partition, nodes = self.app.account_ring.get_nodes(self.account_name)
+        shuffle(nodes)
         resp = self.GETorHEAD_base(req, _('Account'), partition, nodes,
                 req.path_info.rstrip('/'), self.app.account_ring.replica_count)
         if resp.status_int == 404 and self.app.account_autocreate:
@@ -1449,6 +1490,8 @@
             int(conf.get('recheck_account_existence', 60))
         self.allow_account_management = \
             conf.get('allow_account_management', 'no').lower() in TRUE_VALUES
+        self.object_post_as_copy = \
+            conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES
         self.resellers_conf = ConfigParser()
         self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
         self.object_ring = object_ring or \
 
=== modified file 'test/functional/swift.py'
--- test/functional/swift.py	2011-02-23 04:25:38 +0000
+++ test/functional/swift.py	2011-06-14 22:21:02 +0000
@@ -668,7 +668,7 @@
 
             self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg)
 
-            if self.conn.response.status != 202:
+            if self.conn.response.status not in (201, 202):
                 raise ResponseError(self.conn.response)
 
         return True
 
=== modified file 'test/functional/tests.py'
--- test/functional/tests.py	2011-06-10 18:49:41 +0000
+++ test/functional/tests.py	2011-06-14 22:21:02 +0000
@@ -1032,7 +1032,7 @@
                 self.assert_(file.write())
                 self.assert_status(201)
                 self.assert_(file.sync_metadata())
-                self.assert_status(202)
+                self.assert_status((201, 202))
             else:
                 self.assertRaises(ResponseError, file.write)
                 self.assert_status(400)
@@ -1245,7 +1245,7 @@
 
             file.metadata = metadata
             self.assert_(file.sync_metadata())
-            self.assert_status(202)
+            self.assert_status((201, 202))
 
             file = self.env.container.file(file.name)
             self.assert_(file.initialize())
 
=== modified file 'test/probe/test_account_failures.py'
--- test/probe/test_account_failures.py	2011-01-04 23:34:43 +0000
+++ test/probe/test_account_failures.py	2011-06-14 22:21:02 +0000
@@ -20,7 +20,7 @@
 from subprocess import Popen
 from time import sleep
 
-from swift.common import client
+from swift.common import client, direct_client
 from test.probe.common import get_to_final_state, kill_pids, reset_environment
 
 
@@ -146,7 +146,8 @@
         sleep(2)
         # This is the earlier counts and bytes because the first node doesn't
         # have the newest udpates yet.
-        headers, containers = client.get_account(self.url, self.token)
+        headers, containers = \
+            direct_client.direct_get_account(anodes[0], apart, self.account)
         self.assertEquals(headers['x-account-container-count'], '2')
         self.assertEquals(headers['x-account-object-count'], '1')
         self.assertEquals(headers['x-account-bytes-used'], '4')
@@ -167,7 +168,8 @@
         self.assert_(found2)
 
         get_to_final_state()
-        headers, containers = client.get_account(self.url, self.token)
+        headers, containers = \
+            direct_client.direct_get_account(anodes[0], apart, self.account)
         self.assertEquals(headers['x-account-container-count'], '1')
         self.assertEquals(headers['x-account-object-count'], '2')
         self.assertEquals(headers['x-account-bytes-used'], '9')
 
=== modified file 'test/probe/test_container_failures.py'
--- test/probe/test_container_failures.py	2011-04-13 17:57:59 +0000
+++ test/probe/test_container_failures.py	2011-06-14 22:21:02 +0000
@@ -24,7 +24,7 @@
 import eventlet
 import sqlite3
 
-from swift.common import client
+from swift.common import client, direct_client
 from swift.common.utils import hash_path, readconf
 
 from test.probe.common import get_to_final_state, kill_pids, reset_environment
@@ -72,7 +72,8 @@
         # This okay because the first node hasn't got the update that the
         # object was deleted yet.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
 
         # Unfortunately, the following might pass or fail, depending on the
         # position of the account server associated with the first container
@@ -88,7 +89,8 @@
         client.put_object(self.url, self.token, container, object2, 'test')
         # First node still doesn't know object1 was deleted yet; this is okay.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
                      client.get_container(self.url, self.token, container)[1]])
@@ -150,7 +152,8 @@
         # server has to indicate the container exists for the put to continue.
         client.put_object(self.url, self.token, container, object2, 'test')
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
                      client.get_container(self.url, self.token, container)[1]])
@@ -201,7 +204,8 @@
         # This okay because the first node hasn't got the update that the
         # object was deleted yet.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -228,7 +232,8 @@
         client.put_object(self.url, self.token, container, object2, 'test')
         # First node still doesn't know object1 was deleted yet; this is okay.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
                      client.get_container(self.url, self.token, container)[1]])
@@ -277,7 +282,8 @@
         self.assert_(container in [c['name'] for c in
                      client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -303,7 +309,8 @@
         # server has to indicate the container exists for the put to continue.
         client.put_object(self.url, self.token, container, object2, 'test')
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)[1]])
+                     direct_client.direct_get_container(cnodes[0], cpart,
+                     self.account, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
                      client.get_container(self.url, self.token, container)[1]])
 
=== modified file 'test/probe/test_object_handoff.py'
--- test/probe/test_object_handoff.py	2011-01-26 22:31:33 +0000
+++ test/probe/test_object_handoff.py	2011-06-14 22:21:02 +0000
@@ -124,47 +124,49 @@
         if not exc:
             raise Exception('Handoff object server still had test object')
 
-        kill(self.pids[self.port2server[onode['port']]], SIGTERM)
-        client.post_object(self.url, self.token, container, obj,
-                           headers={'x-object-meta-probe': 'value'})
-        oheaders = client.head_object(self.url, self.token, container, obj)
-        if oheaders.get('x-object-meta-probe') != 'value':
-            raise Exception('Metadata incorrect, was %s' % repr(oheaders))
-        exc = False
-        try:
-            direct_client.direct_get_object(another_onode, opart, self.account,
-                                            container, obj)
-        except Exception:
-            exc = True
-        if not exc:
-            raise Exception('Handoff server claimed it had the object when '
-                            'it should not have it')
-        self.pids[self.port2server[onode['port']]] = Popen([
-            'swift-object-server',
-            '/etc/swift/object-server/%d.conf' %
-            ((onode['port'] - 6000) / 10)]).pid
-        sleep(2)
-        oheaders = direct_client.direct_get_object(onode, opart, self.account,
-                                                    container, obj)[0]
-        if oheaders.get('x-object-meta-probe') == 'value':
-            raise Exception('Previously downed object server had the new '
-                            'metadata when it should not have it')
-        # Run the extra server last so it'll remove it's extra partition
-        ps = []
-        for n in onodes:
-            ps.append(Popen(['swift-object-replicator',
-                             '/etc/swift/object-server/%d.conf' %
-                             ((n['port'] - 6000) / 10), 'once']))
-        for p in ps:
-            p.wait()
-        call(['swift-object-replicator',
-              '/etc/swift/object-server/%d.conf' %
-              ((another_onode['port'] - 6000) / 10), 'once'])
-        oheaders = direct_client.direct_get_object(onode, opart, self.account,
-                                                    container, obj)[0]
-        if oheaders.get('x-object-meta-probe') != 'value':
-            raise Exception(
-                'Previously downed object server did not have the new metadata')
+# Because POST has changed to a COPY by default, POSTs will succeed on all up
+# nodes now if at least one up node has the object.
+#       kill(self.pids[self.port2server[onode['port']]], SIGTERM)
+#       client.post_object(self.url, self.token, container, obj,
+#                          headers={'x-object-meta-probe': 'value'})
+#       oheaders = client.head_object(self.url, self.token, container, obj)
+#       if oheaders.get('x-object-meta-probe') != 'value':
+#           raise Exception('Metadata incorrect, was %s' % repr(oheaders))
+#       exc = False
+#       try:
+#           direct_client.direct_get_object(another_onode, opart, self.account,
+#                                           container, obj)
+#       except Exception:
+#           exc = True
+#       if not exc:
+#           raise Exception('Handoff server claimed it had the object when '
+#                           'it should not have it')
+#       self.pids[self.port2server[onode['port']]] = Popen([
+#           'swift-object-server',
+#           '/etc/swift/object-server/%d.conf' %
+#           ((onode['port'] - 6000) / 10)]).pid
+#       sleep(2)
+#       oheaders = direct_client.direct_get_object(onode, opart, self.account,
+#                                                   container, obj)[0]
+#       if oheaders.get('x-object-meta-probe') == 'value':
+#           raise Exception('Previously downed object server had the new '
+#                           'metadata when it should not have it')
+#       # Run the extra server last so it'll remove it's extra partition
+#       ps = []
+#       for n in onodes:
+#           ps.append(Popen(['swift-object-replicator',
+#                            '/etc/swift/object-server/%d.conf' %
+#                            ((n['port'] - 6000) / 10), 'once']))
+#       for p in ps:
+#           p.wait()
+#       call(['swift-object-replicator',
+#             '/etc/swift/object-server/%d.conf' %
+#             ((another_onode['port'] - 6000) / 10), 'once'])
+#       oheaders = direct_client.direct_get_object(onode, opart, self.account,
+#                                                   container, obj)[0]
+#       if oheaders.get('x-object-meta-probe') != 'value':
+#           raise Exception(
+#               'Previously downed object server did not have the new metadata')
 
         kill(self.pids[self.port2server[onode['port']]], SIGTERM)
         client.delete_object(self.url, self.token, container, obj)
 
=== modified file 'test/unit/proxy/test_server.py'
--- test/unit/proxy/test_server.py	2011-06-11 04:57:04 +0000
+++ test/unit/proxy/test_server.py	2011-06-14 22:21:02 +0000
@@ -150,7 +150,7 @@
 
     class FakeConn(object):
 
-        def __init__(self, status, etag=None, body=''):
+        def __init__(self, status, etag=None, body='', timestamp='1'):
             self.status = status
             self.reason = 'Fake'
             self.host = '1.2.3.4'
@@ -159,6 +159,7 @@
             self.received = 0
             self.etag = etag
             self.body = body
+            self.timestamp = timestamp
 
         def getresponse(self):
             if kwargs.get('raise_exc'):
@@ -173,7 +174,8 @@
         def getheaders(self):
             headers = {'content-length': len(self.body),
                        'content-type': 'x-application/test',
-                       'x-timestamp': '1',
+                       'x-timestamp': self.timestamp,
+                       'last-modified': self.timestamp,
                        'x-object-meta-test': 'testing',
                        'etag':
                             self.etag or '"68b329da9893e34099c7d8ad5cb9c940"',
@@ -209,6 +211,7 @@
         def getheader(self, name, default=None):
             return dict(self.getheaders()).get(name.lower(), default)
 
+    timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))
     etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
     x = kwargs.get('missing_container', [False] * len(code_iter))
     if not isinstance(x, (tuple, list)):
@@ -226,9 +229,11 @@
             kwargs['give_connect'](*args, **ckwargs)
         status = code_iter.next()
         etag = etag_iter.next()
+        timestamp = timestamps_iter.next()
         if status == -1:
             raise HTTPException()
-        return FakeConn(status, etag, body=kwargs.get('body', ''))
+        return FakeConn(status, etag, body=kwargs.get('body', ''),
+                        timestamp=timestamp)
 
     return connect
 
@@ -962,6 +967,7 @@
 
     def test_POST(self):
         with save_globals():
+            self.app.object_post_as_copy = False
             controller = proxy_server.ObjectController(self.app, 'account',
                 'container', 'object')
 
@@ -982,6 +988,28 @@
             test_status_map((200, 200, 404, 500, 500), 503)
             test_status_map((200, 200, 404, 404, 404), 404)
 
+    def test_POST_as_copy(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+
+            def test_status_map(statuses, expected):
+                proxy_server.http_connect = fake_http_connect(*statuses)
+                self.app.memcache.store = {}
+                req = Request.blank('/a/c/o', {}, headers={
+                                                'Content-Type': 'foo/bar'})
+                self.app.update_request(req)
+                res = controller.POST(req)
+                expected = str(expected)
+                self.assertEquals(res.status[:len(expected)], expected)
+            test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202)
+            test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202)
+            test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503)
+            test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503)
+            test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404)
+            test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503)
+            test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404)
+
     def test_DELETE(self):
         with save_globals():
             controller = proxy_server.ObjectController(self.app, 'account',
@@ -1028,8 +1056,77 @@
             test_status_map((404, 404, 500), 404)
             test_status_map((500, 500, 500), 503)
 
+    def test_HEAD_newest(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+
+            def test_status_map(statuses, expected, timestamps,
+                                expected_timestamp):
+                proxy_server.http_connect = \
+                    fake_http_connect(*statuses, timestamps=timestamps)
+                self.app.memcache.store = {}
+                req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
+                self.app.update_request(req)
+                res = controller.HEAD(req)
+                self.assertEquals(res.status[:len(str(expected))],
+                                  str(expected))
+                self.assertEquals(res.headers.get('last-modified'),
+                                  expected_timestamp)
+
+            test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3')
+            test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3')
+            test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3')
+            test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
+
+    def test_GET_newest(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+
+            def test_status_map(statuses, expected, timestamps,
+                                expected_timestamp):
+                proxy_server.http_connect = \
+                    fake_http_connect(*statuses, timestamps=timestamps)
+                self.app.memcache.store = {}
+                req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'})
+                self.app.update_request(req)
+                res = controller.GET(req)
+                self.assertEquals(res.status[:len(str(expected))],
+                                  str(expected))
+                self.assertEquals(res.headers.get('last-modified'),
+                                  expected_timestamp)
+
+            test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3')
+            test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3')
+            test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3')
+            test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
+
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+
+            def test_status_map(statuses, expected, timestamps,
+                                expected_timestamp):
+                proxy_server.http_connect = \
+                    fake_http_connect(*statuses, timestamps=timestamps)
+                self.app.memcache.store = {}
+                req = Request.blank('/a/c/o', {})
+                self.app.update_request(req)
+                res = controller.HEAD(req)
+                self.assertEquals(res.status[:len(str(expected))],
+                                  str(expected))
+                self.assertEquals(res.headers.get('last-modified'),
+                                  expected_timestamp)
+
+            test_status_map((200, 200, 200), 200, ('1', '2', '3'), '1')
+            test_status_map((200, 200, 200), 200, ('1', '3', '2'), '1')
+            test_status_map((200, 200, 200), 200, ('1', '3', '1'), '1')
+            test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3')
+
     def test_POST_meta_val_len(self):
         with save_globals():
+            self.app.object_post_as_copy = False
             controller = proxy_server.ObjectController(self.app, 'account',
                 'container', 'object')
             proxy_server.http_connect = \
@@ -1049,8 +1146,30 @@
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
+    def test_POST_as_copy_meta_val_len(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
+                #                 acct cont objc objc objc obj  obj  obj
+            req = Request.blank('/a/c/o', {}, headers={
+                                            'Content-Type': 'foo/bar',
+                                            'X-Object-Meta-Foo': 'x' * 256})
+            self.app.update_request(req)
+            res = controller.POST(req)
+            self.assertEquals(res.status_int, 202)
+            proxy_server.http_connect = fake_http_connect(202, 202, 202)
+            req = Request.blank('/a/c/o', {}, headers={
+                                            'Content-Type': 'foo/bar',
+                                            'X-Object-Meta-Foo': 'x' * 257})
+            self.app.update_request(req)
+            res = controller.POST(req)
+            self.assertEquals(res.status_int, 400)
+
     def test_POST_meta_key_len(self):
         with save_globals():
+            self.app.object_post_as_copy = False
             controller = proxy_server.ObjectController(self.app, 'account',
                 'container', 'object')
             proxy_server.http_connect = \
@@ -1070,6 +1189,27 @@
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
+    def test_POST_as_copy_meta_key_len(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'account',
+                'container', 'object')
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202)
+                #                 acct cont objc objc objc obj  obj  obj
+            req = Request.blank('/a/c/o', {}, headers={
+                'Content-Type': 'foo/bar',
+                ('X-Object-Meta-' + 'x' * 128): 'x'})
+            self.app.update_request(req)
+            res = controller.POST(req)
+            self.assertEquals(res.status_int, 202)
+            proxy_server.http_connect = fake_http_connect(202, 202, 202)
+            req = Request.blank('/a/c/o', {}, headers={
+                'Content-Type': 'foo/bar',
+                ('X-Object-Meta-' + 'x' * 129): 'x'})
+            self.app.update_request(req)
+            res = controller.POST(req)
+            self.assertEquals(res.status_int, 400)
+
     def test_POST_meta_count(self):
         with save_globals():
             controller = proxy_server.ObjectController(self.app, 'account',
@@ -1344,7 +1484,8 @@
             self.assert_status_map(controller.HEAD, (200, 200, 200), 503)
             self.assert_('last_error' in controller.app.object_ring.devs[0])
             self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503)
-            self.assert_status_map(controller.POST, (200, 202, 202, 202), 503)
+            self.assert_status_map(controller.POST,
+                                   (200, 200, 200, 200, 202, 202, 202), 503)
             self.assert_status_map(controller.DELETE,
                                    (200, 204, 204, 204), 503)
             self.app.error_suppression_interval = -300
@@ -1437,18 +1578,41 @@
 
     def test_PUT_POST_requires_container_exist(self):
         with save_globals():
-            self.app.memcache = FakeMemcacheReturnsNone()
-            controller = proxy_server.ObjectController(self.app, 'account',
-                                                       'container', 'object')
-            proxy_server.http_connect = \
-                fake_http_connect(404, 404, 404, 200, 200, 200)
-            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
-            self.app.update_request(req)
-            resp = controller.PUT(req)
-            self.assertEquals(resp.status_int, 404)
-
-            proxy_server.http_connect = \
-                fake_http_connect(404, 404, 404, 200, 200, 200)
+            self.app.object_post_as_copy = False
+            self.app.memcache = FakeMemcacheReturnsNone()
+            controller = proxy_server.ObjectController(self.app, 'account',
+                                                       'container', 'object')
+
+            proxy_server.http_connect = \
+                fake_http_connect(200, 404, 404, 404, 200, 200, 200)
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
+            self.app.update_request(req)
+            resp = controller.PUT(req)
+            self.assertEquals(resp.status_int, 404)
+
+            proxy_server.http_connect = \
+                fake_http_connect(200, 404, 404, 404, 200, 200)
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
+                                headers={'Content-Type': 'text/plain'})
+            self.app.update_request(req)
+            resp = controller.POST(req)
+            self.assertEquals(resp.status_int, 404)
+
+    def test_PUT_POST_as_copy_requires_container_exist(self):
+        with save_globals():
+            self.app.memcache = FakeMemcacheReturnsNone()
+            controller = proxy_server.ObjectController(self.app, 'account',
+                                                       'container', 'object')
+            proxy_server.http_connect = \
+                fake_http_connect(200, 404, 404, 404, 200, 200, 200)
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
+            self.app.update_request(req)
+            resp = controller.PUT(req)
+            self.assertEquals(resp.status_int, 404)
+
+            proxy_server.http_connect = \
+                fake_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200,
+                                  200)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                 headers={'Content-Type': 'text/plain'})
             self.app.update_request(req)
@@ -1568,8 +1732,10 @@
                                           'X-Copy-From': 'c/o'})
             self.app.update_request(req)
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1581,8 +1747,8 @@
                                           'X-Copy-From': 'c/o'})
             self.app.update_request(req)
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200)
-                #                 acct cont acct cont objc
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200)
+                #                 acct cont acct cont objc objc objc
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 400)
@@ -1593,8 +1759,10 @@
                                           'X-Copy-From': 'c/o/o2'})
             req.account = 'a'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1606,8 +1774,10 @@
                                           'X-Copy-From': 'c/o%20o2'})
             req.account = 'a'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1619,8 +1789,10 @@
                                           'X-Copy-From': '/c/o'})
             self.app.update_request(req)
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1631,8 +1803,10 @@
                                           'X-Copy-From': '/c/o/o2'})
             req.account = 'a'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1692,8 +1866,8 @@
                                           'X-Object-Meta-Ours': 'okay'})
             self.app.update_request(req)
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 201, 201, 201)
-                #                 acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
+                #                 acct cont objc objc objc obj  obj  obj
             self.app.memcache.store = {}
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
@@ -1717,8 +1891,10 @@
                                 headers={'Destination': 'c/o'})
             req.account = 'a'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.COPY(req)
             self.assertEquals(resp.status_int, 201)
@@ -1730,8 +1906,10 @@
             req.account = 'a'
             controller.object_name = 'o/o2'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.COPY(req)
             self.assertEquals(resp.status_int, 201)
@@ -1742,8 +1920,10 @@
             req.account = 'a'
             controller.object_name = 'o'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.COPY(req)
             self.assertEquals(resp.status_int, 201)
@@ -1755,8 +1935,10 @@
             req.account = 'a'
             controller.object_name = 'o/o2'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
-                #                 acct cont acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201,
+                    201)
+                #                 acct cont acct cont objc objc objc obj  obj
+                #   obj
             self.app.memcache.store = {}
             resp = controller.COPY(req)
             self.assertEquals(resp.status_int, 201)
@@ -1812,8 +1994,8 @@
             req.account = 'a'
             controller.object_name = 'o'
             proxy_server.http_connect = \
-                fake_http_connect(200, 200, 200, 201, 201, 201)
-                #                 acct cont objc obj  obj  obj
+                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
+                #                 acct cont objc objc objc obj  obj  obj
             self.app.memcache.store = {}
             resp = controller.COPY(req)
             self.assertEquals(resp.status_int, 201)
@@ -1821,6 +2003,23 @@
                               'testing')
             self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')
 
+    def test_COPY_newest(self):
+        with save_globals():
+            controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'},
+                                headers={'Destination': '/c/o'})
+            req.account = 'a'
+            controller.object_name = 'o'
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201,
+                      timestamps=('1', '1', '1', '3', '2', '4', '4', '4'))
+                #                 acct cont objc objc objc obj  obj  obj
+            self.app.memcache.store = {}
+            resp = controller.COPY(req)
+            self.assertEquals(resp.status_int, 201)
+            self.assertEquals(resp.headers['x-copied-from-last-modified'],
+                              '3')
+
     def test_chunked_put(self):
 
         class ChunkedFile():
@@ -2596,6 +2795,7 @@
             called[0] = True
             return HTTPUnauthorized(request=req)
         with save_globals():
+            self.app.object_post_as_copy = False
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 201, 201, 201)
             controller = proxy_server.ObjectController(self.app, 'account',
@@ -2607,6 +2807,24 @@
             res = controller.POST(req)
         self.assert_(called[0])
 
+    def test_POST_as_copy_calls_authorize(self):
+        called = [False]
+
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
+            controller = proxy_server.ObjectController(self.app, 'account',
+                            'container', 'object')
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
+                                headers={'Content-Length': '5'}, body='12345')
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.POST(req)
+        self.assert_(called[0])
+
     def test_PUT_calls_authorize(self):
         called = [False]
 
@@ -2814,6 +3032,7 @@
 
     def test_error_limiting(self):
         with save_globals():
+            proxy_server.shuffle = lambda l: None
             controller = proxy_server.ContainerController(self.app, 'account',
                                                           'container')
             self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200,