Merge lp:~garethwatts/goamz/goamz into lp:goamz

[Gareth Watts]

Adds support for deleting up to 1,000 objects from S3 per request using S3's Delete Multiple Objects interface (http://goo.gl/W5ICVJ )

I used this to selectively delete over 4 billion objects from S3 last night fwiw

Unmerged revisions

44. By Gareth Watts on 2013-11-23

Add s3.DelMulti() to delete multiple objects in one request

=== modified file 's3/responses_test.go'
--- s3/responses_test.go	2013-02-12 04:50:11 +0000
+++ s3/responses_test.go	2013-11-23 00:42:56 +0000
@@ -196,3 +196,17 @@
   <HostId>kjhwqk</HostId>
 </Error>
 `
+
+var DelMultiResultDump = `
+<?xml version="1.0" encoding="UTF-8"?>
+<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Deleted>
+    <Key>Nelson</Key>
+  </Deleted>
+  <Error>
+    <Key>Neo</Key>
+    <Code>AccessDenied</Code>
+    <Message>Access Denied</Message>
+  </Error>
+</DeleteResult>
+`

=== modified file 's3/s3.go'
--- s3/s3.go	2013-08-15 15:33:14 +0000
+++ s3/s3.go	2013-11-23 00:42:56 +0000
@@ -220,6 +220,97 @@
 	return b.S3.query(req, nil)
 }
 
+// The DelObject type holds the key name and optional version identifier of an object
+// to be deleted from S3.
+type DelObject struct {
+	Key       string
+	VersionId string `xml:",omitempty"`
+}
+
+// The DelErr type holds the deletion error for a specific object returned as part of
+// the DelResp response from DelMulti.
+type DelErr struct {
+	Key       string
+	VersionId string
+	Code      string
+	Message   string
+}
+
+// The DelSuccess type holds details of a successful object deletion fom S3 as part of
+// the DelResp response from DelMulti.
+type DelSuccess struct {
+	Key                   string
+	VersionId             string
+	DeleteMarker          bool
+	DeleteMarkerVersionId string
+}
+
+// The DelResp type holds the result of a DelMulti operation.
+type DelResp struct {
+	// All objects that were successfully deleted.
+	// Note this will always be empty if DelMulti is called with quiet=True.
+	Deleted []DelSuccess
+	// All objects that failed to be deleted.
+	Error []DelErr
+}
+
+type delRequest struct {
+	XMLName xml.Name `xml:"Delete"`
+	Quiet   bool
+	Object  []DelObject
+}
+
+// DelMultiLimit defines the maximum number of objects S3 supports for a DelMulti operation.
+const DelMultiLimit = 1000
+
+// DelMulti deletes up to 1,000 objects from the S3 bucket.
+//
+// Set quiet to true to return only errors in the result from S3.
+//
+// See http://goo.gl/W5ICVJ for details.
+func (b *Bucket) DelMulti(objects []DelObject, quiet bool) (result *DelResp, err error) {
+	if len(objects) > DelMultiLimit {
+		// this is a programming error really; perhaps it should be a panic.
+		return nil, fmt.Errorf("attempted to delete %d objects.  maximum is %d.", len(objects), DelMultiLimit)
+	}
+	req := &delRequest{Object: objects, Quiet: quiet}
+
+	reqXml, err := xml.Marshal(req)
+	if err != nil {
+		return nil, err
+	}
+
+	reqXml = append([]byte("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"), reqXml...)
+
+	headers := http.Header{
+		"Content-Type": []string{"text/xml"},
+	}
+
+	params := map[string][]string{
+		"delete": {""},
+	}
+
+	result = &DelResp{}
+	for attempt := attempts.Start(); attempt.Next(); {
+		req := &request{
+			method:      "POST",
+			bucket:      b.Name,
+			path:        "/",
+			params:      params,
+			headers:     headers,
+			signpayload: reqXml,
+		}
+		err = b.S3.query(req, result)
+		if !shouldRetry(err) {
+			break
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
 // The ListResp type holds the results of a List bucket operation.
 type ListResp struct {
 	Name      string
@@ -370,15 +461,16 @@
 }
 
 type request struct {
-	method   string
-	bucket   string
-	path     string
-	signpath string
-	params   url.Values
-	headers  http.Header
-	baseurl  string
-	payload  io.Reader
-	prepared bool
+	method      string
+	bucket      string
+	path        string
+	signpath    string
+	params      url.Values
+	headers     http.Header
+	baseurl     string
+	payload     io.Reader
+	signpayload []byte // use instead of payload to auto-set Content-MD5
+	prepared    bool
 }
 
 func (req *request) url() (*url.URL, error) {
@@ -428,6 +520,10 @@
 		}
 		req.params = params
 		req.headers = headers
+		if req.signpayload != nil {
+			// Add Content-MD5 and Content-Length headers
+			signBody(req.signpayload, headers)
+		}
 		if !strings.HasPrefix(req.path, "/") {
 			req.path = "/" + req.path
 		}
@@ -485,10 +581,17 @@
 		hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64)
 		delete(req.headers, "Content-Length")
 	}
+	if req.signpayload != nil {
+		req.payload = bytes.NewBuffer(req.signpayload)
+	}
 	if req.payload != nil {
 		hreq.Body = ioutil.NopCloser(req.payload)
 	}
 
+	if debug {
+		dump, _ := httputil.DumpRequestOut(&hreq, true)
+		log.Printf("{ <- %s\n", dump)
+	}
 	hresp, err := http.DefaultClient.Do(&hreq)
 	if err != nil {
 		return nil, err

=== modified file 's3/s3_test.go'
--- s3/s3_test.go	2013-08-15 13:18:02 +0000
+++ s3/s3_test.go	2013-11-23 00:42:56 +0000
@@ -4,6 +4,7 @@
 	"bytes"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"testing"
 
 	"launchpad.net/goamz/aws"
@@ -206,6 +207,41 @@
 	c.Assert(req.Header["Date"], Not(Equals), "")
 }
 
+// Delete Multiple Objects docs: http://goo.gl/W5ICVJ
+
+func (s *S) TestDelMultiObjects(c *C) {
+	testServer.Response(200, nil, DelMultiResultDump)
+
+	b := s.s3.Bucket("quotes")
+
+	entries := []s3.DelObject{
+		s3.DelObject{"Nelson", ""},
+		s3.DelObject{"Neo", "12345"},
+	}
+	data, err := b.DelMulti(entries, false)
+	c.Assert(err, IsNil)
+	c.Assert(data, NotNil)
+
+	req := testServer.WaitRequest()
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<Delete><Quiet>false</Quiet><Object><Key>Nelson</Key></Object><Object><Key>Neo</Key><VersionId>12345</VersionId></Object></Delete>`
+
+	c.Assert(req.Method, Equals, "POST")
+	c.Assert(req.URL.Path, Equals, "/quotes/")
+	c.Assert(req.URL.RawQuery, Equals, "delete=")
+	c.Assert(req.Header["Date"], Not(Equals), "")
+	c.Assert(req.Header["Content-MD5"], Not(Equals), "")
+	c.Assert(req.Header["Content-Length"], DeepEquals, []string{strconv.Itoa(len(expected))})
+
+	reqbody, err := ioutil.ReadAll(req.Body)
+	c.Assert(string(reqbody), Equals, expected)
+
+	c.Assert(data.Deleted, HasLen, 1)
+	c.Assert(data.Error, HasLen, 1)
+	c.Assert(data.Deleted[0], DeepEquals, s3.DelSuccess{"Nelson", "", false, ""})
+	c.Assert(data.Error[0], DeepEquals, s3.DelErr{"Neo", "", "AccessDenied", "Access Denied"})
+}
+
 // Bucket List Objects docs: http://goo.gl/YjQTc
 
 func (s *S) TestList(c *C) {

=== modified file 's3/sign.go'
--- s3/sign.go	2012-07-19 13:51:13 +0000
+++ s3/sign.go	2013-11-23 00:42:56 +0000
@@ -2,11 +2,13 @@
 
 import (
 	"crypto/hmac"
+	"crypto/md5"
 	"crypto/sha1"
 	"encoding/base64"
 	"launchpad.net/goamz/aws"
 	"log"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -17,6 +19,7 @@
 
 var s3ParamsToSign = map[string]bool{
 	"acl":                          true,
+	"delete":                       true,
 	"location":                     true,
 	"logging":                      true,
 	"notification":                 true,
@@ -110,3 +113,10 @@
 		log.Printf("Signature: %q", signature)
 	}
 }
+
+func signBody(body []byte, headers map[string][]string) {
+	digest := md5.New()
+	digest.Write(body)
+	headers["Content-MD5"] = []string{base64.StdEncoding.EncodeToString(digest.Sum(nil))}
+	headers["Content-Length"] = []string{strconv.Itoa(len(body))}
+}