1
=== modified file 'example/storage/run.go'
2
--- example/storage/run.go	2013-04-17 10:11:07 +0000
3
+++ example/storage/run.go	2013-04-22 16:24:27 +0000
4
@@ -10,15 +10,11 @@
5
10
package main
10
package main
6
11
11
7
12
import (
12
import (
8
13
    "bytes"
9
14
    "flag"
13
    "flag"
10
15
    "fmt"
14
    "fmt"
11
16
    "io"
12
17
    "io/ioutil"
15
    "io/ioutil"
13
18
    "launchpad.net/gwacl"
16
    "launchpad.net/gwacl"
14
19
    "log"
15
20
    "os"
17
    "os"
16
21
    "strconv"
17
22
)
18
)
18
23
19
19
24
var account string
20
var account string
20
@@ -125,35 +121,11 @@
21
125
    }
121
    }
22
126
    defer file.Close()
122
    defer file.Close()
23
127
123
47
128
    buffer := make([]byte, 1024*1024) // 1MB buffer
124
    err = storage.UploadBlockBlob(container, filename, file)
48
129
    blockList := &gwacl.BlockList{}
125
    if err != nil {
49
130
126
        dumpError(err)
50
131
    // Upload the file in chunks.
127
        return
28
132
    for blockNum := int64(0); ; blockNum++ {
29
133
        blockSize, err := file.Read(buffer)
30
134
        if err == io.EOF {
31
135
            break
32
136
        }
33
137
        if err != nil {
34
138
            dumpError(err)
35
139
            return
36
140
        }
37
141
        block := bytes.NewReader(buffer[:blockSize])
38
142
        blockID := strconv.FormatInt(blockNum, 36)
39
143
        log.Printf("Uploading block %d (size=%d, id=%s).\n",
40
144
            blockNum, blockSize, blockID)
41
145
        err = storage.PutBlock(container, filename, blockID, block)
42
146
        if err != nil {
43
147
            dumpError(err)
44
148
            return
45
149
        }
46
150
        blockList.Add(gwacl.BlockListLatest, blockID)
51
151
    }
128
    }
52
152
53
153
    // Commit those blocks by writing the block list.
54
154
    log.Printf("Committing %d blocks.\n", len(blockList.Items))
55
155
    err = storage.PutBlockList(container, filename, blockList)
56
156
    dumpError(err)
57
157
}
129
}
58
158
130
59
159
func deleteblob(storage gwacl.StorageContext) {
131
func deleteblob(storage gwacl.StorageContext) {
60
160
132
61
=== added directory 'logging'
62
=== added file 'logging/logging.go'
63
--- logging/logging.go	1970-01-01 00:00:00 +0000
64
+++ logging/logging.go	2013-04-22 16:24:27 +0000
65
@@ -0,0 +1,83 @@
66
1
// Copyright 2013 Canonical Ltd.  This software is licensed under the
67
2
// GNU Lesser General Public License version 3 (see the file COPYING).
68
3
69
4
package logging
70
5
71
6
import (
72
7
    "log"
73
8
    "os"
74
9
)
75
10
76
11
const (
77
12
    DEBUG = 10 * (iota + 1)
78
13
    INFO
79
14
    WARN
80
15
    ERROR
81
16
)
82
17
83
18
var level = WARN
84
19
85
20
func init() {
86
21
    setLevel(os.Getenv("LOGLEVEL"))
87
22
}
88
23
89
24
func setLevel(levelName string) {
90
25
    switch levelName {
91
26
    case "DEBUG":
92
27
        level = DEBUG
93
28
    case "INFO":
94
29
        level = INFO
95
30
    case "WARN":
96
31
        level = WARN
97
32
    case "ERROR":
98
33
        level = ERROR
99
34
    }
100
35
}
101
36
102
37
func Debug(args ...interface{}) {
103
38
    if level <= DEBUG {
104
39
        log.Println(args...)
105
40
    }
106
41
}
107
42
108
43
func Debugf(format string, args ...interface{}) {
109
44
    if level <= DEBUG {
110
45
        log.Printf(format, args...)
111
46
    }
112
47
}
113
48
114
49
func Info(args ...interface{}) {
115
50
    if level <= INFO {
116
51
        log.Println(args...)
117
52
    }
118
53
}
119
54
120
55
func Infof(format string, args ...interface{}) {
121
56
    if level <= INFO {
122
57
        log.Printf(format, args...)
123
58
    }
124
59
}
125
60
126
61
func Warn(args ...interface{}) {
127
62
    if level <= WARN {
128
63
        log.Println(args...)
129
64
    }
130
65
}
131
66
132
67
func Warnf(format string, args ...interface{}) {
133
68
    if level <= WARN {
134
69
        log.Printf(format, args...)
135
70
    }
136
71
}
137
72
138
73
func Error(args ...interface{}) {
139
74
    if level <= ERROR {
140
75
        log.Println(args...)
141
76
    }
142
77
}
143
78
144
79
func Errorf(format string, args ...interface{}) {
145
80
    if level <= ERROR {
146
81
        log.Printf(format, args...)
147
82
    }
148
83
}
149
0
84
150
=== added file 'logging/logging_test.go'
151
--- logging/logging_test.go	1970-01-01 00:00:00 +0000
152
+++ logging/logging_test.go	2013-04-22 16:24:27 +0000
153
@@ -0,0 +1,39 @@
154
1
// Copyright 2013 Canonical Ltd.  This software is licensed under the
155
2
// GNU Lesser General Public License version 3 (see the file COPYING).
156
3
157
4
package logging
158
5
159
6
import (
160
7
    . "launchpad.net/gocheck"
161
8
)
162
9
163
10
var originalLevel = level
164
11
165
12
func restoreLevel() {
166
13
    level = originalLevel
167
14
}
168
15
169
16
type testLogging struct{}
170
17
171
18
var _ = Suite(&testLogging{})
172
19
173
20
func (suite *testLogging) TestSetLevel(c *C) {
174
21
    defer restoreLevel()
175
22
    // The names of the logging constants are recognised arguments to
176
23
    // setLevel().
177
24
    level = -1
178
25
    setLevel("DEBUG")
179
26
    c.Check(level, Equals, DEBUG)
180
27
    setLevel("INFO")
181
28
    c.Check(level, Equals, INFO)
182
29
    setLevel("WARN")
183
30
    c.Check(level, Equals, WARN)
184
31
    setLevel("ERROR")
185
32
    c.Check(level, Equals, ERROR)
186
33
    // Unrecognised arguments are ignored.
187
34
    level = -1
188
35
    setLevel("FOOBAR")
189
36
    c.Check(level, Equals, -1)
190
37
    setLevel("123")
191
38
    c.Check(level, Equals, -1)
192
39
}
193
0
40
194
=== added file 'storage.go'
195
--- storage.go	1970-01-01 00:00:00 +0000
196
+++ storage.go	2013-04-22 16:24:27 +0000
197
@@ -0,0 +1,47 @@
198
1
// Copyright 2013 Canonical Ltd.  This software is licensed under the
199
2
// GNU Lesser General Public License version 3 (see the file COPYING).
200
3
201
4
package gwacl
202
5
203
6
// This file contains higher level operations necessary to work with the Azure
204
7
// file storage API.
205
8
206
9
import (
207
10
    "bytes"
208
11
    "io"
209
12
    . "launchpad.net/gwacl/logging"
210
13
    "strconv"
211
14
)
212
15
213
16
// UploadBlockBlob uses PutBlock and PutBlockList API operations to upload
214
17
// arbitrarily large files, 1MB at a time.
215
18
func (context *StorageContext) UploadBlockBlob(
216
19
    container, filename string, data io.Reader) error {
217
20
218
21
    buffer := make([]byte, 1024*1024) // 1MB buffer
219
22
    blockList := &BlockList{}
220
23
221
24
    // Upload the file in chunks.
222
25
    for blockNum := int64(0); ; blockNum++ {
223
26
        blockSize, err := data.Read(buffer)
224
27
        if err == io.EOF {
225
28
            break
226
29
        }
227
30
        if err != nil {
228
31
            return err
229
32
        }
230
33
        block := bytes.NewReader(buffer[:blockSize])
231
34
        blockID := strconv.FormatInt(blockNum, 36)
232
35
        Debugf("Uploading block %d (size=%d, id=%s).\n",
233
36
            blockNum, blockSize, blockID)
234
37
        err = context.PutBlock(container, filename, blockID, block)
235
38
        if err != nil {
236
39
            return err
237
40
        }
238
41
        blockList.Add(BlockListLatest, blockID)
239
42
    }
240
43
241
44
    // Commit those blocks by writing the block list.
242
45
    Debugf("Committing %d blocks.\n", len(blockList.Items))
243
46
    return context.PutBlockList(container, filename, blockList)
244
47
}
245
0
48
246
=== renamed file 'storage.go' => 'storage_base.go'
247
=== renamed file 'storage_test.go' => 'storage_base_test.go'
248
=== added file 'storage_test.go'
249
--- storage_test.go	1970-01-01 00:00:00 +0000
250
+++ storage_test.go	2013-04-22 16:24:27 +0000
251
@@ -0,0 +1,130 @@
252
1
// Copyright 2013 Canonical Ltd.  This software is licensed under the
253
2
// GNU Lesser General Public License version 3 (see the file COPYING).
254
3
255
4
package gwacl
256
5
257
6
import (
258
7
    "bytes"
259
8
    "encoding/base64"
260
9
    "fmt"
261
10
    "io/ioutil"
262
11
    . "launchpad.net/gocheck"
263
12
    "net/http"
264
13
    "net/url"
265
14
)
266
15
267
16
func b64(s string) string {
268
17
    return base64.StdEncoding.EncodeToString([]byte(s))
269
18
}
270
19
271
20
type TestTransport2Exchange struct {
272
21
    Request  *http.Request
273
22
    Response *http.Response
274
23
    Error    error
275
24
}
276
25
277
26
// TestTransport2 is used as an http.Client.Transport for testing. The only
278
27
// requirement is that it adhere to the http.RoundTripper interface.
279
28
type TestTransport2 struct {
280
29
    Exchanges     []*TestTransport2Exchange
281
30
    ExchangeCount int
282
31
}
283
32
284
33
func (t *TestTransport2) AddExchange(response *http.Response, error error) {
285
34
    exchange := TestTransport2Exchange{Response: response, Error: error}
286
35
    t.Exchanges = append(t.Exchanges, &exchange)
287
36
}
288
37
289
38
func (t *TestTransport2) RoundTrip(req *http.Request) (resp *http.Response, err error) {
290
39
    exchange := t.Exchanges[t.ExchangeCount]
291
40
    t.ExchangeCount += 1
292
41
    exchange.Request = req
293
42
    return exchange.Response, exchange.Error
294
43
}
295
44
296
45
type testUploadBlockBlob struct{}
297
46
298
47
var _ = Suite(&testUploadBlockBlob{})
299
48
300
49
func (suite *testUploadBlockBlob) TestSmallFile(c *C) {
301
50
    context := makeStorageContext()
302
51
    transport := &TestTransport2{}
303
52
    // UploadBlockBlob uses PutBlock to upload the data.
304
53
    transport.AddExchange(makeFakeCreatedResponse(), nil)
305
54
    // UploadBlockBlob then sends the list of blocks with PutBlockList.
306
55
    transport.AddExchange(makeFakeCreatedResponse(), nil)
307
56
    // Plug in the test transport.
308
57
    context.client = &http.Client{Transport: transport}
309
58
    // Upload a random blob of data.
310
59
    data := uploadRandomBlob(c, context, 10)
311
60
    // There were two exchanges.
312
61
    c.Assert(transport.ExchangeCount, Equals, 2)
313
62
    // The first request is a Put Block with the block data.
314
63
    assertBlockSent(c, context, data, b64("0"), transport.Exchanges[0])
315
64
    // The second request is Put Block List to commit the block above.
316
65
    assertBlockListSent(c, context, []string{b64("0")}, transport.Exchanges[1])
317
66
}
318
67
319
68
func (suite *testUploadBlockBlob) TestLargeFile(c *C) {
320
69
    context := makeStorageContext()
321
70
    transport := &TestTransport2{}
322
71
    // UploadBlockBlob uses PutBlock twice to upload the data.
323
72
    transport.AddExchange(makeFakeCreatedResponse(), nil)
324
73
    transport.AddExchange(makeFakeCreatedResponse(), nil)
325
74
    // UploadBlockBlob then sends the list of blocks with PutBlockList.
326
75
    transport.AddExchange(makeFakeCreatedResponse(), nil)
327
76
    // Plug in the test transport.
328
77
    context.client = &http.Client{Transport: transport}
329
78
    // Upload a large random blob of data.
330
79
    data := uploadRandomBlob(c, context, 1348*1024)
331
80
    // There were three exchanges.
332
81
    c.Assert(transport.ExchangeCount, Equals, 3)
333
82
    // The first two requests are Put Block with chunks of the block data. The
334
83
    // weird looking block IDs are base64 encodings of the strings "0" and "1".
335
84
    assertBlockSent(c, context, data[:1024*1024], b64("0"), transport.Exchanges[0])
336
85
    assertBlockSent(c, context, data[1024*1024:], b64("1"), transport.Exchanges[1])
337
86
    // The second request is Put Block List to commit the block above.
338
87
    assertBlockListSent(c, context, []string{b64("0"), b64("1")}, transport.Exchanges[2])
339
88
}
340
89
341
90
func makeFakeCreatedResponse() *http.Response {
342
91
    return &http.Response{
343
92
        Status:     fmt.Sprintf("%d", http.StatusCreated),
344
93
        StatusCode: http.StatusCreated,
345
94
        Body:       Empty,
346
95
    }
347
96
}
348
97
349
98
func uploadRandomBlob(c *C, context *StorageContext, size int) []byte {
350
99
    data := MakeRandomByteSlice(size)
351
100
    err := context.UploadBlockBlob(
352
101
        "MyContainer", "MyFile", bytes.NewReader(data))
353
102
    c.Assert(err, IsNil)
354
103
    return data
355
104
}
356
105
357
106
func assertBlockSent(
358
107
    c *C, context *StorageContext, data []byte, blockID string, exchange *TestTransport2Exchange) {
359
108
    c.Check(exchange.Request.URL.String(), Equals, fmt.Sprintf(
360
109
        "http://%s.blob.core.windows.net/MyContainer/MyFile"+
361
110
            "?comp=block&blockid=%s",
362
111
        context.Account, url.QueryEscape(blockID)))
363
112
    body, err := ioutil.ReadAll(exchange.Request.Body)
364
113
    c.Check(err, IsNil)
365
114
    c.Check(body, DeepEquals, data)
366
115
}
367
116
368
117
func assertBlockListSent(
369
118
    c *C, context *StorageContext, blockIDs []string, exchange *TestTransport2Exchange) {
370
119
    c.Check(exchange.Request.URL.String(), Equals, fmt.Sprintf(
371
120
        "http://%s.blob.core.windows.net/MyContainer/MyFile"+
372
121
            "?comp=blocklist", context.Account))
373
122
    body, err := ioutil.ReadAll(exchange.Request.Body)
374
123
    c.Check(err, IsNil)
375
124
    expected := "\n<BlockList>\n"
376
125
    for _, blockID := range blockIDs {
377
126
        expected += "  <Latest>" + blockID + "</Latest>\n"
378
127
    }
379
128
    expected += "</BlockList>"
380
129
    c.Check(string(body), Equals, expected)
381
130
}
382
0
131
383
=== modified file 'test_helpers.go'
384
--- test_helpers.go	2013-04-02 08:51:18 +0000
385
+++ test_helpers.go	2013-04-22 16:24:27 +0000
386
@@ -4,7 +4,10 @@
387
4
package gwacl
4
package gwacl
388
5
5
389
6
import (
6
import (
390
7
    "bytes"
391
7
    "fmt"
8
    "fmt"
392
9
    "io"
393
10
    "io/ioutil"
394
8
    "math/rand"
11
    "math/rand"
395
9
    "net/http"
12
    "net/http"
396
10
    "strings"
13
    "strings"
397
@@ -15,6 +18,9 @@
398
15
// Perhaps we can add it to gocheck, or start a testtools-like project.
18
// Perhaps we can add it to gocheck, or start a testtools-like project.
399
16
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890abcdefghijklmnopqrstuvwxyz"
19
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890abcdefghijklmnopqrstuvwxyz"
400
17
20
401
21
// A Reader and ReadCloser that EOFs immediately.
402
22
var Empty io.ReadCloser = ioutil.NopCloser(bytes.NewReader(nil))
403
23
404
18
func MakeRandomString(length int) string {
24
func MakeRandomString(length int) string {
405
19
    return string(MakeRandomByteSlice(length))
25
    return string(MakeRandomByteSlice(length))
406
20
}
26
}
407
21
27
408
=== modified file 'x509dispatcher.go'
409
--- x509dispatcher.go	2013-03-26 06:50:50 +0000
410
+++ x509dispatcher.go	2013-04-22 16:24:27 +0000
411
@@ -5,7 +5,7 @@
412
5
    "bytes"
5
    "bytes"
413
6
    "fmt"
6
    "fmt"
414
7
    curl "github.com/andelf/go-curl"
7
    curl "github.com/andelf/go-curl"
416
8
    "log"
8
    . "launchpad.net/gwacl/logging"
417
9
    "net/http"
9
    "net/http"
418
10
    "net/textproto"
10
    "net/textproto"
419
11
)
11
)
420
@@ -143,10 +143,10 @@
421
143
143
422
144
func performX509CurlRequest(session *x509Session, request *x509Request) (*x509Response, error) {
144
func performX509CurlRequest(session *x509Session, request *x509Request) (*x509Response, error) {
423
145
    if verbose {
145
    if verbose {
428
146
        log.Println("Performing request")
146
        Debug("Performing request")
429
147
        log.Println("Request url: " + request.URL)
147
        Debug("Request url: " + request.URL)
430
148
        log.Println("Request method: " + request.Method)
148
        Debug("Request method: " + request.Method)
431
149
        log.Printf("Request body: %s\n", request.Payload)
149
        Debugf("Request body: %s\n", request.Payload)
432
150
    }
150
    }
433
151
    response := newX509Response()
151
    response := newX509Response()
434
152
    ch := request.makeCurlRequest(session, response)
152
    ch := request.makeCurlRequest(session, response)
435
@@ -170,10 +170,10 @@
436
170
    }
170
    }
437
171
    response.StatusCode = status.(int)
171
    response.StatusCode = status.(int)
438
172
    if verbose {
172
    if verbose {
443
173
        log.Println("Got response")
173
        Debug("Got response")
444
174
        log.Printf("Response status: %d\n", response.StatusCode)
174
        Debugf("Response status: %d\n", response.StatusCode)
445
175
        log.Printf("Response headers: %s\n", response.RawHeader)
175
        Debugf("Response headers: %s\n", response.RawHeader)
446
176
        log.Printf("Response body: %s\n", response.Body)
176
        Debugf("Response body: %s\n", response.Body)
447
177
    }
177
    }
448
178
    return response, nil
178
    return response, nil
449
179
}
179
}
Status:	Merged
Approved by:	Gavin Panella on 2013-04-22
Approved revision:	118
Merged at revision:	93
Proposed branch:	lp:~allenap/gwacl/chunky-blobs-operation
Merge into:	lp:gwacl
Diff against target:	449 lines (+318/-41) 7 files modified example/storage/run.go (+4/-32) logging/logging.go (+83/-0) logging/logging_test.go (+39/-0) storage.go (+47/-0) storage_test.go (+130/-0) test_helpers.go (+6/-0) x509dispatcher.go (+9/-9)
To merge this branch:	bzr merge lp:~allenap/gwacl/chunky-blobs-operation
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Julian Edwards (community)		2013-04-17	Approve on 2013-04-21
Review via email: mp+159512@code.launchpad.net