1
=== modified file 'http/client.go'
2
--- http/client.go	2012-12-21 04:10:14 +0000
3
+++ http/client.go	2013-01-29 01:46:23 +0000
4
@@ -25,9 +25,9 @@
5
25
25
6
26
type Client struct {
26
type Client struct {
7
27
	http.Client
27
	http.Client
11
28
	logger     *log.Logger
28
	logger          *log.Logger
12
29
	authToken  string
29
	authToken       string
13
30
	maxRetries int
30
	maxSendAttempts int
14
31
}
31
}
15
32
32
16
33
type ErrorResponse struct {
33
type ErrorResponse struct {
17
@@ -55,11 +55,17 @@
18
55
	RespReader     io.ReadCloser
55
	RespReader     io.ReadCloser
19
56
}
56
}
20
57
57
21
58
const (
22
59
	// The maximum number of times to try sending a request before we give up
23
60
	// (assuming any unsuccessful attempts can be sensibly tried again).
24
61
	MaxSendAttempts = 3
25
62
)
26
63
27
58
func New(httpClient http.Client, logger *log.Logger, token string) *Client {
64
func New(httpClient http.Client, logger *log.Logger, token string) *Client {
28
59
	if logger == nil {
65
	if logger == nil {
29
60
		logger = log.New(os.Stderr, "", log.LstdFlags)
66
		logger = log.New(os.Stderr, "", log.LstdFlags)
30
61
	}
67
	}
32
62
	return &Client{httpClient, logger, token, 3}
68
	return &Client{httpClient, logger, token, MaxSendAttempts}
33
63
}
69
}
34
64
70
35
65
// JsonRequest JSON encodes and sends the supplied object (if any) to the specified URL.
71
// JsonRequest JSON encodes and sends the supplied object (if any) to the specified URL.
36
@@ -187,7 +193,7 @@
37
187
}
193
}
38
188
194
39
189
func (c *Client) sendRateLimitedRequest(method, URL string, headers http.Header, reqData []byte) (resp *http.Response, err error) {
195
func (c *Client) sendRateLimitedRequest(method, URL string, headers http.Header, reqData []byte) (resp *http.Response, err error) {
41
190
	for i := 0; i < c.maxRetries; i++ {
196
	for i := 0; i < c.maxSendAttempts; i++ {
42
191
		var reqReader io.Reader
197
		var reqReader io.Reader
43
192
		if reqData != nil {
198
		if reqData != nil {
44
193
			reqReader = bytes.NewReader(reqData)
199
			reqReader = bytes.NewReader(reqData)
45
@@ -209,17 +215,17 @@
46
209
		if resp.StatusCode != http.StatusRequestEntityTooLarge {
215
		if resp.StatusCode != http.StatusRequestEntityTooLarge {
47
210
			return resp, nil
216
			return resp, nil
48
211
		}
217
		}
50
212
		retryAfter, err := strconv.Atoi(resp.Header.Get("Retry-After"))
218
		retryAfter, err := strconv.ParseFloat(resp.Header.Get("Retry-After"), 32)
51
213
		if err != nil {
219
		if err != nil {
52
214
			return nil, errors.Newf(err, URL, "Invalid Retry-After header")
220
			return nil, errors.Newf(err, URL, "Invalid Retry-After header")
53
215
		}
221
		}
54
216
		if retryAfter == 0 {
222
		if retryAfter == 0 {
55
217
			return nil, errors.Newf(err, URL, "Resource limit exeeded at URL %s.", URL)
223
			return nil, errors.Newf(err, URL, "Resource limit exeeded at URL %s.", URL)
56
218
		}
224
		}
58
219
		c.logger.Printf("Too many requests, retrying in %d seconds.", retryAfter)
225
		c.logger.Printf("Too many requests, retrying in %dms.", int(retryAfter*1000))
59
220
		time.Sleep(time.Duration(retryAfter) * time.Second)
226
		time.Sleep(time.Duration(retryAfter) * time.Second)
60
221
	}
227
	}
62
222
	return nil, errors.Newf(err, URL, "Maximum number of retries (%d) reached sending request to %s.", c.maxRetries, URL)
228
	return nil, errors.Newf(err, URL, "Maximum number of attempts (%d) reached sending request to %s.", c.maxSendAttempts, URL)
63
223
}
229
}
64
224
230
65
225
type HttpError struct {
231
type HttpError struct {
66
226
232
67
=== modified file 'identity/userpass.go'
68
--- identity/userpass.go	2013-01-27 13:37:53 +0000
69
+++ identity/userpass.go	2013-01-29 01:46:23 +0000
70
@@ -105,10 +105,16 @@
71
105
				service.Endpoints = append(service.Endpoints[:i], service.Endpoints[i+1:]...)
105
				service.Endpoints = append(service.Endpoints[:i], service.Endpoints[i+1:]...)
72
106
			}
106
			}
73
107
		}
107
		}
74
108
<<<<<<< TREE
75
108
		if len(service.Endpoints) == 0 {
109
		if len(service.Endpoints) == 0 {
76
109
			fmt.Fprintf(os.Stderr, "Found no endpoints for %v\n", service.Type)
110
			fmt.Fprintf(os.Stderr, "Found no endpoints for %v\n", service.Type)
77
110
		}
111
		}
78
111
		details.ServiceURLs[service.Type] = service.Endpoints[0].PublicURL
112
		details.ServiceURLs[service.Type] = service.Endpoints[0].PublicURL
79
113
=======
80
114
		if len(service.Endpoints) > 0 {
81
115
			details.ServiceURLs[service.Type] = service.Endpoints[0].PublicURL
82
116
		}
83
117
>>>>>>> MERGE-SOURCE
84
112
	}
118
	}
85
113
119
86
114
	return details, nil
120
	return details, nil
87
115
121
88
=== modified file 'nova/live_test.go'
89
--- nova/live_test.go	2013-01-21 11:18:33 +0000
90
+++ nova/live_test.go	2013-01-29 01:46:23 +0000
91
@@ -1,11 +1,14 @@
92
1
package nova_test
1
package nova_test
93
2
2
94
3
import (
3
import (
95
4
	"bytes"
96
4
	. "launchpad.net/gocheck"
5
	. "launchpad.net/gocheck"
97
5
	"launchpad.net/goose/client"
6
	"launchpad.net/goose/client"
98
6
	"launchpad.net/goose/errors"
7
	"launchpad.net/goose/errors"
99
7
	"launchpad.net/goose/identity"
8
	"launchpad.net/goose/identity"
100
8
	"launchpad.net/goose/nova"
9
	"launchpad.net/goose/nova"
101
10
	"log"
102
11
	"strings"
103
9
	"time"
12
	"time"
104
10
)
13
)
105
11
14
106
@@ -419,3 +422,36 @@
107
419
	c.Check(fip.FixedIP, IsNil)
422
	c.Check(fip.FixedIP, IsNil)
108
420
	c.Check(fip.InstanceId, IsNil)
423
	c.Check(fip.InstanceId, IsNil)
109
421
}
424
}
110
425
111
426
// TestRateLimitRetry checks that when we make too many requests and receive a Retry-After response, the retry
112
427
// occurs and the request ultimately succeeds.
113
428
func (s *LiveTests) TestRateLimitRetry(c *C) {
114
429
	// Capture the logged output so we can check for retry messages.
115
430
	var logout bytes.Buffer
116
431
	logger := log.New(&logout, "", log.LstdFlags)
117
432
	client := client.NewClient(s.cred, identity.AuthUserPass, logger)
118
433
	nova := nova.New(client)
119
434
	// Delete the artifact if it already exists.
120
435
	testGroup, err := nova.SecurityGroupByName("test_group")
121
436
	if err != nil {
122
437
		c.Assert(errors.IsNotFound(err), Equals, true)
123
438
	} else {
124
439
		nova.DeleteSecurityGroup(testGroup.Id)
125
440
		c.Assert(err, IsNil)
126
441
	}
127
442
	// Create some artifacts a number of times in succession and ensure each time is successful,
128
443
	// even with retries being required. As soon as we see a retry message, the test has passed
129
444
	// and we exit.
130
445
	for i := 0; i < 50; i++ {
131
446
		testGroup, err = nova.CreateSecurityGroup("test_group", "test")
132
447
		c.Assert(err, IsNil)
133
448
		nova.DeleteSecurityGroup(testGroup.Id)
134
449
		c.Assert(err, IsNil)
135
450
		output := logout.String()
136
451
		if strings.Contains(output, "Too many requests, retrying in") == true {
137
452
			return
138
453
		}
139
454
	}
140
455
	// No retry message logged so test has failed.
141
456
	c.Fail()
142
457
}
143
422
458
144
=== modified file 'nova/local_test.go'
145
--- nova/local_test.go	2013-01-29 01:46:22 +0000
146
+++ nova/local_test.go	2013-01-29 01:46:23 +0000
147
@@ -2,11 +2,14 @@
148
2
2
149
3
import (
3
import (
150
4
	"bytes"
4
	"bytes"
151
5
	"fmt"
152
5
	. "launchpad.net/gocheck"
6
	. "launchpad.net/gocheck"
153
6
	"launchpad.net/goose/client"
7
	"launchpad.net/goose/client"
154
7
	"launchpad.net/goose/errors"
8
	"launchpad.net/goose/errors"
155
9
	goosehttp "launchpad.net/goose/http"
156
8
	"launchpad.net/goose/identity"
10
	"launchpad.net/goose/identity"
157
9
	"launchpad.net/goose/nova"
11
	"launchpad.net/goose/nova"
158
12
	"launchpad.net/goose/testservices"
159
10
	"launchpad.net/goose/testservices/openstackservice"
13
	"launchpad.net/goose/testservices/openstackservice"
160
11
	"log"
14
	"log"
161
12
	"net/http"
15
	"net/http"
162
@@ -23,9 +26,12 @@
163
23
type localLiveSuite struct {
26
type localLiveSuite struct {
164
24
	LiveTests
27
	LiveTests
165
25
	// The following attributes are for using testing doubles.
28
	// The following attributes are for using testing doubles.
169
26
	Server     *httptest.Server
29
	Server                *httptest.Server
170
27
	Mux        *http.ServeMux
30
	Mux                   *http.ServeMux
171
28
	oldHandler http.Handler
31
	oldHandler            http.Handler
172
32
	openstack             *openstackservice.Openstack
173
33
	retryErrorCount       int // The current retry error count.
174
34
	retryErrorCountToSend int // The number of retry errors to send.
175
29
}
35
}
176
30
36
177
31
func (s *localLiveSuite) SetUpSuite(c *C) {
37
func (s *localLiveSuite) SetUpSuite(c *C) {
178
@@ -45,8 +51,8 @@
179
45
		Region:     "some region",
51
		Region:     "some region",
180
46
		TenantName: "tenant",
52
		TenantName: "tenant",
181
47
	}
53
	}
184
48
	openstack := openstackservice.New(s.cred)
54
	s.openstack = openstackservice.New(s.cred)
185
49
	openstack.SetupHTTP(s.Mux)
55
	s.openstack.SetupHTTP(s.Mux)
186
50
56
187
51
	s.LiveTests.SetUpSuite(c)
57
	s.LiveTests.SetUpSuite(c)
188
52
}
58
}
189
@@ -59,6 +65,7 @@
190
59
}
65
}
191
60
66
192
61
func (s *localLiveSuite) SetUpTest(c *C) {
67
func (s *localLiveSuite) SetUpTest(c *C) {
193
68
	s.retryErrorCount = 0
194
62
	s.LiveTests.SetUpTest(c)
69
	s.LiveTests.SetUpTest(c)
195
63
}
70
}
196
64
71
197
@@ -68,12 +75,18 @@
198
68
75
199
69
// Additional tests to be run against the service double only go here.
76
// Additional tests to be run against the service double only go here.
200
70
77
207
71
// TestRateLimitRetry checks that when we make too many requests and receive a Retry-After response, the retry
78
func (s *localLiveSuite) retryLimitHook(sc testservices.ServiceControl) testservices.ControlProcessor {
208
72
// occurs and the request ultimately succeeds.
79
	return func(sc testservices.ServiceControl, args ...interface{}) error {
209
73
func (s *localLiveSuite) TestRateLimitRetry(c *C) {
80
		sendError := s.retryErrorCount < s.retryErrorCountToSend
210
74
	// Capture the logged output so we can check for retry messages.
81
		if sendError {
211
75
	var logout bytes.Buffer
82
			s.retryErrorCount++
212
76
	logger := log.New(&logout, "", log.LstdFlags)
83
			return &testservices.RateLimitExceededError{fmt.Errorf("retry limit exceeded")}
213
84
		}
214
85
		return nil
215
86
	}
216
87
}
217
88
218
89
func (s *localLiveSuite) setupRetryErrorTest(c *C, logger *log.Logger) (*nova.Client, *nova.SecurityGroup) {
219
77
	client := client.NewClient(s.cred, identity.AuthUserPass, logger)
90
	client := client.NewClient(s.cred, identity.AuthUserPass, logger)
220
78
	nova := nova.New(client)
91
	nova := nova.New(client)
221
79
	// Delete the artifact if it already exists.
92
	// Delete the artifact if it already exists.
222
@@ -84,11 +97,35 @@
223
84
		nova.DeleteSecurityGroup(testGroup.Id)
97
		nova.DeleteSecurityGroup(testGroup.Id)
224
85
		c.Assert(err, IsNil)
98
		c.Assert(err, IsNil)
225
86
	}
99
	}
227
87
	testGroup, err = nova.CreateSecurityGroup("test_group", "test rate limit")
100
	testGroup, err = nova.CreateSecurityGroup("test_group", "test")
228
88
	c.Assert(err, IsNil)
101
	c.Assert(err, IsNil)
230
89
	nova.DeleteSecurityGroup(testGroup.Id)
102
	return nova, testGroup
231
103
}
232
104
233
105
// TestRateLimitRetry checks that when we make too many requests and receive a Retry-After response, the retry
234
106
// occurs and the request ultimately succeeds.
235
107
func (s *localLiveSuite) TestRateLimitRetry(c *C) {
236
108
	// Capture the logged output so we can check for retry messages.
237
109
	var logout bytes.Buffer
238
110
	logger := log.New(&logout, "", log.LstdFlags)
239
111
	nova, testGroup := s.setupRetryErrorTest(c, logger)
240
112
	s.retryErrorCountToSend = goosehttp.MaxSendAttempts - 1
241
113
	s.openstack.Nova.RegisterControlPoint("removeSecurityGroup", s.retryLimitHook(s.openstack.Nova))
242
114
	defer s.openstack.Nova.RegisterControlPoint("removeSecurityGroup", nil)
243
115
	err := nova.DeleteSecurityGroup(testGroup.Id)
244
90
	c.Assert(err, IsNil)
116
	c.Assert(err, IsNil)
245
91
	// Ensure we got at least one retry message.
117
	// Ensure we got at least one retry message.
246
92
	output := logout.String()
118
	output := logout.String()
247
93
	c.Assert(strings.Contains(output, "Too many requests, retrying in"), Equals, true)
119
	c.Assert(strings.Contains(output, "Too many requests, retrying in"), Equals, true)
248
94
}
120
}
249
121
250
122
// TestRateLimitRetryExceeded checks that an error is raised if too many retry responses are received from the server.
251
123
func (s *localLiveSuite) TestRateLimitRetryExceeded(c *C) {
252
124
	nova, testGroup := s.setupRetryErrorTest(c, nil)
253
125
	s.retryErrorCountToSend = goosehttp.MaxSendAttempts
254
126
	s.openstack.Nova.RegisterControlPoint("removeSecurityGroup", s.retryLimitHook(s.openstack.Nova))
255
127
	defer s.openstack.Nova.RegisterControlPoint("removeSecurityGroup", nil)
256
128
	err := nova.DeleteSecurityGroup(testGroup.Id)
257
129
	c.Assert(err, Not(IsNil))
258
130
	c.Assert(err.Error(), Matches, ".*Maximum number of attempts.*")
259
131
}
260
95
132
261
=== modified file 'testservices/novaservice/service.go'
262
--- testservices/novaservice/service.go	2013-01-29 01:46:22 +0000
263
+++ testservices/novaservice/service.go	2013-01-29 01:46:23 +0000
264
@@ -18,17 +18,16 @@
265
18
// contains the service double's internal state.
18
// contains the service double's internal state.
266
19
type Nova struct {
19
type Nova struct {
267
20
	testservices.ServiceInstance
20
	testservices.ServiceInstance
279
21
	flavors                   map[string]nova.FlavorDetail
21
	flavors      map[string]nova.FlavorDetail
280
22
	servers                   map[string]nova.ServerDetail
22
	servers      map[string]nova.ServerDetail
281
23
	groups                    map[int]nova.SecurityGroup
23
	groups       map[int]nova.SecurityGroup
282
24
	rules                     map[int]nova.SecurityGroupRule
24
	rules        map[int]nova.SecurityGroupRule
283
25
	floatingIPs               map[int]nova.FloatingIP
25
	floatingIPs  map[int]nova.FloatingIP
284
26
	serverGroups              map[string][]int
26
	serverGroups map[string][]int
285
27
	serverIPs                 map[string][]int
27
	serverIPs    map[string][]int
286
28
	nextGroupId               int
28
	nextGroupId  int
287
29
	nextRuleId                int
29
	nextRuleId   int
288
30
	nextIPId                  int
30
	nextIPId     int
278
31
	sendFakeRateLimitResponse bool
289
32
}
31
}
290
33
32
291
34
// endpoint returns either a versioned or non-versioned service
33
// endpoint returns either a versioned or non-versioned service
292
@@ -82,9 +81,6 @@
293
82
		floatingIPs:  make(map[int]nova.FloatingIP),
81
		floatingIPs:  make(map[int]nova.FloatingIP),
294
83
		serverGroups: make(map[string][]int),
82
		serverGroups: make(map[string][]int),
295
84
		serverIPs:    make(map[string][]int),
83
		serverIPs:    make(map[string][]int),
296
85
		// The following attribute controls whether rate limit responses are sent back to the caller.
297
86
		// This is switched off when we want to ensure the client eventually gets a proper response.
298
87
		sendFakeRateLimitResponse: true,
299
88
		ServiceInstance: testservices.ServiceInstance{
84
		ServiceInstance: testservices.ServiceInstance{
300
89
			IdentityService: identityService,
85
			IdentityService: identityService,
301
90
			Hostname:        hostname,
86
			Hostname:        hostname,
302
91
87
303
=== modified file 'testservices/novaservice/service_http.go'
304
--- testservices/novaservice/service_http.go	2013-01-23 23:37:38 +0000
305
+++ testservices/novaservice/service_http.go	2013-01-29 01:46:23 +0000
306
@@ -9,6 +9,7 @@
307
9
	"io"
9
	"io"
308
10
	"io/ioutil"
10
	"io/ioutil"
309
11
	"launchpad.net/goose/nova"
11
	"launchpad.net/goose/nova"
310
12
	"launchpad.net/goose/testservices"
311
12
	"launchpad.net/goose/testservices/identityservice"
13
	"launchpad.net/goose/testservices/identityservice"
312
13
	"net/http"
14
	"net/http"
313
14
	"path"
15
	"path"
314
@@ -212,7 +213,8 @@
315
212
		"",
213
		"",
316
213
		"text/plain; charset=UTF-8",
214
		"text/plain; charset=UTF-8",
317
214
		"too many requests",
215
		"too many requests",
319
215
		map[string]string{"Retry-After": "1"},
216
		// RFC says that Retry-After should be an int, but we don't want to wait an entire second during the test suite.
320
217
		map[string]string{"Retry-After": "0.001"},
321
216
		nil,
218
		nil,
322
217
	}
219
	}
323
218
)
220
)
324
@@ -286,15 +288,20 @@
325
286
	if err == nil {
288
	if err == nil {
326
287
		return
289
		return
327
288
	}
290
	}
337
289
	resp, _ := err.(http.Handler)
291
	var resp http.Handler
338
290
	if resp == nil {
292
	if _, ok := err.(*testservices.RateLimitExceededError); ok {
339
291
		resp = &errorResponse{
293
		resp = errRateLimitExceeded
340
292
			http.StatusInternalServerError,
294
	} else {
341
293
			`{"internalServerError":{"message":"$ERROR$",code:500}}`,
295
		resp, _ = err.(http.Handler)
342
294
			"application/json",
296
		if resp == nil {
343
295
			err.Error(),
297
			resp = &errorResponse{
344
296
			nil,
298
				http.StatusInternalServerError,
345
297
			h.n,
299
				`{"internalServerError":{"message":"$ERROR$",code:500}}`,
346
300
				"application/json",
347
301
				err.Error(),
348
302
				nil,
349
303
				h.n,
350
304
			}
351
298
		}
305
		}
352
299
	}
306
	}
353
300
	resp.ServeHTTP(w, r)
307
	resp.ServeHTTP(w, r)
354
@@ -782,12 +789,6 @@
355
782
			if err == nil {
789
			if err == nil {
356
783
				return errBadRequestDuplicateValue
790
				return errBadRequestDuplicateValue
357
784
			}
791
			}
358
785
			if req.Group.Description == "test rate limit" && n.sendFakeRateLimitResponse {
359
786
				n.sendFakeRateLimitResponse = false
360
787
				return errRateLimitExceeded
361
788
			} else {
362
789
				n.sendFakeRateLimitResponse = true
363
790
			}
364
791
			n.nextGroupId++
792
			n.nextGroupId++
365
792
			nextId := n.nextGroupId
793
			nextId := n.nextGroupId
366
793
			err = n.addSecurityGroup(nova.SecurityGroup{
794
			err = n.addSecurityGroup(nova.SecurityGroup{
367
794
795
368
=== modified file 'testservices/service.go'
369
--- testservices/service.go	2013-01-29 01:46:22 +0000
370
+++ testservices/service.go	2013-01-29 01:46:23 +0000
371
@@ -25,6 +25,11 @@
372
25
	ControlHooks map[string]ControlProcessor
25
	ControlHooks map[string]ControlProcessor
373
26
}
26
}
374
27
27
375
28
// Internal Openstack errors.
376
29
type RateLimitExceededError struct {
377
30
	error
378
31
}
379
32
380
28
// ControlProcessor defines a function that is run when a specified control point is reached in the service
33
// ControlProcessor defines a function that is run when a specified control point is reached in the service
381
29
// business logic. The function receives the service instance so internal state can be inspected, plus for any
34
// business logic. The function receives the service instance so internal state can be inspected, plus for any
382
30
// arguments passed to the currently executing service function.
35
// arguments passed to the currently executing service function.
Status:	Merged
Merged at revision:	55
Proposed branch:	lp:~wallyworld/goose/rate-limit-retry-tests
Merge into:	lp:~gophers/goose/trunk
Prerequisite:	lp:~wallyworld/goose/control-test-doubles
Diff against target:	382 lines (+138/-51) (has conflicts) 7 files modified http/client.go (+14/-8) identity/userpass.go (+6/-0) nova/live_test.go (+36/-0) nova/local_test.go (+50/-13) testservices/novaservice/service.go (+10/-14) testservices/novaservice/service_http.go (+17/-16) testservices/service.go (+5/-0) Text conflict in identity/userpass.go
To merge this branch:	bzr merge lp:~wallyworld/goose/rate-limit-retry-tests
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
The Go Language Gophers		2013-01-25	Pending
Review via email: mp+144849@code.launchpad.net