Merge lp:~david-goetz/swift/ratelimit_no_memcache_bug into lp:~hudson-openstack/swift/trunk

Proposed by David Goetz
Status: Merged
Approved by: Mike Barton
Approved revision: 225
Merged at revision: 225
Proposed branch: lp:~david-goetz/swift/ratelimit_no_memcache_bug
Merge into: lp:~hudson-openstack/swift/trunk
Diff against target: 196 lines (+62/-22)
4 files modified
swift/common/memcached.py (+7/-0)
swift/common/middleware/ratelimit.py (+26/-22)
test/unit/common/middleware/test_ratelimit.py (+19/-0)
test/unit/common/test_memcached.py (+10/-0)
To merge this branch: bzr merge lp:~david-goetz/swift/ratelimit_no_memcache_bug
Reviewer Review Type Date Requested Status
Swift Core security contacts Pending
Review via email: mp+51205@code.launchpad.net

Description of the change

ratelimiting does not handle memcache restart

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'swift/common/memcached.py'
2--- swift/common/memcached.py 2011-01-04 23:34:43 +0000
3+++ swift/common/memcached.py 2011-02-24 20:42:51 +0000
4@@ -45,6 +45,10 @@
5 return md5(key).hexdigest()
6
7
8+class MemcacheConnectionError(Exception):
9+ pass
10+
11+
12 class MemcacheRing(object):
13 """
14 Simple, consistent-hashed memcache client.
15@@ -180,6 +184,7 @@
16 :param delta: amount to add to the value of key (or set as the value
17 if the key is not found) will be cast to an int
18 :param timeout: ttl in memcache
19+ :raises MemcacheConnectionError:
20 """
21 key = md5hash(key)
22 command = 'incr'
23@@ -209,6 +214,7 @@
24 return ret
25 except Exception, e:
26 self._exception_occurred(server, e)
27+ raise MemcacheConnectionError("No Memcached connections succeeded.")
28
29 def decr(self, key, delta=1, timeout=0):
30 """
31@@ -220,6 +226,7 @@
32 value to 0 if the key is not found) will be cast to
33 an int
34 :param timeout: ttl in memcache
35+ :raises MemcacheConnectionError:
36 """
37 self.incr(key, delta=-delta, timeout=timeout)
38
39
40=== modified file 'swift/common/middleware/ratelimit.py'
41--- swift/common/middleware/ratelimit.py 2011-02-02 21:39:08 +0000
42+++ swift/common/middleware/ratelimit.py 2011-02-24 20:42:51 +0000
43@@ -18,6 +18,7 @@
44
45 from swift.common.utils import split_path, cache_from_env, get_logger
46 from swift.proxy.server import get_container_memcache_key
47+from swift.common.memcached import MemcacheConnectionError
48
49
50 class MaxSleepTimeHitError(Exception):
51@@ -136,28 +137,31 @@
52 :param max_rate: maximum rate allowed in requests per second
53 :raises: MaxSleepTimeHitError if max sleep time is exceeded.
54 '''
55- now_m = int(round(time.time() * self.clock_accuracy))
56- time_per_request_m = int(round(self.clock_accuracy / max_rate))
57- running_time_m = self.memcache_client.incr(key,
58- delta=time_per_request_m)
59- need_to_sleep_m = 0
60- if (now_m - running_time_m >
61- self.rate_buffer_seconds * self.clock_accuracy):
62- next_avail_time = int(now_m + time_per_request_m)
63- self.memcache_client.set(key, str(next_avail_time),
64- serialize=False)
65- else:
66- need_to_sleep_m = \
67- max(running_time_m - now_m - time_per_request_m, 0)
68-
69- max_sleep_m = self.max_sleep_time_seconds * self.clock_accuracy
70- if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01:
71- # treat as no-op decrement time
72- self.memcache_client.decr(key, delta=time_per_request_m)
73- raise MaxSleepTimeHitError("Max Sleep Time Exceeded: %s" %
74- need_to_sleep_m)
75-
76- return float(need_to_sleep_m) / self.clock_accuracy
77+ try:
78+ now_m = int(round(time.time() * self.clock_accuracy))
79+ time_per_request_m = int(round(self.clock_accuracy / max_rate))
80+ running_time_m = self.memcache_client.incr(key,
81+ delta=time_per_request_m)
82+ need_to_sleep_m = 0
83+ if (now_m - running_time_m >
84+ self.rate_buffer_seconds * self.clock_accuracy):
85+ next_avail_time = int(now_m + time_per_request_m)
86+ self.memcache_client.set(key, str(next_avail_time),
87+ serialize=False)
88+ else:
89+ need_to_sleep_m = \
90+ max(running_time_m - now_m - time_per_request_m, 0)
91+
92+ max_sleep_m = self.max_sleep_time_seconds * self.clock_accuracy
93+ if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01:
94+ # treat as no-op decrement time
95+ self.memcache_client.decr(key, delta=time_per_request_m)
96+ raise MaxSleepTimeHitError("Max Sleep Time Exceeded: %s" %
97+ need_to_sleep_m)
98+
99+ return float(need_to_sleep_m) / self.clock_accuracy
100+ except MemcacheConnectionError:
101+ return 0
102
103 def handle_ratelimit(self, req, account_name, container_name, obj_name):
104 '''
105
106=== modified file 'test/unit/common/middleware/test_ratelimit.py'
107--- test/unit/common/middleware/test_ratelimit.py 2011-01-26 22:38:13 +0000
108+++ test/unit/common/middleware/test_ratelimit.py 2011-02-24 20:42:51 +0000
109@@ -21,12 +21,14 @@
110
111 from swift.common.middleware import ratelimit
112 from swift.proxy.server import get_container_memcache_key
113+from swift.common.memcached import MemcacheConnectionError
114
115
116 class FakeMemcache(object):
117
118 def __init__(self):
119 self.store = {}
120+ self.error_on_incr = False
121
122 def get(self, key):
123 return self.store.get(key)
124@@ -36,6 +38,8 @@
125 return True
126
127 def incr(self, key, delta=1, timeout=0):
128+ if self.error_on_incr:
129+ raise MemcacheConnectionError('Memcache restarting')
130 self.store[key] = int(self.store.setdefault(key, 0)) + int(delta)
131 if self.store[key] < 0:
132 self.store[key] = 0
133@@ -403,6 +407,21 @@
134 start_response)
135 self._run(make_app_call, num_calls, current_rate)
136
137+ def test_restarting_memcache(self):
138+ current_rate = 2
139+ num_calls = 5
140+ conf_dict = {'account_ratelimit': current_rate}
141+ self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
142+ ratelimit.http_connect = mock_http_connect(204)
143+ req = Request.blank('/v/a')
144+ req.environ['swift.cache'] = FakeMemcache()
145+ req.environ['swift.cache'].error_on_incr = True
146+ make_app_call = lambda: self.test_ratelimit(req.environ,
147+ start_response)
148+ begin = time.time()
149+ self._run(make_app_call, num_calls, current_rate, check_time=False)
150+ time_took = time.time() - begin
151+ self.assert_(round(time_took, 1) == 0) # no memcache, no limiting
152
153 if __name__ == '__main__':
154 unittest.main()
155
156=== modified file 'test/unit/common/test_memcached.py'
157--- test/unit/common/test_memcached.py 2011-01-04 23:34:43 +0000
158+++ test/unit/common/test_memcached.py 2011-02-24 20:42:51 +0000
159@@ -50,6 +50,7 @@
160 self.cache = {}
161 self.down = False
162 self.exc_on_delete = False
163+ self.read_return_none = False
164
165 def sendall(self, string):
166 if self.down:
167@@ -110,6 +111,8 @@
168 else:
169 self.outbuf += 'NOT_FOUND\r\n'
170 def readline(self):
171+ if self.read_return_none:
172+ return None
173 if self.down:
174 raise Exception('mock is down')
175 if '\n' in self.outbuf:
176@@ -166,6 +169,9 @@
177 self.assertEquals(memcache_client.get('some_key'), '6')
178 memcache_client.incr('some_key', delta=-15)
179 self.assertEquals(memcache_client.get('some_key'), '0')
180+ mock.read_return_none = True
181+ self.assertRaises(memcached.MemcacheConnectionError,
182+ memcache_client.incr, 'some_key', delta=-15)
183
184 def test_decr(self):
185 memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'])
186@@ -179,6 +185,10 @@
187 self.assertEquals(memcache_client.get('some_key'), '11')
188 memcache_client.decr('some_key', delta=15)
189 self.assertEquals(memcache_client.get('some_key'), '0')
190+ mock.read_return_none = True
191+ self.assertRaises(memcached.MemcacheConnectionError,
192+ memcache_client.decr, 'some_key', delta=15)
193+
194
195 def test_retry(self):
196 logging.getLogger().addHandler(NullLoggingHandler())