Merge lp:~eday/burrow/backend-api-cleanup into lp:burrow
- backend-api-cleanup
- Merge into trunk
Proposed by
Eric Day
Status: | Merged |
---|---|
Approved by: | Eric Day |
Approved revision: | 11 |
Merged at revision: | 4 |
Proposed branch: | lp:~eday/burrow/backend-api-cleanup |
Merge into: | lp:burrow |
Diff against target: |
1944 lines (+980/-436) 17 files modified
ChangeLog (+40/-0) bin/burrow (+154/-0) bin/burrowd (+4/-4) burrow/__init__.py (+6/-77) burrow/backend/__init__.py (+52/-50) burrow/backend/http.py (+103/-0) burrow/backend/memory.py (+217/-99) burrow/backend/sqlite.py (+160/-123) burrow/client.py (+63/-0) burrow/config.py (+1/-1) burrow/frontend/__init__.py (+6/-6) burrow/frontend/wsgi.py (+52/-52) burrow/server.py (+95/-0) etc/burrowd.conf (+9/-9) locale/burrow.pot (+8/-8) setup.py (+4/-1) test/frontend/test_wsgi.py (+6/-6) |
To merge this branch: | bzr merge lp:~eday/burrow/backend-api-cleanup |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Burrow Core Team | Pending | ||
Review via email: mp+56510@code.launchpad.net |
Commit message
Description of the change
A number of cleanup tasks and client support.
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'ChangeLog' | |||
2 | --- ChangeLog 2011-03-18 00:47:26 +0000 | |||
3 | +++ ChangeLog 2011-04-06 05:49:23 +0000 | |||
4 | @@ -1,3 +1,43 @@ | |||
5 | 1 | 2011-03-30 Eric Day <eday@oddments.org> | ||
6 | 2 | |||
7 | 3 | Merged burrow and burrowd packages, the interfaces for backend/client can be shared, and this will also allow modular in-process queues. | ||
8 | 4 | |||
9 | 5 | 2011-03-18 Eric Day <eday@oddments.org> | ||
10 | 6 | |||
11 | 7 | Started SQLite backend cleanup. | ||
12 | 8 | |||
13 | 9 | 2011-03-18 Eric Day <eday@oddments.org> | ||
14 | 10 | |||
15 | 11 | More backend API cleanup. | ||
16 | 12 | |||
17 | 13 | 2011-03-18 Eric Day <eday@oddments.org> | ||
18 | 14 | |||
19 | 15 | Further backend API cleanup, started cleanup in memory backend. | ||
20 | 16 | |||
21 | 17 | 2011-03-17 Eric Day <eday@oddments.org> | ||
22 | 18 | |||
23 | 19 | Removed the need to 'queue_exists' decorator and backend methods. | ||
24 | 20 | |||
25 | 21 | 2011-03-17 Eric Day <eday@oddments.org> | ||
26 | 22 | |||
27 | 23 | Switched to pass attributes as a dict instead of positional arguments. | ||
28 | 24 | |||
29 | 25 | 2011-03-17 Eric Day <eday@oddments.org> | ||
30 | 26 | |||
31 | 27 | Switched to pass filters as a dict instead of positional arguments. | ||
32 | 28 | |||
33 | 29 | 2011-03-18 Eric Day <eday@oddments.org> | ||
34 | 30 | |||
35 | 31 | Started locale support. | ||
36 | 32 | |||
37 | 33 | 2011-03-17 Eric Day <eday@oddments.org> | ||
38 | 34 | |||
39 | 35 | Started locale support. | ||
40 | 36 | |||
41 | 37 | 2011-03-17 Eric Day <eday@oddments.org> | ||
42 | 38 | |||
43 | 39 | Python prototype conversion to get new trunk started. | ||
44 | 40 | |||
45 | 1 | 2011-03-17 Eric Day <eday@oddments.org> | 41 | 2011-03-17 Eric Day <eday@oddments.org> |
46 | 2 | 42 | ||
47 | 3 | First chunk of code from prototype. Beyond the prototype, configuration, module loading, and log handling was added. | 43 | First chunk of code from prototype. Beyond the prototype, configuration, module loading, and log handling was added. |
48 | 4 | 44 | ||
49 | === modified file 'bin/burrow' | |||
50 | --- bin/burrow 2011-03-17 23:42:41 +0000 | |||
51 | +++ bin/burrow 2011-04-06 05:49:23 +0000 | |||
52 | @@ -16,3 +16,157 @@ | |||
53 | 16 | ''' | 16 | ''' |
54 | 17 | Burrow command line client. | 17 | Burrow command line client. |
55 | 18 | ''' | 18 | ''' |
56 | 19 | |||
57 | 20 | import pwd | ||
58 | 21 | import optparse | ||
59 | 22 | import os | ||
60 | 23 | import sys | ||
61 | 24 | |||
62 | 25 | # If ../burrow/__init__.py exists, add ../ to the Python search path so | ||
63 | 26 | # that it will override whatever may be installed in the default Python | ||
64 | 27 | # search path. | ||
65 | 28 | BASE_DIRECTORY = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir) | ||
66 | 29 | BASE_DIRECTORY = os.path.normpath(BASE_DIRECTORY) | ||
67 | 30 | if os.path.exists(os.path.join(BASE_DIRECTORY, 'burrow', '__init__.py')): | ||
68 | 31 | sys.path.insert(0, BASE_DIRECTORY) | ||
69 | 32 | |||
70 | 33 | import burrow | ||
71 | 34 | |||
72 | 35 | |||
73 | 36 | class Burrow(object): | ||
74 | 37 | |||
75 | 38 | sections = [ | ||
76 | 39 | dict(name='Global', | ||
77 | 40 | filters=True, | ||
78 | 41 | args=[], | ||
79 | 42 | commands=['delete_accounts', 'get_accounts']), | ||
80 | 43 | dict(name='Account', | ||
81 | 44 | account=True, | ||
82 | 45 | filters=True, | ||
83 | 46 | args=[], | ||
84 | 47 | commands=['delete_queues', 'get_queues']), | ||
85 | 48 | dict(name='Queue', | ||
86 | 49 | account=True, | ||
87 | 50 | filters=True, | ||
88 | 51 | args=['queue'], | ||
89 | 52 | commands=['delete_messages', 'get_messages', 'update_messages']), | ||
90 | 53 | dict(name='Message', | ||
91 | 54 | account=True, | ||
92 | 55 | args=['queue', 'message'], | ||
93 | 56 | commands=[ | ||
94 | 57 | 'create_message', | ||
95 | 58 | 'delete_message', | ||
96 | 59 | 'get_message', | ||
97 | 60 | 'update_message'])] | ||
98 | 61 | |||
99 | 62 | attribute_commands = [ | ||
100 | 63 | 'update_messages', | ||
101 | 64 | 'create_message', | ||
102 | 65 | 'update_message'] | ||
103 | 66 | |||
104 | 67 | stdin_commands = ['create_message'] | ||
105 | 68 | |||
106 | 69 | def __init__(self): | ||
107 | 70 | self.parser = optparse.OptionParser(version=burrow.__version__) | ||
108 | 71 | self.parser.add_option('-c', '--commands', action='store_true', | ||
109 | 72 | help=_('Print help for the available commands')) | ||
110 | 73 | self.parser.add_option('-u', '--url', default='http://localhost:8080', | ||
111 | 74 | help=_('Backend URL to use')) | ||
112 | 75 | rcfile = os.path.expanduser('~') | ||
113 | 76 | rcfile = os.path.join(rcfile, '.burrowrc') | ||
114 | 77 | if not os.path.exists(rcfile): | ||
115 | 78 | rcfile = None | ||
116 | 79 | self.parser.add_option('-f', '--files', default=rcfile, | ||
117 | 80 | help=_('Configuration file(s) to use (comma separated)')) | ||
118 | 81 | user = pwd.getpwuid(os.getuid())[0] | ||
119 | 82 | self.parser.add_option('-a', '--account', default=user, | ||
120 | 83 | help=_('Account to use for queue and message commands')) | ||
121 | 84 | self.parser.add_option('-w', '--wait', | ||
122 | 85 | help=_('Number of seconds to wait if no messages match')) | ||
123 | 86 | |||
124 | 87 | attributes = optparse.OptionGroup(self.parser, | ||
125 | 88 | _('Messages attribute options')) | ||
126 | 89 | attributes.add_option('-t', '--ttl', | ||
127 | 90 | help=_('TTL attribute in seconds to set for message(s)')) | ||
128 | 91 | attributes.add_option('-H', '--hide', | ||
129 | 92 | help=_('Hidden time attribute in seconds to set for message(s)')) | ||
130 | 93 | self.parser.add_option_group(attributes) | ||
131 | 94 | |||
132 | 95 | filters = optparse.OptionGroup(self.parser, _('Filtering options')) | ||
133 | 96 | filters.add_option('-l', '--limit', | ||
134 | 97 | help=_('Limit the number of messages to match')) | ||
135 | 98 | filters.add_option('-m', '--marker', | ||
136 | 99 | help=_('Only match messages that were inserted after this id')) | ||
137 | 100 | filters.add_option('-A', '--all', action='store_true', | ||
138 | 101 | help=_('Match all messages, including those that are hidden')) | ||
139 | 102 | choices = ['none', 'id', 'attributes', 'all'] | ||
140 | 103 | filters.add_option('-d', '--detail', type='choice', choices=choices, | ||
141 | 104 | help=_('What message information to return. Options are: %s') % | ||
142 | 105 | ', '.join(choices)) | ||
143 | 106 | self.parser.add_option_group(filters) | ||
144 | 107 | |||
145 | 108 | def run(self): | ||
146 | 109 | (self.options, args) = self.parser.parse_args() | ||
147 | 110 | if self.options.commands: | ||
148 | 111 | self.print_help() | ||
149 | 112 | if len(args) == 0: | ||
150 | 113 | self.print_help(_('No command given')) | ||
151 | 114 | if self.options.files is None: | ||
152 | 115 | files = [] | ||
153 | 116 | else: | ||
154 | 117 | files = self.options.files.split(', ') | ||
155 | 118 | self.client = burrow.Client(url=self.options.url, config_files=files) | ||
156 | 119 | for section in self.sections: | ||
157 | 120 | if args[0] in section['commands']: | ||
158 | 121 | self.run_command(section, args[0], args[1:]) | ||
159 | 122 | self.print_help(_('Command not found')) | ||
160 | 123 | |||
161 | 124 | def run_command(self, section, command, args): | ||
162 | 125 | if len(args) != len(section['args']): | ||
163 | 126 | self.print_help(_('Wrong number of arguments')) | ||
164 | 127 | if section.get('account', None): | ||
165 | 128 | args.insert(0, self.options.account) | ||
166 | 129 | if command in self.stdin_commands: | ||
167 | 130 | args.append(sys.stdin.read()) | ||
168 | 131 | if command in self.attribute_commands: | ||
169 | 132 | attributes = {} | ||
170 | 133 | if self.options.ttl is not None: | ||
171 | 134 | attributes['ttl'] = self.options.ttl | ||
172 | 135 | if self.options.hide is not None: | ||
173 | 136 | attributes['hide'] = self.options.hide | ||
174 | 137 | args.append(attributes) | ||
175 | 138 | if section.get('filters', None): | ||
176 | 139 | filters = {} | ||
177 | 140 | if self.options.limit is not None: | ||
178 | 141 | filters['limit'] = self.options.limit | ||
179 | 142 | if self.options.marker is not None: | ||
180 | 143 | filters['marker'] = self.options.marker | ||
181 | 144 | if self.options.all is not None: | ||
182 | 145 | filters['match_hidden'] = self.options.all | ||
183 | 146 | args.append(filters) | ||
184 | 147 | getattr(self.client.backend, command)(*args) | ||
185 | 148 | sys.exit(0) | ||
186 | 149 | |||
187 | 150 | def print_help(self, message=None): | ||
188 | 151 | if message: | ||
189 | 152 | print message | ||
190 | 153 | |||
191 | 154 | self.parser.print_help() | ||
192 | 155 | |||
193 | 156 | for section in self.sections: | ||
194 | 157 | print '%s commands:' % section['name'] | ||
195 | 158 | for command in section['commands']: | ||
196 | 159 | help = '' | ||
197 | 160 | if section.get('filters', None): | ||
198 | 161 | help += ' [filters]' | ||
199 | 162 | if command in self.attribute_commands: | ||
200 | 163 | help += ' [attributes]' | ||
201 | 164 | for arg in section['args']: | ||
202 | 165 | help += ' <%s>' % arg | ||
203 | 166 | print ' %s%s' % (command, help) | ||
204 | 167 | |||
205 | 168 | sys.exit(1) | ||
206 | 169 | |||
207 | 170 | |||
208 | 171 | if __name__ == '__main__': | ||
209 | 172 | Burrow().run() | ||
210 | 19 | 173 | ||
211 | === modified file 'bin/burrowd' | |||
212 | --- bin/burrowd 2011-03-17 23:42:41 +0000 | |||
213 | +++ bin/burrowd 2011-04-06 05:49:23 +0000 | |||
214 | @@ -20,15 +20,15 @@ | |||
215 | 20 | import os | 20 | import os |
216 | 21 | import sys | 21 | import sys |
217 | 22 | 22 | ||
219 | 23 | # If ../burrowd/__init__.py exists, add ../ to the Python search path so | 23 | # If ../burrow/__init__.py exists, add ../ to the Python search path so |
220 | 24 | # that it will override whatever may be installed in the default Python | 24 | # that it will override whatever may be installed in the default Python |
221 | 25 | # search path. | 25 | # search path. |
222 | 26 | BASE_DIRECTORY = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir) | 26 | BASE_DIRECTORY = os.path.join(os.path.abspath(__file__), os.pardir, os.pardir) |
223 | 27 | BASE_DIRECTORY = os.path.normpath(BASE_DIRECTORY) | 27 | BASE_DIRECTORY = os.path.normpath(BASE_DIRECTORY) |
225 | 28 | if os.path.exists(os.path.join(BASE_DIRECTORY, 'burrowd', '__init__.py')): | 28 | if os.path.exists(os.path.join(BASE_DIRECTORY, 'burrow', '__init__.py')): |
226 | 29 | sys.path.insert(0, BASE_DIRECTORY) | 29 | sys.path.insert(0, BASE_DIRECTORY) |
227 | 30 | 30 | ||
229 | 31 | import burrowd | 31 | import burrow |
230 | 32 | 32 | ||
231 | 33 | if __name__ == '__main__': | 33 | if __name__ == '__main__': |
233 | 34 | burrowd.Burrowd(sys.argv[1:]).run() | 34 | burrow.Server(sys.argv[1:]).run() |
234 | 35 | 35 | ||
235 | === removed directory 'burrow' | |||
236 | === renamed directory 'burrowd' => 'burrow' | |||
237 | === removed file 'burrow/__init__.py' | |||
238 | --- burrow/__init__.py 2011-03-17 23:42:41 +0000 | |||
239 | +++ burrow/__init__.py 1970-01-01 00:00:00 +0000 | |||
240 | @@ -1,19 +0,0 @@ | |||
241 | 1 | # Copyright (C) 2011 OpenStack LLC. | ||
242 | 2 | # | ||
243 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
244 | 4 | # you may not use this file except in compliance with the License. | ||
245 | 5 | # You may obtain a copy of the License at | ||
246 | 6 | # | ||
247 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
248 | 8 | # | ||
249 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
250 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
251 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
252 | 12 | # See the License for the specific language governing permissions and | ||
253 | 13 | # limitations under the License. | ||
254 | 14 | |||
255 | 15 | '''Main client module for burrow.''' | ||
256 | 16 | |||
257 | 17 | |||
258 | 18 | class Burrow(object): | ||
259 | 19 | pass | ||
260 | 20 | 0 | ||
261 | === modified file 'burrow/__init__.py' | |||
262 | --- burrowd/__init__.py 2011-03-18 00:47:26 +0000 | |||
263 | +++ burrow/__init__.py 2011-04-06 05:49:23 +0000 | |||
264 | @@ -12,99 +12,28 @@ | |||
265 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
266 | 13 | # limitations under the License. | 13 | # limitations under the License. |
267 | 14 | 14 | ||
269 | 15 | '''Main server module for burrow.''' | 15 | '''Main module for burrow.''' |
270 | 16 | 16 | ||
271 | 17 | import ConfigParser | ||
272 | 18 | import gettext | 17 | import gettext |
273 | 19 | import logging | 18 | import logging |
274 | 20 | import logging.config | ||
275 | 21 | import sys | 19 | import sys |
276 | 22 | 20 | ||
278 | 23 | import eventlet | 21 | from burrow.client import Client |
279 | 22 | from burrow.server import Server | ||
280 | 23 | import burrow.config | ||
281 | 24 | 24 | ||
283 | 25 | import burrowd.config | 25 | __version__ = '0.1' |
284 | 26 | 26 | ||
285 | 27 | # This installs the _(...) function as a built-in so all other modules | 27 | # This installs the _(...) function as a built-in so all other modules |
286 | 28 | # don't need to. | 28 | # don't need to. |
287 | 29 | gettext.install('burrow') | 29 | gettext.install('burrow') |
288 | 30 | 30 | ||
289 | 31 | # Default configuration values for this module. | ||
290 | 32 | DEFAULT_BACKEND = 'burrowd.backend.sqlite' | ||
291 | 33 | DEFAULT_FRONTENDS = 'burrowd.frontend.wsgi' | ||
292 | 34 | DEFAULT_THREAD_POOL_SIZE = 1000 | ||
293 | 35 | |||
294 | 36 | |||
295 | 37 | class Burrowd(object): | ||
296 | 38 | '''Server class for burrow.''' | ||
297 | 39 | |||
298 | 40 | def __init__(self, config_files=[], add_default_log_handler=True): | ||
299 | 41 | '''Initialize a server using the config files from the given | ||
300 | 42 | list. This is passed directly to ConfigParser.read(), so | ||
301 | 43 | files should be in ConfigParser format. This will load all | ||
302 | 44 | frontend and backend classes from the configuration.''' | ||
303 | 45 | if len(config_files) > 0: | ||
304 | 46 | logging.config.fileConfig(config_files) | ||
305 | 47 | self._config = ConfigParser.ConfigParser() | ||
306 | 48 | self._config.read(config_files) | ||
307 | 49 | self.config = burrowd.config.Config(self._config, 'burrowd') | ||
308 | 50 | self.log = get_logger(self.config) | ||
309 | 51 | if add_default_log_handler: | ||
310 | 52 | self._add_default_log_handler() | ||
311 | 53 | self._import_backend() | ||
312 | 54 | self._import_frontends() | ||
313 | 55 | |||
314 | 56 | def _add_default_log_handler(self): | ||
315 | 57 | '''Add a default log handler it one has not been set.''' | ||
316 | 58 | root_log = logging.getLogger() | ||
317 | 59 | if len(root_log.handlers) > 0 or len(self.log.handlers) > 0: | ||
318 | 60 | return | ||
319 | 61 | handler = logging.StreamHandler() | ||
320 | 62 | handler.setLevel(logging.DEBUG) | ||
321 | 63 | log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' | ||
322 | 64 | handler.setFormatter(logging.Formatter(log_format)) | ||
323 | 65 | root_log.addHandler(handler) | ||
324 | 66 | |||
325 | 67 | def _import_backend(self): | ||
326 | 68 | '''Load backend given in the 'backend' option.''' | ||
327 | 69 | backend = self.config.get('backend', DEFAULT_BACKEND) | ||
328 | 70 | config = (self._config, backend) | ||
329 | 71 | self.backend = import_class(backend, 'Backend')(config) | ||
330 | 72 | |||
331 | 73 | def _import_frontends(self): | ||
332 | 74 | '''Load frontends given in the 'frontends' option.''' | ||
333 | 75 | self.frontends = [] | ||
334 | 76 | frontends = self.config.get('frontends', DEFAULT_FRONTENDS) | ||
335 | 77 | for frontend in frontends.split(','): | ||
336 | 78 | frontend = frontend.split(':') | ||
337 | 79 | if len(frontend) == 1: | ||
338 | 80 | frontend.append(None) | ||
339 | 81 | config = (self._config, frontend[0], frontend[1]) | ||
340 | 82 | frontend = import_class(frontend[0], 'Frontend') | ||
341 | 83 | frontend = frontend(config, self.backend) | ||
342 | 84 | self.frontends.append(frontend) | ||
343 | 85 | |||
344 | 86 | def run(self): | ||
345 | 87 | '''Create the thread pool and start the main server loop. Wait | ||
346 | 88 | for the pool to complete, but possibly run forever if the | ||
347 | 89 | frontends and backend never remove threads.''' | ||
348 | 90 | thread_pool_size = self.config.getint('thread_pool_size', | ||
349 | 91 | DEFAULT_THREAD_POOL_SIZE) | ||
350 | 92 | thread_pool = eventlet.GreenPool(size=int(thread_pool_size)) | ||
351 | 93 | self.backend.run(thread_pool) | ||
352 | 94 | for frontend in self.frontends: | ||
353 | 95 | frontend.run(thread_pool) | ||
354 | 96 | self.log.info(_('Waiting for all threads to exit')) | ||
355 | 97 | try: | ||
356 | 98 | thread_pool.waitall() | ||
357 | 99 | except KeyboardInterrupt: | ||
358 | 100 | pass | ||
359 | 101 | |||
360 | 102 | 31 | ||
361 | 103 | class Module(object): | 32 | class Module(object): |
362 | 104 | '''Common module class for burrow.''' | 33 | '''Common module class for burrow.''' |
363 | 105 | 34 | ||
364 | 106 | def __init__(self, config): | 35 | def __init__(self, config): |
366 | 107 | self.config = burrowd.config.Config(*config) | 36 | self.config = burrow.config.Config(*config) |
367 | 108 | self.log = get_logger(self.config) | 37 | self.log = get_logger(self.config) |
368 | 109 | self.log.debug(_('Module created')) | 38 | self.log.debug(_('Module created')) |
369 | 110 | 39 | ||
370 | 111 | 40 | ||
371 | === modified file 'burrow/backend/__init__.py' | |||
372 | --- burrowd/backend/__init__.py 2011-03-17 23:42:41 +0000 | |||
373 | +++ burrow/backend/__init__.py 2011-04-06 05:49:23 +0000 | |||
374 | @@ -12,14 +12,14 @@ | |||
375 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
376 | 13 | # limitations under the License. | 13 | # limitations under the License. |
377 | 14 | 14 | ||
379 | 15 | '''Backends for the burrow server.''' | 15 | '''Backends for burrow.''' |
380 | 16 | 16 | ||
381 | 17 | import eventlet | 17 | import eventlet |
382 | 18 | 18 | ||
387 | 19 | import burrowd | 19 | import burrow |
388 | 20 | 20 | ||
389 | 21 | 21 | ||
390 | 22 | class Backend(burrowd.Module): | 22 | class Backend(burrow.Module): |
391 | 23 | '''Interface that backend implementations must provide.''' | 23 | '''Interface that backend implementations must provide.''' |
392 | 24 | 24 | ||
393 | 25 | def __init__(self, config): | 25 | def __init__(self, config): |
394 | @@ -27,62 +27,52 @@ | |||
395 | 27 | self.queues = {} | 27 | self.queues = {} |
396 | 28 | 28 | ||
397 | 29 | def run(self, thread_pool): | 29 | def run(self, thread_pool): |
398 | 30 | '''Run the backend. This should start any periodic tasks in | ||
399 | 31 | separate threads and should never block.''' | ||
400 | 30 | thread_pool.spawn_n(self._clean) | 32 | thread_pool.spawn_n(self._clean) |
401 | 31 | 33 | ||
439 | 32 | def _clean(self): | 34 | def delete_accounts(self, filters={}): |
440 | 33 | while True: | 35 | pass |
441 | 34 | self.clean() | 36 | |
442 | 35 | eventlet.sleep(1) | 37 | def get_accounts(self, filters={}): |
443 | 36 | 38 | return [] | |
444 | 37 | def delete_accounts(self): | 39 | |
445 | 38 | pass | 40 | def delete_queues(self, account, filters={}): |
446 | 39 | 41 | pass | |
447 | 40 | def get_accounts(self): | 42 | |
448 | 41 | return [] | 43 | def get_queues(self, account, filters={}): |
449 | 42 | 44 | return [] | |
450 | 43 | def delete_account(self, account): | 45 | |
451 | 44 | pass | 46 | def delete_messages(self, account, queue, filters={}): |
452 | 45 | 47 | return [] | |
453 | 46 | def get_queues(self, account): | 48 | |
454 | 47 | return [] | 49 | def get_messages(self, account, queue, filters={}): |
455 | 48 | 50 | return [] | |
456 | 49 | def queue_exists(self, account, queue): | 51 | |
457 | 50 | return False | 52 | def update_messages(self, account, queue, attributes={}, filters={}): |
458 | 51 | 53 | return [] | |
459 | 52 | def delete_messages(self, account, queue, limit, marker, match_hidden): | 54 | |
460 | 53 | return [] | 55 | def create_message(self, account, queue, message, body, attributes={}): |
424 | 54 | |||
425 | 55 | def get_messages(self, account, queue, limit, marker, match_hidden): | ||
426 | 56 | return [] | ||
427 | 57 | |||
428 | 58 | def update_messages(self, account, queue, limit, marker, match_hidden, ttl, | ||
429 | 59 | hide): | ||
430 | 60 | return [] | ||
431 | 61 | |||
432 | 62 | def delete_message(self, account, queue, message_id): | ||
433 | 63 | return None | ||
434 | 64 | |||
435 | 65 | def get_message(self, account, queue, message_id): | ||
436 | 66 | return None | ||
437 | 67 | |||
438 | 68 | def put_message(self, account, queue, message_id, ttl, hide, body): | ||
461 | 69 | return True | 56 | return True |
462 | 70 | 57 | ||
471 | 71 | def update_message(self, account, queue, message_id, ttl, hide): | 58 | def delete_message(self, account, queue, message): |
472 | 72 | return None | 59 | return None |
473 | 73 | 60 | ||
474 | 74 | def clean(self): | 61 | def get_message(self, account, queue, message): |
475 | 75 | '''This method should remove all messages with an expired | 62 | return None |
476 | 76 | TTL and make hidden messages that have an expired hide time | 63 | |
477 | 77 | visible again.''' | 64 | def update_message(self, account, queue, message, attributes={}): |
478 | 78 | pass | 65 | return None |
479 | 79 | 66 | ||
480 | 80 | def notify(self, account, queue): | 67 | def notify(self, account, queue): |
481 | 68 | '''Notify any waiting callers that the account/queue has | ||
482 | 69 | a visible message.''' | ||
483 | 81 | queue = '%s/%s' % (account, queue) | 70 | queue = '%s/%s' % (account, queue) |
484 | 82 | if queue in self.queues: | 71 | if queue in self.queues: |
485 | 83 | self.queues[queue].put(0) | 72 | self.queues[queue].put(0) |
486 | 84 | 73 | ||
487 | 85 | def wait(self, account, queue, seconds): | 74 | def wait(self, account, queue, seconds): |
488 | 75 | '''Wait for a message to appear in the account/queue.''' | ||
489 | 86 | queue = '%s/%s' % (account, queue) | 76 | queue = '%s/%s' % (account, queue) |
490 | 87 | if queue not in self.queues: | 77 | if queue not in self.queues: |
491 | 88 | self.queues[queue] = eventlet.Queue() | 78 | self.queues[queue] = eventlet.Queue() |
492 | @@ -92,3 +82,15 @@ | |||
493 | 92 | pass | 82 | pass |
494 | 93 | if self.queues[queue].getting() == 0: | 83 | if self.queues[queue].getting() == 0: |
495 | 94 | del self.queues[queue] | 84 | del self.queues[queue] |
496 | 85 | |||
497 | 86 | def clean(self): | ||
498 | 87 | '''This method should remove all messages with an expired | ||
499 | 88 | TTL and make hidden messages that have an expired hide time | ||
500 | 89 | visible again.''' | ||
501 | 90 | pass | ||
502 | 91 | |||
503 | 92 | def _clean(self): | ||
504 | 93 | '''Thread to run the clean method periodically.''' | ||
505 | 94 | while True: | ||
506 | 95 | self.clean() | ||
507 | 96 | eventlet.sleep(1) | ||
508 | 95 | 97 | ||
509 | === added file 'burrow/backend/http.py' | |||
510 | --- burrow/backend/http.py 1970-01-01 00:00:00 +0000 | |||
511 | +++ burrow/backend/http.py 2011-04-06 05:49:23 +0000 | |||
512 | @@ -0,0 +1,103 @@ | |||
513 | 1 | # Copyright (C) 2011 OpenStack LLC. | ||
514 | 2 | # | ||
515 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
516 | 4 | # you may not use this file except in compliance with the License. | ||
517 | 5 | # You may obtain a copy of the License at | ||
518 | 6 | # | ||
519 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
520 | 8 | # | ||
521 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
522 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
523 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
524 | 12 | # See the License for the specific language governing permissions and | ||
525 | 13 | # limitations under the License. | ||
526 | 14 | |||
527 | 15 | '''HTTP backend for burrow using httplib.''' | ||
528 | 16 | |||
529 | 17 | import httplib | ||
530 | 18 | |||
531 | 19 | import burrow.backend | ||
532 | 20 | |||
533 | 21 | |||
534 | 22 | class Backend(burrow.backend.Backend): | ||
535 | 23 | '''This backend forwards all requests via HTTP using the httplib | ||
536 | 24 | module. It is used for clients and proxies.''' | ||
537 | 25 | |||
538 | 26 | def __init__(self, config): | ||
539 | 27 | super(Backend, self).__init__(config) | ||
540 | 28 | self.server = ('localhost', 8080) | ||
541 | 29 | |||
542 | 30 | def delete_accounts(self, filters={}): | ||
543 | 31 | url = self._add_parameters('/', filters=filters) | ||
544 | 32 | self._request('DELETE', url) | ||
545 | 33 | |||
546 | 34 | def get_accounts(self, filters={}): | ||
547 | 35 | url = self._add_parameters('/', filters=filters) | ||
548 | 36 | self._request('GET', url) | ||
549 | 37 | |||
550 | 38 | def delete_queues(self, account, filters={}): | ||
551 | 39 | url = self._add_parameters('/%s' % account, filters=filters) | ||
552 | 40 | self._request('DELETE', url) | ||
553 | 41 | |||
554 | 42 | def get_queues(self, account, filters={}): | ||
555 | 43 | url = self._add_parameters('/%s' % account, filters=filters) | ||
556 | 44 | self._request('GET', url) | ||
557 | 45 | |||
558 | 46 | def delete_messages(self, account, queue, filters={}): | ||
559 | 47 | url = '/%s/%s' % (account, queue) | ||
560 | 48 | url = self._add_parameters(url, filters=filters) | ||
561 | 49 | self._request('DELETE', url) | ||
562 | 50 | |||
563 | 51 | def get_messages(self, account, queue, filters={}): | ||
564 | 52 | url = '/%s/%s' % (account, queue) | ||
565 | 53 | url = self._add_parameters(url, filters=filters) | ||
566 | 54 | self._request('GET', url) | ||
567 | 55 | |||
568 | 56 | def update_messages(self, account, queue, attributes={}, filters={}): | ||
569 | 57 | url = '/%s/%s' % (account, queue) | ||
570 | 58 | url = self._add_parameters(url, attributes, filters) | ||
571 | 59 | self._request('POST', url) | ||
572 | 60 | |||
573 | 61 | def create_message(self, account, queue, message, body, attributes={}): | ||
574 | 62 | url = '/%s/%s/%s' % (account, queue, message) | ||
575 | 63 | url = self._add_parameters(url, attributes) | ||
576 | 64 | self._request('PUT', url, body=body) | ||
577 | 65 | |||
578 | 66 | def delete_message(self, account, queue, message): | ||
579 | 67 | url = '/%s/%s/%s' % (account, queue, message) | ||
580 | 68 | self._request('DELETE', url) | ||
581 | 69 | |||
582 | 70 | def get_message(self, account, queue, message): | ||
583 | 71 | url = '/%s/%s/%s' % (account, queue, message) | ||
584 | 72 | self._request('GET', url) | ||
585 | 73 | |||
586 | 74 | def update_message(self, account, queue, message, attributes={}): | ||
587 | 75 | url = '/%s/%s/%s' % (account, queue, message) | ||
588 | 76 | url = self._add_parameters(url, attributes) | ||
589 | 77 | self._request('POST', url) | ||
590 | 78 | |||
591 | 79 | def clean(self): | ||
592 | 80 | pass | ||
593 | 81 | |||
594 | 82 | def _add_parameters(self, url, attributes={}, filters={}): | ||
595 | 83 | separator = '?' | ||
596 | 84 | for attribute in ['ttl', 'hide']: | ||
597 | 85 | value = attributes.get(attribute, None) | ||
598 | 86 | if value is not None: | ||
599 | 87 | url += '%s%s=%s' % (separator, attribute, value) | ||
600 | 88 | separator = '&' | ||
601 | 89 | for filter in ['marker', 'limit', 'match_hidden', 'detail']: | ||
602 | 90 | value = filters.get(filter, None) | ||
603 | 91 | if value is not None: | ||
604 | 92 | url += '%s%s=%s' % (separator, filter, value) | ||
605 | 93 | separator = '&' | ||
606 | 94 | return url | ||
607 | 95 | |||
608 | 96 | def _request(self, *args, **kwargs): | ||
609 | 97 | connection = httplib.HTTPConnection(*self.server) | ||
610 | 98 | connection.request(*args, **kwargs) | ||
611 | 99 | response = connection.getresponse() | ||
612 | 100 | if response.status == 200: | ||
613 | 101 | print response.read() | ||
614 | 102 | if response.status >= 400: | ||
615 | 103 | print response.reason | ||
616 | 0 | 104 | ||
617 | === modified file 'burrow/backend/memory.py' | |||
618 | --- burrowd/backend/memory.py 2011-03-17 23:42:41 +0000 | |||
619 | +++ burrow/backend/memory.py 2011-04-06 05:49:23 +0000 | |||
620 | @@ -12,115 +12,117 @@ | |||
621 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
622 | 13 | # limitations under the License. | 13 | # limitations under the License. |
623 | 14 | 14 | ||
625 | 15 | '''Memory backend for the burrow server.''' | 15 | '''Memory backend for burrow.''' |
626 | 16 | 16 | ||
627 | 17 | import time | 17 | import time |
628 | 18 | 18 | ||
633 | 19 | import burrowd.backend | 19 | import burrow.backend |
634 | 20 | 20 | ||
635 | 21 | 21 | ||
636 | 22 | class Backend(burrowd.backend.Backend): | 22 | class Backend(burrow.backend.Backend): |
637 | 23 | '''This backend stores all data using native Python data | ||
638 | 24 | structures. It uses a linked list of tuples to store data | ||
639 | 25 | (accounts, queues, and messages) with a dict as a secondary index | ||
640 | 26 | into this list. This is required so we can have O(1) appends, | ||
641 | 27 | deletes, and lookups by name, along with easy traversal starting | ||
642 | 28 | anywhere in the list.''' | ||
643 | 23 | 29 | ||
644 | 24 | def __init__(self, config): | 30 | def __init__(self, config): |
645 | 25 | super(Backend, self).__init__(config) | 31 | super(Backend, self).__init__(config) |
709 | 26 | self.accounts = {} | 32 | self.accounts = Accounts() |
710 | 27 | 33 | ||
711 | 28 | def delete_accounts(self): | 34 | def delete_accounts(self, filters={}): |
712 | 29 | self.accounts.clear() | 35 | self.accounts.delete(filters) |
713 | 30 | 36 | ||
714 | 31 | def get_accounts(self): | 37 | def get_accounts(self, filters={}): |
715 | 32 | return self.accounts.keys() | 38 | for account in self.accounts.iter(filters): |
716 | 33 | 39 | yield account[0] | |
717 | 34 | def delete_account(self, account): | 40 | |
718 | 35 | del self.accounts[account] | 41 | def delete_queues(self, account, filters={}): |
719 | 36 | 42 | self.accounts.delete_queues(account, filters) | |
720 | 37 | def get_queues(self, account): | 43 | |
721 | 38 | if account not in self.accounts: | 44 | def get_queues(self, account, filters={}): |
722 | 39 | return [] | 45 | for queue in self.accounts.iter_queues(account, filters): |
723 | 40 | return self.accounts[account].keys() | 46 | yield queue[0] |
724 | 41 | 47 | ||
725 | 42 | def queue_exists(self, account, queue): | 48 | def delete_messages(self, account, queue, filters={}): |
726 | 43 | return account in self.accounts and queue in self.accounts[account] | 49 | return self._scan_queue(account, queue, filters, delete=True) |
727 | 44 | 50 | ||
728 | 45 | def delete_messages(self, account, queue, limit, marker, match_hidden): | 51 | def get_messages(self, account, queue, filters={}): |
729 | 46 | messages = self._scan_queue(account, queue, limit, marker, | 52 | return self._scan_queue(account, queue, filters) |
730 | 47 | match_hidden, delete=True) | 53 | |
731 | 48 | if len(self.accounts[account][queue]) == 0: | 54 | def update_messages(self, account, queue, attributes={}, filters={}): |
732 | 49 | del self.accounts[account][queue] | 55 | return self._scan_queue(account, queue, filters, attributes) |
733 | 50 | if len(self.accounts[account]) == 0: | 56 | |
734 | 51 | del self.accounts[account] | 57 | def create_message(self, account, queue, message, body, attributes={}): |
735 | 52 | return messages | 58 | account, queue = self.accounts.get_queue(account, queue, True) |
736 | 53 | 59 | ttl = attributes.get('ttl', 0) | |
737 | 54 | def get_messages(self, account, queue, limit, marker, match_hidden): | 60 | hide = attributes.get('hide', 0) |
738 | 55 | return self._scan_queue(account, queue, limit, marker, match_hidden) | 61 | for index in xrange(0, len(queue[3])): |
739 | 56 | 62 | if queue[3][index]['id'] == message: | |
740 | 57 | def update_messages(self, account, queue, limit, marker, match_hidden, ttl, | 63 | message = queue[3][index] |
678 | 58 | hide): | ||
679 | 59 | return self._scan_queue(account, queue, limit, marker, match_hidden, | ||
680 | 60 | ttl=ttl, hide=hide) | ||
681 | 61 | |||
682 | 62 | def delete_message(self, account, queue, message_id): | ||
683 | 63 | for index in range(0, len(self.accounts[account][queue])): | ||
684 | 64 | message = self.accounts[account][queue][index] | ||
685 | 65 | if message['id'] == message_id: | ||
686 | 66 | del self.accounts[account][queue][index] | ||
687 | 67 | if len(self.accounts[account][queue]) == 0: | ||
688 | 68 | del self.accounts[account][queue] | ||
689 | 69 | if len(self.accounts[account]) == 0: | ||
690 | 70 | del self.accounts[account] | ||
691 | 71 | return message | ||
692 | 72 | return None | ||
693 | 73 | |||
694 | 74 | def get_message(self, account, queue, message_id): | ||
695 | 75 | for index in range(0, len(self.accounts[account][queue])): | ||
696 | 76 | message = self.accounts[account][queue][index] | ||
697 | 77 | if message['id'] == message_id: | ||
698 | 78 | return message | ||
699 | 79 | return None | ||
700 | 80 | |||
701 | 81 | def put_message(self, account, queue, message_id, ttl, hide, body): | ||
702 | 82 | if account not in self.accounts: | ||
703 | 83 | self.accounts[account] = {} | ||
704 | 84 | if queue not in self.accounts[account]: | ||
705 | 85 | self.accounts[account][queue] = [] | ||
706 | 86 | for index in range(0, len(self.accounts[account][queue])): | ||
707 | 87 | message = self.accounts[account][queue][index] | ||
708 | 88 | if message['id'] == message_id: | ||
741 | 89 | message['ttl'] = ttl | 64 | message['ttl'] = ttl |
742 | 90 | message['hide'] = hide | 65 | message['hide'] = hide |
743 | 91 | message['body'] = body | 66 | message['body'] = body |
744 | 92 | if hide == 0: | 67 | if hide == 0: |
746 | 93 | self.notify(account, queue) | 68 | self.notify(account[0], queue[0]) |
747 | 94 | return False | 69 | return False |
751 | 95 | message = dict(id=message_id, ttl=ttl, hide=hide, body=body) | 70 | message = dict(id=message, ttl=ttl, hide=hide, body=body) |
752 | 96 | self.accounts[account][queue].append(message) | 71 | queue[3].append(message) |
753 | 97 | self.notify(account, queue) | 72 | self.notify(account[0], queue[0]) |
754 | 98 | return True | 73 | return True |
755 | 99 | 74 | ||
760 | 100 | def update_message(self, account, queue, message_id, ttl, hide): | 75 | def delete_message(self, account, queue, message): |
761 | 101 | for index in range(0, len(self.accounts[account][queue])): | 76 | account, queue = self.accounts.get_queue(account, queue) |
762 | 102 | message = self.accounts[account][queue][index] | 77 | if queue is None: |
763 | 103 | if message['id'] == message_id: | 78 | return None |
764 | 79 | for index in xrange(0, len(queue[3])): | ||
765 | 80 | if queue[3][index]['id'] == message: | ||
766 | 81 | message = queue[3][index] | ||
767 | 82 | del queue[3][index] | ||
768 | 83 | if len(queue[3]) == 0: | ||
769 | 84 | self.accounts.delete_queue(account[0], queue[0]) | ||
770 | 85 | return message | ||
771 | 86 | return None | ||
772 | 87 | |||
773 | 88 | def get_message(self, account, queue, message): | ||
774 | 89 | account, queue = self.accounts.get_queue(account, queue) | ||
775 | 90 | if queue is None: | ||
776 | 91 | return None | ||
777 | 92 | for index in xrange(0, len(queue[3])): | ||
778 | 93 | if queue[3][index]['id'] == message: | ||
779 | 94 | return queue[3][index] | ||
780 | 95 | return None | ||
781 | 96 | |||
782 | 97 | def update_message(self, account, queue, message, attributes={}): | ||
783 | 98 | account, queue = self.accounts.get_queue(account, queue) | ||
784 | 99 | if queue is None: | ||
785 | 100 | return None | ||
786 | 101 | ttl = attributes.get('ttl', None) | ||
787 | 102 | hide = attributes.get('hide', None) | ||
788 | 103 | for index in xrange(0, len(queue[3])): | ||
789 | 104 | if queue[3][index]['id'] == message: | ||
790 | 105 | message = queue[3][index] | ||
791 | 104 | if ttl is not None: | 106 | if ttl is not None: |
792 | 105 | message['ttl'] = ttl | 107 | message['ttl'] = ttl |
793 | 106 | if hide is not None: | 108 | if hide is not None: |
794 | 107 | message['hide'] = hide | 109 | message['hide'] = hide |
795 | 108 | if hide == 0: | 110 | if hide == 0: |
797 | 109 | self.notify(account, queue) | 111 | self.notify(account[0], queue[0]) |
798 | 110 | return message | 112 | return message |
799 | 111 | return None | 113 | return None |
800 | 112 | 114 | ||
801 | 113 | def clean(self): | 115 | def clean(self): |
802 | 114 | now = int(time.time()) | 116 | now = int(time.time()) |
805 | 115 | for account in self.accounts.keys(): | 117 | for account in self.accounts.iter(): |
806 | 116 | for queue in self.accounts[account].keys(): | 118 | for queue in account[3].iter(): |
807 | 117 | index = 0 | 119 | index = 0 |
808 | 118 | notify = False | 120 | notify = False |
810 | 119 | total = len(self.accounts[account][queue]) | 121 | total = len(queue[3]) |
811 | 120 | while index < total: | 122 | while index < total: |
813 | 121 | message = self.accounts[account][queue][index] | 123 | message = queue[3][index] |
814 | 122 | if 0 < message['ttl'] <= now: | 124 | if 0 < message['ttl'] <= now: |
816 | 123 | del self.accounts[account][queue][index] | 125 | del queue[3][index] |
817 | 124 | total -= 1 | 126 | total -= 1 |
818 | 125 | else: | 127 | else: |
819 | 126 | if 0 < message['hide'] <= now: | 128 | if 0 < message['hide'] <= now: |
820 | @@ -128,30 +130,35 @@ | |||
821 | 128 | notify = True | 130 | notify = True |
822 | 129 | index += 1 | 131 | index += 1 |
823 | 130 | if notify: | 132 | if notify: |
829 | 131 | self.notify(account, queue) | 133 | self.notify(account[0], queue[0]) |
830 | 132 | if len(self.accounts[account][queue]) == 0: | 134 | if len(queue[3]) == 0: |
831 | 133 | del self.accounts[account][queue] | 135 | self.accounts.delete_queue(account[0], queue[0]) |
827 | 134 | if len(self.accounts[account]) == 0: | ||
828 | 135 | del self.accounts[account] | ||
832 | 136 | 136 | ||
835 | 137 | def _scan_queue(self, account, queue, limit, marker, match_hidden, | 137 | def _scan_queue(self, account, queue, filters, attributes={}, |
836 | 138 | ttl=None, hide=None, delete=False): | 138 | delete=False): |
837 | 139 | account, queue = self.accounts.get_queue(account, queue) | ||
838 | 140 | if queue is None: | ||
839 | 141 | return | ||
840 | 139 | index = 0 | 142 | index = 0 |
841 | 140 | notify = False | 143 | notify = False |
843 | 141 | if marker is not None: | 144 | if 'marker' in filters and filters['marker'] is not None: |
844 | 142 | found = False | 145 | found = False |
848 | 143 | for index in range(0, len(self.accounts[account][queue])): | 146 | for index in xrange(0, len(queue[3])): |
849 | 144 | message = self.accounts[account][queue][index] | 147 | message = queue[3][index] |
850 | 145 | if message['id'] == marker: | 148 | if message['id'] == filters['marker']: |
851 | 146 | index += 1 | 149 | index += 1 |
852 | 147 | found = True | 150 | found = True |
853 | 148 | break | 151 | break |
854 | 149 | if not found: | 152 | if not found: |
855 | 150 | index = 0 | 153 | index = 0 |
856 | 151 | messages = [] | 154 | messages = [] |
858 | 152 | total = len(self.accounts[account][queue]) | 155 | total = len(queue[3]) |
859 | 156 | limit = filters.get('limit', None) | ||
860 | 157 | match_hidden = filters.get('match_hidden', False) | ||
861 | 158 | ttl = attributes.get('ttl', None) | ||
862 | 159 | hide = attributes.get('hide', None) | ||
863 | 153 | while index < total: | 160 | while index < total: |
865 | 154 | message = self.accounts[account][queue][index] | 161 | message = queue[3][index] |
866 | 155 | if not match_hidden and message['hide'] != 0: | 162 | if not match_hidden and message['hide'] != 0: |
867 | 156 | index += 1 | 163 | index += 1 |
868 | 157 | continue | 164 | continue |
869 | @@ -162,15 +169,126 @@ | |||
870 | 162 | if hide == 0: | 169 | if hide == 0: |
871 | 163 | notify = True | 170 | notify = True |
872 | 164 | if delete: | 171 | if delete: |
874 | 165 | del self.accounts[account][queue][index] | 172 | del queue[3][index] |
875 | 166 | total -= 1 | 173 | total -= 1 |
876 | 167 | else: | 174 | else: |
877 | 168 | index += 1 | 175 | index += 1 |
879 | 169 | messages.append(message) | 176 | yield message |
880 | 170 | if limit: | 177 | if limit: |
881 | 171 | limit -= 1 | 178 | limit -= 1 |
882 | 172 | if limit == 0: | 179 | if limit == 0: |
883 | 173 | break | 180 | break |
884 | 174 | if notify: | 181 | if notify: |
887 | 175 | self.notify(account, queue) | 182 | self.notify(account[0], queue[0]) |
888 | 176 | return messages | 183 | if len(queue[3]) == 0: |
889 | 184 | self.accounts.delete_queue(account[0], queue[0]) | ||
890 | 185 | |||
891 | 186 | |||
892 | 187 | class IndexedTupleList(object): | ||
893 | 188 | '''Class for managing an indexed tuple list. The tuple must be at | ||
894 | 189 | least three elements and must reserve the first three for (name, | ||
895 | 190 | next, previous).''' | ||
896 | 191 | |||
897 | 192 | def __init__(self): | ||
898 | 193 | self.first = None | ||
899 | 194 | self.last = None | ||
900 | 195 | self.index = {} | ||
901 | 196 | |||
902 | 197 | def add(self, item): | ||
903 | 198 | item = (item[0], None, self.last) + item[3:] | ||
904 | 199 | if self.first is None: | ||
905 | 200 | self.first = item | ||
906 | 201 | self.last = item | ||
907 | 202 | self.index[item[0]] = item | ||
908 | 203 | return item | ||
909 | 204 | |||
910 | 205 | def count(self): | ||
911 | 206 | return len(self.index) | ||
912 | 207 | |||
913 | 208 | def delete(self, filters): | ||
914 | 209 | if len(filters) == 0: | ||
915 | 210 | self.first = None | ||
916 | 211 | self.last = None | ||
917 | 212 | self.index.clear() | ||
918 | 213 | return | ||
919 | 214 | for item in self.iter(filters): | ||
920 | 215 | self.delete_item(item[0]) | ||
921 | 216 | |||
922 | 217 | def delete_item(self, name): | ||
923 | 218 | if name not in self.index: | ||
924 | 219 | return | ||
925 | 220 | item = self.index[name][1] | ||
926 | 221 | if item is not None: | ||
927 | 222 | prev_item = self.index[name][2] | ||
928 | 223 | self.index[item[0]] = (item[0], item[1], prev_item) + item[3:] | ||
929 | 224 | item = self.index[name][2] | ||
930 | 225 | if item is not None: | ||
931 | 226 | next_item = self.index[name][1] | ||
932 | 227 | self.index[item[0]] = (item[0], next_item, item[2]) + item[3:] | ||
933 | 228 | if self.first == self.index[name]: | ||
934 | 229 | self.first = self.index[name][1] | ||
935 | 230 | if self.last == self.index[name]: | ||
936 | 231 | self.last = self.index[name][2] | ||
937 | 232 | del self.index[name] | ||
938 | 233 | |||
939 | 234 | def get(self, name): | ||
940 | 235 | if name in self.index: | ||
941 | 236 | return self.index[name] | ||
942 | 237 | return None | ||
943 | 238 | |||
944 | 239 | def iter(self, filters={}): | ||
945 | 240 | marker = filters.get('marker', None) | ||
946 | 241 | if marker is not None and marker in self.index: | ||
947 | 242 | item = self.index[marker] | ||
948 | 243 | else: | ||
949 | 244 | item = self.first | ||
950 | 245 | limit = filters.get('limit', None) | ||
951 | 246 | while item is not None: | ||
952 | 247 | yield item | ||
953 | 248 | if limit: | ||
954 | 249 | limit -= 1 | ||
955 | 250 | if limit == 0: | ||
956 | 251 | break | ||
957 | 252 | item = item[1] | ||
958 | 253 | |||
959 | 254 | |||
960 | 255 | class Accounts(IndexedTupleList): | ||
961 | 256 | |||
962 | 257 | def delete_queue(self, account, queue): | ||
963 | 258 | account = self.get(account) | ||
964 | 259 | if account is not None: | ||
965 | 260 | account[3].delete_item(queue) | ||
966 | 261 | if account[3].count() == 0: | ||
967 | 262 | self.delete_item(account[0]) | ||
968 | 263 | |||
969 | 264 | def delete_queues(self, account, filters): | ||
970 | 265 | account = self.get(account) | ||
971 | 266 | if account is not None: | ||
972 | 267 | account[3].delete(filters) | ||
973 | 268 | if account[3].count() == 0: | ||
974 | 269 | self.delete_item(account[0]) | ||
975 | 270 | |||
976 | 271 | def get_queue(self, account, queue, create=False): | ||
977 | 272 | if account in self.index: | ||
978 | 273 | account = self.index[account] | ||
979 | 274 | elif create: | ||
980 | 275 | account = self.add((account, None, None, Queues())) | ||
981 | 276 | else: | ||
982 | 277 | return None, None | ||
983 | 278 | return account, account[3].get(queue, create) | ||
984 | 279 | |||
985 | 280 | def iter_queues(self, account, filters={}): | ||
986 | 281 | account = self.get(account) | ||
987 | 282 | if account is not None: | ||
988 | 283 | for queue in account[3].iter(filters): | ||
989 | 284 | yield queue | ||
990 | 285 | |||
991 | 286 | |||
992 | 287 | class Queues(IndexedTupleList): | ||
993 | 288 | |||
994 | 289 | def get(self, queue, create=False): | ||
995 | 290 | if queue in self.index: | ||
996 | 291 | return self.index[queue] | ||
997 | 292 | elif create: | ||
998 | 293 | return self.add((queue, None, None, [])) | ||
999 | 294 | return None | ||
1000 | 177 | 295 | ||
1001 | === modified file 'burrow/backend/sqlite.py' | |||
1002 | --- burrowd/backend/sqlite.py 2011-03-17 23:42:41 +0000 | |||
1003 | +++ burrow/backend/sqlite.py 2011-04-06 05:49:23 +0000 | |||
1004 | @@ -12,23 +12,24 @@ | |||
1005 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
1006 | 13 | # limitations under the License. | 13 | # limitations under the License. |
1007 | 14 | 14 | ||
1009 | 15 | '''Memory backend for the burrow server.''' | 15 | '''SQLite backend for burrow.''' |
1010 | 16 | 16 | ||
1011 | 17 | import sqlite3 | 17 | import sqlite3 |
1012 | 18 | import time | 18 | import time |
1013 | 19 | 19 | ||
1015 | 20 | import burrowd.backend | 20 | import burrow.backend |
1016 | 21 | 21 | ||
1017 | 22 | # Default configuration values for this module. | 22 | # Default configuration values for this module. |
1018 | 23 | DEFAULT_DATABASE = ':memory:' | 23 | DEFAULT_DATABASE = ':memory:' |
1019 | 24 | 24 | ||
1020 | 25 | 25 | ||
1022 | 26 | class Backend(burrowd.backend.Backend): | 26 | class Backend(burrow.backend.Backend): |
1023 | 27 | 27 | ||
1024 | 28 | def __init__(self, config): | 28 | def __init__(self, config): |
1025 | 29 | super(Backend, self).__init__(config) | 29 | super(Backend, self).__init__(config) |
1026 | 30 | database = self.config.get('database', DEFAULT_DATABASE) | 30 | database = self.config.get('database', DEFAULT_DATABASE) |
1027 | 31 | self.db = sqlite3.connect(database) | 31 | self.db = sqlite3.connect(database) |
1028 | 32 | self.db.isolation_level = None | ||
1029 | 32 | queries = [ | 33 | queries = [ |
1030 | 33 | 'CREATE TABLE queues (' | 34 | 'CREATE TABLE queues (' |
1031 | 34 | 'account VARCHAR(255) NOT NULL,' | 35 | 'account VARCHAR(255) NOT NULL,' |
1032 | @@ -44,129 +45,97 @@ | |||
1033 | 44 | for query in queries: | 45 | for query in queries: |
1034 | 45 | self.db.execute(query) | 46 | self.db.execute(query) |
1035 | 46 | 47 | ||
1048 | 47 | def delete_accounts(self): | 48 | def delete_accounts(self, filters={}): |
1049 | 48 | self.db.execute("DELETE FROM queues") | 49 | if len(filters) == 0: |
1050 | 49 | self.db.execute("DELETE FROM messages") | 50 | self.db.execute('DELETE FROM queues') |
1051 | 50 | 51 | self.db.execute('DELETE FROM messages') | |
1040 | 51 | def get_accounts(self): | ||
1041 | 52 | result = self.db.execute("SELECT account FROM queues").fetchall() | ||
1042 | 53 | return [row[0] for row in result] | ||
1043 | 54 | |||
1044 | 55 | def delete_account(self, account): | ||
1045 | 56 | query = "SELECT rowid FROM queues WHERE account='%s'" % account | ||
1046 | 57 | result = self.db.execute(query).fetchall() | ||
1047 | 58 | if len(result) == 0: | ||
1052 | 59 | return | 52 | return |
1103 | 60 | queues = [str(queue[0]) for queue in result] | 53 | |
1104 | 61 | query = "DELETE FROM messages WHERE queue IN (%s)" % (','.join(queues)) | 54 | def get_accounts(self, filters={}): |
1105 | 62 | self.db.execute(query) | 55 | query = 'SELECT DISTINCT account FROM queues' |
1106 | 63 | self.db.execute("DELETE FROM queues WHERE account='%s'" % account) | 56 | values = tuple() |
1107 | 64 | 57 | marker = filters.get('marker', None) | |
1108 | 65 | def get_queues(self, account): | 58 | if marker is not None: |
1109 | 66 | query = "SELECT queue FROM queues WHERE account='%s'" % account | 59 | query += ' WHERE account > ?' |
1110 | 67 | result = self.db.execute(query).fetchall() | 60 | values += (marker,) |
1111 | 68 | return [row[0] for row in result] | 61 | limit = filters.get('limit', None) |
1062 | 69 | |||
1063 | 70 | def queue_exists(self, account, queue): | ||
1064 | 71 | query = "SELECT COUNT(*) FROM queues " \ | ||
1065 | 72 | "WHERE account='%s' AND queue='%s'" % \ | ||
1066 | 73 | (account, queue) | ||
1067 | 74 | result = self.db.execute(query).fetchall() | ||
1068 | 75 | if len(result) == 0: | ||
1069 | 76 | return False | ||
1070 | 77 | self.rowid = result[0][0] | ||
1071 | 78 | return True | ||
1072 | 79 | |||
1073 | 80 | def delete_messages(self, account, queue, limit, marker, match_hidden): | ||
1074 | 81 | messages = self.get_messages(account, queue, limit, marker, | ||
1075 | 82 | match_hidden) | ||
1076 | 83 | ids = [message['id'] for message in messages] | ||
1077 | 84 | query = "DELETE FROM messages WHERE queue=%d AND name IN (%s)" % \ | ||
1078 | 85 | (self.rowid, ','.join(ids)) | ||
1079 | 86 | self.db.execute(query) | ||
1080 | 87 | query = "SELECT rowid FROM messages WHERE queue=%d LIMIT 1" % \ | ||
1081 | 88 | self.rowid | ||
1082 | 89 | if len(self.db.execute(query).fetchall()) == 0: | ||
1083 | 90 | query = "DELETE FROM queues WHERE rowid=%d" % self.rowid | ||
1084 | 91 | self.db.execute(query) | ||
1085 | 92 | return messages | ||
1086 | 93 | |||
1087 | 94 | def get_messages(self, account, queue, limit, marker, match_hidden): | ||
1088 | 95 | if marker is not None: | ||
1089 | 96 | query = "SELECT rowid FROM messages " \ | ||
1090 | 97 | "WHERE queue=%d AND name='%s'" % \ | ||
1091 | 98 | (self.rowid, marker) | ||
1092 | 99 | result = self.db.execute(query).fetchall() | ||
1093 | 100 | if len(result) == 0: | ||
1094 | 101 | marker = None | ||
1095 | 102 | else: | ||
1096 | 103 | marker = result[0][0] | ||
1097 | 104 | query = "SELECT name,ttl,hide,body FROM messages WHERE queue=%d" % \ | ||
1098 | 105 | self.rowid | ||
1099 | 106 | if match_hidden is False: | ||
1100 | 107 | query += " AND hide == 0" | ||
1101 | 108 | if marker is not None: | ||
1102 | 109 | query += " AND rowid > %d" % marker | ||
1112 | 110 | if limit is not None: | 62 | if limit is not None: |
1137 | 111 | query += " LIMIT %d" % limit | 63 | query += ' LIMIT ?' |
1138 | 112 | result = self.db.execute(query).fetchall() | 64 | values += (limit,) |
1139 | 113 | messages = [] | 65 | for row in self.db.execute(query, values): |
1140 | 114 | for row in result: | 66 | yield row[0] |
1141 | 115 | messages.append(dict(id=row[0], ttl=row[1], hide=row[2], | 67 | |
1142 | 116 | body=row[3])) | 68 | def delete_queues(self, account, filters={}): |
1143 | 117 | return messages | 69 | query = 'SELECT rowid FROM queues WHERE account=?' |
1144 | 118 | 70 | ids = [] | |
1145 | 119 | def update_messages(self, account, queue, limit, marker, match_hidden, ttl, | 71 | for row in self.db.execute(query, (account,)): |
1146 | 120 | hide): | 72 | ids.append(str(row[0])) |
1147 | 121 | messages = self.get_messages(account, queue, limit, marker, | 73 | if len(ids) == 0: |
1148 | 122 | match_hidden) | 74 | return |
1149 | 123 | query = "UPDATE messages SET" | 75 | query = 'DELETE FROM messages WHERE queue IN (%s)' |
1150 | 124 | comma = '' | 76 | self.db.execute(query % ','.join(ids)) |
1151 | 125 | if ttl is not None: | 77 | self.db.execute('DELETE FROM queues WHERE account=?', (account,)) |
1152 | 126 | query += "%s ttl=%d" % (comma, ttl) | 78 | |
1153 | 127 | comma = ',' | 79 | def get_queues(self, account, filters={}): |
1154 | 128 | if hide is not None: | 80 | query = 'SELECT queue FROM queues WHERE account=?' |
1155 | 129 | query += "%s hide=%d" % (comma, hide) | 81 | for row in self.db.execute(query, (account,)): |
1156 | 130 | comma = ',' | 82 | yield row[0] |
1157 | 131 | if comma == '': | 83 | |
1158 | 132 | return (False, message) | 84 | def delete_messages(self, account, queue, filters={}): |
1159 | 133 | ids = [] | 85 | result = self._get_messages(account, queue, filters) |
1160 | 134 | for message in messages: | 86 | rowid = result.next() |
1161 | 87 | ids = [] | ||
1162 | 88 | for message in result: | ||
1163 | 89 | ids.append(message['id']) | ||
1164 | 90 | yield message | ||
1165 | 91 | if len(ids) == 0: | ||
1166 | 92 | return | ||
1167 | 93 | query = 'DELETE FROM messages WHERE queue=%d AND name IN (%s)' | ||
1168 | 94 | self.db.execute(query % (rowid, ','.join(ids))) | ||
1169 | 95 | query = 'SELECT rowid FROM messages WHERE queue=? LIMIT 1' | ||
1170 | 96 | if len(self.db.execute(query, (rowid,)).fetchall()) == 0: | ||
1171 | 97 | query = 'DELETE FROM queues WHERE rowid=?' | ||
1172 | 98 | self.db.execute(query, (rowid,)) | ||
1173 | 99 | |||
1174 | 100 | def get_messages(self, account, queue, filters={}): | ||
1175 | 101 | result = self._get_messages(account, queue, filters) | ||
1176 | 102 | rowid = result.next() | ||
1177 | 103 | return result | ||
1178 | 104 | |||
1179 | 105 | def update_messages(self, account, queue, attributes={}, filters={}): | ||
1180 | 106 | result = self._get_messages(account, queue, filters) | ||
1181 | 107 | rowid = result.next() | ||
1182 | 108 | ids = [] | ||
1183 | 109 | ttl = attributes.get('ttl', None) | ||
1184 | 110 | hide = attributes.get('hide', None) | ||
1185 | 111 | for message in result: | ||
1186 | 135 | ids.append(message['id']) | 112 | ids.append(message['id']) |
1187 | 136 | if ttl is not None: | 113 | if ttl is not None: |
1188 | 137 | message['ttl'] = ttl | 114 | message['ttl'] = ttl |
1189 | 138 | if hide is not None: | 115 | if hide is not None: |
1190 | 139 | message['hide'] = hide | 116 | message['hide'] = hide |
1194 | 140 | query += " WHERE queue=%d AND name IN (%s)" % \ | 117 | yield message |
1195 | 141 | (self.rowid, ','.join(ids)) | 118 | if len(ids) == 0: |
1196 | 142 | self.db.execute(query) | 119 | return |
1197 | 120 | query = 'UPDATE messages SET' | ||
1198 | 121 | comma = '' | ||
1199 | 122 | values = tuple() | ||
1200 | 123 | if ttl is not None: | ||
1201 | 124 | query += '%s ttl=?' % comma | ||
1202 | 125 | values += (ttl,) | ||
1203 | 126 | comma = ',' | ||
1204 | 127 | if hide is not None: | ||
1205 | 128 | query += '%s hide=?' % comma | ||
1206 | 129 | values += (hide,) | ||
1207 | 130 | comma = ',' | ||
1208 | 131 | if comma == '': | ||
1209 | 132 | return | ||
1210 | 133 | values += (rowid,) | ||
1211 | 134 | query += ' WHERE queue=? AND name IN (%s)' | ||
1212 | 135 | self.db.execute(query % ','.join(ids), values) | ||
1213 | 143 | self.notify(account, queue) | 136 | self.notify(account, queue) |
1240 | 144 | return messages | 137 | |
1241 | 145 | 138 | def create_message(self, account, queue, message, body, attributes): | |
1216 | 146 | def delete_message(self, account, queue, message_id): | ||
1217 | 147 | message = self.get_message(account, queue, message_id) | ||
1218 | 148 | if message is None: | ||
1219 | 149 | return None | ||
1220 | 150 | query = "DELETE FROM messages WHERE queue=%d AND name='%s'" % \ | ||
1221 | 151 | (self.rowid, message_id) | ||
1222 | 152 | self.db.execute(query) | ||
1223 | 153 | query = "SELECT rowid FROM messages WHERE queue=%d LIMIT 1" % \ | ||
1224 | 154 | self.rowid | ||
1225 | 155 | if len(self.db.execute(query).fetchall()) == 0: | ||
1226 | 156 | query = "DELETE FROM queues WHERE rowid=%d" % self.rowid | ||
1227 | 157 | self.db.execute(query) | ||
1228 | 158 | return message | ||
1229 | 159 | |||
1230 | 160 | def get_message(self, account, queue, message_id): | ||
1231 | 161 | query = "SELECT name,ttl,hide,body FROM messages " \ | ||
1232 | 162 | "WHERE queue=%d AND name='%s'" % (self.rowid, message_id) | ||
1233 | 163 | result = self.db.execute(query).fetchall() | ||
1234 | 164 | if len(result) == 0: | ||
1235 | 165 | return None | ||
1236 | 166 | row = result[0] | ||
1237 | 167 | return dict(id=row[0], ttl=row[1], hide=row[2], body=row[3]) | ||
1238 | 168 | |||
1239 | 169 | def put_message(self, account, queue, message_id, ttl, hide, body): | ||
1242 | 170 | query = "SELECT rowid FROM queues " \ | 139 | query = "SELECT rowid FROM queues " \ |
1243 | 171 | "WHERE account='%s' AND queue='%s'" % (account, queue) | 140 | "WHERE account='%s' AND queue='%s'" % (account, queue) |
1244 | 172 | result = self.db.execute(query).fetchall() | 141 | result = self.db.execute(query).fetchall() |
1245 | @@ -176,11 +145,13 @@ | |||
1246 | 176 | else: | 145 | else: |
1247 | 177 | rowid = result[0][0] | 146 | rowid = result[0][0] |
1248 | 178 | query = "SELECT rowid FROM messages WHERE queue=%d AND name='%s'" % \ | 147 | query = "SELECT rowid FROM messages WHERE queue=%d AND name='%s'" % \ |
1250 | 179 | (rowid, message_id) | 148 | (rowid, message) |
1251 | 180 | result = self.db.execute(query).fetchall() | 149 | result = self.db.execute(query).fetchall() |
1252 | 150 | ttl = attributes.get('ttl', 0) | ||
1253 | 151 | hide = attributes.get('hide', 0) | ||
1254 | 181 | if len(result) == 0: | 152 | if len(result) == 0: |
1255 | 182 | query = "INSERT INTO messages VALUES (%d, '%s', %d, %d, '%s')" % \ | 153 | query = "INSERT INTO messages VALUES (%d, '%s', %d, %d, '%s')" % \ |
1257 | 183 | (rowid, message_id, ttl, hide, body) | 154 | (rowid, message, ttl, hide, body) |
1258 | 184 | self.db.execute(query) | 155 | self.db.execute(query) |
1259 | 185 | self.notify(account, queue) | 156 | self.notify(account, queue) |
1260 | 186 | return True | 157 | return True |
1261 | @@ -191,12 +162,45 @@ | |||
1262 | 191 | self.notify(account, queue) | 162 | self.notify(account, queue) |
1263 | 192 | return False | 163 | return False |
1264 | 193 | 164 | ||
1267 | 194 | def update_message(self, account, queue, message_id, ttl, hide): | 165 | def delete_message(self, account, queue, message): |
1268 | 195 | message = self.get_message(account, queue, message_id) | 166 | rowid = self._get_queue(account, queue) |
1269 | 167 | if rowid is None: | ||
1270 | 168 | return None | ||
1271 | 169 | message = self.get_message(account, queue, message) | ||
1272 | 170 | if message is None: | ||
1273 | 171 | return None | ||
1274 | 172 | query = "DELETE FROM messages WHERE queue=%d AND name='%s'" % \ | ||
1275 | 173 | (rowid, message['id']) | ||
1276 | 174 | self.db.execute(query) | ||
1277 | 175 | query = "SELECT rowid FROM messages WHERE queue=%d LIMIT 1" % rowid | ||
1278 | 176 | if len(self.db.execute(query).fetchall()) == 0: | ||
1279 | 177 | query = "DELETE FROM queues WHERE rowid=%d" % rowid | ||
1280 | 178 | self.db.execute(query) | ||
1281 | 179 | return message | ||
1282 | 180 | |||
1283 | 181 | def get_message(self, account, queue, message): | ||
1284 | 182 | rowid = self._get_queue(account, queue) | ||
1285 | 183 | if rowid is None: | ||
1286 | 184 | return None | ||
1287 | 185 | query = "SELECT name,ttl,hide,body FROM messages " \ | ||
1288 | 186 | "WHERE queue=%d AND name='%s'" % (rowid, message) | ||
1289 | 187 | result = self.db.execute(query).fetchall() | ||
1290 | 188 | if len(result) == 0: | ||
1291 | 189 | return None | ||
1292 | 190 | row = result[0] | ||
1293 | 191 | return dict(id=row[0], ttl=row[1], hide=row[2], body=row[3]) | ||
1294 | 192 | |||
1295 | 193 | def update_message(self, account, queue, message, attributes): | ||
1296 | 194 | rowid = self._get_queue(account, queue) | ||
1297 | 195 | if rowid is None: | ||
1298 | 196 | return None | ||
1299 | 197 | message = self.get_message(account, queue, message) | ||
1300 | 196 | if message is None: | 198 | if message is None: |
1301 | 197 | return None | 199 | return None |
1302 | 198 | query = "UPDATE messages SET" | 200 | query = "UPDATE messages SET" |
1303 | 199 | comma = '' | 201 | comma = '' |
1304 | 202 | ttl = attributes.get('ttl', None) | ||
1305 | 203 | hide = attributes.get('hide', None) | ||
1306 | 200 | if ttl is not None: | 204 | if ttl is not None: |
1307 | 201 | query += "%s ttl=%d" % (comma, ttl) | 205 | query += "%s ttl=%d" % (comma, ttl) |
1308 | 202 | comma = ',' | 206 | comma = ',' |
1309 | @@ -205,7 +209,7 @@ | |||
1310 | 205 | comma = ',' | 209 | comma = ',' |
1311 | 206 | if comma == '': | 210 | if comma == '': |
1312 | 207 | return message | 211 | return message |
1314 | 208 | query += " WHERE queue=%d AND name='%s'" % (self.rowid, message_id) | 212 | query += " WHERE queue=%d AND name='%s'" % (rowid, message['id']) |
1315 | 209 | self.db.execute(query) | 213 | self.db.execute(query) |
1316 | 210 | if hide == 0: | 214 | if hide == 0: |
1317 | 211 | self.notify(account, queue) | 215 | self.notify(account, queue) |
1318 | @@ -248,3 +252,36 @@ | |||
1319 | 248 | queue | 252 | queue |
1320 | 249 | result = self.db.execute(query).fetchall()[0] | 253 | result = self.db.execute(query).fetchall()[0] |
1321 | 250 | self.notify(result[0], result[1]) | 254 | self.notify(result[0], result[1]) |
1322 | 255 | |||
1323 | 256 | def _get_queue(self, account, queue): | ||
1324 | 257 | query = "SELECT COUNT(*) FROM queues " \ | ||
1325 | 258 | "WHERE account='%s' AND queue='%s'" % \ | ||
1326 | 259 | (account, queue) | ||
1327 | 260 | result = self.db.execute(query).fetchall() | ||
1328 | 261 | if len(result) == 0: | ||
1329 | 262 | return None | ||
1330 | 263 | return result[0][0] | ||
1331 | 264 | |||
1332 | 265 | def _get_messages(self, account, queue, filters): | ||
1333 | 266 | rowid = self._get_queue(account, queue) | ||
1334 | 267 | yield rowid | ||
1335 | 268 | if rowid is None: | ||
1336 | 269 | return | ||
1337 | 270 | marker = None | ||
1338 | 271 | if 'marker' in filters and filters['marker'] is not None: | ||
1339 | 272 | query = "SELECT rowid FROM messages " \ | ||
1340 | 273 | "WHERE queue=%d AND name='%s'" % (rowid, filters['marker']) | ||
1341 | 274 | result = self.db.execute(query).fetchall() | ||
1342 | 275 | if len(result) > 0: | ||
1343 | 276 | marker = result[0][0] | ||
1344 | 277 | query = "SELECT name,ttl,hide,body FROM messages WHERE queue=%d" % \ | ||
1345 | 278 | rowid | ||
1346 | 279 | if marker is not None: | ||
1347 | 280 | query += " AND rowid > %d" % marker | ||
1348 | 281 | if 'match_hidden' not in filters or filters['match_hidden'] is False: | ||
1349 | 282 | query += " AND hide == 0" | ||
1350 | 283 | if 'limit' in filters and filters['limit'] is not None: | ||
1351 | 284 | query += " LIMIT %d" % filters['limit'] | ||
1352 | 285 | result = self.db.execute(query).fetchall() | ||
1353 | 286 | for row in result: | ||
1354 | 287 | yield dict(id=row[0], ttl=row[1], hide=row[2], body=row[3]) | ||
1355 | 251 | 288 | ||
1356 | === added file 'burrow/client.py' | |||
1357 | --- burrow/client.py 1970-01-01 00:00:00 +0000 | |||
1358 | +++ burrow/client.py 2011-04-06 05:49:23 +0000 | |||
1359 | @@ -0,0 +1,63 @@ | |||
1360 | 1 | # Copyright (C) 2011 OpenStack LLC. | ||
1361 | 2 | # | ||
1362 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
1363 | 4 | # you may not use this file except in compliance with the License. | ||
1364 | 5 | # You may obtain a copy of the License at | ||
1365 | 6 | # | ||
1366 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
1367 | 8 | # | ||
1368 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
1369 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
1370 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
1371 | 12 | # See the License for the specific language governing permissions and | ||
1372 | 13 | # limitations under the License. | ||
1373 | 14 | |||
1374 | 15 | '''Client module for burrow.''' | ||
1375 | 16 | |||
1376 | 17 | import ConfigParser | ||
1377 | 18 | import logging | ||
1378 | 19 | import logging.config | ||
1379 | 20 | |||
1380 | 21 | import burrow | ||
1381 | 22 | import burrow.config | ||
1382 | 23 | |||
1383 | 24 | # Default configuration values for this module. | ||
1384 | 25 | DEFAULT_BACKEND = 'burrow.backend.http' | ||
1385 | 26 | |||
1386 | 27 | |||
1387 | 28 | class Client(object): | ||
1388 | 29 | '''Client class for burrow.''' | ||
1389 | 30 | |||
1390 | 31 | def __init__(self, url=None, config_files=[], | ||
1391 | 32 | add_default_log_handler=True): | ||
1392 | 33 | '''Initialize a client using the URL and config files from the | ||
1393 | 34 | given list. This is passed directly to ConfigParser.read(), | ||
1394 | 35 | so files should be in ConfigParser format. This will load | ||
1395 | 36 | all the backend class from the configuration.''' | ||
1396 | 37 | if len(config_files) > 0: | ||
1397 | 38 | logging.config.fileConfig(config_files) | ||
1398 | 39 | self._config = ConfigParser.ConfigParser() | ||
1399 | 40 | self._config.read(config_files) | ||
1400 | 41 | # TODO: Parse URL if given and overwrite any values in self._config. | ||
1401 | 42 | self.config = burrow.config.Config(self._config, 'burrow.client') | ||
1402 | 43 | self.log = burrow.get_logger(self.config) | ||
1403 | 44 | if add_default_log_handler: | ||
1404 | 45 | self._add_default_log_handler() | ||
1405 | 46 | self._import_backend() | ||
1406 | 47 | |||
1407 | 48 | def _add_default_log_handler(self): | ||
1408 | 49 | '''Add a default log handler it one has not been set.''' | ||
1409 | 50 | root_log = logging.getLogger() | ||
1410 | 51 | if len(root_log.handlers) > 0 or len(self.log.handlers) > 0: | ||
1411 | 52 | return | ||
1412 | 53 | handler = logging.StreamHandler() | ||
1413 | 54 | handler.setLevel(logging.ERROR) | ||
1414 | 55 | log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' | ||
1415 | 56 | handler.setFormatter(logging.Formatter(log_format)) | ||
1416 | 57 | root_log.addHandler(handler) | ||
1417 | 58 | |||
1418 | 59 | def _import_backend(self): | ||
1419 | 60 | '''Load backend given in the 'backend' option.''' | ||
1420 | 61 | backend = self.config.get('backend', DEFAULT_BACKEND) | ||
1421 | 62 | config = (self._config, backend) | ||
1422 | 63 | self.backend = burrow.import_class(backend, 'Backend')(config) | ||
1423 | 0 | 64 | ||
1424 | === modified file 'burrow/config.py' | |||
1425 | --- burrowd/config.py 2011-03-17 23:42:41 +0000 | |||
1426 | +++ burrow/config.py 2011-04-06 05:49:23 +0000 | |||
1427 | @@ -12,7 +12,7 @@ | |||
1428 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
1429 | 13 | # limitations under the License. | 13 | # limitations under the License. |
1430 | 14 | 14 | ||
1432 | 15 | '''Configuration module for the burrow server.''' | 15 | '''Configuration module for burrow.''' |
1433 | 16 | 16 | ||
1434 | 17 | import ConfigParser | 17 | import ConfigParser |
1435 | 18 | 18 | ||
1436 | 19 | 19 | ||
1437 | === modified file 'burrow/frontend/__init__.py' | |||
1438 | --- burrowd/frontend/__init__.py 2011-03-17 23:42:41 +0000 | |||
1439 | +++ burrow/frontend/__init__.py 2011-04-06 05:49:23 +0000 | |||
1440 | @@ -12,12 +12,12 @@ | |||
1441 | 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
1442 | 13 | # limitations under the License. | 13 | # limitations under the License. |
1443 | 14 | 14 | ||
1450 | 15 | '''Frontends for the burrow server.''' | 15 | '''Frontends for burrow.''' |
1451 | 16 | 16 | ||
1452 | 17 | import burrowd | 17 | import burrow |
1453 | 18 | 18 | ||
1454 | 19 | 19 | ||
1455 | 20 | class Frontend(burrowd.Module): | 20 | class Frontend(burrow.Module): |
1456 | 21 | '''Interface that frontend implementations must provide.''' | 21 | '''Interface that frontend implementations must provide.''' |
1457 | 22 | 22 | ||
1458 | 23 | def __init__(self, config, backend): | 23 | def __init__(self, config, backend): |
1459 | 24 | 24 | ||
1460 | === modified file 'burrow/frontend/wsgi.py' | |||
1461 | --- burrowd/frontend/wsgi.py 2011-03-17 23:42:41 +0000 | |||
1462 | +++ burrow/frontend/wsgi.py 2011-04-06 05:49:23 +0000 | |||
1463 | @@ -24,7 +24,7 @@ | |||
1464 | 24 | import webob.dec | 24 | import webob.dec |
1465 | 25 | import webob.exc | 25 | import webob.exc |
1466 | 26 | 26 | ||
1468 | 27 | import burrowd.frontend | 27 | import burrow.frontend |
1469 | 28 | 28 | ||
1470 | 29 | # Default configuration values for this module. | 29 | # Default configuration values for this module. |
1471 | 30 | DEFAULT_HOST = '0.0.0.0' | 30 | DEFAULT_HOST = '0.0.0.0' |
1472 | @@ -38,20 +38,18 @@ | |||
1473 | 38 | DEFAULT_HIDE = 0 | 38 | DEFAULT_HIDE = 0 |
1474 | 39 | 39 | ||
1475 | 40 | 40 | ||
1480 | 41 | def queue_exists(method): | 41 | def wait_on_queue(method): |
1481 | 42 | '''Decorator to ensure an account and queue exists. If the wait | 42 | '''Decorator to wait on an account/queue if the wait option is |
1482 | 43 | option is given, this will block until a message in the queue is | 43 | given. This will block until a message in the queue is ready or |
1483 | 44 | ready or the timeout expires.''' | 44 | the timeout expires.''' |
1484 | 45 | def wrapper(self, req, account, queue, *args, **kwargs): | 45 | def wrapper(self, req, account, queue, *args, **kwargs): |
1485 | 46 | wait = 0 | 46 | wait = 0 |
1486 | 47 | if 'wait' in req.params: | 47 | if 'wait' in req.params: |
1487 | 48 | wait = int(req.params['wait']) | 48 | wait = int(req.params['wait']) |
1488 | 49 | if wait > 0: | 49 | if wait > 0: |
1489 | 50 | wait += time.time() | 50 | wait += time.time() |
1490 | 51 | res = webob.exc.HTTPNotFound() | ||
1491 | 52 | while True: | 51 | while True: |
1494 | 53 | if self.backend.queue_exists(account, queue): | 52 | res = method(self, req, account, queue, *args, **kwargs) |
1493 | 54 | res = method(self, req, account, queue, *args, **kwargs) | ||
1495 | 55 | if wait == 0 or res.status_int != 404: | 53 | if wait == 0 or res.status_int != 404: |
1496 | 56 | break | 54 | break |
1497 | 57 | now = time.time() | 55 | now = time.time() |
1498 | @@ -63,7 +61,7 @@ | |||
1499 | 63 | return wrapper | 61 | return wrapper |
1500 | 64 | 62 | ||
1501 | 65 | 63 | ||
1503 | 66 | class Frontend(burrowd.frontend.Frontend): | 64 | class Frontend(burrow.frontend.Frontend): |
1504 | 67 | 65 | ||
1505 | 68 | def __init__(self, config, backend): | 66 | def __init__(self, config, backend): |
1506 | 69 | super(Frontend, self).__init__(config, backend) | 67 | super(Frontend, self).__init__(config, backend) |
1507 | @@ -73,7 +71,7 @@ | |||
1508 | 73 | mapper.connect('/', action='root') | 71 | mapper.connect('/', action='root') |
1509 | 74 | mapper.connect('/{account}', action='account') | 72 | mapper.connect('/{account}', action='account') |
1510 | 75 | mapper.connect('/{account}/{queue}', action='queue') | 73 | mapper.connect('/{account}/{queue}', action='queue') |
1512 | 76 | mapper.connect('/{account}/{queue}/{message_id}', action='message') | 74 | mapper.connect('/{account}/{queue}/{message}', action='message') |
1513 | 77 | self._routes = routes.middleware.RoutesMiddleware(self._route, mapper) | 75 | self._routes = routes.middleware.RoutesMiddleware(self._route, mapper) |
1514 | 78 | 76 | ||
1515 | 79 | def run(self, thread_pool): | 77 | def run(self, thread_pool): |
1516 | @@ -121,85 +119,86 @@ | |||
1517 | 121 | 119 | ||
1518 | 122 | @webob.dec.wsgify | 120 | @webob.dec.wsgify |
1519 | 123 | def _delete_root(self, req): | 121 | def _delete_root(self, req): |
1521 | 124 | self.backend.delete_accounts() | 122 | filters = self._parse_filters(req) |
1522 | 123 | self.backend.delete_accounts(filters) | ||
1523 | 125 | return webob.exc.HTTPNoContent() | 124 | return webob.exc.HTTPNoContent() |
1524 | 126 | 125 | ||
1525 | 127 | @webob.dec.wsgify | 126 | @webob.dec.wsgify |
1526 | 128 | def _get_root(self, req): | 127 | def _get_root(self, req): |
1528 | 129 | accounts = self.backend.get_accounts() | 128 | filters = self._parse_filters(req) |
1529 | 129 | accounts = [account for account in self.backend.get_accounts(filters)] | ||
1530 | 130 | if len(accounts) == 0: | 130 | if len(accounts) == 0: |
1531 | 131 | return webob.exc.HTTPNotFound() | 131 | return webob.exc.HTTPNotFound() |
1532 | 132 | return webob.exc.HTTPOk(body=json.dumps(accounts, indent=2)) | 132 | return webob.exc.HTTPOk(body=json.dumps(accounts, indent=2)) |
1533 | 133 | 133 | ||
1534 | 134 | @webob.dec.wsgify | 134 | @webob.dec.wsgify |
1535 | 135 | def _delete_account(self, req, account): | 135 | def _delete_account(self, req, account): |
1537 | 136 | self.backend.delete_account(account) | 136 | filters = self._parse_filters(req) |
1538 | 137 | self.backend.delete_queues(account, filters) | ||
1539 | 137 | return webob.exc.HTTPNoContent() | 138 | return webob.exc.HTTPNoContent() |
1540 | 138 | 139 | ||
1541 | 139 | @webob.dec.wsgify | 140 | @webob.dec.wsgify |
1542 | 140 | def _get_account(self, req, account): | 141 | def _get_account(self, req, account): |
1544 | 141 | queues = self.backend.get_queues(account) | 142 | filters = self._parse_filters(req) |
1545 | 143 | queues = [queue for queue in self.backend.get_queues(account, filters)] | ||
1546 | 142 | if len(queues) == 0: | 144 | if len(queues) == 0: |
1547 | 143 | return webob.exc.HTTPNotFound() | 145 | return webob.exc.HTTPNotFound() |
1548 | 144 | return webob.exc.HTTPOk(body=json.dumps(queues, indent=2)) | 146 | return webob.exc.HTTPOk(body=json.dumps(queues, indent=2)) |
1549 | 145 | 147 | ||
1550 | 146 | @webob.dec.wsgify | 148 | @webob.dec.wsgify |
1552 | 147 | @queue_exists | 149 | @wait_on_queue |
1553 | 148 | def _delete_queue(self, req, account, queue): | 150 | def _delete_queue(self, req, account, queue): |
1557 | 149 | limit, marker, match_hidden = self._parse_filters(req) | 151 | filters = self._parse_filters(req) |
1558 | 150 | messages = self.backend.delete_messages(account, queue, limit, marker, | 152 | messages = [message for message in |
1559 | 151 | match_hidden) | 153 | self.backend.delete_messages(account, queue, filters)] |
1560 | 152 | return self._return_messages(req, account, queue, messages, 'none') | 154 | return self._return_messages(req, account, queue, messages, 'none') |
1561 | 153 | 155 | ||
1562 | 154 | @webob.dec.wsgify | 156 | @webob.dec.wsgify |
1564 | 155 | @queue_exists | 157 | @wait_on_queue |
1565 | 156 | def _get_queue(self, req, account, queue): | 158 | def _get_queue(self, req, account, queue): |
1569 | 157 | limit, marker, match_hidden = self._parse_filters(req) | 159 | filters = self._parse_filters(req) |
1570 | 158 | messages = self.backend.get_messages(account, queue, limit, marker, | 160 | messages = [message for message in |
1571 | 159 | match_hidden) | 161 | self.backend.get_messages(account, queue, filters)] |
1572 | 160 | return self._return_messages(req, account, queue, messages, 'all') | 162 | return self._return_messages(req, account, queue, messages, 'all') |
1573 | 161 | 163 | ||
1574 | 162 | @webob.dec.wsgify | 164 | @webob.dec.wsgify |
1576 | 163 | @queue_exists | 165 | @wait_on_queue |
1577 | 164 | def _post_queue(self, req, account, queue): | 166 | def _post_queue(self, req, account, queue): |
1582 | 165 | limit, marker, match_hidden = self._parse_filters(req) | 167 | attributes = self._parse_attributes(req) |
1583 | 166 | ttl, hide = self._parse_metadata(req) | 168 | filters = self._parse_filters(req) |
1584 | 167 | messages = self.backend.update_messages(account, queue, limit, marker, | 169 | messages = [message for message in |
1585 | 168 | match_hidden, ttl, hide) | 170 | self.backend.update_messages(account, queue, attributes, filters)] |
1586 | 169 | return self._return_messages(req, account, queue, messages, 'all') | 171 | return self._return_messages(req, account, queue, messages, 'all') |
1587 | 170 | 172 | ||
1588 | 171 | @webob.dec.wsgify | 173 | @webob.dec.wsgify |
1592 | 172 | @queue_exists | 174 | def _delete_message(self, req, account, queue, message): |
1593 | 173 | def _delete_message(self, req, account, queue, message_id): | 175 | message = self.backend.delete_message(account, queue, message) |
1591 | 174 | message = self.backend.delete_message(account, queue, message_id) | ||
1594 | 175 | if message is None: | 176 | if message is None: |
1595 | 176 | return webob.exc.HTTPNotFound() | 177 | return webob.exc.HTTPNotFound() |
1596 | 177 | return self._return_message(req, account, queue, message, 'none') | 178 | return self._return_message(req, account, queue, message, 'none') |
1597 | 178 | 179 | ||
1598 | 179 | @webob.dec.wsgify | 180 | @webob.dec.wsgify |
1602 | 180 | @queue_exists | 181 | def _get_message(self, req, account, queue, message): |
1603 | 181 | def _get_message(self, req, account, queue, message_id): | 182 | message = self.backend.get_message(account, queue, message) |
1601 | 182 | message = self.backend.get_message(account, queue, message_id) | ||
1604 | 183 | if message is None: | 183 | if message is None: |
1605 | 184 | return webob.exc.HTTPNotFound() | 184 | return webob.exc.HTTPNotFound() |
1606 | 185 | return self._return_message(req, account, queue, message, 'all') | 185 | return self._return_message(req, account, queue, message, 'all') |
1607 | 186 | 186 | ||
1608 | 187 | @webob.dec.wsgify | 187 | @webob.dec.wsgify |
1614 | 188 | @queue_exists | 188 | def _post_message(self, req, account, queue, message): |
1615 | 189 | def _post_message(self, req, account, queue, message_id): | 189 | attributes = self._parse_attributes(req) |
1616 | 190 | ttl, hide = self._parse_metadata(req) | 190 | message = self.backend.update_message(account, queue, message, |
1617 | 191 | message = self.backend.update_message(account, queue, message_id, ttl, | 191 | attributes) |
1613 | 192 | hide) | ||
1618 | 193 | if message is None: | 192 | if message is None: |
1619 | 194 | return webob.exc.HTTPNotFound() | 193 | return webob.exc.HTTPNotFound() |
1620 | 195 | return self._return_message(req, account, queue, message, 'id') | 194 | return self._return_message(req, account, queue, message, 'id') |
1621 | 196 | 195 | ||
1622 | 197 | @webob.dec.wsgify | 196 | @webob.dec.wsgify |
1628 | 198 | def _put_message(self, req, account, queue, message_id): | 197 | def _put_message(self, req, account, queue, message): |
1629 | 199 | (ttl, hide) = self._parse_metadata(req, self.default_ttl, | 198 | attributes = self._parse_attributes(req, self.default_ttl, |
1630 | 200 | self.default_hide) | 199 | self.default_hide) |
1631 | 201 | if self.backend.put_message(account, queue, message_id, ttl, hide, \ | 200 | if self.backend.create_message(account, queue, message, req.body, |
1632 | 202 | req.body): | 201 | attributes): |
1633 | 203 | return webob.exc.HTTPCreated() | 202 | return webob.exc.HTTPCreated() |
1634 | 204 | return webob.exc.HTTPNoContent() | 203 | return webob.exc.HTTPNoContent() |
1635 | 205 | 204 | ||
1636 | @@ -239,31 +238,32 @@ | |||
1637 | 239 | return webob.exc.HTTPOk(body=json.dumps(body, indent=2)) | 238 | return webob.exc.HTTPOk(body=json.dumps(body, indent=2)) |
1638 | 240 | 239 | ||
1639 | 241 | def _parse_filters(self, req): | 240 | def _parse_filters(self, req): |
1641 | 242 | limit = None | 241 | filters = {} |
1642 | 243 | if 'limit' in req.params: | 242 | if 'limit' in req.params: |
1645 | 244 | limit = int(req.params['limit']) | 243 | filters['limit'] = int(req.params['limit']) |
1644 | 245 | marker = None | ||
1646 | 246 | if 'marker' in req.params: | 244 | if 'marker' in req.params: |
1649 | 247 | marker = req.params['marker'] | 245 | filters['marker'] = req.params['marker'] |
1648 | 248 | match_hidden = False | ||
1650 | 249 | if 'hidden' in req.params and req.params['hidden'].lower() == 'true': | 246 | if 'hidden' in req.params and req.params['hidden'].lower() == 'true': |
1653 | 250 | match_hidden = True | 247 | filters['match_hidden'] = True |
1654 | 251 | return limit, marker, match_hidden | 248 | return filters |
1655 | 252 | 249 | ||
1657 | 253 | def _parse_metadata(self, req, default_ttl=None, default_hide=None): | 250 | def _parse_attributes(self, req, default_ttl=None, default_hide=None): |
1658 | 251 | attributes = {} | ||
1659 | 254 | if 'ttl' in req.params: | 252 | if 'ttl' in req.params: |
1660 | 255 | ttl = int(req.params['ttl']) | 253 | ttl = int(req.params['ttl']) |
1661 | 256 | else: | 254 | else: |
1662 | 257 | ttl = default_ttl | 255 | ttl = default_ttl |
1663 | 258 | if ttl is not None and ttl > 0: | 256 | if ttl is not None and ttl > 0: |
1664 | 259 | ttl += int(time.time()) | 257 | ttl += int(time.time()) |
1665 | 258 | attributes['ttl'] = ttl | ||
1666 | 260 | if 'hide' in req.params: | 259 | if 'hide' in req.params: |
1667 | 261 | hide = int(req.params['hide']) | 260 | hide = int(req.params['hide']) |
1668 | 262 | else: | 261 | else: |
1669 | 263 | hide = default_hide | 262 | hide = default_hide |
1670 | 264 | if hide is not None and hide > 0: | 263 | if hide is not None and hide > 0: |
1671 | 265 | hide += int(time.time()) | 264 | hide += int(time.time()) |
1673 | 266 | return ttl, hide | 265 | attributes['hide'] = hide |
1674 | 266 | return attributes | ||
1675 | 267 | 267 | ||
1676 | 268 | 268 | ||
1677 | 269 | class WSGILog(object): | 269 | class WSGILog(object): |
1678 | 270 | 270 | ||
1679 | === added file 'burrow/server.py' | |||
1680 | --- burrow/server.py 1970-01-01 00:00:00 +0000 | |||
1681 | +++ burrow/server.py 2011-04-06 05:49:23 +0000 | |||
1682 | @@ -0,0 +1,95 @@ | |||
1683 | 1 | # Copyright (C) 2011 OpenStack LLC. | ||
1684 | 2 | # | ||
1685 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
1686 | 4 | # you may not use this file except in compliance with the License. | ||
1687 | 5 | # You may obtain a copy of the License at | ||
1688 | 6 | # | ||
1689 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
1690 | 8 | # | ||
1691 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
1692 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
1693 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
1694 | 12 | # See the License for the specific language governing permissions and | ||
1695 | 13 | # limitations under the License. | ||
1696 | 14 | |||
1697 | 15 | '''Server module for burrow.''' | ||
1698 | 16 | |||
1699 | 17 | import ConfigParser | ||
1700 | 18 | import logging | ||
1701 | 19 | import logging.config | ||
1702 | 20 | |||
1703 | 21 | import eventlet | ||
1704 | 22 | |||
1705 | 23 | import burrow | ||
1706 | 24 | import burrow.config | ||
1707 | 25 | |||
1708 | 26 | # Default configuration values for this module. | ||
1709 | 27 | DEFAULT_BACKEND = 'burrow.backend.sqlite' | ||
1710 | 28 | DEFAULT_FRONTENDS = 'burrow.frontend.wsgi' | ||
1711 | 29 | DEFAULT_THREAD_POOL_SIZE = 1000 | ||
1712 | 30 | |||
1713 | 31 | |||
1714 | 32 | class Server(object): | ||
1715 | 33 | '''Server class for burrow.''' | ||
1716 | 34 | |||
1717 | 35 | def __init__(self, config_files=[], add_default_log_handler=True): | ||
1718 | 36 | '''Initialize a server using the config files from the given | ||
1719 | 37 | list. This is passed directly to ConfigParser.read(), so | ||
1720 | 38 | files should be in ConfigParser format. This will load all | ||
1721 | 39 | frontend and backend classes from the configuration.''' | ||
1722 | 40 | if len(config_files) > 0: | ||
1723 | 41 | logging.config.fileConfig(config_files) | ||
1724 | 42 | self._config = ConfigParser.ConfigParser() | ||
1725 | 43 | self._config.read(config_files) | ||
1726 | 44 | self.config = burrow.config.Config(self._config, 'burrow.server') | ||
1727 | 45 | self.log = burrow.get_logger(self.config) | ||
1728 | 46 | if add_default_log_handler: | ||
1729 | 47 | self._add_default_log_handler() | ||
1730 | 48 | self._import_backend() | ||
1731 | 49 | self._import_frontends() | ||
1732 | 50 | |||
1733 | 51 | def _add_default_log_handler(self): | ||
1734 | 52 | '''Add a default log handler it one has not been set.''' | ||
1735 | 53 | root_log = logging.getLogger() | ||
1736 | 54 | if len(root_log.handlers) > 0 or len(self.log.handlers) > 0: | ||
1737 | 55 | return | ||
1738 | 56 | handler = logging.StreamHandler() | ||
1739 | 57 | handler.setLevel(logging.DEBUG) | ||
1740 | 58 | log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' | ||
1741 | 59 | handler.setFormatter(logging.Formatter(log_format)) | ||
1742 | 60 | root_log.addHandler(handler) | ||
1743 | 61 | |||
1744 | 62 | def _import_backend(self): | ||
1745 | 63 | '''Load backend given in the 'backend' option.''' | ||
1746 | 64 | backend = self.config.get('backend', DEFAULT_BACKEND) | ||
1747 | 65 | config = (self._config, backend) | ||
1748 | 66 | self.backend = burrow.import_class(backend, 'Backend')(config) | ||
1749 | 67 | |||
1750 | 68 | def _import_frontends(self): | ||
1751 | 69 | '''Load frontends given in the 'frontends' option.''' | ||
1752 | 70 | self.frontends = [] | ||
1753 | 71 | frontends = self.config.get('frontends', DEFAULT_FRONTENDS) | ||
1754 | 72 | for frontend in frontends.split(','): | ||
1755 | 73 | frontend = frontend.split(':') | ||
1756 | 74 | if len(frontend) == 1: | ||
1757 | 75 | frontend.append(None) | ||
1758 | 76 | config = (self._config, frontend[0], frontend[1]) | ||
1759 | 77 | frontend = burrow.import_class(frontend[0], 'Frontend') | ||
1760 | 78 | frontend = frontend(config, self.backend) | ||
1761 | 79 | self.frontends.append(frontend) | ||
1762 | 80 | |||
1763 | 81 | def run(self): | ||
1764 | 82 | '''Create the thread pool and start the main server loop. Wait | ||
1765 | 83 | for the pool to complete, but possibly run forever if the | ||
1766 | 84 | frontends and backend never remove threads.''' | ||
1767 | 85 | thread_pool_size = self.config.getint('thread_pool_size', | ||
1768 | 86 | DEFAULT_THREAD_POOL_SIZE) | ||
1769 | 87 | thread_pool = eventlet.GreenPool(size=int(thread_pool_size)) | ||
1770 | 88 | self.backend.run(thread_pool) | ||
1771 | 89 | for frontend in self.frontends: | ||
1772 | 90 | frontend.run(thread_pool) | ||
1773 | 91 | self.log.info(_('Waiting for all threads to exit')) | ||
1774 | 92 | try: | ||
1775 | 93 | thread_pool.waitall() | ||
1776 | 94 | except KeyboardInterrupt: | ||
1777 | 95 | pass | ||
1778 | 0 | 96 | ||
1779 | === modified file 'etc/burrowd.conf' | |||
1780 | --- etc/burrowd.conf 2011-03-17 23:42:41 +0000 | |||
1781 | +++ etc/burrowd.conf 2011-04-06 05:49:23 +0000 | |||
1782 | @@ -1,6 +1,6 @@ | |||
1783 | 1 | [DEFAULT] | 1 | [DEFAULT] |
1784 | 2 | 2 | ||
1786 | 3 | # Log level to use. All sections below prefixed with 'burrowd' can define | 3 | # Log level to use. All sections below prefixed with 'burrow' can define |
1787 | 4 | # this to override this default. | 4 | # this to override this default. |
1788 | 5 | log_level = DEBUG | 5 | log_level = DEBUG |
1789 | 6 | 6 | ||
1790 | @@ -11,26 +11,26 @@ | |||
1791 | 11 | default_hide = 0 | 11 | default_hide = 0 |
1792 | 12 | 12 | ||
1793 | 13 | 13 | ||
1795 | 14 | [burrowd] | 14 | [burrow.server] |
1796 | 15 | 15 | ||
1797 | 16 | # Backend to use for storing messages. | 16 | # Backend to use for storing messages. |
1799 | 17 | backend = burrowd.backend.sqlite | 17 | backend = burrow.backend.sqlite |
1800 | 18 | 18 | ||
1801 | 19 | # Comma separated list of frontends to run. | 19 | # Comma separated list of frontends to run. |
1804 | 20 | # frontends = burrowd.frontend.wsgi,burrowd.frontend.wsgi:ssl | 20 | # frontends = burrow.frontend.wsgi,burrow.frontend.wsgi:ssl |
1805 | 21 | frontends = burrowd.frontend.wsgi | 21 | frontends = burrow.frontend.wsgi |
1806 | 22 | 22 | ||
1807 | 23 | # Size of the thread pool to use for the server. | 23 | # Size of the thread pool to use for the server. |
1808 | 24 | thread_pool_size = 1000 | 24 | thread_pool_size = 1000 |
1809 | 25 | 25 | ||
1810 | 26 | 26 | ||
1812 | 27 | [burrowd.backend.sqlite] | 27 | [burrow.backend.sqlite] |
1813 | 28 | 28 | ||
1814 | 29 | # Database file to use, passed to sqlite3.connect. | 29 | # Database file to use, passed to sqlite3.connect. |
1815 | 30 | database = :memory: | 30 | database = :memory: |
1816 | 31 | 31 | ||
1817 | 32 | 32 | ||
1819 | 33 | [burrowd.frontend.wsgi] | 33 | [burrow.frontend.wsgi] |
1820 | 34 | 34 | ||
1821 | 35 | # Host to listen on. | 35 | # Host to listen on. |
1822 | 36 | host = 0.0.0.0 | 36 | host = 0.0.0.0 |
1823 | @@ -51,7 +51,7 @@ | |||
1824 | 51 | ssl_keyfile = example.key | 51 | ssl_keyfile = example.key |
1825 | 52 | 52 | ||
1826 | 53 | # Size of thread pool for the WSGI server. If the size is 0, use the main | 53 | # Size of thread pool for the WSGI server. If the size is 0, use the main |
1828 | 54 | # burrowd thread pool. | 54 | # burrow thread pool. |
1829 | 55 | thread_pool_size = 0 | 55 | thread_pool_size = 0 |
1830 | 56 | 56 | ||
1831 | 57 | # Default expiration time in seconds to set for messages. This overrides | 57 | # Default expiration time in seconds to set for messages. This overrides |
1832 | @@ -63,7 +63,7 @@ | |||
1833 | 63 | # default_hide = 0 | 63 | # default_hide = 0 |
1834 | 64 | 64 | ||
1835 | 65 | 65 | ||
1837 | 66 | [burrowd.frontend.wsgi:ssl] | 66 | [burrow.frontend.wsgi:ssl] |
1838 | 67 | 67 | ||
1839 | 68 | # Port to listen on. | 68 | # Port to listen on. |
1840 | 69 | port = 8443 | 69 | port = 8443 |
1841 | 70 | 70 | ||
1842 | === modified file 'locale/burrow.pot' | |||
1843 | --- locale/burrow.pot 2011-03-18 00:47:26 +0000 | |||
1844 | +++ locale/burrow.pot 2011-04-06 05:49:23 +0000 | |||
1845 | @@ -8,7 +8,7 @@ | |||
1846 | 8 | msgstr "" | 8 | msgstr "" |
1847 | 9 | "Project-Id-Version: burrow 0.1\n" | 9 | "Project-Id-Version: burrow 0.1\n" |
1848 | 10 | "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" | 10 | "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" |
1850 | 11 | "POT-Creation-Date: 2011-03-17 17:43-0700\n" | 11 | "POT-Creation-Date: 2011-04-05 19:00-0700\n" |
1851 | 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" | 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" |
1852 | 13 | "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" | 13 | "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" |
1853 | 14 | "Language-Team: LANGUAGE <LL@li.org>\n" | 14 | "Language-Team: LANGUAGE <LL@li.org>\n" |
1854 | @@ -17,20 +17,20 @@ | |||
1855 | 17 | "Content-Transfer-Encoding: 8bit\n" | 17 | "Content-Transfer-Encoding: 8bit\n" |
1856 | 18 | "Generated-By: Babel 0.9.4\n" | 18 | "Generated-By: Babel 0.9.4\n" |
1857 | 19 | 19 | ||
1863 | 20 | #: burrowd/__init__.py:96 | 20 | #: burrow/__init__.py:38 |
1859 | 21 | msgid "Waiting for all threads to exit" | ||
1860 | 22 | msgstr "" | ||
1861 | 23 | |||
1862 | 24 | #: burrowd/__init__.py:109 | ||
1864 | 25 | msgid "Module created" | 21 | msgid "Module created" |
1865 | 26 | msgstr "" | 22 | msgstr "" |
1866 | 27 | 23 | ||
1868 | 28 | #: burrowd/__init__.py:130 | 24 | #: burrow/__init__.py:59 |
1869 | 29 | #, python-format | 25 | #, python-format |
1870 | 30 | msgid "Class %s.%s cannot be found (%s)" | 26 | msgid "Class %s.%s cannot be found (%s)" |
1871 | 31 | msgstr "" | 27 | msgstr "" |
1872 | 32 | 28 | ||
1874 | 33 | #: burrowd/frontend/wsgi.py:87 | 29 | #: burrow/server.py:91 |
1875 | 30 | msgid "Waiting for all threads to exit" | ||
1876 | 31 | msgstr "" | ||
1877 | 32 | |||
1878 | 33 | #: burrow/frontend/wsgi.py:85 | ||
1879 | 34 | #, python-format | 34 | #, python-format |
1880 | 35 | msgid "Listening on %s:%d" | 35 | msgid "Listening on %s:%d" |
1881 | 36 | msgstr "" | 36 | msgstr "" |
1882 | 37 | 37 | ||
1883 | === modified file 'setup.py' | |||
1884 | --- setup.py 2011-03-17 23:42:41 +0000 | |||
1885 | +++ setup.py 2011-04-06 05:49:23 +0000 | |||
1886 | @@ -17,11 +17,14 @@ | |||
1887 | 17 | from setuptools.command.sdist import sdist | 17 | from setuptools.command.sdist import sdist |
1888 | 18 | import os | 18 | import os |
1889 | 19 | import subprocess | 19 | import subprocess |
1890 | 20 | |||
1891 | 20 | try: | 21 | try: |
1892 | 21 | from babel.messages import frontend | 22 | from babel.messages import frontend |
1893 | 22 | except ImportError: | 23 | except ImportError: |
1894 | 23 | frontend = None | 24 | frontend = None |
1895 | 24 | 25 | ||
1896 | 26 | import burrow | ||
1897 | 27 | |||
1898 | 25 | 28 | ||
1899 | 26 | class local_sdist(sdist): | 29 | class local_sdist(sdist): |
1900 | 27 | """Customized sdist hook - builds the ChangeLog file from VC first""" | 30 | """Customized sdist hook - builds the ChangeLog file from VC first""" |
1901 | @@ -54,7 +57,7 @@ | |||
1902 | 54 | 57 | ||
1903 | 55 | setup( | 58 | setup( |
1904 | 56 | name=name, | 59 | name=name, |
1906 | 57 | version='0.1', | 60 | version=burrow.__version__, |
1907 | 58 | description='Burrow', | 61 | description='Burrow', |
1908 | 59 | license='Apache License (2.0)', | 62 | license='Apache License (2.0)', |
1909 | 60 | author='OpenStack, LLC.', | 63 | author='OpenStack, LLC.', |
1910 | 61 | 64 | ||
1911 | === modified file 'test/frontend/test_wsgi.py' | |||
1912 | --- test/frontend/test_wsgi.py 2011-03-17 23:42:41 +0000 | |||
1913 | +++ test/frontend/test_wsgi.py 2011-04-06 05:49:23 +0000 | |||
1914 | @@ -20,19 +20,19 @@ | |||
1915 | 20 | import eventlet | 20 | import eventlet |
1916 | 21 | import webob | 21 | import webob |
1917 | 22 | 22 | ||
1921 | 23 | import burrowd.backend.memory | 23 | import burrow.backend.memory |
1922 | 24 | import burrowd.backend.sqlite | 24 | import burrow.backend.sqlite |
1923 | 25 | import burrowd.frontend.wsgi | 25 | import burrow.frontend.wsgi |
1924 | 26 | 26 | ||
1925 | 27 | 27 | ||
1926 | 28 | class TestWSGIMemory(unittest.TestCase): | 28 | class TestWSGIMemory(unittest.TestCase): |
1927 | 29 | '''Unittests for the WSGI frontend to SQLite backend.''' | 29 | '''Unittests for the WSGI frontend to SQLite backend.''' |
1929 | 30 | backend_class = burrowd.backend.memory.Backend | 30 | backend_class = burrow.backend.memory.Backend |
1930 | 31 | 31 | ||
1931 | 32 | def setUp(self): | 32 | def setUp(self): |
1932 | 33 | config = (ConfigParser.ConfigParser(), 'test') | 33 | config = (ConfigParser.ConfigParser(), 'test') |
1933 | 34 | self.backend = self.backend_class(config) | 34 | self.backend = self.backend_class(config) |
1935 | 35 | self.frontend = burrowd.frontend.wsgi.Frontend(config, self.backend) | 35 | self.frontend = burrow.frontend.wsgi.Frontend(config, self.backend) |
1936 | 36 | self.frontend.default_ttl = 0 | 36 | self.frontend.default_ttl = 0 |
1937 | 37 | self._get_url('/', status=404) | 37 | self._get_url('/', status=404) |
1938 | 38 | self._get_url('/a', status=404) | 38 | self._get_url('/a', status=404) |
1939 | @@ -310,4 +310,4 @@ | |||
1940 | 310 | 310 | ||
1941 | 311 | class TestWSGISQLite(TestWSGIMemory): | 311 | class TestWSGISQLite(TestWSGIMemory): |
1942 | 312 | '''Unittests for the WSGI frontend to SQLite backend.''' | 312 | '''Unittests for the WSGI frontend to SQLite backend.''' |
1944 | 313 | backend_class = burrowd.backend.sqlite.Backend | 313 | backend_class = burrow.backend.sqlite.Backend |