Merge lp:~free.ekanayaka/rabbitfixture/fix-buildout-and-unittests into lp:rabbitfixture
- fix-buildout-and-unittests
- Merge into trunk
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | 36 | ||||
Proposed branch: | lp:~free.ekanayaka/rabbitfixture/fix-buildout-and-unittests | ||||
Merge into: | lp:rabbitfixture | ||||
Diff against target: |
486 lines (+161/-196) 5 files modified
bootstrap.py (+122/-189) buildout.cfg (+2/-1) rabbitfixture/server.py (+28/-5) rabbitfixture/tests/test_server.py (+8/-0) setup.py (+1/-1) |
||||
To merge this branch: | bzr merge lp:~free.ekanayaka/rabbitfixture/fix-buildout-and-unittests | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+296003@code.launchpad.net |
Commit message
Description of the change
This branch fixes buildout no longer working with latest eggs. It upgrades bootstrap.py to the latest version 2 series (from http://
It also fixes the test_stop_hang unit tests which was failing because latest versions of the rabbitmq-package (from trusty on) don't spawn the erlang runtime (which is actually the process listening to the 5672 port) as top-level process, but rather as a child process of /usr/sbin/
Finally it moves the kill code into a new RabbitServerRun
tests that need to simulate a crash have an handy way to do so.
- 38. By Free Ekanayaka
-
Expose kill()
- 39. By Free Ekanayaka
-
Simpler docstring
William Grant (wgrant) : | # |
Preview Diff
1 | === modified file 'bootstrap.py' |
2 | --- bootstrap.py 2013-04-12 12:47:25 +0000 |
3 | +++ bootstrap.py 2016-05-29 15:05:52 +0000 |
4 | @@ -18,75 +18,17 @@ |
5 | use the -c option to specify an alternate configuration file. |
6 | """ |
7 | |
8 | -import os, shutil, sys, tempfile, urllib, urllib2, subprocess |
9 | +import os |
10 | +import shutil |
11 | +import sys |
12 | +import tempfile |
13 | + |
14 | from optparse import OptionParser |
15 | |
16 | -if sys.platform == 'win32': |
17 | - def quote(c): |
18 | - if ' ' in c: |
19 | - return '"%s"' % c # work around spawn lamosity on windows |
20 | - else: |
21 | - return c |
22 | -else: |
23 | - quote = str |
24 | - |
25 | -# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments. |
26 | -stdout, stderr = subprocess.Popen( |
27 | - [sys.executable, '-Sc', |
28 | - 'try:\n' |
29 | - ' import ConfigParser\n' |
30 | - 'except ImportError:\n' |
31 | - ' print 1\n' |
32 | - 'else:\n' |
33 | - ' print 0\n'], |
34 | - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() |
35 | -has_broken_dash_S = bool(int(stdout.strip())) |
36 | - |
37 | -# In order to be more robust in the face of system Pythons, we want to |
38 | -# run without site-packages loaded. This is somewhat tricky, in |
39 | -# particular because Python 2.6's distutils imports site, so starting |
40 | -# with the -S flag is not sufficient. However, we'll start with that: |
41 | -if not has_broken_dash_S and 'site' in sys.modules: |
42 | - # We will restart with python -S. |
43 | - args = sys.argv[:] |
44 | - args[0:0] = [sys.executable, '-S'] |
45 | - args = map(quote, args) |
46 | - os.execv(sys.executable, args) |
47 | -# Now we are running with -S. We'll get the clean sys.path, import site |
48 | -# because distutils will do it later, and then reset the path and clean |
49 | -# out any namespace packages from site-packages that might have been |
50 | -# loaded by .pth files. |
51 | -clean_path = sys.path[:] |
52 | -import site # imported because of its side effects |
53 | -sys.path[:] = clean_path |
54 | -for k, v in sys.modules.items(): |
55 | - if k in ('setuptools', 'pkg_resources') or ( |
56 | - hasattr(v, '__path__') and |
57 | - len(v.__path__) == 1 and |
58 | - not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))): |
59 | - # This is a namespace package. Remove it. |
60 | - sys.modules.pop(k) |
61 | - |
62 | -is_jython = sys.platform.startswith('java') |
63 | - |
64 | -setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py' |
65 | -distribute_source = 'http://python-distribute.org/distribute_setup.py' |
66 | - |
67 | - |
68 | -# parsing arguments |
69 | -def normalize_to_url(option, opt_str, value, parser): |
70 | - if value: |
71 | - if '://' not in value: # It doesn't smell like a URL. |
72 | - value = 'file://%s' % ( |
73 | - urllib.pathname2url( |
74 | - os.path.abspath(os.path.expanduser(value))),) |
75 | - if opt_str == '--download-base' and not value.endswith('/'): |
76 | - # Download base needs a trailing slash to make the world happy. |
77 | - value += '/' |
78 | - else: |
79 | - value = None |
80 | - name = opt_str[2:].replace('-', '_') |
81 | - setattr(parser.values, name, value) |
82 | +__version__ = '2015-07-01' |
83 | +# See zc.buildout's changelog if this version is up to date. |
84 | + |
85 | +tmpeggs = tempfile.mkdtemp(prefix='bootstrap-') |
86 | |
87 | usage = '''\ |
88 | [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] |
89 | @@ -96,31 +38,14 @@ |
90 | Simply run this script in a directory containing a buildout.cfg, using the |
91 | Python that you want bin/buildout to use. |
92 | |
93 | -Note that by using --setup-source and --download-base to point to |
94 | -local resources, you can keep this script from going over the network. |
95 | +Note that by using --find-links to point to local resources, you can keep |
96 | +this script from going over the network. |
97 | ''' |
98 | |
99 | parser = OptionParser(usage=usage) |
100 | -parser.add_option("-v", "--version", dest="version", |
101 | - help="use a specific zc.buildout version") |
102 | -parser.add_option("-d", "--distribute", |
103 | - action="store_true", dest="use_distribute", default=False, |
104 | - help="Use Distribute rather than Setuptools.") |
105 | -parser.add_option("--setup-source", action="callback", dest="setup_source", |
106 | - callback=normalize_to_url, nargs=1, type="string", |
107 | - help=("Specify a URL or file location for the setup file. " |
108 | - "If you use Setuptools, this will default to " + |
109 | - setuptools_source + "; if you use Distribute, this " |
110 | - "will default to " + distribute_source + ".")) |
111 | -parser.add_option("--download-base", action="callback", dest="download_base", |
112 | - callback=normalize_to_url, nargs=1, type="string", |
113 | - help=("Specify a URL or directory for downloading " |
114 | - "zc.buildout and either Setuptools or Distribute. " |
115 | - "Defaults to PyPI.")) |
116 | -parser.add_option("--eggs", |
117 | - help=("Specify a directory for storing eggs. Defaults to " |
118 | - "a temporary directory that is deleted when the " |
119 | - "bootstrap script completes.")) |
120 | +parser.add_option("--version", |
121 | + action="store_true", default=False, |
122 | + help=("Return bootstrap.py version.")) |
123 | parser.add_option("-t", "--accept-buildout-test-releases", |
124 | dest='accept_buildout_test_releases', |
125 | action="store_true", default=False, |
126 | @@ -130,95 +55,117 @@ |
127 | "extensions for you. If you use this flag, " |
128 | "bootstrap and buildout will get the newest releases " |
129 | "even if they are alphas or betas.")) |
130 | -parser.add_option("-c", None, action="store", dest="config_file", |
131 | - help=("Specify the path to the buildout configuration " |
132 | - "file to be used.")) |
133 | +parser.add_option("-c", "--config-file", |
134 | + help=("Specify the path to the buildout configuration " |
135 | + "file to be used.")) |
136 | +parser.add_option("-f", "--find-links", |
137 | + help=("Specify a URL to search for buildout releases")) |
138 | +parser.add_option("--allow-site-packages", |
139 | + action="store_true", default=False, |
140 | + help=("Let bootstrap.py use existing site packages")) |
141 | +parser.add_option("--buildout-version", |
142 | + help="Use a specific zc.buildout version") |
143 | +parser.add_option("--setuptools-version", |
144 | + help="Use a specific setuptools version") |
145 | +parser.add_option("--setuptools-to-dir", |
146 | + help=("Allow for re-use of existing directory of " |
147 | + "setuptools versions")) |
148 | |
149 | options, args = parser.parse_args() |
150 | - |
151 | -if options.eggs: |
152 | - eggs_dir = os.path.abspath(os.path.expanduser(options.eggs)) |
153 | -else: |
154 | - eggs_dir = tempfile.mkdtemp() |
155 | - |
156 | -if options.setup_source is None: |
157 | - if options.use_distribute: |
158 | - options.setup_source = distribute_source |
159 | - else: |
160 | - options.setup_source = setuptools_source |
161 | - |
162 | -if options.accept_buildout_test_releases: |
163 | - args.insert(0, 'buildout:accept-buildout-test-releases=true') |
164 | +if options.version: |
165 | + print("bootstrap.py version %s" % __version__) |
166 | + sys.exit(0) |
167 | + |
168 | + |
169 | +###################################################################### |
170 | +# load/install setuptools |
171 | |
172 | try: |
173 | - import pkg_resources |
174 | - import setuptools # A flag. Sometimes pkg_resources is installed alone. |
175 | - if not hasattr(pkg_resources, '_distribute'): |
176 | - raise ImportError |
177 | + from urllib.request import urlopen |
178 | except ImportError: |
179 | - ez_code = urllib2.urlopen( |
180 | - options.setup_source).read().replace('\r\n', '\n') |
181 | - ez = {} |
182 | - exec ez_code in ez |
183 | - setup_args = dict(to_dir=eggs_dir, download_delay=0) |
184 | - if options.download_base: |
185 | - setup_args['download_base'] = options.download_base |
186 | - if options.use_distribute: |
187 | - setup_args['no_fake'] = True |
188 | - if sys.version_info[:2] == (2, 4): |
189 | - setup_args['version'] = '0.6.32' |
190 | - ez['use_setuptools'](**setup_args) |
191 | - if 'pkg_resources' in sys.modules: |
192 | - reload(sys.modules['pkg_resources']) |
193 | - import pkg_resources |
194 | - # This does not (always?) update the default working set. We will |
195 | - # do it. |
196 | - for path in sys.path: |
197 | - if path not in pkg_resources.working_set.entries: |
198 | - pkg_resources.working_set.add_entry(path) |
199 | - |
200 | -cmd = [quote(sys.executable), |
201 | - '-c', |
202 | - quote('from setuptools.command.easy_install import main; main()'), |
203 | - '-mqNxd', |
204 | - quote(eggs_dir)] |
205 | - |
206 | -if not has_broken_dash_S: |
207 | - cmd.insert(1, '-S') |
208 | - |
209 | -find_links = options.download_base |
210 | -if not find_links: |
211 | - find_links = os.environ.get('bootstrap-testing-find-links') |
212 | -if not find_links and options.accept_buildout_test_releases: |
213 | - find_links = 'http://downloads.buildout.org/' |
214 | + from urllib2 import urlopen |
215 | + |
216 | +ez = {} |
217 | +if os.path.exists('ez_setup.py'): |
218 | + exec(open('ez_setup.py').read(), ez) |
219 | +else: |
220 | + exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez) |
221 | + |
222 | +if not options.allow_site_packages: |
223 | + # ez_setup imports site, which adds site packages |
224 | + # this will remove them from the path to ensure that incompatible versions |
225 | + # of setuptools are not in the path |
226 | + import site |
227 | + # inside a virtualenv, there is no 'getsitepackages'. |
228 | + # We can't remove these reliably |
229 | + if hasattr(site, 'getsitepackages'): |
230 | + for sitepackage_path in site.getsitepackages(): |
231 | + # Strip all site-packages directories from sys.path that |
232 | + # are not sys.prefix; this is because on Windows |
233 | + # sys.prefix is a site-package directory. |
234 | + if sitepackage_path != sys.prefix: |
235 | + sys.path[:] = [x for x in sys.path |
236 | + if sitepackage_path not in x] |
237 | + |
238 | +setup_args = dict(to_dir=tmpeggs, download_delay=0) |
239 | + |
240 | +if options.setuptools_version is not None: |
241 | + setup_args['version'] = options.setuptools_version |
242 | +if options.setuptools_to_dir is not None: |
243 | + setup_args['to_dir'] = options.setuptools_to_dir |
244 | + |
245 | +ez['use_setuptools'](**setup_args) |
246 | +import setuptools |
247 | +import pkg_resources |
248 | + |
249 | +# This does not (always?) update the default working set. We will |
250 | +# do it. |
251 | +for path in sys.path: |
252 | + if path not in pkg_resources.working_set.entries: |
253 | + pkg_resources.working_set.add_entry(path) |
254 | + |
255 | +###################################################################### |
256 | +# Install buildout |
257 | + |
258 | +ws = pkg_resources.working_set |
259 | + |
260 | +setuptools_path = ws.find( |
261 | + pkg_resources.Requirement.parse('setuptools')).location |
262 | + |
263 | +# Fix sys.path here as easy_install.pth added before PYTHONPATH |
264 | +cmd = [sys.executable, '-c', |
265 | + 'import sys; sys.path[0:0] = [%r]; ' % setuptools_path + |
266 | + 'from setuptools.command.easy_install import main; main()', |
267 | + '-mZqNxd', tmpeggs] |
268 | + |
269 | +find_links = os.environ.get( |
270 | + 'bootstrap-testing-find-links', |
271 | + options.find_links or |
272 | + ('http://downloads.buildout.org/' |
273 | + if options.accept_buildout_test_releases else None) |
274 | + ) |
275 | if find_links: |
276 | - cmd.extend(['-f', quote(find_links)]) |
277 | - |
278 | -if options.use_distribute: |
279 | - setup_requirement = 'distribute' |
280 | -else: |
281 | - setup_requirement = 'setuptools' |
282 | -ws = pkg_resources.working_set |
283 | -setup_requirement_path = ws.find( |
284 | - pkg_resources.Requirement.parse(setup_requirement)).location |
285 | -env = dict( |
286 | - os.environ, |
287 | - PYTHONPATH=setup_requirement_path) |
288 | + cmd.extend(['-f', find_links]) |
289 | |
290 | requirement = 'zc.buildout' |
291 | -version = options.version |
292 | +version = options.buildout_version |
293 | if version is None and not options.accept_buildout_test_releases: |
294 | # Figure out the most recent final version of zc.buildout. |
295 | import setuptools.package_index |
296 | _final_parts = '*final-', '*final' |
297 | |
298 | def _final_version(parsed_version): |
299 | - for part in parsed_version: |
300 | - if (part[:1] == '*') and (part not in _final_parts): |
301 | - return False |
302 | - return True |
303 | + try: |
304 | + return not parsed_version.is_prerelease |
305 | + except AttributeError: |
306 | + # Older setuptools |
307 | + for part in parsed_version: |
308 | + if (part[:1] == '*') and (part not in _final_parts): |
309 | + return False |
310 | + return True |
311 | + |
312 | index = setuptools.package_index.PackageIndex( |
313 | - search_path=[setup_requirement_path]) |
314 | + search_path=[setuptools_path]) |
315 | if find_links: |
316 | index.add_find_links((find_links,)) |
317 | req = pkg_resources.Requirement.parse(requirement) |
318 | @@ -227,8 +174,6 @@ |
319 | bestv = None |
320 | for dist in index[req.project_name]: |
321 | distv = dist.parsed_version |
322 | - if distv >= pkg_resources.parse_version('2dev'): |
323 | - continue |
324 | if _final_version(distv): |
325 | if bestv is None or distv > bestv: |
326 | best = [dist] |
327 | @@ -238,40 +183,28 @@ |
328 | if best: |
329 | best.sort() |
330 | version = best[-1].version |
331 | - |
332 | if version: |
333 | - requirement += '=='+version |
334 | -else: |
335 | - requirement += '<2dev' |
336 | - |
337 | + requirement = '=='.join((requirement, version)) |
338 | cmd.append(requirement) |
339 | |
340 | -if is_jython: |
341 | - import subprocess |
342 | - exitcode = subprocess.Popen(cmd, env=env).wait() |
343 | -else: # Windows prefers this, apparently; otherwise we would prefer subprocess |
344 | - exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env])) |
345 | -if exitcode != 0: |
346 | - sys.stdout.flush() |
347 | - sys.stderr.flush() |
348 | - print ("An error occurred when trying to install zc.buildout. " |
349 | - "Look above this message for any errors that " |
350 | - "were output by easy_install.") |
351 | - sys.exit(exitcode) |
352 | - |
353 | -ws.add_entry(eggs_dir) |
354 | +import subprocess |
355 | +if subprocess.call(cmd) != 0: |
356 | + raise Exception( |
357 | + "Failed to execute command:\n%s" % repr(cmd)[1:-1]) |
358 | + |
359 | +###################################################################### |
360 | +# Import and run buildout |
361 | + |
362 | +ws.add_entry(tmpeggs) |
363 | ws.require(requirement) |
364 | import zc.buildout.buildout |
365 | |
366 | -# If there isn't already a command in the args, add bootstrap |
367 | if not [a for a in args if '=' not in a]: |
368 | args.append('bootstrap') |
369 | |
370 | - |
371 | -# if -c was provided, we push it back into args for buildout's main function |
372 | +# if -c was provided, we push it back into args for buildout' main function |
373 | if options.config_file is not None: |
374 | args[0:0] = ['-c', options.config_file] |
375 | |
376 | zc.buildout.buildout.main(args) |
377 | -if not options.eggs: # clean up temporary egg directory |
378 | - shutil.rmtree(eggs_dir) |
379 | +shutil.rmtree(tmpeggs) |
380 | |
381 | === modified file 'buildout.cfg' |
382 | --- buildout.cfg 2013-04-12 12:47:27 +0000 |
383 | +++ buildout.cfg 2016-05-29 15:05:52 +0000 |
384 | @@ -16,6 +16,7 @@ |
385 | defaults = '--tests-pattern ^tests --exit-with-status'.split() |
386 | |
387 | [interpreter] |
388 | -recipe = z3c.recipe.scripts |
389 | +recipe = zc.recipe.egg:scripts |
390 | +dependent-scripts = true |
391 | interpreter = py |
392 | eggs = rabbitfixture |
393 | |
394 | === modified file 'rabbitfixture/server.py' |
395 | --- rabbitfixture/server.py 2015-04-24 12:32:53 +0000 |
396 | +++ rabbitfixture/server.py 2016-05-29 15:05:52 +0000 |
397 | @@ -42,6 +42,9 @@ |
398 | # Revert Python's handling of SIGPIPE. See |
399 | # http://bugs.python.org/issue1652 for more info. |
400 | signal.signal(signal.SIGPIPE, signal.SIG_DFL) |
401 | + # Create a new process group, so we can send signals to both |
402 | + # rabbitmq and its child processes. |
403 | + os.setsid() |
404 | |
405 | |
406 | def get_port(socket): |
407 | @@ -292,6 +295,21 @@ |
408 | else: |
409 | raise Exception("RabbitMQ server is not running.") |
410 | |
411 | + def kill(self): |
412 | + """Kill the RabbitMQ server process. |
413 | + |
414 | + This will send a SIGKILL to the server process and its children, it is |
415 | + used as last resort if both 'rabbitmqctl stop' and sending SIGTERM |
416 | + haven't managed to shutdown RabbitMQ. |
417 | + |
418 | + It is also useful to test your code against scenarios where the server |
419 | + dies. |
420 | + """ |
421 | + self._signal(signal.SIGKILL) |
422 | + time.sleep(0.5) |
423 | + if self.is_running(): |
424 | + raise Exception("RabbitMQ server just won't die.") |
425 | + |
426 | def _spawn(self): |
427 | """Spawn the RabbitMQ server process.""" |
428 | cmd = os.path.join(RABBITBIN, 'rabbitmq-server') |
429 | @@ -375,15 +393,20 @@ |
430 | while time.time() < timeout: |
431 | if not self.is_running(): |
432 | break |
433 | - self.process.terminate() |
434 | + self._signal(signal.SIGTERM) |
435 | time.sleep(0.1) |
436 | else: |
437 | # Die!!! |
438 | if self.is_running(): |
439 | - self.process.kill() |
440 | - time.sleep(0.5) |
441 | - if self.is_running(): |
442 | - raise Exception("RabbitMQ server just won't die.") |
443 | + self.kill() |
444 | + |
445 | + def _signal(self, code): |
446 | + """Send a signal to the server process and all its children.""" |
447 | + # We need to send the signal to the process group, since on Ubuntu |
448 | + # 14.04 an later /usr/sbin/rabbitmq-server is a shell script wrapper |
449 | + # that spawns the actual Erlang runtime sub-process which what does |
450 | + # actually do the work and listen for connections. |
451 | + os.killpg(os.getpgid(self.process.pid), code) |
452 | |
453 | |
454 | class RabbitServer(Fixture): |
455 | |
456 | === modified file 'rabbitfixture/tests/test_server.py' |
457 | --- rabbitfixture/tests/test_server.py 2015-04-24 12:32:53 +0000 |
458 | +++ rabbitfixture/tests/test_server.py 2016-05-29 15:05:52 +0000 |
459 | @@ -102,6 +102,14 @@ |
460 | self.assertIs(config, fixture.runner.config) |
461 | self.assertIs(config, fixture.runner.environment.config) |
462 | |
463 | + def test_kill(self): |
464 | + # The fixture can kill RabbitMQ even before cleanUp time, if requested. |
465 | + with RabbitServer() as fixture: |
466 | + fixture.runner.kill() |
467 | + # The daemon should be died, even if we didn't run |
468 | + # cleanUp yet. |
469 | + self.assertFalse(fixture.runner.is_running()) |
470 | + |
471 | |
472 | class TestRabbitServerResources(TestCase): |
473 | |
474 | |
475 | === modified file 'setup.py' |
476 | --- setup.py 2015-04-24 13:06:15 +0000 |
477 | +++ setup.py 2016-05-29 15:05:52 +0000 |
478 | @@ -17,7 +17,7 @@ |
479 | |
480 | setup( |
481 | name='rabbitfixture', |
482 | - version="0.3.6", |
483 | + version="0.3.7", |
484 | packages=find_packages('.'), |
485 | package_dir={'': '.'}, |
486 | include_package_data=True, |