Merge lp:~cjwatson/launchpad/simplify-buildout-bin-python-easy into lp:launchpad

Proposed by Colin Watson on 2017-01-18
Status: Merged
Merged at revision: 18314
Proposed branch: lp:~cjwatson/launchpad/simplify-buildout-bin-python-easy
Merge into: lp:launchpad
Diff against target: 453 lines (+149/-140)
9 files modified
Makefile (+1/-1)
lib/lp/app/templates/base-layout-macros.pt (+1/-1)
lib/lp/scripts/utilities/bzr.py (+10/-12)
lib/lp/scripts/utilities/js/combinecss.py (+32/-35)
lib/lp/scripts/utilities/js/watchjsbuild.py (+10/-2)
lib/lp/scripts/utilities/killtestservices.py (+19/-26)
lib/lp/scripts/utilities/spriteutil.py (+53/-47)
lib/lp/testing/utilities/retest.py (+13/-10)
setup.py (+10/-6)
To merge this branch: bzr merge lp:~cjwatson/launchpad/simplify-buildout-bin-python-easy
Reviewer Review Type Date Requested Status
William Grant code 2017-01-18 Approve on 2017-01-18
Review via email: mp+314976@code.launchpad.net

Commit Message

Turn most Python scripts in buildout-templates/bin/ into entry points to utility modules.

Description of the Change

buildout-templates/bin/test.in still remains after this, but that's by far the most complicated and my conversion to a utility module still seems to have a subtle bug in it somewhere, so I'll do that in a follow-up branch once I work out the details.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
18315. By Colin Watson on 2017-01-18

Update path to combinecss.py.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'Makefile'
2--- Makefile 2016-09-21 02:51:58 +0000
3+++ Makefile 2017-01-18 08:50:25 +0000
4@@ -47,7 +47,7 @@
5 # NB: It's important BUILDOUT_BIN only mentions things genuinely produced by
6 # buildout.
7 BUILDOUT_BIN = \
8- $(PY) bin/apiindex bin/combine-css bin/fl-build-report \
9+ $(PY) bin/apiindex bin/bzr bin/combine-css bin/fl-build-report \
10 bin/fl-credential-ctl bin/fl-install-demo bin/fl-monitor-ctl \
11 bin/fl-record bin/fl-run-bench bin/fl-run-test bin/googletestservice \
12 bin/i18ncompile bin/i18nextract bin/i18nmergeall bin/i18nstats \
13
14=== modified file 'lib/lp/app/templates/base-layout-macros.pt'
15--- lib/lp/app/templates/base-layout-macros.pt 2016-09-14 11:13:06 +0000
16+++ lib/lp/app/templates/base-layout-macros.pt 2017-01-18 08:50:25 +0000
17@@ -198,7 +198,7 @@
18 <tal:comment replace="nothing">
19 This macro loads a single css file containing all our stylesheets.
20 If you need to include a new css file here, add it to
21- buildout-templates/bin/combine-css.in instead.
22+ lib/lp/scripts/utilities/js/combinecss.py instead.
23
24 We load the CSS from the same host that served the HTML in order to optimize
25 IE caching over SSL. This is inefficient when you cross subdomains (from
26
27=== renamed file 'buildout-templates/bin/bzr.in' => 'lib/lp/scripts/utilities/bzr.py'
28--- buildout-templates/bin/bzr.in 2010-11-24 17:27:46 +0000
29+++ lib/lp/scripts/utilities/bzr.py 2017-01-18 08:50:25 +0000
30@@ -1,15 +1,13 @@
31-#!${buildout:executable} -S
32-
33-# Initialize our paths.
34-${python-relative-path-setup}
35-import sys
36-sys.path.insert(0, ${scripts:parts-directory|path-repr})
37-import site
38-
39-# Run the script.
40+# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
41+# GNU Affero General Public License version 3 (see the file LICENSE).
42+
43 import pkg_resources
44
45-bzr_distribution = pkg_resources.get_distribution(
46- pkg_resources.Requirement.parse('bzr'))
47
48-bzr_distribution.run_script('bzr', globals().copy())
49+def main():
50+ # Run the script.
51+ bzr_distribution = pkg_resources.get_distribution(
52+ pkg_resources.Requirement.parse('bzr'))
53+ namespace = globals().copy()
54+ namespace['__name__'] = '__main__'
55+ bzr_distribution.run_script('bzr', namespace)
56
57=== renamed file 'buildout-templates/bin/combine-css.in' => 'lib/lp/scripts/utilities/js/combinecss.py'
58--- buildout-templates/bin/combine-css.in 2013-07-04 01:04:50 +0000
59+++ lib/lp/scripts/utilities/js/combinecss.py 2017-01-18 08:50:25 +0000
60@@ -1,20 +1,13 @@
61-#!${buildout:executable} -S
62-
63-# Initialize our paths.
64-${python-relative-path-setup}
65-import sys
66-sys.path.insert(0, ${scripts:parts-directory|path-repr})
67-import site
68+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
69+# GNU Affero General Public License version 3 (see the file LICENSE).
70
71 import os
72
73 from lp.scripts.utilities.js.jsbuild import ComboFile
74 from lp.scripts.utilities.js.combo import combine_files
75-
76-root = os.path.abspath('.')
77-root = os.path.normpath(${buildout:directory|path-repr})
78-icing = os.path.join(root, 'lib/canonical/launchpad/icing')
79-target = os.path.join(icing, 'combo.css')
80+from lp.services.config import config
81+
82+
83 # It'd probably be nice to have this script find all the CSS files we might
84 # need and combine them together, but if we do that we'd certainly end up
85 # including lots of styles that we don't need/want, so keeping this hard-coded
86@@ -58,26 +51,30 @@
87 'css/layout.css',
88 'css/modifiers.css']
89
90-# Get all the component css files so we don't have to edit this file every
91-# time a new component is added
92-component_dir = 'css/components'
93-component_path = os.path.abspath(os.path.join(icing, component_dir))
94-for root, dirs, files in os.walk(component_path):
95- for file in files:
96- if file.endswith('.css'):
97- names.append('%s/%s' % (component_dir, file))
98-
99-absolute_names = []
100-for name in names:
101- full_path_name = os.path.abspath(os.path.join(icing, name))
102- absolute_names.append(full_path_name)
103-
104-combo = ComboFile(absolute_names, target)
105-if combo.needs_update():
106- result = ''
107- for content in combine_files(names, icing):
108- result += content
109-
110- f = open(target, 'w')
111- f.write(result)
112- f.close()
113+
114+def main():
115+ icing = os.path.join(config.root, 'lib/canonical/launchpad/icing')
116+ target = os.path.join(icing, 'combo.css')
117+
118+ # Get all the component css files so we don't have to edit this file
119+ # every time a new component is added.
120+ component_dir = 'css/components'
121+ component_path = os.path.abspath(os.path.join(icing, component_dir))
122+ for root, dirs, files in os.walk(component_path):
123+ for file in files:
124+ if file.endswith('.css'):
125+ names.append('%s/%s' % (component_dir, file))
126+
127+ absolute_names = []
128+ for name in names:
129+ full_path_name = os.path.abspath(os.path.join(icing, name))
130+ absolute_names.append(full_path_name)
131+
132+ combo = ComboFile(absolute_names, target)
133+ if combo.needs_update():
134+ result = ''
135+ for content in combine_files(names, icing):
136+ result += content
137+
138+ with open(target, 'w') as f:
139+ f.write(result)
140
141=== renamed file 'buildout-templates/bin/watch_jsbuild.in' => 'lib/lp/scripts/utilities/js/watchjsbuild.py'
142--- buildout-templates/bin/watch_jsbuild.in 2012-02-28 19:40:50 +0000
143+++ lib/lp/scripts/utilities/js/watchjsbuild.py 2017-01-18 08:50:25 +0000
144@@ -1,8 +1,16 @@
145-#!/usr/bin/env python
146+# Copyright 2016 Canonical Ltd. This software is licensed under the
147+# GNU Affero General Public License version 3 (see the file LICENSE).
148+
149+"""Automatically rebuild the JavaScript bundle when source files change."""
150+
151+from __future__ import absolute_import, print_function
152+
153 import os
154 import re
155+
156 from jsautobuild import YUIBuilder
157
158+
159 # Using ionotify we watch our sources of JavaScript in order to know we should
160 # build when the files change.
161
162@@ -18,7 +26,7 @@
163 return os.path.join(JSDIR, RENAME.sub(js_dir, changed_path))
164
165
166-if __name__ == "__main__":
167+def main():
168 build_dir = 'build/js/lp'
169 meta_name = 'LP_MODULES'
170 watch_dir = 'lib'
171
172=== renamed file 'buildout-templates/bin/kill-test-services.in' => 'lib/lp/scripts/utilities/killtestservices.py'
173--- buildout-templates/bin/kill-test-services.in 2011-12-30 01:48:17 +0000
174+++ lib/lp/scripts/utilities/killtestservices.py 2017-01-18 08:50:25 +0000
175@@ -1,40 +1,33 @@
176-#!${buildout:executable} -S
177-#
178-# Copyright 2009 Canonical Ltd. This software is licensed under the
179+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
180 # GNU Affero General Public License version 3 (see the file LICENSE).
181+
182 """Kill all the test services that may persist between test runs."""
183
184-# Initialize our paths.
185-${python-relative-path-setup}
186+from __future__ import print_function
187+
188 import sys
189-sys.path.insert(0, ${scripts:parts-directory|path-repr})
190-import site
191
192-# Tell lp.services.config to use the testrunner config instance, so that
193-# we don't kill the real services.
194 from lp.services.config import config
195-config.setInstance('testrunner')
196-config.generate_overrides()
197-
198-import sys
199-
200-from lp.services.librarianserver.testing.server import LibrarianTestSetup
201+from lp.services.librarianserver.testing.server import LibrarianServerFixture
202 from lp.services.osutils import kill_by_pidfile
203 from lp.testing.layers import MemcachedLayer
204
205
206-def main(args):
207+def main():
208+ args = sys.argv[1:]
209 if '-h' in args or '--help' in args:
210- print __doc__
211+ print(__doc__)
212 return 0
213- print "Killing Memcached....",
214+ # Tell lp.services.config to use the testrunner config instance, so that
215+ # we don't kill the real services.
216+ config.setInstance('testrunner')
217+ config.generate_overrides()
218+ print("Killing Memcached....", end="")
219 kill_by_pidfile(MemcachedLayer.getPidFile())
220- print "done."
221- print "Killing Librarian....",
222- LibrarianTestSetup().tearDownRoot()
223- print "done."
224+ print("done.")
225+ print("Killing Librarian....", end="")
226+ librarian_fixture = LibrarianServerFixture(None)
227+ kill_by_pidfile(librarian_fixture.pidfile)
228+ librarian_fixture.tearDownRoot()
229+ print("done.")
230 return 0
231-
232-
233-if __name__ == '__main__':
234- sys.exit(main(sys.argv[1:]))
235
236=== renamed file 'buildout-templates/bin/sprite-util.in' => 'lib/lp/scripts/utilities/spriteutil.py'
237--- buildout-templates/bin/sprite-util.in 2012-06-02 12:08:17 +0000
238+++ lib/lp/scripts/utilities/spriteutil.py 2017-01-18 08:50:25 +0000
239@@ -1,59 +1,65 @@
240-#!${buildout:executable} -S
241+# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
242+# GNU Affero General Public License version 3 (see the file LICENSE).
243+
244+"""Create sprites."""
245+
246+from __future__ import absolute_import, print_function
247
248 import os
249 import sys
250
251-# Initialize our paths.
252-${python-relative-path-setup}
253-sys.path.insert(0, ${scripts:parts-directory|path-repr})
254-import site
255-
256+from lp.services.config import config
257 from lp.services.spriteutils import SpriteUtil
258
259+
260 command_options = ('create-image', 'create-css')
261
262+
263 def usage():
264 return " Usage: %s %s" % (sys.argv[0], '|'.join(command_options))
265
266-if len(sys.argv) != 2:
267- print >> sys.stderr, "Expected a single argument."
268- print >> sys.stderr, usage()
269- sys.exit(1)
270-else:
271- command = sys.argv[1]
272- if command not in command_options:
273- print >> sys.stderr, "Unknown argument: %s" % command
274- print >> sys.stderr, usage()
275- sys.exit(2)
276-
277-icing = ${buildout:directory/lib/canonical/launchpad/icing|path-repr}
278-sprite_groups = [
279- file_name.replace('.css.in', '')
280- for file_name in os.listdir(icing) if file_name.endswith('.css.in')]
281-
282-for group_name in sprite_groups:
283- css_template_file = os.path.join(icing, '%s.css.in' % group_name)
284- combined_image_file = os.path.join(icing, '%s.png' % group_name)
285- positioning_file = os.path.join(icing, '%s.positioning' % group_name)
286- css_file = os.path.join(icing, 'build/%s.css' % group_name)
287- if group_name.startswith('block-'):
288- # 3 times the size of inline.
289- margin = 300
290+
291+def main():
292+ if len(sys.argv) != 2:
293+ print("Expected a single argument.", file=sys.stderr)
294+ print(usage(), file=sys.stderr)
295+ sys.exit(1)
296 else:
297- # Inline is 2 lines to h1 text + %50 for zooming. 40px + 40px + 20px,
298- margin = 100
299-
300- sprite_util = SpriteUtil(
301- css_template_file, 'icon-sprites',
302- url_prefix_substitutions={'/@@/': '../images/'},
303- margin=margin)
304-
305- if command == 'create-image':
306- sprite_util.combineImages(icing)
307- sprite_util.savePNG(combined_image_file)
308- sprite_util.savePositioning(positioning_file)
309- elif command == 'create-css':
310- sprite_util.loadPositioning(positioning_file)
311- # The icing/icon-sprites.png file is relative to the css file
312- # in the icing/build/ directory.
313- sprite_util.saveConvertedCSS(css_file, '../%s.png' % group_name)
314+ command = sys.argv[1]
315+ if command not in command_options:
316+ print("Unknown argument: %s" % command, file=sys.stderr)
317+ print(usage(), file=sys.stderr)
318+ sys.exit(2)
319+
320+ icing = os.path.join(config.root, 'lib/canonical/launchpad/icing')
321+ sprite_groups = [
322+ file_name.replace('.css.in', '')
323+ for file_name in os.listdir(icing) if file_name.endswith('.css.in')]
324+
325+ for group_name in sprite_groups:
326+ css_template_file = os.path.join(icing, '%s.css.in' % group_name)
327+ combined_image_file = os.path.join(icing, '%s.png' % group_name)
328+ positioning_file = os.path.join(icing, '%s.positioning' % group_name)
329+ css_file = os.path.join(icing, 'build/%s.css' % group_name)
330+ if group_name.startswith('block-'):
331+ # 3 times the size of inline.
332+ margin = 300
333+ else:
334+ # Inline is 2 lines to h1 text + %50 for zooming.
335+ # 40px + 40px + 20px
336+ margin = 100
337+
338+ sprite_util = SpriteUtil(
339+ css_template_file, 'icon-sprites',
340+ url_prefix_substitutions={'/@@/': '../images/'},
341+ margin=margin)
342+
343+ if command == 'create-image':
344+ sprite_util.combineImages(icing)
345+ sprite_util.savePNG(combined_image_file)
346+ sprite_util.savePositioning(positioning_file)
347+ elif command == 'create-css':
348+ sprite_util.loadPositioning(positioning_file)
349+ # The icing/icon-sprites.png file is relative to the css file
350+ # in the icing/build/ directory.
351+ sprite_util.saveConvertedCSS(css_file, '../%s.png' % group_name)
352
353=== added directory 'lib/lp/testing/utilities'
354=== added file 'lib/lp/testing/utilities/__init__.py'
355=== renamed file 'buildout-templates/bin/retest.in' => 'lib/lp/testing/utilities/retest.py'
356--- buildout-templates/bin/retest.in 2012-01-05 16:22:45 +0000
357+++ lib/lp/testing/utilities/retest.py 2017-01-18 08:50:25 +0000
358@@ -1,6 +1,4 @@
359-#!${buildout:executable}
360-#
361-# Copyright 2009 Canonical Ltd. This software is licensed under the
362+# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
363 # GNU Affero General Public License version 3 (see the file LICENSE).
364
365 """
366@@ -24,16 +22,19 @@
367
368 """
369
370+from __future__ import print_function
371+
372 import fileinput
373+from itertools import takewhile, imap
374 import os
375 import re
376 import sys
377-from itertools import takewhile, imap
378-
379-${python-relative-path-setup}
380+
381+from lp.services.config import config
382+
383
384 # The test script for this branch.
385-TEST = "${buildout:directory/bin/test}"
386+TEST = os.path.join(config.root, "bin/test")
387
388 # Regular expression to match numbered stories.
389 STORY_RE = re.compile("(.*)/\d{2}-.*")
390@@ -66,10 +67,12 @@
391 return (
392 line.startswith('Tests with failures:') or
393 line.startswith('Tests with errors:'))
394+
395 def p_take(line):
396 return not (
397 line.isspace() or
398 line.startswith('Total:'))
399+
400 lines = iter(lines)
401 for line in lines:
402 if p_start(line):
403@@ -88,9 +91,9 @@
404
405 def run_tests(tests):
406 """Given a set of tests, run them as one group."""
407- print "Running tests:"
408+ print("Running tests:")
409 for test in tests:
410- print " %s" % test
411+ print(" %s" % test)
412 args = ['-vvc'] if sys.stdout.isatty() else ['-vv']
413 for test in tests:
414 args.append('-t')
415@@ -98,7 +101,7 @@
416 os.execl(TEST, TEST, *args)
417
418
419-if __name__ == '__main__':
420+def main():
421 lines = imap(decolorize, fileinput.input())
422 tests = extract_tests(lines)
423 if len(tests) >= 1:
424
425=== modified file 'setup.py'
426--- setup.py 2016-11-03 15:19:01 +0000
427+++ setup.py 2017-01-18 08:50:25 +0000
428@@ -164,15 +164,19 @@
429 entry_points=dict(
430 console_scripts=[ # `console_scripts` is a magic name to setuptools
431 'apiindex = lp.scripts.utilities.apiindex:main',
432+ 'bzr = lp.scripts.utilities.bzr:main',
433+ 'combine-css = lp.scripts.utilities.js.combinecss:main',
434+ 'harness = lp.scripts.harness:python',
435+ 'jsbuild = lp.scripts.utilities.js.jsbuild:main',
436+ 'kill-test-services = lp.scripts.utilities.killtestservices:main',
437 'killservice = lp.scripts.utilities.killservice:main',
438- 'jsbuild = lp.scripts.utilities.js.jsbuild:main',
439+ 'retest = lp.testing.utilities.retest:main',
440 'run = lp.scripts.runlaunchpad:start_launchpad',
441- 'run-testapp = '
442- 'lp.scripts.runlaunchpad:start_testapp',
443- 'harness = lp.scripts.harness:python',
444+ 'run-testapp = lp.scripts.runlaunchpad:start_testapp',
445+ 'sprite-util = lp.scripts.utilities.spriteutil:main',
446+ 'start_librarian = lp.scripts.runlaunchpad:start_librarian',
447 'twistd = twisted.scripts.twistd:run',
448- 'start_librarian = '
449- 'lp.scripts.runlaunchpad:start_librarian',
450+ 'watch_jsbuild = lp.scripts.utilities.js.watchjsbuild:main',
451 ]
452 ),
453 )