Merge lp:~cjwatson/launchpad/simplify-buildout-bin-python-easy into lp:launchpad

[Colin Watson]

Turn most Python scripts in buildout-templates/bin/ into entry points to utility modules.

buildout-templates/bin/test.in still remains after this, but that's by far the most complicated and my conversion to a utility module still seems to have a subtle bug in it somewhere, so I'll do that in a follow-up branch once I work out the details.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'Makefile'
--- Makefile	2016-09-21 02:51:58 +0000
+++ Makefile	2017-01-18 08:50:25 +0000
@@ -47,7 +47,7 @@
 # NB: It's important BUILDOUT_BIN only mentions things genuinely produced by
 # buildout.
 BUILDOUT_BIN = \
-    $(PY) bin/apiindex bin/combine-css bin/fl-build-report \
+    $(PY) bin/apiindex bin/bzr bin/combine-css bin/fl-build-report \
     bin/fl-credential-ctl bin/fl-install-demo bin/fl-monitor-ctl \
     bin/fl-record bin/fl-run-bench bin/fl-run-test bin/googletestservice \
     bin/i18ncompile bin/i18nextract bin/i18nmergeall bin/i18nstats \
 
=== modified file 'lib/lp/app/templates/base-layout-macros.pt'
--- lib/lp/app/templates/base-layout-macros.pt	2016-09-14 11:13:06 +0000
+++ lib/lp/app/templates/base-layout-macros.pt	2017-01-18 08:50:25 +0000
@@ -198,7 +198,7 @@
   <tal:comment replace="nothing">
     This macro loads a single css file containing all our stylesheets.
     If you need to include a new css file here, add it to
-    buildout-templates/bin/combine-css.in instead.
+    lib/lp/scripts/utilities/js/combinecss.py instead.
 
     We load the CSS from the same host that served the HTML in order to optimize
     IE caching over SSL. This is inefficient when you cross subdomains (from
 
=== renamed file 'buildout-templates/bin/bzr.in' => 'lib/lp/scripts/utilities/bzr.py'
--- buildout-templates/bin/bzr.in	2010-11-24 17:27:46 +0000
+++ lib/lp/scripts/utilities/bzr.py	2017-01-18 08:50:25 +0000
@@ -1,15 +1,13 @@
-#!${buildout:executable} -S
-
-# Initialize our paths.
-${python-relative-path-setup}
-import sys
-sys.path.insert(0, ${scripts:parts-directory|path-repr})
-import site
-
-# Run the script.
+# Copyright 2010-2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
 import pkg_resources
 
-bzr_distribution = pkg_resources.get_distribution(
-    pkg_resources.Requirement.parse('bzr'))
 
-bzr_distribution.run_script('bzr', globals().copy())
+def main():
+    # Run the script.
+    bzr_distribution = pkg_resources.get_distribution(
+        pkg_resources.Requirement.parse('bzr'))
+    namespace = globals().copy()
+    namespace['__name__'] = '__main__'
+    bzr_distribution.run_script('bzr', namespace)
 
=== renamed file 'buildout-templates/bin/combine-css.in' => 'lib/lp/scripts/utilities/js/combinecss.py'
--- buildout-templates/bin/combine-css.in	2013-07-04 01:04:50 +0000
+++ lib/lp/scripts/utilities/js/combinecss.py	2017-01-18 08:50:25 +0000
@@ -1,20 +1,13 @@
-#!${buildout:executable} -S
-
-# Initialize our paths.
-${python-relative-path-setup}
-import sys
-sys.path.insert(0, ${scripts:parts-directory|path-repr})
-import site
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
 
 import os
 
 from lp.scripts.utilities.js.jsbuild import ComboFile
 from lp.scripts.utilities.js.combo import combine_files
-
-root = os.path.abspath('.')
-root = os.path.normpath(${buildout:directory|path-repr})
-icing = os.path.join(root, 'lib/canonical/launchpad/icing')
-target = os.path.join(icing, 'combo.css')
+from lp.services.config import config
+
+
 # It'd probably be nice to have this script find all the CSS files we might
 # need and combine them together, but if we do that we'd certainly end up
 # including lots of styles that we don't need/want, so keeping this hard-coded
@@ -58,26 +51,30 @@
     'css/layout.css',
     'css/modifiers.css']
 
-# Get all the component css files so we don't have to edit this file every
-# time a new component is added
-component_dir = 'css/components'
-component_path = os.path.abspath(os.path.join(icing, component_dir))
-for root, dirs, files in os.walk(component_path):
-    for file in files:
-        if file.endswith('.css'):
-            names.append('%s/%s' % (component_dir, file))
-
-absolute_names = []
-for name in names:
-    full_path_name = os.path.abspath(os.path.join(icing, name))
-    absolute_names.append(full_path_name)
-
-combo = ComboFile(absolute_names, target)
-if combo.needs_update():
-    result = ''
-    for content in combine_files(names, icing):
-        result += content
-
-    f = open(target, 'w')
-    f.write(result)
-    f.close()
+
+def main():
+    icing = os.path.join(config.root, 'lib/canonical/launchpad/icing')
+    target = os.path.join(icing, 'combo.css')
+
+    # Get all the component css files so we don't have to edit this file
+    # every time a new component is added.
+    component_dir = 'css/components'
+    component_path = os.path.abspath(os.path.join(icing, component_dir))
+    for root, dirs, files in os.walk(component_path):
+        for file in files:
+            if file.endswith('.css'):
+                names.append('%s/%s' % (component_dir, file))
+
+    absolute_names = []
+    for name in names:
+        full_path_name = os.path.abspath(os.path.join(icing, name))
+        absolute_names.append(full_path_name)
+
+    combo = ComboFile(absolute_names, target)
+    if combo.needs_update():
+        result = ''
+        for content in combine_files(names, icing):
+            result += content
+
+        with open(target, 'w') as f:
+            f.write(result)
 
=== renamed file 'buildout-templates/bin/watch_jsbuild.in' => 'lib/lp/scripts/utilities/js/watchjsbuild.py'
--- buildout-templates/bin/watch_jsbuild.in	2012-02-28 19:40:50 +0000
+++ lib/lp/scripts/utilities/js/watchjsbuild.py	2017-01-18 08:50:25 +0000
@@ -1,8 +1,16 @@
-#!/usr/bin/env python
+# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Automatically rebuild the JavaScript bundle when source files change."""
+
+from __future__ import absolute_import, print_function
+
 import os
 import re
+
 from jsautobuild import YUIBuilder
 
+
 # Using ionotify we watch our sources of JavaScript in order to know we should
 # build when the files change.
 
@@ -18,7 +26,7 @@
     return os.path.join(JSDIR, RENAME.sub(js_dir, changed_path))
 
 
-if __name__ == "__main__":
+def main():
     build_dir = 'build/js/lp'
     meta_name = 'LP_MODULES'
     watch_dir = 'lib'
 
=== renamed file 'buildout-templates/bin/kill-test-services.in' => 'lib/lp/scripts/utilities/killtestservices.py'
--- buildout-templates/bin/kill-test-services.in	2011-12-30 01:48:17 +0000
+++ lib/lp/scripts/utilities/killtestservices.py	2017-01-18 08:50:25 +0000
@@ -1,40 +1,33 @@
-#!${buildout:executable} -S
-#
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
+
 """Kill all the test services that may persist between test runs."""
 
-# Initialize our paths.
-${python-relative-path-setup}
+from __future__ import print_function
+
 import sys
-sys.path.insert(0, ${scripts:parts-directory|path-repr})
-import site
 
-# Tell lp.services.config to use the testrunner config instance, so that
-# we don't kill the real services.
 from lp.services.config import config
-config.setInstance('testrunner')
-config.generate_overrides()
-
-import sys
-
-from lp.services.librarianserver.testing.server import LibrarianTestSetup
+from lp.services.librarianserver.testing.server import LibrarianServerFixture
 from lp.services.osutils import kill_by_pidfile
 from lp.testing.layers import MemcachedLayer
 
 
-def main(args):
+def main():
+    args = sys.argv[1:]
     if '-h' in args or '--help' in args:
-        print __doc__
+        print(__doc__)
         return 0
-    print "Killing Memcached....",
+    # Tell lp.services.config to use the testrunner config instance, so that
+    # we don't kill the real services.
+    config.setInstance('testrunner')
+    config.generate_overrides()
+    print("Killing Memcached....", end="")
     kill_by_pidfile(MemcachedLayer.getPidFile())
-    print "done."
-    print "Killing Librarian....",
-    LibrarianTestSetup().tearDownRoot()
-    print "done."
+    print("done.")
+    print("Killing Librarian....", end="")
+    librarian_fixture = LibrarianServerFixture(None)
+    kill_by_pidfile(librarian_fixture.pidfile)
+    librarian_fixture.tearDownRoot()
+    print("done.")
     return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))
 
=== renamed file 'buildout-templates/bin/sprite-util.in' => 'lib/lp/scripts/utilities/spriteutil.py'
--- buildout-templates/bin/sprite-util.in	2012-06-02 12:08:17 +0000
+++ lib/lp/scripts/utilities/spriteutil.py	2017-01-18 08:50:25 +0000
@@ -1,59 +1,65 @@
-#!${buildout:executable} -S
+# Copyright 2010-2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Create sprites."""
+
+from __future__ import absolute_import, print_function
 
 import os
 import sys
 
-# Initialize our paths.
-${python-relative-path-setup}
-sys.path.insert(0, ${scripts:parts-directory|path-repr})
-import site
-
+from lp.services.config import config
 from lp.services.spriteutils import SpriteUtil
 
+
 command_options = ('create-image', 'create-css')
 
+
 def usage():
     return " Usage: %s %s" % (sys.argv[0], '|'.join(command_options))
 
-if len(sys.argv) != 2:
-    print >> sys.stderr, "Expected a single argument."
-    print >> sys.stderr, usage()
-    sys.exit(1)
-else:
-    command = sys.argv[1]
-    if command not in command_options:
-        print >> sys.stderr, "Unknown argument: %s" % command
-        print >> sys.stderr, usage()
-        sys.exit(2)
-
-icing = ${buildout:directory/lib/canonical/launchpad/icing|path-repr}
-sprite_groups = [
-    file_name.replace('.css.in', '')
-    for file_name in os.listdir(icing) if file_name.endswith('.css.in')]
-
-for group_name in sprite_groups:
-    css_template_file = os.path.join(icing, '%s.css.in' % group_name)
-    combined_image_file = os.path.join(icing, '%s.png' % group_name)
-    positioning_file = os.path.join(icing, '%s.positioning' % group_name)
-    css_file = os.path.join(icing, 'build/%s.css' % group_name)
-    if group_name.startswith('block-'):
-        # 3 times the size of inline.
-        margin = 300
+
+def main():
+    if len(sys.argv) != 2:
+        print("Expected a single argument.", file=sys.stderr)
+        print(usage(), file=sys.stderr)
+        sys.exit(1)
     else:
-        # Inline is 2 lines to h1 text + %50 for zooming. 40px + 40px + 20px,
-        margin = 100
-
-    sprite_util = SpriteUtil(
-        css_template_file, 'icon-sprites',
-        url_prefix_substitutions={'/@@/': '../images/'},
-        margin=margin)
-
-    if command == 'create-image':
-        sprite_util.combineImages(icing)
-        sprite_util.savePNG(combined_image_file)
-        sprite_util.savePositioning(positioning_file)
-    elif command == 'create-css':
-        sprite_util.loadPositioning(positioning_file)
-        # The icing/icon-sprites.png file is relative to the css file
-        # in the icing/build/ directory.
-        sprite_util.saveConvertedCSS(css_file, '../%s.png' % group_name)
+        command = sys.argv[1]
+        if command not in command_options:
+            print("Unknown argument: %s" % command, file=sys.stderr)
+            print(usage(), file=sys.stderr)
+            sys.exit(2)
+
+    icing = os.path.join(config.root, 'lib/canonical/launchpad/icing')
+    sprite_groups = [
+        file_name.replace('.css.in', '')
+        for file_name in os.listdir(icing) if file_name.endswith('.css.in')]
+
+    for group_name in sprite_groups:
+        css_template_file = os.path.join(icing, '%s.css.in' % group_name)
+        combined_image_file = os.path.join(icing, '%s.png' % group_name)
+        positioning_file = os.path.join(icing, '%s.positioning' % group_name)
+        css_file = os.path.join(icing, 'build/%s.css' % group_name)
+        if group_name.startswith('block-'):
+            # 3 times the size of inline.
+            margin = 300
+        else:
+            # Inline is 2 lines to h1 text + %50 for zooming.
+            # 40px + 40px + 20px
+            margin = 100
+
+        sprite_util = SpriteUtil(
+            css_template_file, 'icon-sprites',
+            url_prefix_substitutions={'/@@/': '../images/'},
+            margin=margin)
+
+        if command == 'create-image':
+            sprite_util.combineImages(icing)
+            sprite_util.savePNG(combined_image_file)
+            sprite_util.savePositioning(positioning_file)
+        elif command == 'create-css':
+            sprite_util.loadPositioning(positioning_file)
+            # The icing/icon-sprites.png file is relative to the css file
+            # in the icing/build/ directory.
+            sprite_util.saveConvertedCSS(css_file, '../%s.png' % group_name)
 
=== added directory 'lib/lp/testing/utilities'
=== added file 'lib/lp/testing/utilities/__init__.py'
=== renamed file 'buildout-templates/bin/retest.in' => 'lib/lp/testing/utilities/retest.py'
--- buildout-templates/bin/retest.in	2012-01-05 16:22:45 +0000
+++ lib/lp/testing/utilities/retest.py	2017-01-18 08:50:25 +0000
@@ -1,6 +1,4 @@
-#!${buildout:executable}
-#
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """
@@ -24,16 +22,19 @@
 
 """
 
+from __future__ import print_function
+
 import fileinput
+from itertools import takewhile, imap
 import os
 import re
 import sys
-from itertools import takewhile, imap
-
-${python-relative-path-setup}
+
+from lp.services.config import config
+
 
 # The test script for this branch.
-TEST = "${buildout:directory/bin/test}"
+TEST = os.path.join(config.root, "bin/test")
 
 # Regular expression to match numbered stories.
 STORY_RE = re.compile("(.*)/\d{2}-.*")
@@ -66,10 +67,12 @@
         return (
             line.startswith('Tests with failures:') or
             line.startswith('Tests with errors:'))
+
     def p_take(line):
         return not (
             line.isspace() or
             line.startswith('Total:'))
+
     lines = iter(lines)
     for line in lines:
         if p_start(line):
@@ -88,9 +91,9 @@
 
 def run_tests(tests):
     """Given a set of tests, run them as one group."""
-    print "Running tests:"
+    print("Running tests:")
     for test in tests:
-        print "  %s" % test
+        print("  %s" % test)
     args = ['-vvc'] if sys.stdout.isatty() else ['-vv']
     for test in tests:
         args.append('-t')
@@ -98,7 +101,7 @@
     os.execl(TEST, TEST, *args)
 
 
-if __name__ == '__main__':
+def main():
     lines = imap(decolorize, fileinput.input())
     tests = extract_tests(lines)
     if len(tests) >= 1:
 
=== modified file 'setup.py'
--- setup.py	2016-11-03 15:19:01 +0000
+++ setup.py	2017-01-18 08:50:25 +0000
@@ -164,15 +164,19 @@
     entry_points=dict(
         console_scripts=[  # `console_scripts` is a magic name to setuptools
             'apiindex = lp.scripts.utilities.apiindex:main',
+            'bzr = lp.scripts.utilities.bzr:main',
+            'combine-css = lp.scripts.utilities.js.combinecss:main',
+            'harness = lp.scripts.harness:python',
+            'jsbuild = lp.scripts.utilities.js.jsbuild:main',
+            'kill-test-services = lp.scripts.utilities.killtestservices:main',
             'killservice = lp.scripts.utilities.killservice:main',
-            'jsbuild = lp.scripts.utilities.js.jsbuild:main',
+            'retest = lp.testing.utilities.retest:main',
             'run = lp.scripts.runlaunchpad:start_launchpad',
-            'run-testapp = '
-                'lp.scripts.runlaunchpad:start_testapp',
-            'harness = lp.scripts.harness:python',
+            'run-testapp = lp.scripts.runlaunchpad:start_testapp',
+            'sprite-util = lp.scripts.utilities.spriteutil:main',
+            'start_librarian = lp.scripts.runlaunchpad:start_librarian',
             'twistd = twisted.scripts.twistd:run',
-            'start_librarian = '
-                'lp.scripts.runlaunchpad:start_librarian',
+            'watch_jsbuild = lp.scripts.utilities.js.watchjsbuild:main',
         ]
     ),
 )