Merge lp:~cjwatson/launchpad-buildd/git-recipes into lp:launchpad-buildd

[Colin Watson]

Use git-build-recipe to run git-based recipe builds (LP: #1453022).

Use git-build-recipe to run git-based recipe builds.

git-build-recipe is in ppa:cjwatson/launchpad, and will need to be copied into the appropriate PPAs and installed via the launchpad-buildd-image-modifier charm before this is deployed. It still needs various bits of work, but it works well enough for simple recipes.

The Launchpad webapp will also need to be taught about git-based recipes.

[William Grant (wgrant)]

review: Approve (code)

[lp:~cjwatson/launchpad-buildd/git-recipes]

188. By Colin Watson on 2016-01-18

Print git-build-recipe version.

189. By Colin Watson on 2016-01-18

Merge trunk.

=== modified file 'buildrecipe'
--- buildrecipe	2015-07-04 14:29:24 +0000
+++ buildrecipe	2016-01-18 11:41:14 +0000
@@ -7,6 +7,7 @@
 __metaclass__ = type
 
 
+from optparse import OptionParser
 import os
 import pwd
 import socket
@@ -15,6 +16,7 @@
     Popen,
     call,
     check_call,
+    check_output,
     )
 import sys
 from textwrap import dedent
@@ -47,13 +49,16 @@
     """Builds a package from a recipe."""
 
     def __init__(self, build_id, author_name, author_email,
-                 suite, distroseries_name, component, archive_purpose):
+                 suite, distroseries_name, component, archive_purpose,
+                 git=False):
         """Constructor.
 
         :param build_id: The id of the build (a str).
         :param author_name: The name of the author (a str).
         :param author_email: The email address of the author (a str).
         :param suite: The suite the package should be built for (a str).
+        :param git: If True, build a git-based recipe; if False, build a
+            bzr-based recipe.
         """
         self.build_id = build_id
         self.author_name = author_name.decode('utf-8')
@@ -62,7 +67,7 @@
         self.component = component
         self.distroseries_name = distroseries_name
         self.suite = suite
-        self.base_branch = None
+        self.git = git
         self.chroot_path = get_build_path(build_id, 'chroot-autobuild')
         self.work_dir_relative = os.environ['HOME'] + '/work'
         self.work_dir = os.path.join(self.chroot_path,
@@ -86,7 +91,7 @@
         """Build the recipe into a source tree.
 
         As a side-effect, sets self.source_dir_relative.
-        :return: a retcode from `bzr dailydeb`.
+        :return: a retcode from `bzr dailydeb` or `git-build-recipe`.
         """
         assert not os.path.exists(self.tree_path)
         recipe_path = os.path.join(self.work_dir, 'recipe')
@@ -99,16 +104,23 @@
         # As of bzr 2.2, a defined identity is needed.  In this case, we're
         # using buildd@<hostname>.
         hostname = socket.gethostname()
-        bzr_email = 'buildd@%s' % hostname
+        email = 'buildd@%s' % hostname
         lsb_release = Popen(['/usr/bin/sudo',
             '/usr/sbin/chroot', self.chroot_path, 'lsb_release',
             '-r', '-s'], stdout=PIPE)
         distroseries_version = lsb_release.communicate()[0].rstrip()
         assert lsb_release.returncode == 0
 
-        print 'Bazaar versions:'
-        check_call(['bzr', 'version'])
-        check_call(['bzr', 'plugins'])
+        if self.git:
+            print 'Git version:'
+            check_call(['git', '--version'])
+            print check_output(
+                ['dpkg-query', '-W', 'git-build-recipe']).rstrip(
+                    '\n').replace('\t', ' ')
+        else:
+            print 'Bazaar versions:'
+            check_call(['bzr', 'version'])
+            check_call(['bzr', 'plugins'])
 
         print 'Building recipe:'
         print recipe
@@ -116,17 +128,22 @@
         env = {
             'DEBEMAIL': self.author_email,
             'DEBFULLNAME': self.author_name.encode('utf-8'),
-            'BZR_EMAIL': bzr_email,
+            'EMAIL': email,
             'LANG': 'C.UTF-8',
             }
-        retcode = call_report_rusage([
-            'bzr',
-            '-Derror',
-            'dailydeb', '--safe', '--no-build', recipe_path,
-            self.tree_path, '--manifest', manifest_path,
+        if self.git:
+            cmd = ['git-build-recipe']
+        else:
+            cmd = ['bzr', '-Derror', 'dailydeb']
+        cmd.extend([
+            '--safe', '--no-build',
+            '--manifest', manifest_path,
             '--distribution', self.distroseries_name,
-            '--allow-fallback-to-native', '--append-version',
-            '~ubuntu%s.1' % distroseries_version], env=env)
+            '--allow-fallback-to-native',
+            '--append-version', '~ubuntu%s.1' % distroseries_version,
+            recipe_path, self.tree_path,
+            ])
+        retcode = call_report_rusage(cmd, env=env)
         if retcode != 0:
             return retcode
         (source,) = [name for name in os.listdir(self.tree_path)
@@ -295,14 +312,26 @@
         os.environ["HOME"], "build-" + build_id, *extra)
 
 
-if __name__ == '__main__':
-    builder = RecipeBuilder(*sys.argv[1:])
+def main():
+    parser = OptionParser(usage=(
+        "usage: %prog BUILD-ID AUTHOR-NAME AUTHOR-EMAIL SUITE "
+        "DISTROSERIES-NAME COMPONENT ARCHIVE-PURPOSE"))
+    parser.add_option(
+        "--git", default=False, action="store_true",
+        help="build a git recipe (default: bzr)")
+    options, args = parser.parse_args()
+
+    builder = RecipeBuilder(*args, git=options.git)
     if builder.install() != 0:
-        sys.exit(RETCODE_FAILURE_INSTALL)
+        return RETCODE_FAILURE_INSTALL
     if builder.buildTree() != 0:
-        sys.exit(RETCODE_FAILURE_BUILD_TREE)
+        return RETCODE_FAILURE_BUILD_TREE
     if builder.installBuildDeps() != 0:
-        sys.exit(RETCODE_FAILURE_INSTALL_BUILD_DEPS)
+        return RETCODE_FAILURE_INSTALL_BUILD_DEPS
     if builder.buildSourcePackage() != 0:
-        sys.exit(RETCODE_FAILURE_BUILD_SOURCE_PACKAGE)
-    sys.exit(RETCODE_SUCCESS)
+        return RETCODE_FAILURE_BUILD_SOURCE_PACKAGE
+    return RETCODE_SUCCESS
+
+
+if __name__ == '__main__':
+    sys.exit(main())

=== modified file 'debian/changelog'
--- debian/changelog	2016-01-05 17:48:42 +0000
+++ debian/changelog	2016-01-18 11:41:14 +0000
@@ -8,6 +8,8 @@
     /etc/launchpad-buildd/default.
   * Try to load the nbd module when starting launchpad-buildd
     (LP: #1531171).
+  * buildrecipe: Add option parsing framework.
+  * Use git-build-recipe to run git-based recipe builds (LP: #1453022).
 
  -- Colin Watson <[e-mail]>  Mon, 16 Nov 2015 18:12:29 +0000
 

=== modified file 'lpbuildd/sourcepackagerecipe.py'
--- lpbuildd/sourcepackagerecipe.py	2015-05-11 05:39:25 +0000
+++ lpbuildd/sourcepackagerecipe.py	2016-01-18 11:41:14 +0000
@@ -76,6 +76,7 @@
         self.author_email = extra_args['author_email']
         self.archive_purpose = extra_args['archive_purpose']
         self.distroseries_name = extra_args['distroseries_name']
+        self.git = extra_args.get('git', False)
 
         super(SourcePackageRecipeBuildManager, self).initiate(
             files, chroot, extra_args)
@@ -85,10 +86,13 @@
         os.makedirs(get_chroot_path(self.home, self._buildid, 'work'))
         recipe_path = get_chroot_path(self.home, self._buildid, 'work/recipe')
         splat_file(recipe_path, self.recipe_text)
-        args = [
-            "buildrecipe", self._buildid, self.author_name.encode('utf-8'),
+        args = ["buildrecipe"]
+        if self.git:
+            args.append("--git")
+        args.extend([
+            self._buildid, self.author_name.encode('utf-8'),
             self.author_email, self.suite, self.distroseries_name,
-            self.component, self.archive_purpose]
+            self.component, self.archive_purpose])
         self.runSubProcess(self.build_recipe_path, args)
 
     def iterate_BUILD_RECIPE(self, retcode):

=== modified file 'lpbuildd/tests/test_sourcepackagerecipe.py'
--- lpbuildd/tests/test_sourcepackagerecipe.py	2015-05-11 05:55:24 +0000
+++ lpbuildd/tests/test_sourcepackagerecipe.py	2016-01-18 11:41:14 +0000
@@ -54,7 +54,7 @@
         """Retrieve build manager's state."""
         return self.buildmanager._state
 
-    def startBuild(self):
+    def startBuild(self, git=False):
         # The build manager's iterate() kicks off the consecutive states
         # after INIT.
         extra_args = {
@@ -73,6 +73,8 @@
                     'ubuntu main',
                 ],
             }
+        if git:
+            extra_args['git'] = True
         self.buildmanager.initiate({}, 'chroot.tar.gz', extra_args)
 
         # Skip states that are done in DebianBuildManager to the state
@@ -84,10 +86,14 @@
         self.assertEqual(
             SourcePackageRecipeBuildState.BUILD_RECIPE, self.getState())
         expected_command = [
-            'sharepath/slavebin/buildrecipe', 'buildrecipe', self.buildid,
+            'sharepath/slavebin/buildrecipe', 'buildrecipe']
+        if git:
+            expected_command.append('--git')
+        expected_command.extend([
+            self.buildid,
             'Steve\u1234'.encode('utf-8'), '[e-mail]',
             'maverick', 'maverick', 'universe', 'puppies',
-            ]
+            ])
         self.assertEqual(expected_command, self.buildmanager.commands[-1])
         self.assertEqual(
             self.buildmanager.iterate, self.buildmanager.iterators[-1])
@@ -213,3 +219,9 @@
         self.assertEqual(expected_command, self.buildmanager.commands[-1])
         self.assertEqual(
             self.buildmanager.iterate, self.buildmanager.iterators[-1])
+
+    def test_iterate_git(self):
+        # Starting a git-based recipe build passes the correct option.  (The
+        # rest of the build is identical to bzr-based recipe builds from the
+        # build manager's point of view.)
+        self.startBuild(git=True)