diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/buildd-slave.tac launchpad-buildd-139/buildd-slave.tac --- launchpad-buildd-133~164~ubuntu14.04.1/buildd-slave.tac 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/buildd-slave.tac 2015-08-04 08:59:14.000000000 +0000 @@ -23,6 +23,7 @@ from lpbuildd.livefs import LiveFilesystemBuildManager from lpbuildd.log import RotatableFileLogObserver from lpbuildd.slave import XMLRPCBuildDSlave +from lpbuildd.snap import SnapBuildManager from lpbuildd.sourcepackagerecipe import SourcePackageRecipeBuildManager from lpbuildd.translationtemplates import TranslationTemplatesBuildManager @@ -41,6 +42,7 @@ slave.registerBuilder( TranslationTemplatesBuildManager, 'translation-templates') slave.registerBuilder(LiveFilesystemBuildManager, "livefs") +slave.registerBuilder(SnapBuildManager, "snap") application = service.Application('BuildDSlave') application.addComponent( diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/buildlivefs launchpad-buildd-139/buildlivefs --- launchpad-buildd-133~164~ubuntu14.04.1/buildlivefs 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/buildlivefs 2015-08-04 08:59:14.000000000 +0000 @@ -8,11 +8,15 @@ from optparse import OptionParser import os -import re import subprocess import sys import traceback +from lpbuildd.util import ( + set_personality, + shell_escape, + ) + RETCODE_SUCCESS = 0 RETCODE_FAILURE_INSTALL = 200 @@ -29,41 +33,6 @@ return os.path.join(os.environ["HOME"], "build-" + build_id, *extra) -non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$') - -def shell_escape(arg): - if non_meta_re.match(arg): - return arg - else: - return "'%s'" % arg.replace("'", "'\\''") - - -linux32_arches = [ - "armel", - "armhf", - "hppa", - "i386", - "lpia", - "mips", - "mipsel", - "powerpc", - "s390", - "sparc", - ] -linux64_arches = [ - "alpha", - "amd64", - "arm64", - "hppa64", - "ia64", - "ppc64", - "ppc64el", - "s390x", - "sparc64", - "x32", - ] - - class LiveFSBuilder: """Builds a live file system.""" @@ -77,10 +46,7 @@ :param args: the command and arguments to run. """ - if self.options.arch in linux32_arches: - args = ["linux32"] + args - elif self.options.arch in linux64_arches: - args = ["linux64"] + args + args = set_personality(self.options.arch, args) if echo: print "Running in chroot: %s" % ' '.join( "'%s'" % arg for arg in args) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/buildrecipe launchpad-buildd-139/buildrecipe --- launchpad-buildd-133~164~ubuntu14.04.1/buildrecipe 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/buildrecipe 2016-01-18 11:42:57.000000000 +0000 @@ -7,6 +7,7 @@ __metaclass__ = type +from optparse import OptionParser import os import pwd import socket @@ -15,6 +16,7 @@ Popen, call, check_call, + check_output, ) import sys from textwrap import dedent @@ -47,13 +49,16 @@ """Builds a package from a recipe.""" def __init__(self, build_id, author_name, author_email, - suite, distroseries_name, component, archive_purpose): + suite, distroseries_name, component, archive_purpose, + git=False): """Constructor. :param build_id: The id of the build (a str). :param author_name: The name of the author (a str). :param author_email: The email address of the author (a str). :param suite: The suite the package should be built for (a str). + :param git: If True, build a git-based recipe; if False, build a + bzr-based recipe. """ self.build_id = build_id self.author_name = author_name.decode('utf-8') @@ -62,7 +67,7 @@ self.component = component self.distroseries_name = distroseries_name self.suite = suite - self.base_branch = None + self.git = git self.chroot_path = get_build_path(build_id, 'chroot-autobuild') self.work_dir_relative = os.environ['HOME'] + '/work' self.work_dir = os.path.join(self.chroot_path, @@ -86,7 +91,7 @@ """Build the recipe into a source tree. As a side-effect, sets self.source_dir_relative. - :return: a retcode from `bzr dailydeb`. + :return: a retcode from `bzr dailydeb` or `git-build-recipe`. """ assert not os.path.exists(self.tree_path) recipe_path = os.path.join(self.work_dir, 'recipe') @@ -99,16 +104,23 @@ # As of bzr 2.2, a defined identity is needed. In this case, we're # using buildd@. hostname = socket.gethostname() - bzr_email = 'buildd@%s' % hostname + email = 'buildd@%s' % hostname lsb_release = Popen(['/usr/bin/sudo', '/usr/sbin/chroot', self.chroot_path, 'lsb_release', '-r', '-s'], stdout=PIPE) distroseries_version = lsb_release.communicate()[0].rstrip() assert lsb_release.returncode == 0 - print 'Bazaar versions:' - check_call(['bzr', 'version']) - check_call(['bzr', 'plugins']) + if self.git: + print 'Git version:' + check_call(['git', '--version']) + print check_output( + ['dpkg-query', '-W', 'git-build-recipe']).rstrip( + '\n').replace('\t', ' ') + else: + print 'Bazaar versions:' + check_call(['bzr', 'version']) + check_call(['bzr', 'plugins']) print 'Building recipe:' print recipe @@ -116,17 +128,22 @@ env = { 'DEBEMAIL': self.author_email, 'DEBFULLNAME': self.author_name.encode('utf-8'), - 'BZR_EMAIL': bzr_email, + 'EMAIL': email, 'LANG': 'C.UTF-8', } - retcode = call_report_rusage([ - 'bzr', - '-Derror', - 'dailydeb', '--safe', '--no-build', recipe_path, - self.tree_path, '--manifest', manifest_path, + if self.git: + cmd = ['git-build-recipe'] + else: + cmd = ['bzr', '-Derror', 'dailydeb'] + cmd.extend([ + '--safe', '--no-build', + '--manifest', manifest_path, '--distribution', self.distroseries_name, - '--allow-fallback-to-native', '--append-version', - '~ubuntu%s.1' % distroseries_version], env=env) + '--allow-fallback-to-native', + '--append-version', '~ubuntu%s.1' % distroseries_version, + recipe_path, self.tree_path, + ]) + retcode = call_report_rusage(cmd, env=env) if retcode != 0: return retcode (source,) = [name for name in os.listdir(self.tree_path) @@ -295,14 +312,26 @@ os.environ["HOME"], "build-" + build_id, *extra) -if __name__ == '__main__': - builder = RecipeBuilder(*sys.argv[1:]) +def main(): + parser = OptionParser(usage=( + "usage: %prog BUILD-ID AUTHOR-NAME AUTHOR-EMAIL SUITE " + "DISTROSERIES-NAME COMPONENT ARCHIVE-PURPOSE")) + parser.add_option( + "--git", default=False, action="store_true", + help="build a git recipe (default: bzr)") + options, args = parser.parse_args() + + builder = RecipeBuilder(*args, git=options.git) if builder.install() != 0: - sys.exit(RETCODE_FAILURE_INSTALL) + return RETCODE_FAILURE_INSTALL if builder.buildTree() != 0: - sys.exit(RETCODE_FAILURE_BUILD_TREE) + return RETCODE_FAILURE_BUILD_TREE if builder.installBuildDeps() != 0: - sys.exit(RETCODE_FAILURE_INSTALL_BUILD_DEPS) + return RETCODE_FAILURE_INSTALL_BUILD_DEPS if builder.buildSourcePackage() != 0: - sys.exit(RETCODE_FAILURE_BUILD_SOURCE_PACKAGE) - sys.exit(RETCODE_SUCCESS) + return RETCODE_FAILURE_BUILD_SOURCE_PACKAGE + return RETCODE_SUCCESS + + +if __name__ == '__main__': + sys.exit(main()) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/buildsnap launchpad-buildd-139/buildsnap --- launchpad-buildd-133~164~ubuntu14.04.1/buildsnap 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/buildsnap 2016-04-01 13:35:01.000000000 +0000 @@ -0,0 +1,206 @@ +#! /usr/bin/python -u +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +"""A script that builds a snap.""" + +from __future__ import print_function + +__metaclass__ = type + +import base64 +from optparse import OptionParser +import os +import subprocess +import sys +import traceback +import urllib2 +from urlparse import urlparse + +from lpbuildd.util import ( + set_personality, + shell_escape, + ) + + +RETCODE_SUCCESS = 0 +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +def get_build_path(build_id, *extra): + """Generate a path within the build directory. + + :param build_id: the build id to use. + :param extra: the extra path segments within the build directory. + :return: the generated path. + """ + return os.path.join(os.environ["HOME"], "build-" + build_id, *extra) + + +class SnapBuilder: + """Builds a snap.""" + + def __init__(self, options, name): + self.options = options + self.name = name + self.chroot_path = get_build_path( + self.options.build_id, 'chroot-autobuild') + # Set to False for local testing if your chroot doesn't have an + # appropriate certificate for your codehosting system. + self.ssl_verify = True + + def chroot(self, args, echo=False): + """Run a command in the chroot. + + :param args: the command and arguments to run. + :param echo: if True, print the command before executing it. + """ + args = set_personality(self.options.arch, args) + if echo: + print( + "Running in chroot: %s" % ' '.join( + "'%s'" % arg for arg in args)) + sys.stdout.flush() + subprocess.check_call([ + "/usr/bin/sudo", "/usr/sbin/chroot", self.chroot_path] + args) + + def run_build_command(self, args, path="/build", env=None, echo=False): + """Run a build command in the chroot. + + This is unpleasant because we need to run it in /build under sudo + chroot, and there's no way to do this without either a helper + program in the chroot or unpleasant quoting. We go for the + unpleasant quoting. + + :param args: the command and arguments to run. + :param path: the working directory to use in the chroot. + :param env: dictionary of additional environment variables to set. + :param echo: if True, print the command before executing it. + """ + args = [shell_escape(arg) for arg in args] + path = shell_escape(path) + full_env = { + "LANG": "C.UTF-8", + } + if env: + full_env.update(env) + args = ["env"] + [ + "%s=%s" % (key, shell_escape(value)) + for key, value in full_env.items()] + args + command = "cd %s && %s" % (path, " ".join(args)) + self.chroot(["/bin/sh", "-c", command], echo=echo) + + def install(self): + print("Running install phase...") + deps = ["snapcraft"] + if self.options.branch is not None: + deps.append("bzr") + else: + deps.append("git") + self.chroot(["apt-get", "-y", "install"] + deps) + + def repo(self): + """Collect git or bzr branch.""" + print("Running repo phase...") + if self.options.branch is not None: + self.run_build_command(['ls', '/build']) + cmd = ["bzr", "branch", self.options.branch, self.name] + if not self.ssl_verify: + cmd.insert(1, "-Ossl.cert_reqs=none") + self.run_build_command(cmd) + else: + assert self.options.git_repository is not None + assert self.options.git_path is not None + if not self.ssl_verify: + env = {"GIT_SSL_NO_VERIFY": "1"} + else: + env = None + self.run_build_command([ + "git", "clone", "-b", self.options.git_path, + self.options.git_repository, self.name], + env=env) + + def pull(self): + """Run pull phase.""" + print("Running pull phase...") + env = {"SNAPCRAFT_LOCAL_SOURCES": "1"} + if self.options.proxy_url: + env["http_proxy"] = self.options.proxy_url + https_url = self.options.proxy_url.replace('http://', 'https://') + env["https_proxy"] = https_url + self.run_build_command( + ["snapcraft", "pull"], + path=os.path.join("/build", self.name), + env=env) + + def build(self): + """Run all build, stage and snap phases.""" + print("Running build phase...") + self.run_build_command( + ["snapcraft"], path=os.path.join("/build", self.name)) + + def revoke_token(self): + """Revoke builder proxy token.""" + print("Revoking proxy token...") + url = urlparse(self.options.proxy_url) + auth = '{}:{}'.format(url.username, url.password) + headers = { + 'Authorization': 'Basic {}'.format(base64.b64encode(auth)) + } + req = urllib2.Request(self.options.revocation_endpoint, None, headers) + req.get_method = lambda: 'DELETE' + try: + urllib2.urlopen(req) + except urllib2.HTTPError: + print('Unable to revoke token for %s' % url.username) + + +def main(): + parser = OptionParser("%prog [options] NAME") + parser.add_option("--build-id", help="build identifier") + parser.add_option( + "--arch", metavar="ARCH", help="build for architecture ARCH") + parser.add_option( + "--branch", metavar="BRANCH", help="build from this Bazaar branch") + parser.add_option( + "--git-repository", metavar="REPOSITORY", + help="build from this Git repository") + parser.add_option( + "--git-path", metavar="REF-PATH", + help="build from this ref path in REPOSITORY") + parser.add_option("--proxy-url", help="builder proxy url") + parser.add_option("--revocation-endpoint", + help="builder proxy token revocation endpoint") + options, args = parser.parse_args() + if (options.git_repository is None) != (options.git_path is None): + parser.error( + "must provide both --git-repository and --git-path or neither") + if (options.branch is None) == (options.git_repository is None): + parser.error( + "must provide exactly one of --branch and --git-repository") + if len(args) != 1: + parser.error( + "must provide a package name and no other positional arguments") + [name] = args + builder = SnapBuilder(options, name) + try: + builder.install() + except Exception: + traceback.print_exc() + return RETCODE_FAILURE_INSTALL + try: + builder.repo() + builder.pull() + builder.build() + except Exception: + traceback.print_exc() + return RETCODE_FAILURE_BUILD + finally: + if options.revocation_endpoint is not None: + builder.revoke_token() + return RETCODE_SUCCESS + + +if __name__ == "__main__": + sys.exit(main()) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/bzr-builder.manifest launchpad-buildd-139/debian/bzr-builder.manifest --- launchpad-buildd-133~164~ubuntu14.04.1/debian/bzr-builder.manifest 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/bzr-builder.manifest 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -# bzr-builder format 0.3 deb-version {debupstream}~164 -lp:launchpad-buildd revid:cjwatson@canonical.com-20150716130335-l89w5itjou90xspj diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/changelog launchpad-buildd-139/debian/changelog --- launchpad-buildd-133~164~ubuntu14.04.1/debian/changelog 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/changelog 2016-04-05 15:11:02.000000000 +0000 @@ -1,18 +1,92 @@ -launchpad-buildd (133~164~ubuntu14.04.1) trusty; urgency=low +launchpad-buildd (139) trusty; urgency=medium - * Auto build. + * buildsnap: Set LANG=C.UTF-8 when running snapcraft. - -- Launchpad Package Builder Thu, 16 Jul 2015 13:16:23 +0000 + -- Colin Watson Tue, 05 Apr 2016 16:11:01 +0100 -launchpad-buildd (133) UNRELEASED; urgency=medium +launchpad-buildd (138) trusty; urgency=medium + + [ Colin Watson ] + * slave-prep: Output current versions of git-build-recipe, git, and + qemu-user-static. + * Always raise exception instances rather than using the two-argument form + of raise. + * buildsnap: Run just "snapcraft" rather than "snapcraft all"; works with + snapcraft << 2.0 and >= 2.3. + * sbuild-package: Default LANG/LC_ALL to C.UTF-8 (LP: #1552791). + + [ Kit Randel ] + * Add http/s proxy support for snap builds. + * Refactor buildsnap into distinct repo, pull, build and proxy token + revocation phases. + + -- Colin Watson Thu, 03 Mar 2016 15:44:12 -0300 + +launchpad-buildd (137) trusty; urgency=medium + + * Remove handling for pre-karmic versions of Twisted that didn't support + --umask. + * Stop cleaning /var/log/launchpad-buildd/ from cron; logrotate handles + that now. + * Configure systemd-timesyncd to use the NTP server configured in + /etc/launchpad-buildd/default. + * Try to load the nbd module when starting launchpad-buildd + (LP: #1531171). + * buildrecipe: Add option parsing framework. + * Use git-build-recipe to run git-based recipe builds (LP: #1453022). + + -- Colin Watson Mon, 18 Jan 2016 11:50:05 +0000 + +launchpad-buildd (136) trusty; urgency=medium + + * Use twisted.python.log.msg rather than print to write to the log file. + Twisted 15.2's stdio wrapper tries to decode as ASCII and thus breaks on + sbuild's UTF-8 section markers, and this seems to be the simplest way to + fix that while preserving source compatibility with earlier versions of + Twisted. + * Add Python packaging files so that Launchpad's test suite can + incorporate this as a Python dependency rather than requiring + python-lpbuildd to be installed on the test system. + + -- Colin Watson Fri, 13 Nov 2015 13:59:48 +0000 + +launchpad-buildd (135) trusty; urgency=medium + + * debian/control: Require python-debian (>= 0.1.23), needed for changes in + launchpad-buildd 133. + * debian/control: Drop dependency on linux32. It's been in util-linux for + ages, which is Essential, and utopic dropped the Provides. + * debian/launchpad-buildd.init: Set "Should-Start: cloud-init" to ensure + that launchpad-buildd is started after the hostname is set. + * Simplify BuilddSlaveTestSetup slightly. + + -- Colin Watson Thu, 29 Oct 2015 17:49:57 +0000 + +launchpad-buildd (134) trusty; urgency=medium + + * buildsnap: Drop explicit installation of sudo, now fixed in snapcraft. + * Rewrite debian/rules in modern dh style. + * buildsnap: Pass SNAPCRAFT_LOCAL_SOURCES=1 to snapcraft so that it uses + the build's sources.list. + + -- Colin Watson Thu, 24 Sep 2015 10:20:13 +0100 + +launchpad-buildd (133) trusty; urgency=medium * Liberalise dep-wait matching regexes slightly so that they match multi-line output properly, as in the case where multiple build-dependencies are uninstallable. * If there is a mix of definite and dubious dep-wait output, then analyse the situation rather than trusting just the definite information. + * Handle architecture restrictions, architecture qualifications, and + restriction formulas (build profiles) in build-dependencies. + * Add support for building snaps (LP: #1476405). + * slave-prep: Output current python-debian version, useful for debugging + build-dependency parsing problems. + * Strip qualifications and restrictions even from dep-waits derived solely + from sbuild output. - -- Colin Watson Thu, 16 Jul 2015 14:00:16 +0100 + -- Colin Watson Tue, 04 Aug 2015 22:58:47 +0100 launchpad-buildd (132) trusty; urgency=medium diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/clean launchpad-buildd-139/debian/clean --- launchpad-buildd-133~164~ubuntu14.04.1/debian/clean 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/debian/clean 2015-08-25 09:13:06.000000000 +0000 @@ -0,0 +1 @@ +buildd-slave-example.conf diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/compat launchpad-buildd-139/debian/compat --- launchpad-buildd-133~164~ubuntu14.04.1/debian/compat 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/compat 2015-08-25 09:21:40.000000000 +0000 @@ -1 +1 @@ -5 +7 diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/control launchpad-buildd-139/debian/control --- launchpad-buildd-133~164~ubuntu14.04.1/debian/control 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/control 2015-10-20 09:36:52.000000000 +0000 @@ -3,13 +3,13 @@ Priority: extra Maintainer: Adam Conrad Standards-Version: 3.9.5 -Build-Depends: debhelper (>= 5), python +Build-Depends: debhelper (>= 7.0.50~), python Package: launchpad-buildd Section: misc Architecture: all Depends: python-lpbuildd (=${source:Version}), python, debootstrap, dpkg-dev, - linux32, file, bzip2, sudo, ntpdate, adduser, apt-transport-https, lsb-release, + file, bzip2, sudo, ntpdate, adduser, apt-transport-https, lsb-release, pristine-tar, python-apt, sbuild, ${misc:Depends} Recommends: qemu-user-static Description: Launchpad buildd slave @@ -22,7 +22,7 @@ Package: python-lpbuildd Section: python Architecture: all -Depends: python, python-twisted-core, python-twisted-web, python-zope.interface, python-apt, python-debian, apt-utils, ${misc:Depends} +Depends: python, python-twisted-core, python-twisted-web, python-zope.interface, python-apt, python-debian (>= 0.1.23), apt-utils, ${misc:Depends} Breaks: launchpad-buildd (<< 88) Replaces: launchpad-buildd (<< 88) Description: Python libraries for a Launchpad buildd slave diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.cron.daily launchpad-buildd-139/debian/launchpad-buildd.cron.daily --- launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.cron.daily 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/launchpad-buildd.cron.daily 2015-11-24 13:35:54.000000000 +0000 @@ -4,7 +4,6 @@ # GNU Affero General Public License version 3 (see the file LICENSE). CLEANDIRS="" -CLEANDIRS="$CLEANDIRS /var/log/launchpad-buildd/" CLEANDIRS="$CLEANDIRS /home/buildd/filecache-default/" for cleandir in $CLEANDIRS; do diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.init launchpad-buildd-139/debian/launchpad-buildd.init --- launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.init 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/launchpad-buildd.init 2016-01-08 01:18:19.000000000 +0000 @@ -10,6 +10,7 @@ # Provides: launchpad_buildd # Required-Start: $local_fs $network $syslog $time $remote_fs # Required-Stop: $local_fs $network $syslog $time $remote_fs +# Should-Start: cloud-init # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # X-Interactive: false @@ -74,13 +75,11 @@ CONF=$1 PIDFILE="$PIDROOT"/"$CONF".pid LOGFILE="$LOGROOT"/"$CONF".log - # prior to karmic, twisted didn't support --umask, and defaulted it well. - # we need it to be 022, not 077. - case $(lsb_release -sc) in - [a-j]*) UMASK="";; - [k-z]*) UMASK="--umask 022";; - esac - su - buildd -c "BUILDD_SLAVE_CONFIG=$CONFROOT/$CONF PYTHONPATH=/usr/lib/launchpad-buildd twistd --no_save --pidfile $PIDFILE --python $TACFILE --logfile $LOGFILE $UMASK" + + # Useful for certain kinds of image builds. + modprobe nbd || true + + su - buildd -c "BUILDD_SLAVE_CONFIG=$CONFROOT/$CONF PYTHONPATH=/usr/lib/launchpad-buildd twistd --no_save --pidfile $PIDFILE --python $TACFILE --logfile $LOGFILE --umask 022" } # diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.install launchpad-buildd-139/debian/launchpad-buildd.install --- launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.install 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/launchpad-buildd.install 2015-08-25 09:12:52.000000000 +0000 @@ -1,2 +1,19 @@ -default/launchpad-buildd etc/default -buildd-genconfig usr/share/launchpad-buildd +default/launchpad-buildd etc/default +buildd-genconfig usr/share/launchpad-buildd +debian/upgrade-config usr/share/launchpad-buildd +sbuildrc usr/share/launchpad-buildd +template-buildd-slave.conf usr/share/launchpad-buildd +buildlivefs usr/share/launchpad-buildd/slavebin +buildrecipe usr/share/launchpad-buildd/slavebin +buildsnap usr/share/launchpad-buildd/slavebin +generate-translation-templates usr/share/launchpad-buildd/slavebin +mount-chroot usr/share/launchpad-buildd/slavebin +override-sources-list usr/share/launchpad-buildd/slavebin +remove-build usr/share/launchpad-buildd/slavebin +sbuild-package usr/share/launchpad-buildd/slavebin +scan-for-processes usr/share/launchpad-buildd/slavebin +slave-prep usr/share/launchpad-buildd/slavebin +sudo-wrapper usr/share/launchpad-buildd/slavebin +umount-chroot usr/share/launchpad-buildd/slavebin +unpack-chroot usr/share/launchpad-buildd/slavebin +update-debian-chroot usr/share/launchpad-buildd/slavebin diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.links launchpad-buildd-139/debian/launchpad-buildd.links --- launchpad-buildd-133~164~ubuntu14.04.1/debian/launchpad-buildd.links 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/debian/launchpad-buildd.links 2015-08-25 09:14:49.000000000 +0000 @@ -0,0 +1 @@ +usr/lib/launchpad-buildd/lpbuildd/check_implicit_pointer_functions.py usr/share/launchpad-buildd/slavebin/check-implicit-pointer-functions diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/postinst launchpad-buildd-139/debian/postinst --- launchpad-buildd-133~164~ubuntu14.04.1/debian/postinst 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/postinst 2015-11-25 14:58:33.000000000 +0000 @@ -93,6 +93,18 @@ fi fi + # Configure systemd-timesyncd to use the buildd NTP service + if which systemd >/dev/null 2>&1; then + eval `grep ntphost /etc/launchpad-buildd/default | sed 's/ //g'` + if [ "$ntphost" ]; then + mkdir -p /etc/systemd/timesyncd.conf.d + cat << EOF > /etc/systemd/timesyncd.conf.d/00-launchpad-buildd.conf +[Time] +NTP=$ntphost +EOF + fi + fi + ;; abort-upgrade|abort-remove|abort-deconfigure) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/postrm launchpad-buildd-139/debian/postrm --- launchpad-buildd-133~164~ubuntu14.04.1/debian/postrm 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/postrm 2015-11-25 14:58:32.000000000 +0000 @@ -4,4 +4,8 @@ if [ "$1" = purge ]; then rm -f /etc/launchpad-buildd/* + rm -f /etc/systemd/timesyncd.conf.d/00-launchpad-buildd.conf + if [ -d /etc/systemd/timesyncd.conf.d ]; then + rmdir -p --ignore-fail-on-non-empty /etc/systemd/timesyncd.conf.d + fi fi diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/debian/rules launchpad-buildd-139/debian/rules --- launchpad-buildd-133~164~ubuntu14.04.1/debian/rules 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/debian/rules 2015-08-25 09:20:13.000000000 +0000 @@ -1,78 +1,23 @@ #!/usr/bin/make -f # -# Copyright 2009, 2010, 2011 Canonical Ltd. +# Copyright 2009-2015 Canonical Ltd. # # This software is licensed under the GNU Affero General Public License version # 3 (see the file LICENSE). -export DH_OPTIONS +%: + dh $@ -# This is an incomplete debian rules file for making the launchpad-buildd deb -# Only ever invoke this as debian/rules package, which will build the source -# package in the parent directory, after copying in the files that live above -# this directory, so that the source package is valid. -# after that, build the source package found in the parent directory. - -target = debian/launchpad-buildd -topdir = . - -slavebins = unpack-chroot mount-chroot update-debian-chroot sbuild-package \ - scan-for-processes umount-chroot remove-build override-sources-list \ - buildrecipe generate-translation-templates slave-prep buildlivefs \ - sudo-wrapper - -BUILDDUID=65500 -BUILDDGID=65500 - -install: - dh_testdir - dh_clean - dh_testroot - dh_install - dh_installdirs - dh_installexamples - - export DH_OPTIONS=-plaunchpad-buildd - dh_install $(slavebins) usr/share/launchpad-buildd/slavebin - dh_install template-buildd-slave.conf \ - sbuildrc \ - debian/upgrade-config \ - usr/share/launchpad-buildd - dh_link usr/lib/launchpad-buildd/lpbuildd/check_implicit_pointer_functions.py \ - usr/share/launchpad-buildd/slavebin/check-implicit-pointer-functions - -binary-indep: install - dh_installdocs - dh_installchangelogs - dh_installinit - dh_installcron - dh_installlogrotate - dh_strip - dh_compress - dh_fixperms - dh_installdeb - dh_gencontrol - dh_md5sums - dh_builddeb -- -Zgzip - -binary: binary-indep - -.PHONY: binary binary-indep binary-arch install clean build - -clean: - rm -f buildd-slave-example.conf - find -name \*.pyc -print0 | xargs -0r rm -f - dh_clean - -prepare: - -package: prepare - debuild -uc -us -S - -build: +override_dh_auto_build: python buildd-genconfig --template=template-buildd-slave.conf \ --arch=i386 --port=8221 --name=default --host=buildd.buildd \ > buildd-slave-example.conf -build-arch: build -build-indep: build +override_dh_auto_test: + : + +override_dh_auto_clean: + find -name \*.pyc -print0 | xargs -0r rm -f + +override_dh_builddeb: + dh_builddeb -- -Zgzip diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/binarypackage.py launchpad-buildd-139/lpbuildd/binarypackage.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/binarypackage.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/binarypackage.py 2016-02-18 14:43:10.000000000 +0000 @@ -6,6 +6,7 @@ from collections import defaultdict import os import re +import subprocess import traceback import apt_pkg @@ -61,6 +62,25 @@ } +class DpkgArchitectureCache: + """Cache the results of asking questions of dpkg-architecture.""" + + def __init__(self): + self._matches = {} + + def match(self, arch, wildcard): + if (arch, wildcard) not in self._matches: + command = ["dpkg-architecture", "-i%s" % wildcard] + env = dict(os.environ) + env["DEB_HOST_ARCH"] = arch + ret = (subprocess.call(command, env=env) == 0) + self._matches[(arch, wildcard)] = ret + return self._matches[(arch, wildcard)] + + +dpkg_architecture = DpkgArchitectureCache() + + class BinaryPackageBuildState(DebianBuildState): SBUILD = "SBUILD" @@ -87,7 +107,7 @@ if f.endswith(".dsc"): self._dscfile = f if self._dscfile is None: - raise ValueError, files + raise ValueError(files) self.archive_purpose = extra_args.get('archive_purpose') self.distribution = extra_args['distribution'] @@ -180,18 +200,42 @@ def relationMatches(self, dep, available): """Return True iff a dependency matches an available package. - :param dep: A dictionary with at least a "name" key, perhaps also a - "version" key, and optionally other keys, of the kind returned - in a list of lists by + :param dep: A dictionary with at least a "name" key, perhaps also + "version", "arch", and "restrictions" keys, and optionally other + keys, of the kind returned in a list of lists by `debian.deb822.PkgRelation.parse_relations`. :param available: A dictionary mapping package names to a list of the available versions of each package. """ + dep_arch = dep.get("arch") + if dep_arch is not None: + arch_match = False + for enabled, arch_wildcard in dep_arch: + if dpkg_architecture.match(self.arch_tag, arch_wildcard): + arch_match = enabled + break + elif not enabled: + # Any !other-architecture restriction implies that this + # architecture is allowed, unless it's specifically + # excluded by some other restriction. + arch_match = True + if not arch_match: + # This dependency "matches" in the sense that it's ignored + # on this architecture. + return True + dep_restrictions = dep.get("restrictions") + if dep_restrictions is not None: + if all( + any(restriction.enabled for restriction in restrlist) + for restrlist in dep_restrictions): + # This dependency "matches" in the sense that it's ignored + # when no build profiles are enabled. + return True if dep["name"] not in available: return False - if dep.get("version") is None: - return True dep_version = dep.get("version") + if dep_version is None: + return True operator_map = { "<<": (lambda a, b: a < b), "<=": (lambda a, b: a <= b), @@ -207,6 +251,34 @@ return True return False + def stripDependencies(self, deps): + """Return a stripped and stringified representation of a dependency. + + The build master can't handle the various qualifications and + restrictions that may be present in control-format + build-dependencies (e.g. ":any", "[amd64]", or ""), so we + strip these out before returning them. + + :param deps: Build-dependencies in the form returned by + `debian.deb822.PkgRelation.parse_relations`. + :return: A stripped dependency relation string, or None if deps is + empty. + """ + stripped_deps = [] + for or_dep in deps: + stripped_or_dep = [] + for simple_dep in or_dep: + stripped_simple_dep = dict(simple_dep) + stripped_simple_dep["arch"] = None + stripped_simple_dep["archqual"] = None + stripped_simple_dep["restrictions"] = None + stripped_or_dep.append(stripped_simple_dep) + stripped_deps.append(stripped_or_dep) + if stripped_deps: + return PkgRelation.str(stripped_deps) + else: + return None + def analyseDepWait(self, deps, avail): """Work out the correct dep-wait for a failed build, if any. @@ -231,13 +303,12 @@ for or_dep in deps: if not any(self.relationMatches(dep, avail) for dep in or_dep): unsat_deps.append(or_dep) - if unsat_deps: - return PkgRelation.str(unsat_deps) + return self.stripDependencies(unsat_deps) except Exception: self._slave.log("Failed to analyse dep-wait:\n") for line in traceback.format_exc().splitlines(True): self._slave.log(line) - return None + return None def iterate_SBUILD(self, success): """Finished the sbuild run.""" @@ -296,6 +367,8 @@ elif rx in BuildLogRegexes.DEPFAIL: # A depwait match forces depwait. missing_dep = mo.expand(BuildLogRegexes.DEPFAIL[rx]) + missing_dep = self.stripDependencies( + PkgRelation.parse_relations(missing_dep)) else: # Otherwise it was a givenback pattern, so leave it # in givenback. diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/debian.py launchpad-buildd-139/lpbuildd/debian.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/debian.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/debian.py 2016-02-18 14:43:16.000000000 +0000 @@ -12,6 +12,8 @@ import re import signal +from twisted.python import log + from lpbuildd.slave import ( BuildManager, ) @@ -121,20 +123,20 @@ # We may have been aborted in between subprocesses; pretend that # we were terminated by a signal, which is close enough. success = 128 + signal.SIGKILL - print ("Iterating with success flag %s against stage %s" - % (success, self._state)) + log.msg("Iterating with success flag %s against stage %s" + % (success, self._state)) func = getattr(self, "iterate_" + self._state, None) if func is None: - raise ValueError, "Unknown internal state " + self._state + raise ValueError("Unknown internal state " + self._state) func(success) def iterateReap(self, state, success): - print ("Iterating with success flag %s against stage %s after reaping " - "processes" - % (success, state)) + log.msg("Iterating with success flag %s against stage %s after " + "reaping processes" + % (success, state)) func = getattr(self, "iterateReap_" + state, None) if func is None: - raise ValueError, "Unknown internal post-reap state " + state + raise ValueError("Unknown internal post-reap state " + state) func(success) def iterate_INIT(self, success): @@ -200,10 +202,10 @@ stop_regexes = [ re.compile(pattern, flags) for pattern, flags in stop_patterns_and_flags] - log = open(os.path.join(self._cachepath, "buildlog")) + buildlog = open(os.path.join(self._cachepath, "buildlog")) try: window = "" - chunk = log.read(chunk_size) + chunk = buildlog.read(chunk_size) while chunk: window += chunk for regex in regexes: @@ -215,9 +217,9 @@ return None, None if len(window) > chunk_size: window = window[chunk_size:] - chunk = log.read(chunk_size) + chunk = buildlog.read(chunk_size) finally: - log.close() + buildlog.close() return None, None def iterate_SOURCES(self, success): diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/slave.py launchpad-buildd-139/lpbuildd/slave.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/slave.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/slave.py 2016-02-05 15:47:41.000000000 +0000 @@ -20,6 +20,7 @@ from twisted.internet import protocol from twisted.internet import reactor as default_reactor from twisted.internet import process +from twisted.python import log from twisted.web import xmlrpc # XXX cjwatson 2013-10-07: Remove when all builders are on Ubuntu 10.04 LTS @@ -122,7 +123,11 @@ self.home = os.environ['HOME'] self.abort_timeout = 120 - def runSubProcess(self, command, args, iterate=None): + @property + def needs_sanitized_logs(self): + return self.is_archive_private + + def runSubProcess(self, command, args, iterate=None, env=None): """Run a sub process capturing the results in the log.""" if iterate is None: iterate = self.iterate @@ -130,7 +135,7 @@ self._slave.log("RUN: %s %r\n" % (command, args)) childfds = {0: devnull.fileno(), 1: "r", 2: "r"} self._reactor.spawnProcess( - self._subprocess, command, args, env=os.environ, + self._subprocess, command, args, env=env, path=self.home, childFDs=childfds) def doUnpack(self): @@ -164,7 +169,7 @@ # Sanitize the URLs in the buildlog file if this is a build # in a private archive. - if self.is_archive_private: + if self.needs_sanitized_logs: self._slave.sanitizeBuildlog(self._slave.cachePath("buildlog")) def doMounting(self): @@ -450,7 +455,7 @@ self._log.flush() if data.endswith("\n"): data = data[:-1] - print "Build log: " + data + log.msg("Build log: " + data) def getLogTail(self): """Return the tail of the log. @@ -491,7 +496,7 @@ if rlog is not None: rlog.close() - if self.manager.is_archive_private: + if self.manager.needs_sanitized_logs: # This is a build in a private archive. We need to scrub # the URLs contained in the buildlog excerpt in order to # avoid leaking passwords. @@ -642,7 +647,7 @@ self._version = cache["python-lpbuildd"].installedVersion except KeyError: self._version = None - print "Initialized" + log.msg("Initialized") def registerBuilder(self, builderclass, buildertag): self._builders[buildertag] = builderclass diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/snap.py launchpad-buildd-139/lpbuildd/snap.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/snap.py 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/snap.py 2016-02-05 15:47:41.000000000 +0000 @@ -0,0 +1,105 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +__metaclass__ = type + +import os +import shutil + +from lpbuildd.debian import ( + DebianBuildManager, + DebianBuildState, + get_build_path, + ) + + +RETCODE_SUCCESS = 0 +RETCODE_FAILURE_INSTALL = 200 +RETCODE_FAILURE_BUILD = 201 + + +class SnapBuildState(DebianBuildState): + BUILD_SNAP = "BUILD_SNAP" + + +class SnapBuildManager(DebianBuildManager): + """Build a snap.""" + + initial_build_state = SnapBuildState.BUILD_SNAP + + def __init__(self, slave, buildid, **kwargs): + super(SnapBuildManager, self).__init__(slave, buildid, **kwargs) + self.build_snap_path = os.path.join(self._slavebin, "buildsnap") + + @property + def needs_sanitized_logs(self): + return True + + def initiate(self, files, chroot, extra_args): + """Initiate a build with a given set of files and chroot.""" + self.build_path = get_build_path( + self.home, self._buildid, "chroot-autobuild", "build") + if os.path.isdir(self.build_path): + shutil.rmtree(self.build_path) + + self.name = extra_args["name"] + self.branch = extra_args.get("branch") + self.git_repository = extra_args.get("git_repository") + self.git_path = extra_args.get("git_path") + self.proxy_url = extra_args.get("proxy_url") + self.revocation_endpoint = extra_args.get("revocation_endpoint") + + super(SnapBuildManager, self).initiate(files, chroot, extra_args) + + def doRunBuild(self): + """Run the process to build the snap.""" + args = [ + "buildsnap", + "--build-id", self._buildid, + "--arch", self.arch_tag, + ] + if self.proxy_url: + args.extend(["--proxy-url", self.proxy_url]) + if self.revocation_endpoint: + args.extend(["--revocation-endpoint", self.revocation_endpoint]) + if self.branch is not None: + args.extend(["--branch", self.branch]) + if self.git_repository is not None: + args.extend(["--git-repository", self.git_repository]) + if self.git_path is not None: + args.extend(["--git-path", self.git_path]) + args.append(self.name) + self.runSubProcess(self.build_snap_path, args) + + def iterate_BUILD_SNAP(self, retcode): + """Finished building the snap.""" + if retcode == RETCODE_SUCCESS: + self.gatherResults() + print("Returning build status: OK") + elif (retcode >= RETCODE_FAILURE_INSTALL and + retcode <= RETCODE_FAILURE_BUILD): + if not self.alreadyfailed: + self._slave.buildFail() + print("Returning build status: Build failed.") + self.alreadyfailed = True + else: + if not self.alreadyfailed: + self._slave.builderFail() + print("Returning build status: Builder failed.") + self.alreadyfailed = True + self.doReapProcesses(self._state) + + def iterateReap_BUILD_SNAP(self, retcode): + """Finished reaping after building the snap.""" + self._state = DebianBuildState.UMOUNT + self.doUnmounting() + + def gatherResults(self): + """Gather the results of the build and add them to the file cache.""" + output_path = os.path.join(self.build_path, self.name) + if not os.path.exists(output_path): + return + for entry in sorted(os.listdir(output_path)): + path = os.path.join(output_path, entry) + if entry.endswith(".snap") and not os.path.islink(path): + self._slave.addWaitingFile(path) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/sourcepackagerecipe.py launchpad-buildd-139/lpbuildd/sourcepackagerecipe.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/sourcepackagerecipe.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/sourcepackagerecipe.py 2016-01-18 11:42:57.000000000 +0000 @@ -76,6 +76,7 @@ self.author_email = extra_args['author_email'] self.archive_purpose = extra_args['archive_purpose'] self.distroseries_name = extra_args['distroseries_name'] + self.git = extra_args.get('git', False) super(SourcePackageRecipeBuildManager, self).initiate( files, chroot, extra_args) @@ -85,10 +86,13 @@ os.makedirs(get_chroot_path(self.home, self._buildid, 'work')) recipe_path = get_chroot_path(self.home, self._buildid, 'work/recipe') splat_file(recipe_path, self.recipe_text) - args = [ - "buildrecipe", self._buildid, self.author_name.encode('utf-8'), + args = ["buildrecipe"] + if self.git: + args.append("--git") + args.extend([ + self._buildid, self.author_name.encode('utf-8'), self.author_email, self.suite, self.distroseries_name, - self.component, self.archive_purpose] + self.component, self.archive_purpose]) self.runSubProcess(self.build_recipe_path, args) def iterate_BUILD_RECIPE(self, retcode): diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/harness.py launchpad-buildd-139/lpbuildd/tests/harness.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/harness.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/tests/harness.py 2016-02-21 18:00:54.000000000 +0000 @@ -25,10 +25,14 @@ class MockBuildManager(object): """Mock BuildManager class. - Only implements 'is_archive_private' as False. + Only implements 'is_archive_private' and 'needs_sanitized_logs' as False. """ is_archive_private = False + @property + def needs_sanitized_logs(self): + return self.is_archive_private + class BuilddTestCase(unittest.TestCase): """Unit tests for logtail mechanisms.""" @@ -133,10 +137,7 @@ def logfile(self): return '/var/tmp/build-slave.log' - def _hasDaemonStarted(self): - """Called by the superclass to check if the daemon is listening. - - The slave is ready when it's accepting connections. - """ + @property + def daemon_port(self): # This must match buildd-slave-test.conf. - return self._isPortListening('localhost', 8221) + return 8221 diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/__init__.py launchpad-buildd-139/lpbuildd/tests/__init__.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/__init__.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/tests/__init__.py 2015-08-04 17:50:44.000000000 +0000 @@ -1,4 +0,0 @@ -# Copyright 2009 Canonical Ltd. This software is licensed under the -# GNU Affero General Public License version 3 (see the file LICENSE). - -from harness import BuilddSlaveTestSetup diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_binarypackage.py launchpad-buildd-139/lpbuildd/tests/test_binarypackage.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_binarypackage.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/tests/test_binarypackage.py 2015-10-06 11:21:52.000000000 +0000 @@ -426,6 +426,48 @@ "debhelper (>= 9~), foo (>= 1), bar (<< 1) | bar (>= 2)"), {"debhelper": set(["9"]), "bar": set(["1", "1.5"])})) + def test_analyseDepWait_strips_arch_restrictions(self): + # analyseDepWait removes architecture restrictions (e.g. "[amd64]") + # from the unsatisfied build-dependencies it returns, and only + # returns those relevant to the current architecture. + self.buildmanager.initiate( + {'foo_1.dsc': ''}, 'chroot.tar.gz', + {'distribution': 'ubuntu', 'suite': 'warty', + 'ogrecomponent': 'main', 'arch_tag': 'i386'}) + self.assertEqual( + "foo (>= 1)", + self.buildmanager.analyseDepWait( + PkgRelation.parse_relations( + "foo (>= 1) [any-i386], bar (>= 1) [amd64]"), + {})) + + def test_analyseDepWait_strips_arch_qualifications(self): + # analyseDepWait removes architecture qualifications (e.g. ":any") + # from the unsatisfied build-dependencies it returns. + self.buildmanager.initiate( + {'foo_1.dsc': ''}, 'chroot.tar.gz', + {'distribution': 'ubuntu', 'suite': 'warty', + 'ogrecomponent': 'main', 'arch_tag': 'i386'}) + self.assertEqual( + "foo", + self.buildmanager.analyseDepWait( + PkgRelation.parse_relations("foo:any, bar:any"), + {"bar": set(["1"])})) + + def test_analyseDepWait_strips_restrictions(self): + # analyseDepWait removes restrictions (e.g. "") from the + # unsatisfied build-dependencies it returns, and only returns those + # that evaluate to true when no build profiles are active. + self.buildmanager.initiate( + {'foo_1.dsc': ''}, 'chroot.tar.gz', + {'distribution': 'ubuntu', 'suite': 'warty', + 'ogrecomponent': 'main', 'arch_tag': 'i386'}) + self.assertEqual( + "foo", + self.buildmanager.analyseDepWait( + PkgRelation.parse_relations("foo , bar "), + {})) + def startDepFail(self, error, dscname=''): self.startBuild(dscname=dscname) write_file( @@ -462,6 +504,12 @@ self.assertMatchesDepfail( "ebadver (< 2.0) but 3.0 is installed", "ebadver (< 2.0)") + def test_strips_depfail(self): + # The build manager strips qualifications and restrictions from + # dependency installation failures. + self.assertMatchesDepfail( + "ebadver:any (>= 3.0) but 2.0 is installed", "ebadver (>= 3.0)") + def test_uninstallable_deps_analysis_failure(self): # If there are uninstallable build-dependencies and analysis can't # find any missing direct build-dependencies, the build manager diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_snap.py launchpad-buildd-139/lpbuildd/tests/test_snap.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_snap.py 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/tests/test_snap.py 2016-02-05 15:47:41.000000000 +0000 @@ -0,0 +1,121 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +__metaclass__ = type + +import os +import shutil +import tempfile + +from testtools import TestCase + +from lpbuildd.snap import ( + SnapBuildManager, + SnapBuildState, + ) +from lpbuildd.tests.fakeslave import FakeSlave + + +class MockBuildManager(SnapBuildManager): + def __init__(self, *args, **kwargs): + super(MockBuildManager, self).__init__(*args, **kwargs) + self.commands = [] + self.iterators = [] + + def runSubProcess(self, path, command, iterate=None, env=None): + self.commands.append([path] + command) + if iterate is None: + iterate = self.iterate + self.iterators.append(iterate) + return 0 + + +class TestSnapBuildManagerIteration(TestCase): + """Run SnapBuildManager through its iteration steps.""" + def setUp(self): + super(TestSnapBuildManagerIteration, self).setUp() + self.working_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.working_dir)) + slave_dir = os.path.join(self.working_dir, "slave") + home_dir = os.path.join(self.working_dir, "home") + for dir in (slave_dir, home_dir): + os.mkdir(dir) + self.slave = FakeSlave(slave_dir) + self.buildid = "123" + self.buildmanager = MockBuildManager(self.slave, self.buildid) + self.buildmanager.home = home_dir + self.buildmanager._cachepath = self.slave._cachepath + self.build_dir = os.path.join( + home_dir, "build-%s" % self.buildid, "chroot-autobuild", "build") + + def getState(self): + """Retrieve build manager's state.""" + return self.buildmanager._state + + def startBuild(self): + # The build manager's iterate() kicks off the consecutive states + # after INIT. + extra_args = { + "arch_tag": "i386", + "name": "test-snap", + "git_repository": "https://git.launchpad.dev/~example/+git/snap", + "git_path": "master", + } + self.buildmanager.initiate({}, "chroot.tar.gz", extra_args) + + # Skip states that are done in DebianBuildManager to the state + # directly before BUILD_SNAP. + self.buildmanager._state = SnapBuildState.UPDATE + + # BUILD_SNAP: Run the slave's payload to build the snap package. + self.buildmanager.iterate(0) + self.assertEqual(SnapBuildState.BUILD_SNAP, self.getState()) + expected_command = [ + "sharepath/slavebin/buildsnap", "buildsnap", + "--build-id", self.buildid, "--arch", "i386", + "--git-repository", "https://git.launchpad.dev/~example/+git/snap", + "--git-path", "master", + "test-snap", + ] + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.slave.wasCalled("chrootFail")) + + def test_iterate(self): + # The build manager iterates a normal build from start to finish. + self.startBuild() + + log_path = os.path.join(self.buildmanager._cachepath, "buildlog") + log = open(log_path, "w") + log.write("I am a build log.") + log.close() + + output_dir = os.path.join(self.build_dir, "test-snap") + os.makedirs(output_dir) + snap_path = os.path.join(output_dir, "test-snap_0_all.snap") + with open(snap_path, "w") as snap: + snap.write("I am a snap package.") + + # After building the package, reap processes. + self.buildmanager.iterate(0) + expected_command = [ + "sharepath/slavebin/scan-for-processes", "scan-for-processes", + self.buildid, + ] + self.assertEqual(SnapBuildState.BUILD_SNAP, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertNotEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.slave.wasCalled("buildFail")) + self.assertEqual([((snap_path,), {})], self.slave.addWaitingFile.calls) + + # Control returns to the DebianBuildManager in the UMOUNT state. + self.buildmanager.iterateReap(self.getState(), 0) + expected_command = [ + "sharepath/slavebin/umount-chroot", "umount-chroot", self.buildid] + self.assertEqual(SnapBuildState.UMOUNT, self.getState()) + self.assertEqual(expected_command, self.buildmanager.commands[-1]) + self.assertEqual( + self.buildmanager.iterate, self.buildmanager.iterators[-1]) + self.assertFalse(self.slave.wasCalled("buildFail")) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_sourcepackagerecipe.py launchpad-buildd-139/lpbuildd/tests/test_sourcepackagerecipe.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/tests/test_sourcepackagerecipe.py 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/tests/test_sourcepackagerecipe.py 2016-01-18 11:42:57.000000000 +0000 @@ -54,7 +54,7 @@ """Retrieve build manager's state.""" return self.buildmanager._state - def startBuild(self): + def startBuild(self, git=False): # The build manager's iterate() kicks off the consecutive states # after INIT. extra_args = { @@ -73,6 +73,8 @@ 'ubuntu main', ], } + if git: + extra_args['git'] = True self.buildmanager.initiate({}, 'chroot.tar.gz', extra_args) # Skip states that are done in DebianBuildManager to the state @@ -84,10 +86,14 @@ self.assertEqual( SourcePackageRecipeBuildState.BUILD_RECIPE, self.getState()) expected_command = [ - 'sharepath/slavebin/buildrecipe', 'buildrecipe', self.buildid, + 'sharepath/slavebin/buildrecipe', 'buildrecipe'] + if git: + expected_command.append('--git') + expected_command.extend([ + self.buildid, 'Steve\u1234'.encode('utf-8'), 'stevea@example.org', 'maverick', 'maverick', 'universe', 'puppies', - ] + ]) self.assertEqual(expected_command, self.buildmanager.commands[-1]) self.assertEqual( self.buildmanager.iterate, self.buildmanager.iterators[-1]) @@ -213,3 +219,9 @@ self.assertEqual(expected_command, self.buildmanager.commands[-1]) self.assertEqual( self.buildmanager.iterate, self.buildmanager.iterators[-1]) + + def test_iterate_git(self): + # Starting a git-based recipe build passes the correct option. (The + # rest of the build is identical to bzr-based recipe builds from the + # build manager's point of view.) + self.startBuild(git=True) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/util.py launchpad-buildd-139/lpbuildd/util.py --- launchpad-buildd-133~164~ubuntu14.04.1/lpbuildd/util.py 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/lpbuildd/util.py 2015-08-04 08:59:14.000000000 +0000 @@ -0,0 +1,50 @@ +# Copyright 2015 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +__metaclass__ = type + +import re + + +non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$') + +def shell_escape(arg): + if non_meta_re.match(arg): + return arg + else: + return "'%s'" % arg.replace("'", "'\\''") + + +linux32_arches = [ + "armel", + "armhf", + "hppa", + "i386", + "lpia", + "mips", + "mipsel", + "powerpc", + "s390", + "sparc", + ] +linux64_arches = [ + "alpha", + "amd64", + "arm64", + "hppa64", + "ia64", + "ppc64", + "ppc64el", + "s390x", + "sparc64", + "x32", + ] + + +def set_personality(arch, args): + if arch in linux32_arches: + return ["linux32"] + args + elif arch in linux64_arches: + return ["linux64"] + args + else: + return args diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/Makefile launchpad-buildd-139/Makefile --- launchpad-buildd-133~164~ubuntu14.04.1/Makefile 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/Makefile 2015-08-04 08:59:14.000000000 +0000 @@ -33,5 +33,6 @@ lpbuildd.tests.test_check_implicit_pointer_functions \ lpbuildd.tests.test_harness \ lpbuildd.tests.test_livefs \ + lpbuildd.tests.test_snap \ lpbuildd.tests.test_sourcepackagerecipe \ lpbuildd.tests.test_translationtemplatesbuildmanager diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/MANIFEST.in launchpad-buildd-139/MANIFEST.in --- launchpad-buildd-133~164~ubuntu14.04.1/MANIFEST.in 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/MANIFEST.in 2015-11-11 08:57:39.000000000 +0000 @@ -0,0 +1,24 @@ +include LICENSE +include Makefile +include buildd-genconfig +include buildd-slave.tac +include buildlivefs +include buildrecipe +include buildsnap +include debian/changelog +include generate-translation-templates +include mount-chroot +include override-sources-list +include remove-build +include sbuild-package +include sbuildrc +include scan-for-processes +include slave-prep +include sudo-wrapper +include template-buildd-slave.conf +include test_buildd_generatetranslationtemplates +include test_buildd_recipe +include umount-chroot +include unpack-chroot +include update-debian-chroot +recursive-include lpbuildd/tests *.diff *.tar.gz buildd-slave-test.conf buildlog buildlog.long diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/sbuild-package launchpad-buildd-139/sbuild-package --- launchpad-buildd-133~164~ubuntu14.04.1/sbuild-package 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/sbuild-package 2016-03-03 16:27:33.000000000 +0000 @@ -40,7 +40,7 @@ # Force values for these, since otherwise sudo/pam_env may fill in unwanted # values from /etc/default/locale. -export LANG=C LC_ALL=C LANGUAGE= +export LANG=C.UTF-8 LC_ALL=C.UTF-8 LANGUAGE= # A number of build systems (e.g. automake, Linux) use this as an indication # that they should be more verbose. diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/setup.py launchpad-buildd-139/setup.py --- launchpad-buildd-133~164~ubuntu14.04.1/setup.py 1970-01-01 00:00:00.000000000 +0000 +++ launchpad-buildd-139/setup.py 2015-11-11 08:57:39.000000000 +0000 @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# Copyright 2015 Canonical Ltd. All rights reserved. +# +# This file is part of launchpad-buildd. +# +# launchpad-buildd is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, version 3 of the License. +# +# launchpad-buildd is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public +# License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with launchpad-buildd. If not, see . + +import re +from textwrap import dedent + +from setuptools import ( + find_packages, + setup, + ) + + +changelog_heading = re.compile(r'\w[-+0-9a-z.]* \(([^\(\) \t]+)\)') + +with open('debian/changelog') as changelog: + line = changelog.readline() + match = changelog_heading.match(line) + if match is None: + raise ValueError( + "Failed to parse first line of debian/changelog: '%s'" % line) + version = match.group(1) + + +setup( + name='launchpad-buildd', + version=version, + description='Launchpad buildd slave', + long_description=dedent(""" + The Launchpad buildd slave libraries. The PyPI version of this + package will not produce a complete installation on its own, and is + mostly useful for testing other pieces of software against + launchpad-buildd; for a real Launchpad buildd slave, install the + launchpad-buildd package from ppa:launchpad/ubuntu/ppa. + """).strip(), + url='https://launchpad.net/launchpad-buildd', + packages=find_packages(), + package_data={ + 'lpbuildd': [ + 'tests/buildd-slave-test.conf', + ], + }, + maintainer='Launchpad Developers', + maintainer_email='launchpad-dev@lists.launchpad.net', + license='Affero GPL v3', + install_requires=[ + 'bzr', + # XXX cjwatson 2015-11-04: This does in fact require python-apt, but + # that's normally shipped as a system package and specifying it here + # causes problems for Launchpad's build system. + #'python-apt', + 'python-debian', + 'Twisted', + 'zope.interface', + ], + data_files=[ + ('', ['buildd-slave.tac', 'template-buildd-slave.conf']), + ], + test_suite='lpbuildd.tests', + tests_require=[ + 'fixtures', + 'testtools', + 'txfixtures', + ], + ) diff -Nru launchpad-buildd-133~164~ubuntu14.04.1/slave-prep launchpad-buildd-139/slave-prep --- launchpad-buildd-133~164~ubuntu14.04.1/slave-prep 2015-07-16 13:16:23.000000000 +0000 +++ launchpad-buildd-139/slave-prep 2016-01-19 23:25:39.000000000 +0000 @@ -9,7 +9,7 @@ NTPDATE=ntpdate SUDO=sudo -PACKAGES="launchpad-buildd python-lpbuildd sbuild bzr-builder bzr dpkg-dev" +PACKAGES="launchpad-buildd python-lpbuildd sbuild bzr-builder bzr git-build-recipe git dpkg-dev python-debian qemu-user-static" KERNEL=$(uname -snrvm) echo "Forking launchpad-buildd slave process..."