Merge lp:~cjwatson/launchpad-buildd/reduce-arch-hardcoding into lp:launchpad-buildd

[Colin Watson]

Remove most architecture hardcoding from lpbuildd.util, relying on
dpkg-architecture instead.

[lp:~cjwatson/launchpad-buildd/reduce-arch-hardcoding]

236. By Colin Watson on 2017-07-28

Merge trunk.

237. By Colin Watson on 2017-07-28

Use test discovery rather than hardcoding a list of test modules.

238. By Colin Watson on 2017-07-28

Remove most architecture hardcoding from lpbuildd.util, relying on
dpkg-architecture instead.

239. By Colin Watson on 2017-07-28

Merge trunk.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'Makefile'
--- Makefile	2017-07-28 10:39:50 +0000
+++ Makefile	2017-07-28 14:08:19 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 all: deb
@@ -22,16 +22,5 @@
 
 PYTHON=python
 check:
-	PYTHONPATH=$(CURDIR):$(PYTHONPATH) $(PYTHON) -m testtools.run -v \
-		   lpbuildd.pottery.tests.test_generate_translation_templates \
-		   lpbuildd.pottery.tests.test_intltool \
-		   lpbuildd.tests.test_binarypackage \
-		   lpbuildd.tests.test_buildd_slave \
-		   lpbuildd.tests.test_buildrecipe \
-		   lpbuildd.tests.test_check_implicit_pointer_functions \
-		   lpbuildd.tests.test_debian \
-		   lpbuildd.tests.test_harness \
-		   lpbuildd.tests.test_livefs \
-		   lpbuildd.tests.test_snap \
-		   lpbuildd.tests.test_sourcepackagerecipe \
-		   lpbuildd.tests.test_translationtemplatesbuildmanager
+	PYTHONPATH=$(CURDIR):$(PYTHONPATH) $(PYTHON) -m testtools.run \
+		discover -v

=== modified file 'bin/buildlivefs'
--- bin/buildlivefs	2017-07-25 22:11:19 +0000
+++ bin/buildlivefs	2017-07-28 14:08:19 +0000
@@ -1,5 +1,5 @@
 #! /usr/bin/python -u
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2013-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """A script that builds a live file system."""
@@ -48,7 +48,8 @@
 
         :param args: the command and arguments to run.
         """
-        args = set_personality(self.options.arch, args)
+        args = set_personality(
+            args, self.options.arch, series=self.options.series)
         if echo:
             print("Running in chroot: %s" %
                   ' '.join("'%s'" % arg for arg in args))

=== modified file 'bin/buildsnap'
--- bin/buildsnap	2017-07-25 22:11:19 +0000
+++ bin/buildsnap	2017-07-28 14:08:19 +0000
@@ -1,5 +1,5 @@
 #! /usr/bin/python -u
-# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """A script that builds a snap."""
@@ -58,7 +58,7 @@
         :param echo: if True, print the command before executing it.
         :param get_output: if True, return the output from the command.
         """
-        args = set_personality(self.options.arch, args)
+        args = set_personality(args, self.options.arch)
         if echo:
             print(
                 "Running in chroot: %s" % ' '.join(

=== modified file 'debian/changelog'
--- debian/changelog	2017-07-28 11:10:52 +0000
+++ debian/changelog	2017-07-28 14:08:19 +0000
@@ -6,6 +6,8 @@
     ppa:launchpad/ubuntu/ppa (and >= zesty).
   * Drop qemu emulation support.  It was quite unreliable, and we've had
     real hardware for a while.
+  * Remove most architecture hardcoding from lpbuildd.util, relying on
+    dpkg-architecture instead.
 
  -- Colin Watson <[e-mail]>  Tue, 25 Jul 2017 23:07:58 +0100
 

=== added file 'lpbuildd/tests/test_util.py'
--- lpbuildd/tests/test_util.py	1970-01-01 00:00:00 +0000
+++ lpbuildd/tests/test_util.py	2017-07-28 14:08:19 +0000
@@ -0,0 +1,59 @@
+# Copyright 2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+from testtools import TestCase
+
+from lpbuildd.util import (
+    get_arch_bits,
+    set_personality,
+    shell_escape,
+    )
+
+
+class TestShellEscape(TestCase):
+
+    def test_plain(self):
+        self.assertEqual("foo", shell_escape("foo"))
+
+    def test_whitespace(self):
+        self.assertEqual("'  '", shell_escape("  "))
+
+    def test_single_quotes(self):
+        self.assertEqual("'shell'\\''s great'", shell_escape("shell's great"))
+
+
+class TestGetArchBits(TestCase):
+
+    def test_x32(self):
+        self.assertEqual(64, get_arch_bits("x32"))
+
+    def test_32bit(self):
+        self.assertEqual(32, get_arch_bits("armhf"))
+        self.assertEqual(32, get_arch_bits("i386"))
+
+    def test_64bit(self):
+        self.assertEqual(64, get_arch_bits("amd64"))
+        self.assertEqual(64, get_arch_bits("arm64"))
+
+
+class TestSetPersonality(TestCase):
+
+    def test_32bit(self):
+        self.assertEqual(
+            ["linux32", "sbuild"], set_personality(["sbuild"], "i386"))
+
+    def test_64bit(self):
+        self.assertEqual(
+            ["linux64", "sbuild"], set_personality(["sbuild"], "amd64"))
+
+    def test_uname_26(self):
+        self.assertEqual(
+            ["linux64", "--uname-2.6", "sbuild"],
+            set_personality(["sbuild"], "amd64", series="precise"))
+
+    def test_no_uname_26(self):
+        self.assertEqual(
+            ["linux64", "sbuild"],
+            set_personality(["sbuild"], "amd64", series="trusty"))

=== modified file 'lpbuildd/util.py'
--- lpbuildd/util.py	2015-07-31 11:54:07 +0000
+++ lpbuildd/util.py	2017-07-28 14:08:19 +0000
@@ -1,9 +1,10 @@
-# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
 
 import re
+import subprocess
 
 
 non_meta_re = re.compile(r'^[a-zA-Z0-9+,./:=@_-]+$')
@@ -15,36 +16,34 @@
         return "'%s'" % arg.replace("'", "'\\''")
 
 
-linux32_arches = [
-    "armel",
-    "armhf",
-    "hppa",
-    "i386",
-    "lpia",
-    "mips",
-    "mipsel",
-    "powerpc",
-    "s390",
-    "sparc",
-    ]
-linux64_arches = [
-    "alpha",
-    "amd64",
-    "arm64",
-    "hppa64",
-    "ia64",
-    "ppc64",
-    "ppc64el",
-    "s390x",
-    "sparc64",
-    "x32",
-    ]
-
-
-def set_personality(arch, args):
-    if arch in linux32_arches:
-        return ["linux32"] + args
-    elif arch in linux64_arches:
-        return ["linux64"] + args
-    else:
-        return args
+def get_arch_bits(arch):
+    if arch == "x32":
+        # x32 is an exception: the userspace is 32-bit, but it expects to be
+        # running on a 64-bit kernel.
+        return 64
+    else:
+        bits = subprocess.check_output(
+            ["dpkg-architecture", "-a%s" % arch,
+             "-qDEB_HOST_ARCH_BITS"]).rstrip("\n")
+        if bits == "32":
+            return 32
+        elif bits == "64":
+            return 64
+        else:
+            raise RuntimeError(
+                "Don't know how to deal with architecture %s "
+                "(DEB_HOST_ARCH_BITS=%s)" % (arch, bits))
+
+
+def set_personality(args, arch, series=None):
+    bits = get_arch_bits(arch)
+    assert bits in (32, 64)
+    if bits == 32:
+        setarch_cmd = ["linux32"]
+    else:
+        setarch_cmd = ["linux64"]
+
+    if series in ("hardy", "lucid", "maverick", "natty", "oneiric", "precise"):
+        setarch_cmd.append("--uname-2.6")
+
+    return setarch_cmd + args