Merge lp:~james-w/linaro-image-tools/architecture-support into lp:linaro-image-tools/11.11

[James Westby]

Hi,

Here's a rather simple branch to add architecture support through the code
we have now.

I just updated the spec to state that ARCHITECTURE is a required field in
the metadata, so that's now a required argument to the constructor of the
corresponding class, and it is output in the __str__ method.

Also the Package classes get architecture as an ivar, such that we can
track which architecture they are for, which leads to the change in the
get_packages_file function, which can now stop hardcoding "all".

The dummy test package object defaults to "all" as that way the tests
will work on multiple architectures without updating.

However, that leaves a gap, so I added a couple of new tests to ensure
that we fetch packages from the right architecture.

In order to support cross-build of hardware packs I stole a trick from
the chdist tool, which allows us to tell apt to pretend it is on a
different architecture. I added a test to peek inside the implementation
and check we were writing the config correctly, but the two added tests
check that it has the desired effect.

There's still a gap, as noted in the spec, about how we specify the
architectures to build for, as I'm not sure whether it should be in
the configuration file or on the command line, or some combination
of the two. This will mainly depend on how we want to hook it in to
lexbuilder I think. At least now all the infrastructure is in place
to build for any arch that we like.

Thanks,

James

[lp:~james-w/linaro-image-tools/architecture-support]

105. By James Westby on 2010-09-02

Merge add-packages-to-hwpack in to architecture-support.

[Michael Hudson-Doyle (mwhudson)]

Looks fine to me!

review: Approve

=== modified file 'hwpack/hardwarepack.py'
--- hwpack/hardwarepack.py	2010-09-02 23:07:45 +0000
+++ hwpack/hardwarepack.py	2010-09-02 23:07:46 +0000
@@ -26,8 +26,8 @@
     :type support: str or None
     """
 
-    def __init__(self, name, version, origin=None, maintainer=None,
-                 support=None):
+    def __init__(self, name, version, architecture, origin=None,
+                 maintainer=None, support=None):
         """Create the Metadata for a hardware pack.
 
         See the instance variables for a description of the arguments.
@@ -37,11 +37,13 @@
         self.origin = origin
         self.maintainer = maintainer
         self.support = support
+        self.architecture = architecture
 
     def __str__(self):
         """Get the contents of the metadata file."""
         metadata = "NAME=%s\n" % self.name
         metadata += "VERSION=%s\n" % self.version
+        metadata += "ARCHITECTURE=%s\n" % self.architecture
         if self.origin is not None:
             metadata += "ORIGIN=%s\n" % self.origin
         if self.maintainer is not None:

=== modified file 'hwpack/packages.py'
--- hwpack/packages.py	2010-09-02 23:07:45 +0000
+++ hwpack/packages.py	2010-09-02 23:07:46 +0000
@@ -23,7 +23,7 @@
         parts.append('Filename: %s' % package.filename)
         parts.append('Size: %d' % package.size)
         # TODO: architecture support
-        parts.append('Architecture: all')
+        parts.append('Architecture: %s' % package.architecture)
         parts.append('MD5sum: %s' % package.md5)
         content += "\n".join(parts)
         content += "\n\n"
@@ -75,9 +75,13 @@
     :ivar md5: the hex representation of the md5sum of the contents of
         the package.
     :type md5: str
+    :ivar architecture: the architecture that the package is for, may be
+        'all'.
+    :type architecture: str
     """
 
-    def __init__(self, name, version, filename, content, size, md5):
+    def __init__(self, name, version, filename, content, size, md5,
+                 architecture):
         """Create a FetchedPackage.
 
         See the instance variables for the arguments.
@@ -88,6 +92,7 @@
         self.content = content
         self.size = size
         self.md5 = md5
+        self.architecture = architecture
 
     def __eq__(self, other):
         return (self.name == other.name
@@ -95,7 +100,8 @@
                 and self.filename == other.filename
                 and self.content.read() == other.content.read()
                 and self.size == other.size
-                and self.md5 == other.md5)
+                and self.md5 == other.md5
+                and self.architecture == other.architecture)
 
     def __hash__(self):
         return hash(
@@ -105,7 +111,7 @@
 class PackageFetcher(object):
     """A class to fetch packages from a defined list of sources."""
 
-    def __init__(self, sources):
+    def __init__(self, sources, architecture=None):
         """Create a PackageFetcher.
 
         Once created a PackageFetcher should have its `prepare` method
@@ -114,8 +120,11 @@
         :param sources: a list of sources such that they can be prefixed
             with "deb " and fed to apt.
         :type sources: an iterable of str
+        :param architecture: the architecture to fetch packages for.
+        :type architecture: str
         """
         self.sources = sources
+        self.architecture = architecture
         self.tempdir = None
 
     def prepare(self):
@@ -143,6 +152,10 @@
         with open(sources_list, 'w') as f:
             for source in self.sources:
                 f.write("deb %s\n" % source)
+        if self.architecture is not None:
+            apt_conf = os.path.join(self.tempdir, "etc", "apt", "apt.conf")
+            with open(apt_conf, 'w') as f:
+                f.write('Apt {\nArchitecture "%s";\n}\n' % self.architecture)
         self.cache = Cache(rootdir=self.tempdir, memonly=True)
         self.cache.update()
         self.cache.open()
@@ -183,6 +196,7 @@
                     (acqfile.destfile, acqfile.error_text))
             result_package = FetchedPackage(
                 candidate.package.name, candidate.version, base,
-                open(destfile), candidate.size, candidate.md5)
+                open(destfile), candidate.size, candidate.md5,
+                candidate.architecture)
             results.append(result_package)
         return results

=== modified file 'hwpack/testing.py'
--- hwpack/testing.py	2010-09-02 23:07:45 +0000
+++ hwpack/testing.py	2010-09-02 23:07:46 +0000
@@ -52,16 +52,10 @@
     See FetchedPackage for the instance variables.
     """
 
-    def __init__(self, name, version):
-        """Create a DummyFetchedPackage.
-
-        :param name: the name of the package.
-        :type name: str
-        :param version: the version of the package.
-        :type version: str
-        """
+    def __init__(self, name, version, architecture="all"):
         self.name = name
         self.version = version
+        self.architecture = architecture
 
     @property
     def filename(self):

=== modified file 'hwpack/tests/test_hardwarepack.py'
--- hwpack/tests/test_hardwarepack.py	2010-09-02 23:07:45 +0000
+++ hwpack/tests/test_hardwarepack.py	2010-09-02 23:07:46 +0000
@@ -12,56 +12,67 @@
 class MetadataTests(TestCase):
 
     def test_name(self):
-        metadata = Metadata("ahwpack", "3")
+        metadata = Metadata("ahwpack", "3", "armel")
         self.assertEqual("ahwpack", metadata.name)
 
     def test_version(self):
-        metadata = Metadata("ahwpack", "3")
+        metadata = Metadata("ahwpack", "3", "armel")
         self.assertEqual("3", metadata.version)
 
+    def test_architecture(self):
+        metadata = Metadata("ahwpack", "3", "armel")
+        self.assertEqual("armel", metadata.architecture)
+
     def test_default_origin_is_None(self):
-        metadata = Metadata("ahwpack", "4")
+        metadata = Metadata("ahwpack", "4", "armel")
         self.assertEqual(None, metadata.origin)
 
     def test_origin(self):
-        metadata = Metadata("ahwpack", "4", origin="linaro")
+        metadata = Metadata("ahwpack", "4", "armel", origin="linaro")
         self.assertEqual("linaro", metadata.origin)
 
     def test_default_maintainer_is_None(self):
-        metadata = Metadata("ahwpack", "4")
+        metadata = Metadata("ahwpack", "4", "armel")
         self.assertEqual(None, metadata.maintainer)
 
     def test_maintainer(self):
-        metadata = Metadata("ahwpack", "4", maintainer="Some maintainer")
+        metadata = Metadata(
+            "ahwpack", "4", "armel", maintainer="Some maintainer")
         self.assertEqual("Some maintainer", metadata.maintainer)
 
     def test_default_support_is_None(self):
-        metadata = Metadata("ahwpack", "4")
+        metadata = Metadata("ahwpack", "4", "armel")
         self.assertEqual(None, metadata.support)
 
     def test_support(self):
-        metadata = Metadata("ahwpack", "4", support="supported")
+        metadata = Metadata("ahwpack", "4", "armel", support="supported")
         self.assertEqual("supported", metadata.support)
 
     def test_str(self):
-        metadata = Metadata("ahwpack", "4")
-        self.assertEqual("NAME=ahwpack\nVERSION=4\n", str(metadata))
+        metadata = Metadata("ahwpack", "4", "armel")
+        self.assertEqual(
+            "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n", str(metadata))
 
     def test_str_with_origin(self):
-        metadata = Metadata("ahwpack", "4", origin="linaro")
+        metadata = Metadata("ahwpack", "4", "armel", origin="linaro")
         self.assertEqual(
-            "NAME=ahwpack\nVERSION=4\nORIGIN=linaro\n", str(metadata))
+            "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\nORIGIN=linaro\n",
+            str(metadata))
 
     def test_str_with_maintainer(self):
-        metadata = Metadata("ahwpack", "4", maintainer="Some Maintainer")
+        metadata = Metadata(
+            "ahwpack", "4", "armel", maintainer="Some Maintainer")
         self.assertEqual(
-            "NAME=ahwpack\nVERSION=4\nMAINTAINER=Some Maintainer\n",
+            "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n"
+            "MAINTAINER=Some Maintainer\n",
             str(metadata))
 
     def test_str_with_support(self):
-        metadata = Metadata("ahwpack", "4", support="unsupported")
+        metadata = Metadata("ahwpack", "4", "armel", support="unsupported")
         self.assertEqual(
-            "NAME=ahwpack\nVERSION=4\nSUPPORT=unsupported\n", str(metadata))
+            "NAME=ahwpack\nVERSION=4\nARCHITECTURE=armel\n"
+            "SUPPORT=unsupported\n",
+            str(metadata))
 
 
 class HardwarePackHasFile(TarfileHasFile):
@@ -111,7 +122,7 @@
 
     def setUp(self):
         super(HardwarePackTests, self).setUp()
-        self.metadata = Metadata("ahwpack", 4)
+        self.metadata = Metadata("ahwpack", 4, "armel")
 
     def test_format_is_1_0(self):
         hwpack = HardwarePack(self.metadata)
@@ -122,7 +133,7 @@
         self.assertEqual("hwpack_ahwpack_4.tar.gz", hwpack.filename())
 
     def test_filename_with_support(self):
-        metadata = Metadata("ahwpack", "4", support="supported")
+        metadata = Metadata("ahwpack", "4", "armel", support="supported")
         hwpack = HardwarePack(metadata)
         self.assertEqual(
             "hwpack_ahwpack_4_supported.tar.gz", hwpack.filename())
@@ -144,7 +155,7 @@
 
     def test_creates_metadata_file(self):
         metadata = Metadata(
-            "ahwpack", "4", origin="linaro",
+            "ahwpack", "4", "armel", origin="linaro",
             maintainer="Some Maintainer", support="unsupported")
         hwpack = HardwarePack(metadata)
         tf = self.get_tarfile(hwpack)

=== modified file 'hwpack/tests/test_packages.py'
--- hwpack/tests/test_packages.py	2010-09-02 23:07:45 +0000
+++ hwpack/tests/test_packages.py	2010-09-02 23:07:46 +0000
@@ -19,13 +19,13 @@
 class GetPackagesFileTests(TestCase):
 
     def test_single_stanza(self):
-        package = DummyFetchedPackage("foo", "1.1")
+        package = DummyFetchedPackage("foo", "1.1", architecture="armel")
         self.assertEqual(textwrap.dedent("""\
             Package: foo
             Version: 1.1
             Filename: %(filename)s
             Size: %(size)d
-            Architecture: all
+            Architecture: armel
             MD5sum: %(md5)s
             \n""" % {
                 'filename': package.filename,
@@ -45,68 +45,77 @@
 
     def test_attributes(self):
         package = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa",
+            "armel")
         self.assertEqual("foo", package.name)
         self.assertEqual("1.1", package.version)
         self.assertEqual("foo_1.1.deb", package.filename)
         self.assertEqual("xxxx", package.content.read())
         self.assertEqual(4, package.size)
         self.assertEqual("aaaa", package.md5)
+        self.assertEqual("armel", package.architecture)
 
     def test_equal(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         self.assertEqual(package1, package2)
 
     def test_not_equal_different_name(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "bar", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "bar", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         self.assertNotEqual(package1, package2)
 
     def test_not_equal_different_version(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.2", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.2", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         self.assertNotEqual(package1, package2)
 
     def test_not_equal_different_filename(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.1", "afoo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "afoo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         self.assertNotEqual(package1, package2)
 
     def test_not_equal_different_content(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("yyyy"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("yyyy"), 4, "aaaa", "armel")
         self.assertNotEqual(package1, package2)
 
     def test_not_equal_different_size(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 5, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 5, "aaaa", "armel")
         self.assertNotEqual(package1, package2)
 
     def test_not_equal_different_md5(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
-        package2 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "bbbb")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
+        package2 = FetchedPackage(
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "bbbb", "armel")
+        self.assertNotEqual(package1, package2)
+
+    def test_not_equal_different_architecture(self):
+        package1 = FetchedPackage(
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
+        package2 = FetchedPackage(
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "i386")
         self.assertNotEqual(package1, package2)
 
     def test_equal_hash_equal(self):
         package1 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         package2 = FetchedPackage(
-            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa")
+            "foo", "1.1", "foo_1.1.deb", StringIO("xxxx"), 4, "aaaa", "armel")
         self.assertEqual(hash(package1), hash(package2))
 
 
@@ -177,8 +186,18 @@
             open(os.path.join(
                 fetcher.tempdir, "etc", "apt", "sources.list")).read())
 
-    def get_fetcher(self, sources):
-        fetcher = PackageFetcher([s.sources_entry for s in sources])
+    def test_prepare_with_arch_creates_etc_apt_apt_conf(self):
+        fetcher = PackageFetcher([], architecture="arch")
+        self.addCleanup(fetcher.cleanup)
+        fetcher.prepare()
+        self.assertEqual(
+            'Apt {\nArchitecture "arch";\n}\n',
+            open(os.path.join(
+                fetcher.tempdir, "etc", "apt", "apt.conf")).read())
+
+    def get_fetcher(self, sources, architecture=None):
+        fetcher = PackageFetcher(
+            [s.sources_entry for s in sources], architecture=architecture)
         self.addCleanup(fetcher.cleanup)
         fetcher.prepare()
         return fetcher
@@ -257,3 +276,22 @@
         fetcher = self.get_fetcher([old_source, new_source])
         fetched = fetcher.fetch_packages(["bar"])
         self.assertEqual(new_source_packages[0], fetched[0])
+
+    def test_fetch_package_records_correct_architecture(self):
+        available_package = DummyFetchedPackage(
+            "foo", "1.0", architecture="nonexistant")
+        source = self.useFixture(AptSourceFixture([available_package]))
+        fetcher = self.get_fetcher([source], architecture="nonexistant")
+        self.assertEqual(
+            "nonexistant", fetcher.fetch_packages(["foo"])[0].architecture)
+
+    def test_fetch_package_fetches_from_correct_architecture(self):
+        wanted_package = DummyFetchedPackage(
+            "foo", "1.0", architecture="arch1")
+        unwanted_package = DummyFetchedPackage(
+            "foo", "1.1", architecture="arch2")
+        source = self.useFixture(
+            AptSourceFixture([wanted_package, unwanted_package]))
+        fetcher = self.get_fetcher([source], architecture="arch1")
+        self.assertEqual(
+            wanted_package, fetcher.fetch_packages(["foo"])[0])