Merge lp:~james-w/linaro-image-tools/hwpack-builder into lp:linaro-image-tools/11.11

[James Westby]

Hi,

A larger branch this time.

This adds the class that orchestrates the different parts in order
to produce the hardware packs. It's fairly simple itself, and the
majority of the additions are test code and test helper code.

Thanks,

James

[Zygmunt Krynicki (zyga)]

I just read the dif, didn't run the code yet. Looks very solid!

I like how matchers work, need to try them out

review: Approve

=== added file 'hwpack/builder.py'
--- hwpack/builder.py	1970-01-01 00:00:00 +0000
+++ hwpack/builder.py	2010-09-10 16:11:07 +0000
@@ -0,0 +1,42 @@
+import errno
+
+from hwpack.config import Config
+from hwpack.hardwarepack import HardwarePack, Metadata
+from hwpack.packages import PackageFetcher
+
+
+class ConfigFileMissing(Exception):
+
+    def __init__(self, filename):
+        self.filename = filename
+        super(ConfigFileMissing, self).__init__(
+            "No such config file: '%s'" % self.filename)
+
+
+class HardwarePackBuilder(object):
+
+    def __init__(self, config_path, version):
+        try:
+            with open(config_path) as fp:
+                self.config = Config(fp)
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise ConfigFileMissing(config_path)
+            raise
+        self.config.validate()
+        self.version = version
+
+    def build(self):
+        for architecture in self.config.architectures:
+            metadata = Metadata.from_config(
+                self.config, self.version, architecture)
+            hwpack = HardwarePack(metadata)
+            sources = self.config.sources
+            hwpack.add_apt_sources(sources)
+            fetcher = PackageFetcher(
+                sources.values(), architecture=architecture)
+            with fetcher:
+                packages = fetcher.fetch_packages(self.config.packages)
+                hwpack.add_packages(packages)
+                with open(hwpack.filename(), 'w') as f:
+                    hwpack.to_file(f)

=== modified file 'hwpack/testing.py'
--- hwpack/testing.py	2010-09-02 15:30:36 +0000
+++ hwpack/testing.py	2010-09-10 16:11:07 +0000
@@ -7,8 +7,10 @@
 import tarfile
 
 from testtools import TestCase
+from testtools.matchers import Matcher, Mismatch
 
 from hwpack.better_tarfile import writeable_tarfile
+from hwpack.tarfile_matchers import TarfileHasFile
 from hwpack.packages import get_packages_file, FetchedPackage
 
 
@@ -136,3 +138,158 @@
         self.addCleanup(fixture.tearDown)
         fixture.setUp()
         return fixture
+
+
+class ConfigFileFixture(object):
+
+    def __init__(self, contents):
+        self.contents = contents
+        self.filename = None
+
+    def setUp(self):
+        fh, self.filename = tempfile.mkstemp(prefix="hwpack-test-config-")
+        with os.fdopen(fh, 'w') as f:
+            f.write(self.contents)
+
+    def tearDown(self):
+        if self.filename is not None and os.path.exists(self.filename):
+            os.unlink(self.filename)
+
+
+class ChdirToTempdirFixture(object):
+
+    def __init__(self):
+        self._orig_dir = None
+        self.tempdir = None
+
+    def setUp(self):
+        self.tearDown()
+        self._orig_dir = os.getcwd()
+        self.tempdir = tempfile.mkdtemp(prefix="hwpack-tests-")
+        os.chdir(self.tempdir)
+
+    def tearDown(self):
+        if self._orig_dir is not None:
+            os.chdir(self._orig_dir)
+            self._orig_dir = None
+        if self.tempdir is not None and os.path.exists(self.tempdir):
+            shutil.rmtree(self.tempdir)
+            self.tempdir = None
+
+
+class MismatchesAll(Mismatch):
+    """A mismatch with many child mismatches."""
+
+    def __init__(self, mismatches):
+        self.mismatches = mismatches
+
+    def describe(self):
+        descriptions = ["Differences: ["]
+        for mismatch in self.mismatches:
+            descriptions.append(mismatch.describe())
+        descriptions.append("]\n")
+        return '\n'.join(descriptions)
+
+
+class MatchesAll(object):
+
+    def __init__(self, *matchers):
+        self.matchers = matchers
+
+    def __str__(self):
+        return 'MatchesAll(%s)' % ', '.join(map(str, self.matchers))
+
+    def match(self, matchee):
+        results = []
+        for matcher in self.matchers:
+            mismatch = matcher.match(matchee)
+            if mismatch is not None:
+                results.append(mismatch)
+        if results:
+            return MismatchesAll(results)
+        else:
+            return None
+
+
+class HardwarePackHasFile(TarfileHasFile):
+    """A subclass of TarfileHasFile specific to hardware packs.
+
+    We default to a set of attributes expected for files in a hardware
+    pack.
+    """
+
+    def __init__(self, path, **kwargs):
+        """Create a HardwarePackHasFile matcher.
+
+        The kwargs are the keyword arguments taken by TarfileHasFile.
+        If they are not given then defaults will be checked:
+            - The type should be a regular file
+            - If the content is given then the size will be checked
+                to ensure it indicates the length of the content
+                correctly.
+            - the mode is appropriate for the type. If the type is
+                regular file this is 0644, otherwise if it is
+                a directory then it is 0755.
+            - the linkname should be the empty string.
+            - the uid and gid should be 1000
+            - the uname and gname should be "user" and "group"
+                respectively.
+
+        :param path: the path that should be present.
+        :type path: str
+        """
+        kwargs.setdefault("type", tarfile.REGTYPE)
+        if "content" in kwargs:
+            kwargs.setdefault("size", len(kwargs["content"]))
+        if kwargs["type"] == tarfile.DIRTYPE:
+            kwargs.setdefault("mode", 0755)
+        else:
+            kwargs.setdefault("mode", 0644)
+        kwargs.setdefault("linkname", "")
+        kwargs.setdefault("uid", 1000)
+        kwargs.setdefault("gid", 1000)
+        kwargs.setdefault("uname", "user")
+        kwargs.setdefault("gname", "group")
+        # TODO: mtime checking
+        super(HardwarePackHasFile, self).__init__(path, **kwargs)
+
+
+class IsHardwarePack(Matcher):
+
+    def __init__(self, metadata, packages, sources):
+        self.metadata = metadata
+        self.packages = packages
+        self.sources = sources
+
+    def match(self, path):
+        tf = tarfile.open(name=path, mode="r:gz")
+        try:
+            matchers = []
+            matchers.append(HardwarePackHasFile("FORMAT", content="1.0\n"))
+            matchers.append(HardwarePackHasFile(
+                "metadata", content=str(self.metadata)))
+            manifest = ""
+            for package in self.packages:
+                manifest += "%s=%s\n" % (package.name, package.version)
+            matchers.append(HardwarePackHasFile("manifest", content=manifest))
+            matchers.append(HardwarePackHasFile("pkgs", type=tarfile.DIRTYPE))
+            for package in self.packages:
+                matchers.append(HardwarePackHasFile(
+                    "pkgs/%s" % package.filename,
+                    content=package.content.read()))
+            matchers.append(HardwarePackHasFile(
+                "pkgs/Packages", content=get_packages_file(self.packages)))
+            matchers.append(HardwarePackHasFile(
+                "sources.list.d", type=tarfile.DIRTYPE))
+            for source_id, sources_entry in self.sources.items():
+                matchers.append(HardwarePackHasFile(
+                    "sources.list.d/%s" % source_id,
+                    content="deb " + sources_entry + "\n"))
+            matchers.append(HardwarePackHasFile(
+                "sources.list.d.gpg", type=tarfile.DIRTYPE))
+            return MatchesAll(*matchers).match(tf)
+        finally:
+            tf.close()
+
+    def __str__(self):
+        return "Is a valid hardware pack."

=== modified file 'hwpack/tests/__init__.py'
--- hwpack/tests/__init__.py	2010-09-01 20:33:33 +0000
+++ hwpack/tests/__init__.py	2010-09-10 16:11:07 +0000
@@ -3,6 +3,7 @@
 def test_suite():
     module_names = ['hwpack.tests.test_config',
                     'hwpack.tests.test_better_tarfile',
+                    'hwpack.tests.test_builder',
                     'hwpack.tests.test_hardwarepack',
                     'hwpack.tests.test_packages',
                     'hwpack.tests.test_tarfile_matchers',

=== added file 'hwpack/tests/test_builder.py'
--- hwpack/tests/test_builder.py	1970-01-01 00:00:00 +0000
+++ hwpack/tests/test_builder.py	2010-09-10 16:11:07 +0000
@@ -0,0 +1,74 @@
+import os
+
+from testtools import TestCase
+
+from hwpack.builder import ConfigFileMissing, HardwarePackBuilder
+from hwpack.config import HwpackConfigError
+from hwpack.hardwarepack import Metadata
+from hwpack.testing import (
+    AptSourceFixture,
+    ChdirToTempdirFixture,
+    ConfigFileFixture,
+    DummyFetchedPackage,
+    IsHardwarePack,
+    TestCaseWithFixtures,
+    )
+
+
+class ConfigFileMissingTests(TestCase):
+
+    def test_str(self):
+        exc = ConfigFileMissing("path")
+        self.assertEqual("No such config file: 'path'", str(exc))
+
+
+class HardwarePackBuilderTests(TestCaseWithFixtures):
+
+    def setUp(self):
+        super(HardwarePackBuilderTests, self).setUp()
+        self.useFixture(ChdirToTempdirFixture())
+
+    def test_raises_on_missing_configuration(self):
+        e = self.assertRaises(
+            ConfigFileMissing, HardwarePackBuilder, "nonexistant", "1.0")
+        self.assertEqual("nonexistant", e.filename)
+
+    def test_validates_configuration(self):
+        config = self.useFixture(ConfigFileFixture(''))
+        self.assertRaises(
+            HwpackConfigError, HardwarePackBuilder, config.filename, "1.0")
+
+    def test_builds_one_pack_per_arch(self):
+        available_package = DummyFetchedPackage("foo", "1.1")
+        source = self.useFixture(AptSourceFixture([available_package]))
+        config = self.useFixture(ConfigFileFixture(
+            '[hwpack]\nname=ahwpack\npackages=foo\narchitectures=i386 armel\n'
+            '\n[ubuntu]\nsources-entry=%s\n' % source.sources_entry))
+        builder = HardwarePackBuilder(config.filename, "1.0")
+        builder.build()
+        self.assertTrue(os.path.isfile("hwpack_ahwpack_1.0_i386.tar.gz"))
+        self.assertTrue(os.path.isfile("hwpack_ahwpack_1.0_armel.tar.gz"))
+
+    def test_builds_correct_contents(self):
+        hwpack_name = "ahwpack"
+        hwpack_version = "1.0"
+        architecture = "armel"
+        package_name = "foo"
+        source_id = "ubuntu"
+        available_package = DummyFetchedPackage(
+            package_name, "1.1", architecture=architecture)
+        source = self.useFixture(AptSourceFixture([available_package]))
+        config = self.useFixture(ConfigFileFixture(
+            '[hwpack]\nname=%s\npackages=%s\narchitectures=%s\n'
+            '\n[%s]\nsources-entry=%s\n'
+            % (hwpack_name, package_name, architecture,
+                source_id, source.sources_entry)))
+        builder = HardwarePackBuilder(config.filename, hwpack_version)
+        builder.build()
+        metadata = Metadata(hwpack_name, hwpack_version, architecture)
+        self.assertThat(
+            "hwpack_%s_%s_%s.tar.gz" % (hwpack_name, hwpack_version,
+                architecture),
+            IsHardwarePack(
+                metadata, [available_package],
+                {source_id: source.sources_entry}))

=== modified file 'hwpack/tests/test_hardwarepack.py'
--- hwpack/tests/test_hardwarepack.py	2010-09-10 16:11:07 +0000
+++ hwpack/tests/test_hardwarepack.py	2010-09-10 16:11:07 +0000
@@ -5,8 +5,7 @@
 
 from hwpack.hardwarepack import HardwarePack, Metadata
 from hwpack.packages import get_packages_file
-from hwpack.tarfile_matchers import TarfileHasFile
-from hwpack.testing import DummyFetchedPackage
+from hwpack.testing import DummyFetchedPackage, HardwarePackHasFile
 
 
 class MetadataTests(TestCase):
@@ -90,49 +89,6 @@
         self.assertEqual("i386", metadata.architecture)
 
 
-class HardwarePackHasFile(TarfileHasFile):
-    """A subclass of TarfileHasFile specific to hardware packs.
-
-    We default to a set of attributes expected for files in a hardware
-    pack.
-    """
-
-    def __init__(self, path, **kwargs):
-        """Create a HardwarePackHasFile matcher.
-
-        The kwargs are the keyword arguments taken by TarfileHasFile.
-        If they are not given then defaults will be checked:
-            - The type should be a regular file
-            - If the content is given then the size will be checked
-                to ensure it indicates the length of the content
-                correctly.
-            - the mode is appropriate for the type. If the type is
-                regular file this is 0644, otherwise if it is
-                a directory then it is 0755.
-            - the linkname should be the empty string.
-            - the uid and gid should be 1000
-            - the uname and gname should be "user" and "group"
-                respectively.
-
-        :param path: the path that should be present.
-        :type path: str
-        """
-        kwargs.setdefault("type", tarfile.REGTYPE)
-        if "content" in kwargs:
-            kwargs.setdefault("size", len(kwargs["content"]))
-        if kwargs["type"] == tarfile.DIRTYPE:
-            kwargs.setdefault("mode", 0755)
-        else:
-            kwargs.setdefault("mode", 0644)
-        kwargs.setdefault("linkname", "")
-        kwargs.setdefault("uid", 1000)
-        kwargs.setdefault("gid", 1000)
-        kwargs.setdefault("uname", "user")
-        kwargs.setdefault("gname", "group")
-        # TODO: mtime checking
-        super(HardwarePackHasFile, self).__init__(path, **kwargs)
-
-
 class HardwarePackTests(TestCase):
 
     def setUp(self):