Merge lp:~milo/linaro-image-tools/hwpack-read into lp:linaro-image-tools/11.11

[Milo Casagrande]

With this merge, there is the initial support for hwpack metadata reading, before creating an image with l-m-c.

What has been added here is:
- A new class for reading hwpack, that makes use of the HwpackHandler to open the tarball files
- New tests for the class
- New command line option --read-hwpack that reads the hwpack metadata and prints out information about the supported boards and bootlaoders

I also fixed a couple of PEP8 issues.
Notes:
- There is one test that will fail, and is related to pyflakes, looks like it is bug 812958 (I used getter and setter)
- Some of the required command line options have been set to False, since if we want to just read the hwpack and see what is supported, we should not require all the information. Since there is still work in progress on this, the bits handling the necessary command line options is still missing.

[lp:~milo/linaro-image-tools/hwpack-read]

542. By Milo Casagrande on 2012-07-25

Added option check.

543. By Milo Casagrande on 2012-07-25

Fixed PEP8 warning.

544. By Milo Casagrande on 2012-07-25

Fixed tests and imports.

[James Tunnicliffe (dooferlad)]

Looks fine now. Thanks for the changes.

review: Approve

[lp:~milo/linaro-image-tools/hwpack-read]

545. By Milo Casagrande on 2012-07-25

Fixed annoying pyflakes warnings.

546. By Milo Casagrande on 2012-07-25

Fixed assertEqual tests.

=== modified file 'linaro-media-create'
--- linaro-media-create	2012-04-19 01:12:10 +0000
+++ linaro-media-create	2012-07-25 13:08:18 +0000
@@ -33,6 +33,10 @@
     install_hwpacks,
     install_packages,
     )
+from linaro_image_tools.hwpack.hwpack_reader import (
+    HwpackReader,
+    HwpackReaderError,
+    )
 from linaro_image_tools.media_create.partitions import (
     Media,
     setup_partitions,
@@ -44,13 +48,15 @@
     )
 from linaro_image_tools.media_create import get_args_parser
 from linaro_image_tools.utils import (
+    additional_option_checks,
+    check_file_integrity_and_log_errors,
+    check_required_args,
     ensure_command,
+    IncompatibleOptions,
     is_arm_host,
-    check_file_integrity_and_log_errors,
+    MissingRequiredOption,
     path_in_tarfile_exists,
-    IncompatibleOptions,
     prep_media_path,
-    additional_option_checks,
     )
 
 # Just define the global variables
@@ -107,6 +113,22 @@
         print >> sys.stderr, "\nError:", e.value
         sys.exit(1)
 
+    if args.readhwpack:
+        try:
+            reader = HwpackReader(args.hwpacks)
+            print reader.get_supported_boards()
+            sys.exit(0)
+        except HwpackReaderError as e:
+            print >> sys.stderr, "\nError:", e.value
+            sys.exit(1)
+
+    try:
+        check_required_args(args)
+    except MissingRequiredOption as e:
+        parser.print_help()
+        print >> sys.stderr, "\nError:", e.value
+        sys.exit(1)
+
     board_config = board_configs[args.board]
     board_config.set_metadata(args.hwpacks)
     board_config.set_board(args.board)
@@ -117,11 +139,12 @@
 
     if media.is_block_device:
         if not board_config.supports_writing_to_mmc:
-            print "The board '%s' does not support the --mmc option. Please use "\
-                  "--image_file to create an image file for this board." % args.board
+            print ("The board '%s' does not support the --mmc option. "
+                    "Please use --image_file to create an image file for this "
+                    "board." % args.board)
             sys.exit(1)
-        if not confirm_device_selection_and_ensure_it_is_ready(args.device,
-                                                               args.nocheck_mmc):
+        if not confirm_device_selection_and_ensure_it_is_ready(
+                args.device, args.nocheck_mmc):
             sys.exit(1)
     elif not args.should_format_rootfs or not args.should_format_bootfs:
         print ("Do not use --no-boot or --no-part in conjunction with "

=== added file 'linaro_image_tools/hwpack/hwpack_reader.py'
--- linaro_image_tools/hwpack/hwpack_reader.py	1970-01-01 00:00:00 +0000
+++ linaro_image_tools/hwpack/hwpack_reader.py	2012-07-25 13:08:18 +0000
@@ -0,0 +1,192 @@
+# Copyright (C) 2012 Linaro
+#
+# Author: Milo Casagrande <[e-mail]>
+#
+# This file is part of Linaro Image Tools.
+#
+# Linaro Image Tools is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Linaro Image Tools is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Linaro Image Tools.  If not, see <http://www.gnu.org/licenses/>.
+
+from linaro_image_tools.media_create.boards import HardwarepackHandler
+from linaro_image_tools.hwpack.hwpack_fields import (
+    FORMAT_FIELD,
+    NAME_FIELD,
+    BOARDS_FIELD,
+    BOOTLOADERS_FIELD,
+)
+
+from os import linesep as LINE_SEP
+
+# Fields necessary for the string representation of the hardware pack supported
+# boards and bootlaoders.
+HALF_SEPARATOR = '+--------------------------------------'
+ENDING = '+'
+SEPARATOR = HALF_SEPARATOR * 2 + ENDING
+FORMAT = '{:<80}'
+CENTER_ALIGN = '{:^80}'
+ELEMENT_FORMAT = '{:<39}| {:<39}'
+
+
+class HwpackReaderError(Exception):
+    """General error raised by HwpackReader."""
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
+class Hwpack(object):
+    """A simple representation of an hardware pack and its value."""
+    def __init__(self):
+        self._hwpack = None
+        self._boards = None
+        self._bootloaders = None
+        self._name = None
+
+    @property
+    def hwpack(self):
+        """The hardware pack it refers to."""
+        return self._hwpack
+
+    def sethwpack(self, value):
+        """Sets the hwpack field."""
+        self._hwpack = value
+
+    @property
+    def boards(self):
+        """The boards field of this hardware pack."""
+        return self._boards
+
+    def setboards(self, value):
+        """Sets the boards field."""
+        self._boards = value
+
+    @property
+    def name(self):
+        """The name field of this hardware pack."""
+        return self._name
+
+    def setname(self, value):
+        """Sets the name field."""
+        self._name = value
+
+    @property
+    def bootloaders(self):
+        """The bootlaoders field of this hardware pack."""
+        return self._bootloaders
+
+    def setbootloaders(self, value):
+        """Sets the bootlaoders field."""
+        self._bootloaders = value
+
+    def __eq__(self, other):
+        """Equality method."""
+        equal = False
+        if isinstance(other, Hwpack):
+            equal = (self.name == other.name and
+                        self.boards == other.boards and
+                        self.hwpack == other.hwpack and
+                        self.bootloaders == other.bootloaders)
+        return equal
+
+    def __hash__(self):
+        return hash(frozenset(self.bootloaders), frozenset(self.boards),
+                    self.name, self.hwpack)
+
+    def __str__(self):
+        """String representation of this hwapack supported elements."""
+        string = FORMAT.format("Read hardware pack: %s" % self.hwpack)
+        string += LINE_SEP
+        string += FORMAT.format(SEPARATOR)
+        string += LINE_SEP
+        string += ELEMENT_FORMAT.format("Supported boards",
+                                        "Supported bootloaders")
+        string += LINE_SEP
+        string += FORMAT.format(SEPARATOR)
+        string += LINE_SEP
+
+        if self.boards:
+            for key, value in self.boards.iteritems():
+                if value.get(BOOTLOADERS_FIELD, None) is not None:
+                    bootloaders = value.get(BOOTLOADERS_FIELD)
+                    supported_bootloaders = bootloaders.keys()
+                else:
+                    supported_bootloaders = self.bootloaders.keys()
+                string += ELEMENT_FORMAT.format(key,
+                            ",".join(supported_bootloaders))
+                string += LINE_SEP
+        else:
+            # If we pass a converted file with just a single board, we do not
+            # have the boards section, and we default to the name of the hwpack
+            if self.bootloaders:
+                supported_bootloaders = self.bootloaders.keys()
+                string += ELEMENT_FORMAT.format(self.name,
+                            ",".join(supported_bootloaders))
+                string += LINE_SEP
+            else:
+                string += CENTER_ALIGN.format("No supported boards and "
+                                                "bootloaders")
+                string += LINE_SEP
+        string += FORMAT.format(SEPARATOR)
+        return string + LINE_SEP
+
+
+class HwpackReader(object):
+    """Reads the information contained in a hwpack """
+    def __init__(self, hwpacks):
+        """Create a new instance.
+
+        :param hwpacks: The list of hardware packs to read from."""
+        self.hwpacks = hwpacks
+        # Where we store all the info from the hwpack.
+        self._supported_elements = []
+
+    @property
+    def supported_elements(self):
+        """Gets the supported elements of by all the hardwapare packs."""
+        return self._supported_elements
+
+    def _read_hwpacks_metadata(self):
+        """Reads the hardware pack metadata file, and prints information about
+        the supported boards and bootloaders."""
+        for tarball in self.hwpacks:
+            with HardwarepackHandler([tarball]) as handler:
+                hwpack_format = handler.get_field(FORMAT_FIELD)[0]
+                if hwpack_format.format_as_string == "3.0":
+                    local_hwpack = Hwpack()
+                    local_hwpack.sethwpack(tarball)
+                    local_hwpack.setname(handler.get_field(NAME_FIELD)[0])
+                    local_hwpack.setboards(handler.get_field(BOARDS_FIELD)[0])
+                    local_hwpack.setbootloaders(
+                            handler.get_field(BOOTLOADERS_FIELD)[0])
+                    self.supported_elements.append(local_hwpack)
+                else:
+                    raise HwpackReaderError("Hardwarepack '%s' cannot be "
+                                            "read, unsupported format." %
+                                            (tarball))
+
+    def get_supported_boards(self):
+        """Prints the necessary information.
+
+        :return A string representation of the information."""
+        self._read_hwpacks_metadata()
+        return str(self)
+
+    def __str__(self):
+        """The string representation of this reader. It is a printable
+        representation of the necessary information."""
+        hwpack_reader = ""
+        for element in self.supported_elements:
+            hwpack_reader += str(element)
+        return hwpack_reader

=== modified file 'linaro_image_tools/hwpack/tests/__init__.py'
--- linaro_image_tools/hwpack/tests/__init__.py	2012-07-19 17:57:24 +0000
+++ linaro_image_tools/hwpack/tests/__init__.py	2012-07-25 13:08:18 +0000
@@ -30,6 +30,7 @@
         'linaro_image_tools.hwpack.tests.test_config_v3',
         'linaro_image_tools.hwpack.tests.test_hardwarepack',
         'linaro_image_tools.hwpack.tests.test_hwpack_converter',
+        'linaro_image_tools.hwpack.tests.test_hwpack_reader',
         'linaro_image_tools.hwpack.tests.test_packages',
         'linaro_image_tools.hwpack.tests.test_script',
         'linaro_image_tools.hwpack.tests.test_tarfile_matchers',

=== added file 'linaro_image_tools/hwpack/tests/test_hwpack_reader.py'
--- linaro_image_tools/hwpack/tests/test_hwpack_reader.py	1970-01-01 00:00:00 +0000
+++ linaro_image_tools/hwpack/tests/test_hwpack_reader.py	2012-07-25 13:08:18 +0000
@@ -0,0 +1,114 @@
+# Copyright (C) 2012 Linaro
+#
+# Author: Milo Casagrande <[e-mail]>
+#
+# This file is part of Linaro Image Tools.
+#
+# Linaro Image Tools is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Linaro Image Tools is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Linaro Image Tools.  If not, see <http://www.gnu.org/licenses/>.
+
+import tarfile
+from StringIO import StringIO
+from linaro_image_tools.testing import TestCaseWithFixtures
+from linaro_image_tools.tests.fixtures import (
+    CreateTempDirFixture,
+    )
+
+from linaro_image_tools.media_create.tests.fixtures import (
+    CreateTarballFixture,
+    )
+
+from linaro_image_tools.hwpack.hwpack_reader import (
+    Hwpack,
+    HwpackReader,
+    HwpackReaderError,
+    )
+
+
+class HwpackReaderTests(TestCaseWithFixtures):
+    """Test class for the hwpack reader."""
+
+    def setUp(self):
+        super(HwpackReaderTests, self).setUp()
+        self.metadata = ("format: 3.0\nversion: '1'\nname: test-hwpack\n"
+                            "architecture: armel\norigin: Linaro")
+        self.hwpack = Hwpack()
+        self.hwpack.setname('test-hwpack')
+        self.tar_dir_fixture = CreateTempDirFixture()
+        self.useFixture(self.tar_dir_fixture)
+        self.tarball_fixture = CreateTarballFixture(
+                                self.tar_dir_fixture.get_temp_dir())
+        self.useFixture(self.tarball_fixture)
+
+    def tearDown(self):
+        super(HwpackReaderTests, self).tearDown()
+        self.hwpack = None
+        self.metadata = ""
+
+    def add_to_tarball(self, files, tarball=None):
+        if tarball is None:
+            tarball = self.tarball_fixture.get_tarball()
+        tar_file = tarfile.open(tarball, mode='w:gz')
+        for filename, data in files:
+            tarinfo = tarfile.TarInfo(filename)
+            tarinfo.size = len(data)
+            tar_file.addfile(tarinfo, StringIO(data))
+        tar_file.close()
+        return tarball
+
+    def test_hwpack_class(self):
+        hwpack = Hwpack()
+        hwpack.setname('test-hwpack')
+        hwpack.sethwpack('a_hwpack')
+        self.hwpack.sethwpack('a_hwpack')
+        self.assertEqual(self.hwpack, hwpack)
+
+    def test_hwpack_class_not_equal(self):
+        hwpack = Hwpack()
+        hwpack.setname('test-hwpack')
+        hwpack.sethwpack('a_hwpack')
+        self.hwpack.sethwpack('b_hwpack')
+        self.assertNotEqual(self.hwpack, hwpack)
+
+    def test_hwpack_metadata_read(self):
+        tarball = self.add_to_tarball([('metadata', self.metadata)])
+        reader = HwpackReader([tarball])
+        reader._read_hwpacks_metadata()
+        self.hwpack.sethwpack(tarball)
+        self.assertEqual(self.hwpack, reader.supported_elements[0])
+
+    def test_raise_exception(self):
+        new_metadata = ("format=2.0\nversion=4")
+        tarball = self.add_to_tarball([('metadata', new_metadata)])
+        reader = HwpackReader([tarball])
+        self.assertRaises(HwpackReaderError, reader._read_hwpacks_metadata)
+
+    def test_hwpack_metadata_read_with_boards(self):
+        self.metadata += "\nboards:\n panda:\n  support: supported\n"
+        tarball = self.add_to_tarball([('metadata', self.metadata)])
+        reader = HwpackReader([tarball])
+        reader._read_hwpacks_metadata()
+        self.hwpack.sethwpack(tarball)
+        self.hwpack.setboards({'panda': {'support': 'supported'}})
+        self.assertEqual(self.hwpack, reader.supported_elements[0])
+
+    def test_hwpack_metadata_read_with_bootloaders(self):
+        self.metadata += ("\nboards:\n panda:\n  support: supported\n  "
+                            "bootloaders:\n   u_boot:\n    file: a_file\n")
+        tarball = self.add_to_tarball([('metadata', self.metadata)])
+        reader = HwpackReader([tarball])
+        reader._read_hwpacks_metadata()
+        self.hwpack.sethwpack(tarball)
+        self.hwpack.setboards({'panda': {'support': 'supported', 'bootloaders':
+                                {'u_boot': {'file': 'a_file'}}}})
+        self.assertEqual(self.hwpack, reader.supported_elements[0])

=== modified file 'linaro_image_tools/media_create/__init__.py'
--- linaro_image_tools/media_create/__init__.py	2012-06-13 14:43:59 +0000
+++ linaro_image_tools/media_create/__init__.py	2012-07-25 13:08:18 +0000
@@ -92,7 +92,11 @@
         '--output-directory', dest='directory',
         help='Directory where image and accessories should be written to.')
     parser.add_argument(
-        '--dev', required=True, dest='board', choices=KNOWN_BOARDS,
+        '--read-hwpack', dest='readhwpack', action='store_true',
+        help=('Read the hardware pack and print information about the '
+                'supported boards and bootloaders.'))
+    parser.add_argument(
+        '--dev', dest='board', choices=KNOWN_BOARDS,
         help='Generate an SD card or image for the given board.')
     parser.add_argument(
         '--rootfs', default='ext4', choices=['ext2', 'ext3', 'ext4', 'btrfs'],
@@ -137,7 +141,7 @@
         help=('The image size, specified in mega/giga bytes (e.g. 3000M or '
               '3G); use with --image_file only'))
     parser.add_argument(
-        '--binary', default='binary-tar.tar.gz', required=True,
+        '--binary', default='binary-tar.tar.gz', required=False,
         help=('The tarball containing the rootfs used to create the bootable '
               'system.'))
     parser.add_argument(

=== modified file 'linaro_image_tools/utils.py'
--- linaro_image_tools/utils.py	2012-06-13 14:55:34 +0000
+++ linaro_image_tools/utils.py	2012-07-25 13:08:18 +0000
@@ -306,6 +306,15 @@
     """The hwpack parameter is not a regular file."""
 
 
+class MissingRequiredOption(Exception):
+    """A required option from the command line is missing."""
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
 class IncompatibleOptions(Exception):
     def __init__(self, value):
         self.value = value
@@ -331,3 +340,11 @@
         if not os.path.isfile(hwpack):
             raise InvalidHwpackFile(
                 "--hwpack argument (%s) is not a regular file" % hwpack)
+
+
+def check_required_args(args):
+    """Check that the required args are passed."""
+    if args.board is None:
+        raise MissingRequiredOption("--dev option is required")
+    if args.binary is None:
+        raise MissingRequiredOption("--binary option is required")