Merge lp:~milo/linaro-image-tools/dtb-files-support into lp:linaro-image-tools/11.11

[Milo Casagrande]

Branch adds dtb_files support for hwpack config and metadata.
dtb_files syntax is:

dtb_files:
- destination.dtb : source.dtb

source.dtb can contain basic globbing.

[James Tunnicliffe (dooferlad)]

Looks good.

It would be nice if we could combine all the static files copying code into one place. This is similar to the boot files copying that we do, but not the same as copy_files (since those files stay in their packages).

There is definitely room for tidying up that whole code area, but a bug fix isn't the place for it. If you have nothing else to do I am sure it will be... entertaining :-)

review: Approve

[Milo Casagrande (milo)]

Hey James,

thanks for taking a look.

On Mon, Oct 1, 2012 at 5:14 PM, James Tunnicliffe
<email address hidden> wrote:
> Review: Approve
>
> Looks good.
>
> It would be nice if we could combine all the static files copying code into one place. This is similar to the boot files copying that we do, but not the same as copy_files (since those files stay in their packages).
>
> There is definitely room for tidying up that whole code area, but a bug fix isn't the place for it. If you have nothing else to do I am sure it will be... entertaining :-)

Indeed, I wanted to combine them, but it would have been a little bit
too much for the bug report.
I will indeed try to do it during this maintenance cycle, maybe in a
separate bug.

Ciao.

--
Milo Casagrande
Infrastructure Engineer
Linaro.org <www.linaro.org> │ Open source software for ARM SoCs

=== modified file 'linaro_image_tools/hwpack/config.py'
--- linaro_image_tools/hwpack/config.py	2012-09-11 14:28:20 +0000
+++ linaro_image_tools/hwpack/config.py	2012-10-01 10:21:03 +0000
@@ -44,6 +44,7 @@
     DD_FIELD,
     DTB_ADDR_FIELD,
     DTB_FILE_FIELD,
+    DTB_FILES_FIELD,
     ENV_DD_FIELD,
     EXTRA_BOOT_OPTIONS_FIELD,
     EXTRA_SERIAL_OPTIONS_FIELD,
@@ -269,6 +270,7 @@
             self._validate_vmlinuz()
             self._validate_initrd()
             self._validate_dtb_file()
+            self._validate_dtb_files()
             self._validate_mmc_id()
             self._validate_extra_boot_options()
             self._validate_boot_script()
@@ -821,6 +823,14 @@
         return self._get_option(DTB_FILE_FIELD)
 
     @property
+    def dtb_files(self):
+        """
+        The list of dtb files.
+        :return: A list of dtb files
+        """
+        return self._get_option(DTB_FILES_FIELD)
+
+    @property
     def samsung_bl1_start(self):
         """BL1 start offset for Samsung boards.
 
@@ -953,7 +963,16 @@
             return "No " + thing + " in the [" + v2_section + "] section"
 
     def _validate_dtb_file(self):
-        dtb_file = self.dtb_file
+        self._check_single_dtb_file(self.dtb_file)
+
+    def _validate_dtb_files(self):
+        dtb_files = self.dtb_files
+        if dtb_files:
+            for dtb_file in dtb_files:
+                for _, src in dtb_file.iteritems():
+                    self._check_single_dtb_file(src)
+
+    def _check_single_dtb_file(self, dtb_file):
         if dtb_file is not None:
             self._assert_matches_pattern(
                 self.GLOB_REGEX, dtb_file, "Invalid path: %s" % dtb_file)

=== modified file 'linaro_image_tools/hwpack/hardwarepack.py'
--- linaro_image_tools/hwpack/hardwarepack.py	2012-08-29 10:31:19 +0000
+++ linaro_image_tools/hwpack/hardwarepack.py	2012-10-01 10:21:03 +0000
@@ -43,6 +43,7 @@
     BOOT_SCRIPT_FIELD,
     DTB_ADDR_FIELD,
     DTB_FILE_FIELD,
+    DTB_FILES_FIELD,
     EXTRA_SERIAL_OPTIONS_FIELD,
     FORMAT_FIELD,
     INITRD_ADDR_FIELD,
@@ -166,14 +167,16 @@
         self.samsung_bl2_len = samsung_bl2_len
 
     @classmethod
-    def add_v3_config(self, boards=None, bootloaders=None):
+    def add_v3_config(self, boards=None, bootloaders=None, dtb_files=None):
         """Add fields that are specific to the v3 config format.
         These fields are not present in the earlier config files.
 
         :param boards: The boards section of the hwpack.
-        :param bootloaders: The bootloaders section of the hwpack."""
+        :param bootloaders: The bootloaders section of the hwpack.
+        :param dtb_files: The dtb_files section of the hwpack."""
         self.boards = boards
         self.bootloaders = bootloaders
+        self.dtb_files = dtb_files
 
     @classmethod
     def from_config(cls, config, version, architecture):
@@ -235,7 +238,8 @@
                 )
         if config.format.format_as_string == '3.0':
             metadata.add_v3_config(boards=config.boards,
-                                    bootloaders=config.bootloaders)
+                                   bootloaders=config.bootloaders,
+                                   dtb_files=config.dtb_files)
         return metadata
 
     def __str__(self):
@@ -305,6 +309,8 @@
         if self.dtb_file is not None:
             # XXX In V3 this one should be a list, called dtb_files.
             metadata += dump({DTB_FILE_FIELD: self.dtb_file})
+        if self.dtb_files is not None:
+            metadata += dump({DTB_FILES_FIELD: self.dtb_files})
         if self.boot_script is not None:
             metadata += dump({BOOT_SCRIPT_FIELD: self.boot_script})
         if self.extra_serial_opts is not None:

=== modified file 'linaro_image_tools/hwpack/hwpack_fields.py'
--- linaro_image_tools/hwpack/hwpack_fields.py	2012-09-24 09:07:37 +0000
+++ linaro_image_tools/hwpack/hwpack_fields.py	2012-10-01 10:21:03 +0000
@@ -110,6 +110,7 @@
     ASSUME_INSTALLED_FIELD: None,
     INCLUDE_DEBS_FIELD: None,
     DTB_FILE_FIELD: None,
+    DTB_FILES_FIELD: None,
     DTB_ADDR_FIELD: None,
     SERIAL_TTY_FIELD: None,
     EXTRA_SERIAL_OPTIONS_FIELD: None,

=== modified file 'linaro_image_tools/hwpack/tests/test_config_v3.py'
--- linaro_image_tools/hwpack/tests/test_config_v3.py	2012-09-24 11:29:40 +0000
+++ linaro_image_tools/hwpack/tests/test_config_v3.py	2012-10-01 10:21:03 +0000
@@ -825,3 +825,22 @@
         config = self.get_config(self.valid_start_v3 +
                                  'samsung_wrong_field: 1\n')
         self.assertRaises(HwpackConfigError, config._validate_keys)
+
+    # Tests for dtb_files support
+    def test_dtb_files(self):
+        dtb_files = ('dtb_files:\n' +
+                     ' - adest.dtb : boot/dt-*-linaro-omap/omap4-panda.dtb\n' +
+                     ' - bdest.dtb : ' +
+                     'boot/dt-*-linaro-omap2/omap4-panda2.dtb\n')
+        expected = [{'adest.dtb':'boot/dt-*-linaro-omap/omap4-panda.dtb'},
+                    {'bdest.dtb': 'boot/dt-*-linaro-omap2/omap4-panda2.dtb'}]
+        config = self.get_config(self.valid_complete_v3 + dtb_files)
+        config.validate()
+        self.assertEqual(expected, config.dtb_files)
+
+    def test_dtb_files_one_wrong(self):
+        dtb_files = ('dtb_files:\n' +
+                     ' - adest.dtb : boot/dt-*-linaro-omap/omap4-panda.dtb\n' +
+                     ' - bdest.dtb : ~~~\n')
+        config = self.get_config(self.valid_start_v3 + dtb_files)
+        self.assertRaises(HwpackConfigError, config._validate_dtb_files)

=== modified file 'linaro_image_tools/media_create/boards.py'
--- linaro_image_tools/media_create/boards.py	2012-09-12 10:51:00 +0000
+++ linaro_image_tools/media_create/boards.py	2012-10-01 10:21:03 +0000
@@ -404,6 +404,9 @@
     partition_layout = None
     LOADER_START_S = 1
 
+    # Support for dtb_files as per hwpack v3 format.
+    dtb_files = None
+
     # Samsung v310 implementation notes and terminology
     #
     # * BL0, BL1 etc. are the various bootloaders in order of execution
@@ -495,6 +498,7 @@
             cls.vmlinuz = cls.get_metadata_field('vmlinuz')
             cls.initrd = cls.get_metadata_field('initrd')
             cls.dtb_file = cls.get_metadata_field('dtb_file')
+            cls.dtb_files = cls.get_metadata_field('dtb_files')
             cls.extra_boot_args_options = cls.get_metadata_field(
                 'extra_boot_options')
             cls.boot_script = cls.get_metadata_field('boot_script')
@@ -840,6 +844,53 @@
                 boot_device_or_file, k_img_data, i_img_data, d_img_data)
 
     @classmethod
+    def _copy_dtb_files(cls, dtb_files, dest_dir, search_dir):
+        """Copy the files defined in dtb_files into the boot directory.
+
+        :param dtb_files: The list of dtb files
+        :param dest_dir: The directory where to copy each dtb file.
+        :param search_dir: The directory where to search for the real file.
+        """
+        logger = logging.getLogger("linaro_image_tools")
+        logger.info("Copying dtb files")
+        for dtb_file in dtb_files:
+            if dtb_file:
+                if isinstance(dtb_file, dict):
+                    for key, value in dtb_file.iteritems():
+                        # The name of the dtb file in the new position.
+                        to_file = os.path.basename(key)
+                        # The directory where to copy the dtb file.
+                        to_dir = os.path.join(dest_dir, os.path.dirname(key))
+                        from_file = value
+
+                        # User specified only the directory, without renaming
+                        # the file.
+                        if not to_file:
+                            to_file = os.path.basename(from_file)
+
+                        if not os.path.exists(to_dir):
+                            cmd_runner.run(["mkdir", "-p", to_dir],
+                                           as_root=True).wait()
+                        dtb = _get_file_matching(os.path.join(search_dir,
+                                                              from_file))
+                        if not dtb:
+                            logger.warn('Could not find a valid dtb file, '
+                                        'skipping it.')
+                            continue
+                        else:
+                            dest = os.path.join(to_dir, to_file)
+                            logger.debug('Copying %s into %s' % (dtb, dest))
+                            cmd_runner.run(['cp', dtb, dest],
+                                           as_root=True).wait()
+                else:
+                    # Hopefully we should never get here.
+                    # This should only happen if the hwpack config YAML file is
+                    # wrong
+                    logger.warn('WARNING: Wrong syntax in metadata file. '
+                                'Check the hwpack configuration file used to '
+                                'generate the hwpack archive.')
+
+    @classmethod
     def _dd_file(cls, from_file, to_file, seek, max_size=None):
         assert from_file is not None, "No source file name given."
         if max_size is not None:
@@ -959,6 +1010,10 @@
                 # Handle copy_files field.
                 cls.copy_files(boot_disk)
 
+            # Handle dtb_files field.
+            if cls.dtb_files:
+                cls._copy_dtb_files(cls.dtb_files, boot_disk, chroot_dir)
+
             cls.make_boot_files(
                 bootloader_parts_dir, is_live, is_lowmem, consoles, chroot_dir,
                 rootfs_id, boot_disk, boot_device_or_file)