Merge lp:~salgado/linaro-image-tools/populate-root into lp:linaro-image-tools/11.11

[Guilherme Salgado]

Port populate_rootfs to python.

[lp:~salgado/linaro-image-tools/populate-root]

196. By Guilherme Salgado on 2010-12-16

Fix arg parsing in rootfs.py

197. By Guilherme Salgado on 2010-12-17

merge trunk

[Guilherme Salgado (salgado)]

I may not be around when this is reviewed so if you, dear reviewer, is happy with it, just go ahead and land. Otherwise I'll address any issues you find when I'm back.

[James Westby (james-w)]

27 -unset INITRD_ADDR CREATE_SWAP SWAP_SIZE DEPLOY_STEPS
28 +unset INITRD_ADDR DEPLOY_STEPS
29
30 declare -a HWPACK_FILES
31
32 @@ -30,6 +30,8 @@
33 BOOT_LABEL=boot
34 RFS_LABEL=rootfs
35 IS_LIVE=0
36 +CREATE_SWAP=0
37 +SWAP_SIZE=0

Do we want to unset the variables if we are using them?

Otherwise this looks fine.

Thanks,

James

review: Approve

[Guilherme Salgado (salgado)]

IIUC, we were unsetting the CREATE_SWAP and SWAP_SIZE because further down the code would then be able to just check for their existence ([ "$CREATE_SWAP" ]) instead of checking for a specific value. Given that these things are now passed on to a python function, I want them to always exist, so I don't unset them anymore.

=== modified file 'hwpack/testing.py'
--- hwpack/testing.py	2010-12-17 00:58:17 +0000
+++ hwpack/testing.py	2010-12-17 11:20:41 +0000
@@ -172,12 +172,12 @@
         fixture.setUp()
         return fixture
 
-    def createTempFileAsFixture(self, prefix='tmp'):
+    def createTempFileAsFixture(self, prefix='tmp', dir=None):
         """Create a temp file and make sure it is removed on tearDown.
 
         :return: The filename of the file created.
         """
-        _, filename = tempfile.mkstemp(prefix=prefix)
+        _, filename = tempfile.mkstemp(prefix=prefix, dir=dir)
         self.addCleanup(os.unlink, filename)
         return filename
 

=== modified file 'linaro-media-create'
--- linaro-media-create	2010-12-16 20:27:05 +0000
+++ linaro-media-create	2010-12-17 11:20:41 +0000
@@ -20,7 +20,7 @@
 set -e
 
 unset DEVICE BOOTFS ROOTFS IMAGE_FILE HWPACK_FILES UBOOT_FLAVOR KERNEL_ADDR
-unset INITRD_ADDR CREATE_SWAP SWAP_SIZE DEPLOY_STEPS
+unset INITRD_ADDR DEPLOY_STEPS
 
 declare -a HWPACK_FILES
 
@@ -30,6 +30,8 @@
 BOOT_LABEL=boot
 RFS_LABEL=rootfs
 IS_LIVE=0
+CREATE_SWAP=0
+SWAP_SIZE=0
 IS_LOWMEM=0
 FAT_SIZE=32
 IMAGE_SIZE=2G
@@ -495,47 +497,8 @@
 }
 
 populate_rootfs() {
-  echo ""
-  echo "Populating rootfs Partition"
-  echo "Be patient, this may take a few minutes"
-  echo ""
-  mkdir -p "${ROOT_DISK}"
-  sudo mount ${ROOTFS} "${ROOT_DISK}"
-
-  sudo mv ${DIR}/binary/* "$ROOT_DISK"
-
-  # add fstab entry for rootfs and boot
-  echo "UUID=${RFS_UUID} / ${RFS}  errors=remount-ro 0 1 " | sudo tee -a "${ROOT_DISK}/etc/fstab"
-
-  if [ "$CREATE_SWAP" ] ; then
-
-    echo ""
-    echo "Creating SWAP File"
-    echo ""
-
-    SPACE_LEFT=$(LC_ALL=C df "${ROOT_DISK}" | grep ${ROOTFS} | awk '{print $4}')
-
-    let SIZE=$SWAP_SIZE*1024
-
-    if [ $SPACE_LEFT -ge $SIZE ] ; then
-      sudo dd if=/dev/zero "of=${ROOT_DISK}/SWAP.swap" bs=1M count=$SWAP_SIZE
-      sudo mkswap "${ROOT_DISK}/SWAP.swap"
-      echo "/SWAP.swap  none  swap  sw  0 0" | sudo tee -a "${ROOT_DISK}/etc/fstab"
-    else
-      echo "SWAP file bigger then whats left on partition"
-    fi
-  fi
-
-  echo ""
-  echo "Creating /etc/flash-kernel.conf"
-  echo ""
-
-  TARGET_BOOT_DEV=/dev/mmcblk0p$(( 1 + $MMC_PART_OFFSET ))
-  echo "UBOOT_PART=${TARGET_BOOT_DEV}" | sudo tee "${ROOT_DISK}/etc/flash-kernel.conf" >/dev/null
-
-  sync
-  sync
-  sudo umount "${ROOT_DISK}" || true
+  python -m media_create.rootfs "binary/" "$ROOT_DISK" "$ROOTFS" "$RFS" \
+      "$RFS_UUID" "$CREATE_SWAP" "$SWAP_SIZE" "$MMC_PART_OFFSET"
 }
 
 remove_image_file() {

=== added file 'media_create/rootfs.py'
--- media_create/rootfs.py	1970-01-01 00:00:00 +0000
+++ media_create/rootfs.py	2010-12-17 11:20:41 +0000
@@ -0,0 +1,126 @@
+import glob
+import os
+import sys
+import tempfile
+
+from media_create import cmd_runner
+
+
+def populate_rootfs(content_dir, root_disk, partition, rootfs_type,
+                    rootfs_uuid, should_create_swap, swap_size,
+                    partition_offset):
+    """Populate the rootfs and make the necessary tweaks to make it usable.
+
+    This consists of:
+      1. Create a directory on the path specified by root_disk
+      2. Mount the given partition onto the created directory.
+      3. Move the contents of content_dir to that directory.
+      4. If should_create_swap, then create it with the given size.
+      5. Add fstab entries for the / filesystem and swap (if created).
+      6. Create a /etc/flash-kernel.conf containing the target's boot device.
+      7. Unmount the partition we mounted on step 2.
+    """
+    print "\nPopulating rootfs partition"
+    print "Be patient, this may take a few minutes\n"
+    # Create a directory to mount the rootfs partition.
+    os.makedirs(root_disk)
+
+    cmd_runner.run(['mount', partition, root_disk], as_root=True).wait()
+
+    move_contents(content_dir, root_disk)
+
+    fstab_additions = ["UUID=%s / %s  errors=remount-ro 0 1 " % (
+            rootfs_uuid, rootfs_type)]
+    if should_create_swap:
+        print "\nCreating SWAP File\n"
+        if has_space_left_for_swap(root_disk, swap_size):
+            proc = cmd_runner.run([
+                'dd',
+                'if=/dev/zero',
+                'of=%s/SWAP.swap' % root_disk,
+                'bs=1M',
+                'count=%s' % swap_size], as_root=True)
+            proc.wait()
+            proc = cmd_runner.run(
+                ['mkswap', '%s/SWAP.swap' % root_disk], as_root=True)
+            proc.wait()
+            fstab_additions.append("/SWAP.swap  none  swap  sw  0 0")
+        else:
+            print ("Swap file is bigger than space left on partition; "
+                   "continuing without swap.")
+
+    append_to_fstab(root_disk, fstab_additions)
+
+    print "\nCreating /etc/flash-kernel.conf\n"
+    create_flash_kernel_config(root_disk, 1 + partition_offset)
+
+    cmd_runner.run(['sync']).wait()
+    # The old code used to ignore failures here, but I don't think that's
+    # desirable so I'm using cmd_runner.run()'s standard behaviour, which will
+    # fail on a non-zero return value.
+    cmd_runner.run(['umount', root_disk], as_root=True).wait()
+
+
+def create_flash_kernel_config(root_disk, boot_partition_number):
+    """Create a flash-kernel.conf file under root_disk/etc.
+
+    Uses the given partition number to figure out the boot partition.
+    """
+    target_boot_dev = '/dev/mmcblk0p%s' % boot_partition_number
+    flash_kernel = os.path.join(root_disk, 'etc', 'flash-kernel.conf')
+    write_data_to_protected_file(
+        flash_kernel, "UBOOT_PART=%s" % target_boot_dev)
+
+
+def move_contents(from_, root_disk):
+    """Move everything under from_ to the given root disk.
+
+    Uses sudo for moving.
+    """
+    assert os.path.isdir(from_), "%s is not a directory" % from_
+    files = glob.glob(os.path.join(from_, '*'))
+    mv_cmd = ['mv']
+    mv_cmd.extend(files)
+    mv_cmd.append(root_disk)
+    cmd_runner.run(mv_cmd, as_root=True).wait()
+
+
+def has_space_left_for_swap(root_disk, swap_size_in_mega_bytes):
+    """Is there enough space for a swap file in the given root disk?"""
+    statvfs = os.statvfs(root_disk)
+    free_space = statvfs.f_bavail * statvfs.f_bsize
+    swap_size_in_bytes = int(swap_size_in_mega_bytes) * 1024**2
+    if free_space >= swap_size_in_bytes:
+        return True
+    return False
+
+
+def append_to_fstab(root_disk, fstab_additions):
+    fstab = os.path.join(root_disk, 'etc', 'fstab')
+    data = open(fstab).read() + '\n' + '\n'.join(fstab_additions)
+    write_data_to_protected_file(fstab, data)
+
+
+def write_data_to_protected_file(path, data):
+    """Write data to the file on the given path.
+
+    This is meant to be used when the given file is only writable by root, and
+    we overcome that by writing the data to a tempfile and then moving the
+    tempfile on top of the given one using sudo.
+    """
+    _, tmpfile = tempfile.mkstemp()
+    with open(tmpfile, 'w') as fd:
+        fd.write(data)
+    cmd_runner.run(['mv', '-f', tmpfile, path], as_root=True).wait()
+
+
+if __name__ == '__main__':
+    (contents_dir, root_disk, partition, rootfs_type, rootfs_uuid, create_swap,
+        swap_size, partition_offset) = sys.argv[1:]
+    # The create_swap argument comes as a string containing either 1 or 0, so
+    # we convert it to int and then to boolean.
+    should_create_swap = bool(int(create_swap))
+    partition_offset = int(partition_offset)
+    populate_rootfs(
+        contents_dir, root_disk, partition, rootfs_type, rootfs_uuid,
+        should_create_swap, swap_size, partition_offset)

=== modified file 'media_create/tests/test_media_create.py'
--- media_create/tests/test_media_create.py	2010-12-14 14:14:06 +0000
+++ media_create/tests/test_media_create.py	2010-12-17 11:20:41 +0000
@@ -14,6 +14,7 @@
 from media_create import ensure_command
 from media_create import populate_boot
 from media_create import partitions
+from media_create import rootfs
 from media_create.boot_cmd import create_boot_cmd
 from media_create.partitions import (
     calculate_partition_size_and_offset,
@@ -33,6 +34,13 @@
     _run_mkimage,
     )
 from media_create.remove_binary_dir import remove_binary_dir
+from media_create.rootfs import (
+    create_flash_kernel_config,
+    has_space_left_for_swap,
+    move_contents,
+    populate_rootfs,
+    write_data_to_protected_file,
+    )
 from media_create.unpack_binary_tarball import unpack_binary_tarball
 
 from media_create.tests.fixtures import (
@@ -244,12 +252,11 @@
 
         img = self.createTempFileAsFixture()
         # Use that fake boot script to create a boot loader using mkimage.
-        # Send stdout to file as mkimage will print to stdout and we don't
-        # want that.
+        # Send stdout to /dev/null as mkimage will print to stdout and we
+        # don't want that.
         retval = _run_mkimage(
             'script', '0', '0', 'boot script', filename, img,
-            stdout=open(self.createTempFileAsFixture(), 'w'),
-            as_root=False)
+            stdout=open('/dev/null', 'w'), as_root=False)
 
         self.assertEqual(0, retval)
 
@@ -459,3 +466,115 @@
              ['sudo', 'sfdisk', '-D', '-H', '255', '-S', '63', tempfile],
              ['sync']],
             popen_fixture.mock.calls)
+
+
+class TestPopulateRootFS(TestCaseWithFixtures):
+
+    lines_added_to_fstab = None
+    create_flash_kernel_config_called = False
+
+    def test_populate_rootfs(self):
+        def fake_append_to_fstab(disk, additions):
+            self.lines_added_to_fstab = additions
+
+        def fake_create_flash_kernel_config(disk, partition_offset):
+            self.create_flash_kernel_config_called = True
+
+        # Mock stdout, cmd_runner.Popen(), append_to_fstab and
+        # create_flash_kernel_config.
+        self.useFixture(MockSomethingFixture(
+            sys, 'stdout', open('/dev/null', 'w')))
+        self.useFixture(MockSomethingFixture(
+            rootfs, 'append_to_fstab', fake_append_to_fstab))
+        self.useFixture(MockSomethingFixture(
+            rootfs, 'create_flash_kernel_config',
+            fake_create_flash_kernel_config))
+        popen_fixture = self.useFixture(MockCmdRunnerPopenFixture())
+        # Store a dummy rootdisk and contents_dir in a tempdir.
+        tempdir = self.useFixture(CreateTempDirFixture()).tempdir
+        root_disk = os.path.join(tempdir, 'rootdisk')
+        contents_dir = os.path.join(tempdir, 'contents')
+        contents_bin = os.path.join(contents_dir, 'bin')
+        contents_etc = os.path.join(contents_dir, 'etc')
+        os.makedirs(contents_bin)
+        os.makedirs(contents_etc)
+
+        populate_rootfs(
+            contents_dir, root_disk, partition='/dev/rootfs',
+            rootfs_type='ext3', rootfs_uuid='uuid', should_create_swap=True,
+            swap_size=100, partition_offset=0)
+
+        self.assertEqual(
+            ['UUID=uuid / ext3  errors=remount-ro 0 1 ',
+             '/SWAP.swap  none  swap  sw  0 0'],
+            self.lines_added_to_fstab)
+        self.assertEqual(True, self.create_flash_kernel_config_called)
+        swap_file = os.path.join(root_disk, 'SWAP.swap')
+        expected = [
+            ['sudo', 'mount', '/dev/rootfs', root_disk],
+            ['sudo', 'mv', contents_bin, contents_etc, root_disk],
+            ['sudo', 'dd', 'if=/dev/zero', 'of=%s' % swap_file, 'bs=1M',
+             'count=100'],
+            ['sudo', 'mkswap', swap_file],
+            ['sync'],
+            ['sudo', 'umount', root_disk]]
+        self.assertEqual(expected, popen_fixture.mock.calls)
+
+    def test_create_flash_kernel_config(self):
+        fixture = self.useFixture(MockCmdRunnerPopenFixture())
+        tempdir = self.useFixture(CreateTempDirFixture()).tempdir
+
+        create_flash_kernel_config(tempdir, boot_partition_number=1)
+
+        calls = fixture.mock.calls
+        self.assertEqual(1, len(calls), calls)
+        call = calls[0]
+        # The call writes to a tmpfile and then moves it to the,
+        # /etc/flash-kernel.conf, so the tmpfile is the next to last in the
+        # list of arguments stored.
+        tmpfile = call[-2]
+        self.assertEqual(
+            ['sudo', 'mv', '-f', tmpfile,
+             '%s/etc/flash-kernel.conf' % tempdir],
+            call)
+        self.assertEqual('UBOOT_PART=/dev/mmcblk0p1', open(tmpfile).read())
+
+    def test_move_contents(self):
+        tempdir = self.useFixture(CreateTempDirFixture()).tempdir
+        popen_fixture = self.useFixture(MockCmdRunnerPopenFixture())
+        file1 = self.createTempFileAsFixture(dir=tempdir)
+
+        move_contents(tempdir, '/tmp/')
+
+        self.assertEqual([['sudo', 'mv', file1, '/tmp/']],
+                         popen_fixture.mock.calls)
+
+    def test_has_space_left_for_swap(self):
+        statvfs = os.statvfs('/')
+        space_left = statvfs.f_bavail * statvfs.f_bsize
+        swap_size_in_megs = space_left / (1024**2)
+        self.assertTrue(
+            has_space_left_for_swap('/', swap_size_in_megs))
+
+    def test_has_no_space_left_for_swap(self):
+        statvfs = os.statvfs('/')
+        space_left = statvfs.f_bavail * statvfs.f_bsize
+        swap_size_in_megs = (space_left / (1024**2)) + 1
+        self.assertFalse(
+            has_space_left_for_swap('/', swap_size_in_megs))
+
+    def test_write_data_to_protected_file(self):
+        fixture = self.useFixture(MockCmdRunnerPopenFixture())
+        data = 'foo'
+        path = '/etc/nonexistant'
+
+        write_data_to_protected_file(path, data)
+
+        calls = fixture.mock.calls
+        self.assertEqual(1, len(calls), calls)
+        call = calls[0]
+        # The call moves tmpfile to the given path, so tmpfile is the next to
+        # last in the list of arguments stored.
+        tmpfile = call[-2]
+        self.assertEqual(['sudo', 'mv', '-f', tmpfile, path], call)
+        self.assertEqual(data, open(tmpfile).read())