Merge lp:~wesley-wiedenmeier/curtin/custom-partitioning-layout into lp:~curtin-dev/curtin/trunk

[Wesley Wiedenmeier]

Added support for dname rules https://bugs.launchpad.net/curtin/+bug/1477756

=== modified file 'curtin/commands/block_meta.py'
--- curtin/commands/block_meta.py	2015-08-10 18:27:09 +0000
+++ curtin/commands/block_meta.py	2015-08-12 19:16:20 +0000
@@ -21,6 +21,7 @@
 from curtin.log import LOG
 
 from . import populate_one_subcmd
+from curtin.commands.net_meta import compose_udev_equality
 
 import glob
 import os
@@ -268,6 +269,50 @@
     return partnumber
 
 
+def make_dname(volume, storage_config):
+    state = util.load_command_environment()
+    rules_dir = os.path.join(state['scratch'], "rules.d")
+    vol = storage_config.get(volume)
+    path = get_path_to_storage_volume(volume, storage_config)
+    dname = vol.get('name')
+    if vol.get('type') in ["partition", "disk"]:
+        (out, _err) = util.subp(["blkid", "-o", "export", path], capture=True)
+        for line in out.splitlines():
+            if "PTUUID" in line or "PARTUUID" in line:
+                ptuuid = line.split('=')[-1]
+                break
+    rule = [
+        compose_udev_equality("SUBSYSTEM", "block"),
+        compose_udev_equality("ACTION", "add|change"),
+        ]
+    if vol.get('type') == "disk":
+        rule.append(compose_udev_equality('ENV{DEVTYPE}', "disk"))
+        rule.append(compose_udev_equality('ENV{ID_PART_TABLE_UUID}', ptuuid))
+    elif vol.get('type') == "partition":
+        rule.append(compose_udev_equality('ENV{DEVTYPE}', "partition"))
+        dname = storage_config.get(vol.get('device')).get('name') + \
+            "-part%s" % determine_partition_number(volume, storage_config)
+        rule.append(compose_udev_equality('ENV{ID_PART_ENTRY_UUID}', ptuuid))
+    elif vol.get('type') == "raid":
+        (out, _err) = util.subp(["mdadm", "--detail", "--export", path],
+                                capture=True)
+        for line in out.splitlines():
+            if "MD_UUID" in line:
+                md_uuid = line.split('=')[-1]
+                break
+        rule.append(compose_udev_equality("ENV{MD_UUID}", md_uuid))
+    elif vol.get('type') == "bcache":
+        rule.append(compose_udev_equality("ENV{DEVNAME}", path))
+    elif vol.get('type') == "lvm_partition":
+        volgroup_name = storage_config.get(vol.get('volgroup')).get('name')
+        dname = "%s-%s" % (volgroup_name, dname)
+        rule.append(compose_udev_equality("ENV{DM_NAME}", dname))
+    rule.append("SYMLINK+=\"disk/by-dname/%s\"" % dname)
+    util.ensure_dir(rules_dir)
+    with open(os.path.join(rules_dir, volume), "w") as fp:
+        fp.write(', '.join(rule))
+
+
 def get_path_to_storage_volume(volume, storage_config):
     # Get path to block device for volume. Volume param should refer to id of
     # volume in storage config
@@ -404,6 +449,10 @@
         elif ptable == "msdos":
             util.subp(["parted", disk, "--script", "mklabel", "msdos"])
 
+    # Make the name if needed
+    if info.get('name'):
+        make_dname(info.get('id'), storage_config)
+
 
 def partition_handler(info, storage_config):
     device = info.get('device')
@@ -494,6 +543,9 @@
         wipe_volume(
             get_path_to_storage_volume(info.get('id'), storage_config),
             info.get('wipe'))
+    # Make the name if needed
+    if storage_config.get(device).get('name'):
+        make_dname(info.get('id'), storage_config)
 
 
 def format_handler(info, storage_config):
@@ -687,6 +739,8 @@
         raise ValueError("Partition tables on top of lvm logical volumes is \
                          not supported")
 
+    make_dname(info.get('id'), storage_config)
+
 
 def dm_crypt_handler(info, storage_config):
     state = util.load_command_environment()
@@ -778,6 +832,9 @@
     # Create the raid device
     util.subp(" ".join(cmd), shell=True)
 
+    # Make dname rule for this dev
+    make_dname(info.get('id'), storage_config)
+
     # A mdadm.conf will be created in the same directory as the fstab in the
     # configuration. This will then be copied onto the installed system later.
     # The file must also be written onto the running system to enable it to run
@@ -831,6 +888,10 @@
             fp.write(path)
             fp.close()
 
+    if info.get('name'):
+        # Make dname rule for this dev
+        make_dname(info.get('id'), storage_config)
+
     if info.get('ptable'):
         raise ValueError("Partition tables on top of lvm logical volumes is \
                          not supported")

=== modified file 'curtin/commands/curthooks.py'
--- curtin/commands/curthooks.py	2015-08-06 23:20:36 +0000
+++ curtin/commands/curthooks.py	2015-08-12 19:16:20 +0000
@@ -415,6 +415,16 @@
     shutil.copy(interfaces, eni)
 
 
+def copy_dname_rules(rules_d, target):
+    if not rules_d:
+        LOG.warn("no udev rules directory to copy")
+        return
+    for rule in os.listdir(rules_d):
+        target_file = os.path.join(
+            target, "etc/udev/rules.d", "%s.rules" % rule)
+        shutil.copy(os.path.join(rules_d, rule), target_file)
+
+
 def restore_dist_interfaces(cfg, target):
     # cloud images have a link of /etc/network/interfaces into /run
     eni = os.path.sep.join([target, 'etc/network/interfaces'])
@@ -611,6 +621,11 @@
     if os.path.exists(mdadm_location):
         copy_mdadm_conf(mdadm_location, target)
 
+    # If udev dname rules were created, copy them to target
+    udev_rules_d = os.path.join(state['scratch'], "rules.d")
+    if os.path.isdir(udev_rules_d):
+        copy_dname_rules(udev_rules_d, target)
+
     # As a rule, ARMv7 systems don't use grub. This may change some
     # day, but for now, assume no. They do require the initramfs
     # to be updated, and this also triggers boot loader setup via

=== modified file 'examples/tests/basic.yaml'
--- examples/tests/basic.yaml	2015-08-06 23:20:36 +0000
+++ examples/tests/basic.yaml	2015-08-12 19:16:20 +0000
@@ -6,6 +6,7 @@
         ptable: msdos
         model: QEMU HARDDISK
         path: /dev/vdb
+        name: main_disk
       - id: sda1
         type: partition
         number: 1

=== added file 'examples/tests/lvm.yaml'
--- examples/tests/lvm.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/lvm.yaml	2015-08-12 19:16:20 +0000
@@ -0,0 +1,70 @@
+storage:
+    version: 1
+    config:
+      - id: sda
+        type: disk
+        ptable: msdos
+        model: QEMU HARDDISK
+        path: /dev/vdb
+        name: main_disk
+      - id: sda1
+        type: partition
+        size: 3GB
+        device: sda
+        flag: boot
+      - id: sda_extended
+        type: partition
+        size: 5G
+        flag: extended
+        device: sda
+      - id: sda2
+        type: partition
+        size: 2G
+        flag: logical
+        device: sda
+      - id: sda3
+        type: partition
+        size: 2G
+        flag: logical
+        device: sda
+      - id: volgroup1
+        name: vg1
+        type: lvm_volgroup
+        devices:
+            - sda2
+            - sda3
+      - id: lvmpart1
+        name: lv1
+        size: 1G
+        type: lvm_partition
+        volgroup: volgroup1
+      - id: lvmpart2
+        name: lv2
+        type: lvm_partition
+        volgroup: volgroup1
+      - id: sda1_root
+        type: format
+        fstype: ext4
+        volume: sda1
+      - id: lv1_fs
+        name: storage
+        type: format
+        fstype: fat32
+        volume: lvmpart1
+      - id: lv2_fs
+        name: storage
+        type: format
+        fstype: ext3
+        volume: lvmpart2
+      - id: sda1_mount
+        type: mount
+        path: /
+        device: sda1_root
+      - id: lv1_mount
+        type: mount
+        path: /srv/data
+        device: lv1_fs
+      - id: lv2_mount
+        type: mount
+        path: /srv/backup
+        device: lv2_fs

=== modified file 'examples/tests/mdadm_bcache.yaml'
--- examples/tests/mdadm_bcache.yaml	2015-08-10 16:54:19 +0000
+++ examples/tests/mdadm_bcache.yaml	2015-08-12 19:16:20 +0000
@@ -6,6 +6,7 @@
        ptable: gpt
        model: QEMU HARDDISK
        path: /dev/vdb
+       name: main_disk
      - id: bios_boot_partition
        type: partition
        size: 1MB
@@ -42,6 +43,7 @@
          - sda4
      - id: bcache0
        type: bcache
+       name: cached_array
        backing_device: mddevice
        cache_device: sda5
      - id: sda1_root

=== modified file 'tests/vmtests/test_basic.py'
--- tests/vmtests/test_basic.py	2015-08-06 16:27:22 +0000
+++ tests/vmtests/test_basic.py	2015-08-12 19:16:20 +0000
@@ -7,10 +7,10 @@
 
 class TestBasicAbs(VMBaseClass):
     __test__ = False
+    interactive = False
     conf_file = "examples/tests/basic.yaml"
     install_timeout = 600
     boot_timeout = 120
-    interactive = False
     user_data = textwrap.dedent("""\
         #cloud-config
         password: passw0rd
@@ -23,6 +23,7 @@
           - blkid -o export /dev/vda1 > /media/output/blkid_output_vda1
           - blkid -o export /dev/vda2 > /media/output/blkid_output_vda2
           - cat /etc/fstab > /media/output/fstab
+          - ls /dev/disk/by-dname/ > /media/output/ls_dname
         power_state:
           mode: poweroff
         """)
@@ -30,12 +31,18 @@
     def test_output_files_exist(self):
         self.output_files_exist(
             ["blkid_output_vda", "blkid_output_vda1", "blkid_output_vda2",
-             "fstab"])
+             "fstab", "ls_dname"])
 
     def test_ptable(self):
         blkid_info = self.get_blkid_data("blkid_output_vda")
         self.assertEquals(blkid_info["PTTYPE"], "dos")
 
+    def test_dname(self):
+        with open(os.path.join(self.td.mnt, "ls_dname"), "r") as fp:
+            contents = fp.read().splitlines()
+        for link in ["main_disk", "main_disk-part1", "main_disk-part2"]:
+            self.assertIn(link, contents)
+
     def test_partitions(self):
         with open(os.path.join(self.td.mnt, "fstab")) as fp:
             fstab_lines = fp.readlines()

=== added file 'tests/vmtests/test_lvm.py'
--- tests/vmtests/test_lvm.py	1970-01-01 00:00:00 +0000
+++ tests/vmtests/test_lvm.py	2015-08-12 19:16:20 +0000
@@ -0,0 +1,80 @@
+from . import VMBaseClass
+from unittest import TestCase
+
+import textwrap
+import os
+
+
+class TestMdadmBcacheAbs(VMBaseClass):
+    __test__ = False
+    conf_file = "examples/tests/lvm.yaml"
+    install_timeout = 600
+    boot_timeout = 100
+    interactive = False
+    user_data = textwrap.dedent("""\
+        #cloud-config
+        password: passw0rd
+        chpasswd: { expire: False }
+        bootcmd:
+          - mkdir /media/output
+          - mount /dev/vdb /media/output
+        runcmd:
+          - cat /etc/fstab > /media/output/fstab
+          - ls /dev/disk/by-dname > /media/output/ls_dname
+          - pvdisplay -C --separator = -o vg_name,pv_name --noheadings > \
+                  /media/output/pvs
+          - lvdisplay -C --separator = -o lv_name,vg_name --noheadings > \
+                  /media/output/lvs
+        power_state:
+          mode: poweroff
+        """)
+
+    def test_fstab(self):
+        with open(os.path.join(self.td.mnt, "fstab")) as fp:
+            fstab_lines = fp.readlines()
+        fstab_entry = None
+        for line in fstab_lines:
+            if "/dev/vg1/lv1" in line:
+                fstab_entry = line
+                break
+        self.assertIsNotNone(fstab_entry)
+        self.assertEqual(fstab_entry.split(' ')[1], "/srv/data")
+
+    def test_lvs(self):
+        with open(os.path.join(self.td.mnt, "lvs"), "r") as fp:
+            lv_data = list(i.strip() for i in fp.readlines())
+        self.assertIn("lv1=vg1", lv_data)
+        self.assertIn("lv2=vg1", lv_data)
+
+    def test_pvs(self):
+        with open(os.path.join(self.td.mnt, "pvs"), "r") as fp:
+            lv_data = list(i.strip() for i in fp.readlines())
+        self.assertIn("vg1=/dev/vda5", lv_data)
+        self.assertIn("vg1=/dev/vda6", lv_data)
+
+    def test_output_files_exist(self):
+        self.output_files_exist(
+            ["fstab", "ls_dname"])
+
+    def test_dname(self):
+        with open(os.path.join(self.td.mnt, "ls_dname"), "r") as fp:
+            contents = fp.read().splitlines()
+        for link in list(("main_disk-part%s" % i for i in (1, 2, 5, 6))):
+            self.assertIn(link, contents)
+        self.assertIn("main_disk", contents)
+        self.assertIn("vg1-lv1", contents)
+        self.assertIn("vg1-lv2", contents)
+
+
+class WilyTestLvm(TestMdadmBcacheAbs, TestCase):
+    __test__ = True
+    repo = "maas-daily"
+    release = "wily"
+    arch = "amd64"
+
+
+class VividTestLvm(TestMdadmBcacheAbs, TestCase):
+    __test__ = True
+    repo = "maas-daily"
+    release = "vivid"
+    arch = "amd64"

=== modified file 'tests/vmtests/test_mdadm_bcache.py'
--- tests/vmtests/test_mdadm_bcache.py	2015-08-06 16:27:22 +0000
+++ tests/vmtests/test_mdadm_bcache.py	2015-08-12 19:16:20 +0000
@@ -23,6 +23,7 @@
           - mdadm --detail --scan > /media/output/mdadm_status
           - bcache-super-show /dev/vda6 > /media/output/bcache_super_vda6
           - ls /sys/fs/bcache > /media/output/bcache_ls
+          - ls /dev/disk/by-dname > /media/output/ls_dname
         power_state:
           mode: poweroff
         """)
@@ -47,6 +48,15 @@
         self.output_files_exist(
             ["fstab", "mdadm_status", "bcache_super_vda6", "bcache_ls"])
 
+    def test_dname(self):
+        with open(os.path.join(self.td.mnt, "ls_dname"), "r") as fp:
+            contents = fp.read().splitlines()
+        for link in list(("main_disk-part%s" % i for i in range(1, 6))):
+            self.assertIn(link, contents)
+        self.assertIn("md0", contents)
+        self.assertIn("cached_array", contents)
+        self.assertIn("main_disk", contents)
+
     def test_bcache_status(self):
         bcache_cset_uuid = None
         with open(os.path.join(self.td.mnt, "bcache_super_vda6"), "r") as fp: