Merge lp:~smoser/curtin/add-arm64-support into lp:~newell-jensen/curtin/add-arm64-support

[Scott Moser]

[Newell Jensen (newell-jensen)]

Need to fix merge conflict.

review: Needs Fixing

[lp:~smoser/curtin/add-arm64-support]

153. By Scott Moser on 2014-07-21

merge with Newell's branch

[Scott Moser (smoser)]

fixed merge conflicts.

[Newell Jensen (newell-jensen)]

Looks good.

review: Approve

=== modified file 'curtin/commands/block_meta.py'
--- curtin/commands/block_meta.py	2014-07-17 17:39:59 +0000
+++ curtin/commands/block_meta.py	2014-07-21 19:24:27 +0000
@@ -23,8 +23,9 @@
 from . import populate_one_subcmd
 
 import os
-import tempfile
 
+SIMPLE = 'simple'
+SIMPLE_BOOT = 'simple-boot'
 
 CMD_ARGUMENTS = (
     ((('-D', '--devices'),
@@ -34,16 +35,15 @@
                    'choices': ['ext4', 'ext3'], 'default': 'ext4'}),
      ('--boot-fstype', {'help': 'boot partition filesystem type',
                         'choices': ['ext4', 'ext3'], 'default': None}),
-     ('mode', {'help': 'meta-mode to use', 'choices': ['raid0', 'simple', 'simple-boot']}),
+     ('mode', {'help': 'meta-mode to use',
+               'choices': ['raid0', SIMPLE, SIMPLE_BOOT]}),
      )
 )
 
 
 def block_meta(args):
     # main entry point for the block-meta command.
-    if args.mode == "simple":
-        meta_simple(args)
-    elif args.mode == "simple-boot":
+    if args.mode in (SIMPLE, SIMPLE_BOOT):
         meta_simple(args)
     else:
         raise NotImplementedError("mode=%s is not implemenbed" % args.mode)
@@ -68,6 +68,25 @@
     return block.get_root_device([devname, ])
 
 
+def get_bootpt_cfg(cfg, enabled=False, fstype=None):
+    # 'cfg' looks like:
+    #   enabled: boolean
+    #   fstype: filesystem type (default to 'fstype')
+    #   label:  filesystem label (default to 'boot')
+    #   size:   filesystem size in M (default to 512)
+    # parm enable can enable, but not disable
+    # parm fstype overrides cfg['fstype']
+    ret = {'enabled': False, 'fstype': None, 'size': 512, 'label': 'boot'}
+    ret.update(cfg)
+    if enabled:
+        ret['enabled'] = True
+    if ret['enabled']:
+        if fstype and not ret['fstype']:
+            ret['fstype'] = fstype
+    ret['size'] = int(ret['size'])
+    return ret
+
+
 def meta_simple(args):
     """Creates a root partition. If args.mode == 'simple-boot', it will also
     create a separate /boot partition.
@@ -80,6 +99,10 @@
     if devices is None:
         devices = cfg.get('block-meta', {}).get('devices', [])
 
+    bootpt = get_bootpt_cfg(
+        cfg.get('block-meta', {}).get('boot-partition', {}),
+        enabled=bool(args.mode is SIMPLE_BOOT), fstype=args.boot_fstype)
+
     # Remove duplicates but maintain ordering.
     devices = list(OrderedDict.fromkeys(devices))
 
@@ -110,7 +133,6 @@
     sources = cfg.get('sources', {})
     dd_images = util.get_dd_images(sources)
 
-    # dd images are only for Windows, not Linux
     if len(dd_images):
         # we have at least one dd-able image
         # we will only take the first one
@@ -123,10 +145,9 @@
         logtime(
             "partition --format uefi %s" % devnode,
             util.subp, ("partition", "--format", "uefi", devnode))
-    elif args.mode == 'simple-boot':
-        logtime(
-            "partition %s" % devnode,
-            util.subp, ("partition", "--boot", devnode))
+    elif bootpt['enabled']:
+        logtime("partition %s" % devnode,
+                util.subp, ("partition", "--boot", devnode))
         bootdev = devnode + "1"
         rootdev = devnode + "2"
     else:
@@ -140,25 +161,22 @@
     logtime(' '.join(cmd), util.subp, cmd)
     util.subp(['mount', rootdev, state['target']])
 
-    if args.mode == 'simple-boot':
+    if bootpt['enabled']:
         # create 'boot' directory in state['target']
         boot_dir = os.path.join(state['target'], 'boot')
         util.subp(['mkdir', boot_dir])
         # mkfs for boot partition and mount
-        if args.boot_fstype:
-            cmd = ['mkfs.%s' % args.boot_fstype, '-q', '-L', 'cloudimg-bootfs', bootdev]
-        else:
-            cmd = ['mkfs.%s' % args.fstype, '-q', '-L', 'cloudimg-bootfs', bootdev]    
+        cmd = ['mkfs.%s' % bootpt['fstype'],
+               '-q', '-L', bootpt['label'], bootdev]
         logtime(' '.join(cmd), util.subp, cmd)
         util.subp(['mount', bootdev, boot_dir])
-        
+
     with open(state['fstab'], "w") as fp:
-        if args.mode == 'simple-boot':
-            if args.boot_fstype:
-                fp.write("LABEL=%s /boot %s defaults 0 0\n" % ('cloudimg-bootfs', args.boot_fstype))
-            else:
-                fp.write("LABEL=%s /boot %s defaults 0 0\n" % ('cloudimg-bootfs', args.fstype))
-        fp.write("LABEL=%s / %s defaults 0 0\n" % ('cloudimg-rootfs', args.fstype))
+        if bootpt['enabled']:
+            fp.write("LABEL=%s /boot %s defaults 0 0\n" %
+                     (bootpt['label'], bootpt['fstype']))
+        fp.write("LABEL=%s / %s defaults 0 0\n" %
+                 ('cloudimg-rootfs', args.fstype))
 
     return 0
 

=== modified file 'curtin/net/__init__.py'
--- curtin/net/__init__.py	2014-03-25 23:13:46 +0000
+++ curtin/net/__init__.py	2014-07-21 19:24:27 +0000
@@ -1,6 +1,7 @@
-#   Copyright (C) 2013 Canonical Ltd.
+#   Copyright (C) 2013-2014 Canonical Ltd.
 #
 #   Author: Scott Moser <[e-mail]>
+#   Author: Blake Rouse <[e-mail]>
 #
 #   Curtin is free software: you can redistribute it and/or modify it under
 #   the terms of the GNU Affero General Public License as published by the
@@ -22,6 +23,22 @@
 
 SYS_CLASS_NET = "/sys/class/net/"
 
+NET_CONFIG_OPTIONS = [
+    "address", "netmask", "broadcast", "network", "metric", "gateway",
+    "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
+    "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
+    "netnum", "endpoint", "local", "ttl",
+    ]
+
+NET_CONFIG_COMMANDS = [
+    "pre-up", "up", "post-up", "down", "pre-down", "post-down",
+    ]
+
+NET_CONFIG_BRIDGE_OPTIONS = [
+    "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
+    "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
+    ]
+
 
 def sys_dev_path(devname, path=""):
     return SYS_CLASS_NET + devname + "/" + path
@@ -95,4 +112,100 @@
     return os.listdir(SYS_CLASS_NET)
 
 
+class ParserError(Exception):
+    """Raised when parser has issue parsing the interfaces file."""
+
+
+def parse_deb_config_data(ifaces, contents, path):
+    """Parses the file contents, placing result into ifaces.
+
+    :param ifaces: interface dictionary
+    :param contents: contents of interfaces file
+    :param path: directory interfaces file was located
+    """
+    currif = None
+    src_dir = path
+    for line in contents.splitlines():
+        line = line.strip()
+        if line.startswith('#'):
+            continue
+        split = line.split(' ')
+        option = split[0]
+        if option == "source-directory":
+            src_dir = os.path.join(path, split[1])
+        elif option == "source":
+            src_path = os.path.join(src_dir, split[1])
+            with open(src_path, "r") as fp:
+                src_data = fp.read().strip()
+            parse_deb_config_data(
+                ifaces, src_data,
+                os.path.dirname(os.path.abspath(src_path)))
+        elif option == "auto":
+            for iface in split[1:]:
+                if iface not in ifaces:
+                    ifaces[iface] = {}
+                ifaces[iface]['auto'] = True
+        elif option == "iface":
+            iface, family, method = split[1:4]
+            if iface not in ifaces:
+                ifaces[iface] = {}
+            elif 'family' in ifaces[iface]:
+                raise ParserError("Cannot define %s interface again.")
+            ifaces[iface]['family'] = family
+            ifaces[iface]['method'] = method
+            currif = iface
+        elif option == "hwaddress":
+            ifaces[currif]['hwaddress'] = split[1]
+        elif option in NET_CONFIG_OPTIONS:
+            ifaces[currif][option] = split[1]
+        elif option in NET_CONFIG_COMMANDS:
+            if option not in ifaces[currif]:
+                ifaces[currif][option] = []
+            ifaces[currif][option].append(' '.join(split[1:]))
+        elif option.startswith('dns-'):
+            if 'dns' not in ifaces[currif]:
+                ifaces[currif]['dns'] = {}
+            if option == 'dns-search':
+                ifaces[currif]['dns']['search'] = []
+                for domain in split[1:]:
+                    ifaces[currif]['dns']['search'].append(domain)
+            elif option == 'dns-nameservers':
+                ifaces[currif]['dns']['nameservers'] = []
+                for server in split[1:]:
+                    ifaces[currif]['dns']['nameservers'].append(server)
+        elif option.startswith('bridge_'):
+            if 'bridge' not in ifaces[currif]:
+                ifaces[currif]['bridge'] = {}
+            if option in NET_CONFIG_BRIDGE_OPTIONS:
+                bridge_option = option.replace('bridge_', '')
+                ifaces[currif]['bridge'][bridge_option] = split[1]
+            elif option == "bridge_ports":
+                ifaces[currif]['bridge']['ports'] = []
+                for iface in split[1:]:
+                    ifaces[currif]['bridge']['ports'].append(iface)
+            elif option == "bridge_hw" and split[1].lower() == "mac":
+                ifaces[currif]['bridge']['mac'] = split[2]
+            elif option == "bridge_pathcost":
+                if 'pathcost' not in ifaces[currif]['bridge']:
+                    ifaces[currif]['bridge']['pathcost'] = {}
+                ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
+            elif option == "bridge_portprio":
+                if 'portprio' not in ifaces[currif]['bridge']:
+                    ifaces[currif]['bridge']['portprio'] = {}
+                ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
+    for iface in ifaces.keys():
+        if 'auto' not in ifaces[iface]:
+            ifaces[iface]['auto'] = False
+
+
+def parse_deb_config(path):
+    """Parses a debian network configuration file."""
+    ifaces = {}
+    with open(path, "r") as fp:
+        contents = fp.read().strip()
+    parse_deb_config_data(
+        ifaces, contents,
+        os.path.dirname(os.path.abspath(path)))
+    return ifaces
+
 # vi: ts=4 expandtab syntax=python

=== modified file 'helpers/common'
--- helpers/common	2014-07-21 17:57:28 +0000
+++ helpers/common	2014-07-21 19:24:27 +0000
@@ -76,10 +76,10 @@
     # This is currently the default size we have for the /boot partition
     if [ "$boot" -ge 1 ]; then
         maxend=$((($size/512)-$start-$bootsize))
-    if [ $maxend -lt 0 ]; then
-        { error "Disk is not big enough for /boot partition on $target";
-        return 1; }
-    fi
+        if [ $maxend -lt 0 ]; then
+            error "Disk is not big enough for /boot partition on $target";
+            return 1;
+        fi
     else
         maxend=$((($size/512)-$start))
     fi
@@ -89,13 +89,13 @@
     [ -b "$target" ] && isblk=true
 
     if [ "$boot" -ge 1 ]; then 
-    # Creating 'efi', '/boot' and '/' partitions
+        # Creating 'efi', '/boot' and '/' partitions
         sgdisk --new "15:$start:+1M" --typecode=15:ef02 \
-               --new "1:+1M:+513M" --typecode=1:8300 \
-               --new "2::$end"     --typecode=2:8300 "$target" ||
+            --new "1:+1M:+513M" --typecode=1:8300 \
+            --new "2::$end"     --typecode=2:8300 "$target" ||
             { error "failed to gpt partition $target"; return 1; }
     else
-    # Creating 'efi' and '/' partitions 
+        # Creating 'efi' and '/' partitions 
         sgdisk --new "15:2048:+1M" --typecode=15:ef02 \
                --new "1::$end"     --typecode=1:8300 "$target" ||
             { error "failed to gpt partition $target"; return 1; }

=== added file 'tests/unittests/test_net.py'
--- tests/unittests/test_net.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_net.py	2014-07-21 19:24:27 +0000
@@ -0,0 +1,244 @@
+from unittest import TestCase
+import os
+import shutil
+import tempfile
+
+from curtin import net
+from textwrap import dedent
+
+
+class TestNetParserData(TestCase):
+
+    def test_parse_deb_config_data_ignores_comments(self):
+        contents = dedent("""\
+            # ignore
+            # iface eth0 inet static
+            #  address 192.168.1.1
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({}, ifaces)
+
+    def test_parse_deb_config_data_basic(self):
+        contents = dedent("""\
+            iface eth0 inet static
+            address 192.168.1.2
+            netmask 255.255.255.0
+            hwaddress aa:bb:cc:dd:ee:ff
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({
+            'eth0': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'static',
+                'address': '192.168.1.2',
+                'netmask': '255.255.255.0',
+                'hwaddress': 'aa:bb:cc:dd:ee:ff',
+                },
+            }, ifaces)
+
+    def test_parse_deb_config_data_auto(self):
+        contents = dedent("""\
+            auto eth0 eth1
+            iface eth0 inet manual
+            iface eth1 inet manual
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({
+            'eth0': {
+                'auto': True,
+                'family': 'inet',
+                'method': 'manual',
+                },
+            'eth1': {
+                'auto': True,
+                'family': 'inet',
+                'method': 'manual',
+                },
+            }, ifaces)
+
+    def test_parse_deb_config_data_error_on_redefine(self):
+        contents = dedent("""\
+            iface eth0 inet static
+            address 192.168.1.2
+            iface eth0 inet static
+            address 192.168.1.3
+            """)
+        ifaces = {}
+        self.assertRaises(
+            net.ParserError,
+            net.parse_deb_config_data, ifaces, contents, '')
+
+    def test_parse_deb_config_data_commands(self):
+        contents = dedent("""\
+            iface eth0 inet manual
+            pre-up preup1
+            pre-up preup2
+            up up1
+            post-up postup1
+            pre-down predown1
+            down down1
+            down down2
+            post-down postdown1
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({
+            'eth0': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'manual',
+                'pre-up': ['preup1', 'preup2'],
+                'up': ['up1'],
+                'post-up': ['postup1'],
+                'pre-down': ['predown1'],
+                'down': ['down1', 'down2'],
+                'post-down': ['postdown1'],
+                },
+            }, ifaces)
+
+    def test_parse_deb_config_data_dns(self):
+        contents = dedent("""\
+            iface eth0 inet static
+            dns-nameservers 192.168.1.1 192.168.1.2
+            dns-search curtin local
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({
+            'eth0': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'static',
+                'dns': {
+                    'nameservers': ['192.168.1.1', '192.168.1.2'],
+                    'search': ['curtin', 'local'],
+                    },
+                },
+            }, ifaces)
+
+    def test_parse_deb_config_data_bridge(self):
+        contents = dedent("""\
+            iface eth0 inet manual
+            iface eth1 inet manual
+            iface br0 inet static
+            address 192.168.1.1
+            netmask 255.255.255.0
+            bridge_maxwait 30
+            bridge_ports eth0 eth1
+            bridge_pathcost eth0 1
+            bridge_pathcost eth1 2
+            bridge_portprio eth0 0
+            bridge_portprio eth1 1
+            """)
+        ifaces = {}
+        net.parse_deb_config_data(ifaces, contents, '')
+        self.assertEqual({
+            'eth0': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'manual',
+                },
+            'eth1': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'manual',
+                },
+            'br0': {
+                'auto': False,
+                'family': 'inet',
+                'method': 'static',
+                'address': '192.168.1.1',
+                'netmask': '255.255.255.0',
+                'bridge': {
+                    'maxwait': '30',
+                    'ports': ['eth0', 'eth1'],
+                    'pathcost': {
+                        'eth0': '1',
+                        'eth1': '2',
+                        },
+                    'portprio': {
+                        'eth0': '0',
+                        'eth1': '1'
+                        },
+                    },
+                },
+            }, ifaces)
+
+
+class TestNetParser(TestCase):
+
+    def setUp(self):
+        self.target = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.target)
+
+    def make_config(self, path=None, name=None, contents=None,
+                    parse=True):
+        if path is None:
+            path = self.target
+        if name is None:
+            name = 'interfaces'
+        path = os.path.join(path, name)
+        if contents is None:
+            contents = dedent("""\
+                auto eth0
+                iface eth0 inet static
+                address 192.168.1.2
+                netmask 255.255.255.0
+                hwaddress aa:bb:cc:dd:ee:ff
+                """)
+        with open(path, 'w') as stream:
+            stream.write(contents)
+        ifaces = None
+        if parse:
+            ifaces = {}
+            net.parse_deb_config_data(ifaces, contents, '')
+        return path, ifaces
+
+    def test_parse_deb_config(self):
+        path, data = self.make_config()
+        expected = net.parse_deb_config(path)
+        self.assertEqual(data, expected)
+
+    def test_parse_deb_config_source(self):
+        path, data = self.make_config(name='interfaces2')
+        contents = dedent("""\
+            source interfaces2
+            iface eth1 inet manual
+            """)
+        i_path, _ = self.make_config(
+            contents=contents, parse=False)
+        data['eth1'] = {
+            'auto': False,
+            'family': 'inet',
+            'method': 'manual',
+            }
+        expected = net.parse_deb_config(i_path)
+        self.assertEqual(data, expected)
+
+    def test_parse_deb_config_source_dir(self):
+        subdir = os.path.join(self.target, 'interfaces.d')
+        os.mkdir(subdir)
+        path, data = self.make_config(
+            path=subdir, name='interfaces2')
+        contents = dedent("""\
+            source-directory interfaces.d
+            source interfaces2
+            iface eth1 inet manual
+            """)
+        i_path, _ = self.make_config(
+            contents=contents, parse=False)
+        data['eth1'] = {
+            'auto': False,
+            'family': 'inet',
+            'method': 'manual',
+            }
+        expected = net.parse_deb_config(i_path)
+        self.assertEqual(data, expected)
+
+# vi: ts=4 expandtab syntax=python