Merge ~raharper/cloud-init:curtin-centos into cloud-init:master

[Ryan Harper]

WIP branch with changes for el6/el7 networking rendering

[Scott Moser (smoser)]

I pulled:

Your-commit trunk-comt summary
8da074f831c d1e8eb73ac sysconfig: include GATEWAY value if set in subnet

changes:
i shorted lines in commit messages and made tox tests pass (had to add the newly rendered GATEWAY values) to more than the one test you did it to.

$ git diff 8da074f831c..d1e8eb73ac | pastebinit -f diff
http://paste.ubuntu.com/25121124/

[Ryan Harper (raharper)]

Not sure that's right (I Think it should be IPV6_GATEWAY0)

On Tue, Jul 18, 2017 at 4:09 PM, Scott Moser <email address hidden> wrote:

> I pulled:
>
> Your-commit trunk-comt summary
> 8da074f831c d1e8eb73ac sysconfig: include GATEWAY value if set in subnet
>
> changes:
> i shorted lines in commit messages and made tox tests pass (had to add the
> newly rendered GATEWAY values) to more than the one test you did it to.
>
> $ git diff 8da074f831c..d1e8eb73ac | pastebinit -f diff
> http://paste.ubuntu.com/25121124/
>
> --
> https://code.launchpad.net/~raharper/cloud-init/+git/
> cloud-init/+merge/327648
> You are the owner of ~raharper/cloud-init:curtin-centos.
>

[Scott Moser (smoser)]

i took your commit
  Use prefix for ipv6 and set v4 and v6 gateway keys, use DEFROUTE

and put up
 https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327739

[Scott Moser (smoser)]

your commit
 Templatize systemd service files
is modified and up at
 https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327743

[Scott Moser (smoser)]

merge for your
 Fix sysconfig rendering of virtual interfaces with network configurations
is up at
 https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327821

i plan on adding a vlan and a simple bridge test.

[Scott Moser (smoser)]

merge for
   sysconfig: fix ipv6 gateway routes
at
  https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327828

[Scott Moser (smoser)]

 sysconfig: handle manual type subnets
is at
 https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327828

14f552e... by Ryan Harper on 2017-07-20

sysconfig: use MACADDR on bonds/bridges to configure mac_address

Previously, sysconfig rendered HWADDR for all interface types, but
that value is only used to identify physical devices. Instead use
MACADDR to configure the MAC on virtual devices, like bonds and
bridges.

- Sort bond slave list to ensure consistent ordering in sysconfig
  rendered files.
- Add unittests for sysconfig rendering of bonds/bridges with
  mac_address

LP: #1701417

[Scott Moser (smoser)]

https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/327836
has
 sysconfig: enable mtu set per subnet, including ipv6 mtu
and then
 net: eni route rendering missed ipv6 default route config

8bf9253... by Ryan Harper on 2017-07-20

sysconfig: enable mtu set per subnet, including ipv6 mtu

Render MTU values if present in subnet and route configurations
for v4 and v6.

LP: #1702513

[Scott Moser (smoser)]

marked as merged as we have these all now in trunk.
thanks Ryan

diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index b707146..bb80ec0 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -355,7 +355,7 @@ class Renderer(renderer.Renderer):
             default_gw = " default gw %s" % route['gateway']
             content.append(up + default_gw + or_true)
             content.append(down + default_gw + or_true)
-        elif route['network'] == '::' and route['netmask'] == 0:
+        elif route['network'] == '::' and route['prefix'] == 0:
             # ipv6!
             default_gw = " -A inet6 default gw %s" % route['gateway']
             content.append(up + default_gw + or_true)
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index b0f2ccf..e4490c4 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -264,6 +264,9 @@ class Renderer(renderer.Renderer):
         for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]:
             old_value = iface.get(old_key)
             if old_value is not None:
+                # only set HWADDR on physical interfaces
+                if old_key == 'mac_address' and iface['type'] != 'physical':
+                    continue
                 iface_cfg[new_key] = old_value
 
     @classmethod
@@ -273,6 +276,7 @@ class Renderer(renderer.Renderer):
 
         # modifying base values according to subnets
         for i, subnet in enumerate(subnets, start=len(iface_cfg.children)):
+            mtu_key = 'MTU'
             subnet_type = subnet.get('type')
             if subnet_type == 'dhcp6':
                 iface_cfg['IPV6INIT'] = True
@@ -292,7 +296,13 @@ class Renderer(renderer.Renderer):
                 # if iface_cfg['BOOTPROTO'] == 'none':
                 #    iface_cfg['BOOTPROTO'] = 'static'
                 if subnet_is_ipv6(subnet):
+                    mtu_key = 'IPV6_MTU'
                     iface_cfg['IPV6INIT'] = True
+                if 'mtu' in subnet:
+                    iface_cfg[mtu_key] = subnet['mtu']
+
+            elif subnet_type == 'manual':
+                iface_cfg['ONBOOT'] = False
             else:
                 raise ValueError("Unknown subnet type '%s' found"
                                  " for interface '%s'" % (subnet_type,
@@ -425,6 +435,9 @@ class Renderer(renderer.Renderer):
                 master_cfg['BONDING_MASTER'] = True
                 master_cfg.kind = 'bond'
 
+            if 'mac_address' in iface and iface.get('mac_address'):
+                iface_cfg['MACADDR'] = iface.get('mac_address')
+
             iface_subnets = iface.get("subnets", [])
             route_cfg = iface_cfg.routes
             cls._render_subnets(iface_cfg, iface_subnets)
@@ -436,6 +449,7 @@ class Renderer(renderer.Renderer):
                 [slave_iface['name'] for slave_iface in
                  network_state.iter_interfaces(slave_filter)
                  if slave_iface['bond-master'] == iface_name])
+
             for index, bond_slave in enumerate(bond_slaves):
                 slavestr = 'BONDING_SLAVE%s' % index
                 iface_cfg[slavestr] = bond_slave
@@ -494,6 +508,10 @@ class Renderer(renderer.Renderer):
             for old_key, new_key in cls.bridge_opts_keys:
                 if old_key in iface:
                     iface_cfg[new_key] = iface[old_key]
+
+            if 'mac_address' in iface and iface.get('mac_address'):
+                iface_cfg['MACADDR'] = iface.get('mac_address')
+
             # Is this the right key to get all the connected interfaces?
             for bridged_iface_name in iface.get('bridge_ports', []):
                 # Ensure all bridged interfaces are correctly tagged
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index b5a95a1..e02d7a7 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -442,7 +442,7 @@ class DataSourceAzure(sources.DataSource):
                   self.ds_cfg['agent_command'])
         try:
             fabric_data = metadata_func()
-        except Exception as exc:
+        except Exception:
             LOG.warning(
                 "Error communicating with Azure fabric; You may experience."
                 "connectivity issues.", exc_info=True)
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index e625934..4e05a82 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -422,6 +422,31 @@ NETWORK_CONFIGS = {
                             via: 65.61.151.37
                         set-name: eth99
         """).rstrip(' '),
+        'expected_sysconfig': {
+            'ifcfg-eth1': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        DEVICE=eth1
+        HWADDR=cf:d6:af:48:e8:80
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Ethernet
+        USERCTL=no""").rstrip(' '),
+            'ifcfg-eth99': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=dhcp
+        DEFROUTE=yes
+        DEVICE=eth99
+        GATEWAY=65.61.151.37
+        HWADDR=c0:d6:9f:2c:e8:80
+        IPADDR=192.168.21.3
+        NETMASK=255.255.255.0
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Ethernet
+        USERCTL=no """).rstrip(' ')},
         'yaml': textwrap.dedent("""
             version: 1
             config:
@@ -541,6 +566,8 @@ iface br0 inet static
 # control-alias br0
 iface br0 inet6 static
     address 2001:1::1/64
+    post-up route add -A inet6 default gw 2001:4800:78ff:1b::1 || true
+    pre-down route del -A inet6 default gw 2001:4800:78ff:1b::1 || true
 
 auto bond0.200
 iface bond0.200 inet dhcp
@@ -675,6 +702,9 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                                 eth3: 50
                                 eth4: 75
                             priority: 22
+                        routes:
+                        -   to: ::/0
+                            via: 2001:4800:78ff:1b::1
                 vlans:
                     bond0.200:
                         dhcp4: true
@@ -697,6 +727,146 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                             - sacchromyces.maas
                             - brettanomyces.maas
         """).rstrip(' '),
+        'expected_sysconfig': {
+            'ifcfg-bond0': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BONDING_MASTER=yes
+        BONDING_OPTS="mode=active-backup xmit_hash_policy=layer3+4 miimon=100"
+        BONDING_SLAVE0=eth1
+        BONDING_SLAVE1=eth2
+        BOOTPROTO=dhcp
+        DEVICE=bond0
+        DHCPV6C=yes
+        IPV6INIT=yes
+        MACADDR=aa:bb:cc:dd:ee:ff
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Bond
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-bond0.200': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=dhcp
+        DEVICE=bond0.200
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        PHYSDEV=bond0
+        TYPE=Ethernet
+        USERCTL=no
+        VLAN=yes
+            """).rstrip(' '),
+            'ifcfg-br0': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        AGEING=250
+        BOOTPROTO=none
+        DEFROUTE=yes
+        DEVICE=br0
+        IPADDR=192.168.14.2
+        IPV6ADDR=2001:1::1/64
+        IPV6INIT=yes
+        IPV6_DEFAULTGW=2001:4800:78ff:1b::1
+        NETMASK=255.255.255.0
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        PRIO=22
+        STP=off
+        TYPE=Bridge
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth0': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        DEVICE=eth0
+        HWADDR=c0:d6:9f:2c:e8:80
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth0.101': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        DEFROUTE=yes
+        DEVICE=eth0.101
+        GATEWAY=192.168.0.1
+        IPADDR=192.168.0.2
+        IPADDR1=192.168.2.10
+        MTU=1500
+        NETMASK=255.255.255.0
+        NETMASK1=255.255.255.0
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        PHYSDEV=eth0
+        TYPE=Ethernet
+        USERCTL=no
+        VLAN=yes
+            """).rstrip(' '),
+            'ifcfg-eth1': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        DEVICE=eth1
+        HWADDR=aa:d6:9f:2c:e8:80
+        MASTER=bond0
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        SLAVE=yes
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth2': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        DEVICE=eth2
+        HWADDR=c0:bb:9f:2c:e8:80
+        MASTER=bond0
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        SLAVE=yes
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth3': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        BRIDGE=br0
+        DEVICE=eth3
+        HWADDR=66:bb:9f:2c:e8:80
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth4': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=none
+        BRIDGE=br0
+        DEVICE=eth4
+        HWADDR=98:bb:9f:2c:e8:80
+        NM_CONTROLLED=no
+        ONBOOT=yes
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' '),
+            'ifcfg-eth5': textwrap.dedent("""\
+        # Created by cloud-init on instance boot automatically, do not edit.
+        #
+        BOOTPROTO=dhcp
+        DEVICE=eth5
+        HWADDR=98:bb:9f:2c:e8:8a
+        NM_CONTROLLED=no
+        ONBOOT=no
+        TYPE=Ethernet
+        USERCTL=no
+            """).rstrip(' ')},
         'yaml': textwrap.dedent("""
             version: 1
             config:
@@ -807,6 +977,10 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
                         address: 192.168.14.2/24
                       - type: static
                         address: 2001:1::1/64 # default to /64
+                        routes:
+                          - gateway: 2001:4800:78ff:1b::1
+                            netmask: '::'
+                            network: '::'
                 # A global nameserver.
                 - type: nameserver
                   address: 8.8.8.8
@@ -869,7 +1043,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true
         DEFROUTE=yes
         DEVICE=bond0
         GATEWAY=192.168.0.1
-        HWADDR=aa:bb:cc:dd:e8:ff
+        MACADDR=aa:bb:cc:dd:e8:ff
         IPADDR=192.168.0.2
         IPADDR1=192.168.1.2
         IPV6ADDR=2001:1::1/92
@@ -2004,6 +2178,43 @@ class TestEniRoundTrip(CiTestCase):
             expected, [line for line in found if line])
 
 
+class TestSysconfigRoundTrip(CiTestCase):
+    def _render_and_read(self, network_config=None, state=None, dir=None):
+        if dir is None:
+            dir = self.tmp_dir()
+
+        if network_config:
+            ns = network_state.parse_net_config_data(network_config)
+        elif state:
+            ns = state
+        else:
+            raise ValueError("Expected data or state, got neither")
+
+        renderer = sysconfig.Renderer()
+        renderer.render_network_state(ns, dir)
+        return dir2dict(dir)
+
+    def testsimple_render_small(self):
+        entry = NETWORK_CONFIGS['small']
+        files = self._render_and_read(network_config=yaml.load(entry['yaml']))
+        expected_sysconfig = entry.get('expected_sysconfig')
+        for ifcfg_name in expected_sysconfig:
+            expected_ifcfg = expected_sysconfig.get(ifcfg_name)
+            ifcfg_path = '/etc/sysconfig/network-scripts/' + ifcfg_name
+            self.assertEqual(expected_ifcfg.splitlines(),
+                             files[ifcfg_path].splitlines())
+
+    def testsimple_render_all(self):
+        entry = NETWORK_CONFIGS['all']
+        files = self._render_and_read(network_config=yaml.load(entry['yaml']))
+        expected_sysconfig = entry.get('expected_sysconfig')
+        for ifcfg_name in expected_sysconfig:
+            expected_ifcfg = expected_sysconfig.get(ifcfg_name)
+            ifcfg_path = '/etc/sysconfig/network-scripts/' + ifcfg_name
+            self.assertEqual(expected_ifcfg.splitlines(),
+                             files[ifcfg_path].splitlines())
+
+
 class TestNetRenderers(CiTestCase):
     @mock.patch("cloudinit.net.renderers.sysconfig.available")
     @mock.patch("cloudinit.net.renderers.eni.available")