Merge ~raharper/cloud-init:fix/netplan-ipv6-mtu into cloud-init:master

[Ryan Harper]

net/netplan: use ipv6-mtu key for specifying ipv6 mtu values

netplan introduced an 'info' subcommand which emits yaml describing
implemented features that indicate new or changed fields and values
in the yaml that it accepts. Previously, cloud-init emitted the key
'mtu6' for ipv6 MTU values. This is not correct and netplan will
fail to parse these values. Netplan as of 0.98 supports both the
info subcommand and the ipv6-mtu key.

This branch modifies the netplan renderer to collect the netplan
info output into a 'features' property which is a list of available
feature flags which the renderer can use to modify its output. If
the command is not available, no feature flags are set and
cloud-init will render IPv6 MTU values just as MTU for the subnet.

8f46b8a... by Ryan Harper on 2019-10-23

Drop print debug lines

[Dan Watkins (oddbloke)]

One inline question, but the main implementation LGTM.

review: Needs Information

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:110dd9f865b87746f633210cd8e6f15f54ee32dc
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1228/
Executed test runs:
    SUCCESS: Checkout
    FAILED: Unit & Style Tests

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1228//rebuild

review: Needs Fixing (continuous-integration)

[Dan Watkins (oddbloke)]

review: Approve

[Dan Watkins (oddbloke)]

This looks like it needs rebasing to fix CI.

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:8f46b8a1c633f8a6b112a92ccb2ba34268971dfa
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1230/
Executed test runs:
    FAILED: Checkout

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1230//rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:8f46b8a1c633f8a6b112a92ccb2ba34268971dfa
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1231/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/1231//rebuild

review: Approve (continuous-integration)

diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
index 1ad7e0b..6f513d9 100755
--- a/cloudinit/cmd/devel/net_convert.py
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -116,6 +116,8 @@ def handle_args(name, args):
         config['postcmds'] = False
         # trim leading slash
         config['netplan_path'] = config['netplan_path'][1:]
+        # enable some netplan features
+        config['features'] = ['dhcp-use-domains', 'ipv6-mtu']
     else:
         r_cls = sysconfig.Renderer
         config = distro.renderer_configs.get('sysconfig')
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index e54a34e..1b039ab 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -34,7 +34,7 @@ def _get_params_dict_by_match(config, match):
                 if key.startswith(match))
 
 
-def _extract_addresses(config, entry, ifname):
+def _extract_addresses(config, entry, ifname, features=None):
     """This method parse a cloudinit.net.network_state dictionary (config) and
        maps netstate keys/values into a dictionary (entry) to represent
        netplan yaml.
@@ -66,7 +66,7 @@ def _extract_addresses(config, entry, ifname):
      'match': {'macaddress': '52:54:00:12:34:00'},
      'mtu': 1501,
      'address': ['192.168.1.2/24', '2001:4800:78ff:1b:be76:4eff:fe06:1000"],
-     'mtu6': 1480}
+     'ipv6-mtu': 1480}
 
     """
 
@@ -79,6 +79,8 @@ def _extract_addresses(config, entry, ifname):
         else:
             return [obj, ]
 
+    if features is None:
+        features = []
     addresses = []
     routes = []
     nameservers = []
@@ -108,8 +110,8 @@ def _extract_addresses(config, entry, ifname):
                 searchdomains += _listify(subnet.get('dns_search', []))
             if 'mtu' in subnet:
                 mtukey = 'mtu'
-                if subnet_is_ipv6(subnet):
-                    mtukey += '6'
+                if subnet_is_ipv6(subnet) and 'ipv6-mtu' in features:
+                    mtukey = 'ipv6-mtu'
                 entry.update({mtukey: subnet.get('mtu')})
             for route in subnet.get('routes', []):
                 to_net = "%s/%s" % (route.get('network'),
@@ -179,6 +181,7 @@ class Renderer(renderer.Renderer):
     """Renders network information in a /etc/netplan/network.yaml format."""
 
     NETPLAN_GENERATE = ['netplan', 'generate']
+    NETPLAN_INFO = ['netplan', 'info']
 
     def __init__(self, config=None):
         if not config:
@@ -188,6 +191,22 @@ class Renderer(renderer.Renderer):
         self.netplan_header = config.get('netplan_header', None)
         self._postcmds = config.get('postcmds', False)
         self.clean_default = config.get('clean_default', True)
+        self._features = config.get('features', None)
+
+    @property
+    def features(self):
+        if self._features is None:
+            try:
+                info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True)
+                info = util.load_yaml(info_blob)
+                self._features = info['netplan.io']['features']
+            except util.ProcessExecutionError:
+                # if the info subcommand is not present then we don't have any
+                # new features
+                pass
+            except (TypeError, KeyError) as e:
+                LOG.debug('Failed to list features from netplan info: %s', e)
+        return self._features
 
     def render_network_state(self, network_state, templates=None, target=None):
         # check network state for version
@@ -271,7 +290,7 @@ class Renderer(renderer.Renderer):
                     else:
                         del eth['match']
                         del eth['set-name']
-                _extract_addresses(ifcfg, eth, ifname)
+                _extract_addresses(ifcfg, eth, ifname, self.features)
                 ethernets.update({ifname: eth})
 
             elif if_type == 'bond':
@@ -296,7 +315,7 @@ class Renderer(renderer.Renderer):
                 slave_interfaces = ifcfg.get('bond-slaves')
                 if slave_interfaces == 'none':
                     _extract_bond_slaves_by_name(interfaces, bond, ifname)
-                _extract_addresses(ifcfg, bond, ifname)
+                _extract_addresses(ifcfg, bond, ifname, self.features)
                 bonds.update({ifname: bond})
 
             elif if_type == 'bridge':
@@ -331,7 +350,7 @@ class Renderer(renderer.Renderer):
                     bridge.update({'parameters': br_config})
                 if ifcfg.get('mac_address'):
                     bridge['macaddress'] = ifcfg.get('mac_address').lower()
-                _extract_addresses(ifcfg, bridge, ifname)
+                _extract_addresses(ifcfg, bridge, ifname, self.features)
                 bridges.update({ifname: bridge})
 
             elif if_type == 'vlan':
@@ -343,7 +362,7 @@ class Renderer(renderer.Renderer):
                 macaddr = ifcfg.get('mac_address', None)
                 if macaddr is not None:
                     vlan['macaddress'] = macaddr.lower()
-                _extract_addresses(ifcfg, vlan, ifname)
+                _extract_addresses(ifcfg, vlan, ifname, self.features)
                 vlans.update({ifname: vlan})
 
         # inject global nameserver values under each all interface which
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index d220199..21604b1 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -996,8 +996,8 @@ NETWORK_CONFIGS = {
                         addresses:
                         - 192.168.14.2/24
                         - 2001:1::1/64
+                        ipv6-mtu: 1500
                         mtu: 9000
-                        mtu6: 1500
         """).rstrip(' '),
         'yaml': textwrap.dedent("""\
             version: 1
@@ -3585,7 +3585,9 @@ class TestNetplanPostcommands(CiTestCase):
 
     @mock.patch.object(netplan.Renderer, '_netplan_generate')
     @mock.patch.object(netplan.Renderer, '_net_setup_link')
-    def test_netplan_render_calls_postcmds(self, mock_netplan_generate,
+    @mock.patch('cloudinit.util.subp')
+    def test_netplan_render_calls_postcmds(self, mock_subp,
+                                           mock_netplan_generate,
                                            mock_net_setup_link):
         tmp_dir = self.tmp_dir()
         ns = network_state.parse_net_config_data(self.mycfg,
@@ -3597,6 +3599,7 @@ class TestNetplanPostcommands(CiTestCase):
         render_target = 'netplan.yaml'
         renderer = netplan.Renderer(
             {'netplan_path': render_target, 'postcmds': True})
+        mock_subp.side_effect = iter([util.ProcessExecutionError])
         renderer.render_network_state(ns, target=render_dir)
 
         mock_netplan_generate.assert_called_with(run=True)
@@ -3619,7 +3622,13 @@ class TestNetplanPostcommands(CiTestCase):
         render_target = 'netplan.yaml'
         renderer = netplan.Renderer(
             {'netplan_path': render_target, 'postcmds': True})
+        mock_subp.side_effect = iter([
+            util.ProcessExecutionError,
+            ('', ''),
+            ('', ''),
+        ])
         expected = [
+            mock.call(['netplan', 'info'], capture=True),
             mock.call(['netplan', 'generate'], capture=True),
             mock.call(['udevadm', 'test-builtin', 'net_setup_link',
                        '/sys/class/net/lo'], capture=True),
@@ -3875,6 +3884,20 @@ class TestReadInitramfsConfig(CiTestCase):
 
 
 class TestNetplanRoundTrip(CiTestCase):
+
+    NETPLAN_INFO_OUT = textwrap.dedent("""
+    netplan.io:
+      features:
+        - dhcp-use-domains
+        - ipv6-mtu
+      website: https://netplan.io/
+    """)
+
+    def setUp(self):
+        super(TestNetplanRoundTrip, self).setUp()
+        self.add_patch('cloudinit.net.netplan.util.subp', 'm_subp')
+        self.m_subp.return_value = (self.NETPLAN_INFO_OUT, '')
+
     def _render_and_read(self, network_config=None, state=None,
                          netplan_path=None, target=None):
         if target is None: