Merge ~chad.smith/cloud-init:feature/openstack-network-v2-multi-nic into cloud-init:master

[Chad Smith]

Work in progress for initial review:

openstack: return network v2 from parsed network_data.json

TODO: sort promotion of global dns

Unmerged commits

5fac7a8... by Chad Smith on 2019-08-19

wip: generate openstack v2 networking config

diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
index 1ad7e0b..7b4ae3f 100755
--- a/cloudinit/cmd/devel/net_convert.py
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -81,12 +81,8 @@ def handle_args(name, args):
         pre_ns = yaml.load(net_data)
         if 'network' in pre_ns:
             pre_ns = pre_ns.get('network')
-        if args.debug:
-            sys.stderr.write('\n'.join(
-                ["Input YAML",
-                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
     elif args.kind == 'network_data.json':
-        pre_ns = openstack.convert_net_json(
+        pre_ns = openstack.convert_net_json_v2(
             json.loads(net_data), known_macs=known_macs)
     elif args.kind == 'azure-imds':
         pre_ns = azure.parse_network_config(json.loads(net_data))
@@ -94,6 +90,10 @@ def handle_args(name, args):
         config = ovf.Config(ovf.ConfigFile(args.network_data.name))
         pre_ns = ovf.get_network_config_from_conf(config, False)
 
+    if args.debug:
+        sys.stderr.write('\n'.join(
+            ["Input YAML",
+             yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
     ns = network_state.parse_net_config_data(pre_ns)
     if not ns:
         raise RuntimeError("No valid network_state object created from"
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index c0c415d..a7454f7 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -4,6 +4,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+from collections import defaultdict
 import copy
 import functools
 import logging
@@ -159,6 +160,10 @@ class NetworkState(object):
         return self._version
 
     @property
+    def _global_dns_counts(self):
+        return self._network_state['global_dns_counts']
+
+    @property
     def dns_nameservers(self):
         try:
             return self._network_state['dns']['nameservers']
@@ -234,6 +239,10 @@ class NetworkStateInterpreter(object):
         self._network_state = copy.deepcopy(self.initial_network_state)
         self._network_state['config'] = config
         self._parsed = False
+        # Reference counters to promote to global
+        self._global_dns_refs = {
+            'nameserver': defaultdict(list), 'search': defaultdict(list)}
+        self._network_state['global_dns_refs'] = self._global_dns_refs
 
     @property
     def network_state(self):
@@ -318,7 +327,7 @@ class NetworkStateInterpreter(object):
                                    " command '%s'" % command_type)
             try:
                 handler(self, command)
-                self._v2_common(command)
+                self._maybe_promote_v2_common(command)
             except InvalidCommand:
                 if not skip_broken:
                     raise
@@ -326,6 +335,41 @@ class NetworkStateInterpreter(object):
                     LOG.warning("Skipping invalid command: %s", command,
                                 exc_info=True)
                     LOG.debug(self.dump_network_state())
+        # Post-process v2 dns promotions if needed
+        # count interfaces with ip, compare unpromoted global dns
+        self._cleanup_v2_common_from_interfaces()
+
+    def _cleanup_v2_common_from_interfaces(self):
+        """Strip any promoted global dns/search from specific interfaces."""
+        interfaces = self._network_state.get('interfaces')
+        global_dns = set(self._network_state['dns'].get('nameservers', []))
+        global_search = set(self._network_state['dns'].get('search', []))
+        dns_refs = self._global_dns_refs['nameserver']
+        search_refs = self._global_dns_refs['search']
+        promoted_dns = global_dns.intersection(dns_refs)
+        promoted_search = global_dns.intersection(search_refs)
+        for intf_name, intf_cfg in interfaces.items():
+            for subnet in intf_cfg['subnets']:
+                promote_dns = bool(not promoted_dns and len(interfaces) == 1)
+                subnet_dns = subnet.get('dns_nameservers', [])
+                if promote_dns and (subnet_dns or subnet.get('dns_search')):
+                    name_cmd = {'type': 'nameserver',
+                                'search': subnet.get('dns_search', []),
+                                'address': subnet_dns}
+                    self.handle_nameserver(name_cmd)
+                    subnet.pop('dns_search', None)
+                    subnet.pop('dns_nameservers', None)
+                    continue
+                for dns_ip in subnet_dns:
+                    if dns_ip in promoted_dns:
+                         subnet['dns_nameservers'].remove(dns_ip)
+                         if not subnet['dns_nameservers']:
+                             subnet.pop('dns_nameservers')
+                for search in subnet.get('dns_search', []):
+                    if search in promoted_search:
+                         subnet['dns_search'].remove(search)
+                         if not subnet['dns_search']:
+                             subnet.pop('dns_search')
 
     @ensure_command_keys(['name'])
     def handle_loopback(self, command):
@@ -372,7 +416,6 @@ class NetworkStateInterpreter(object):
             'subnets': subnets,
         })
         self._network_state['interfaces'].update({command.get('name'): iface})
-        self.dump_network_state()
 
     @ensure_command_keys(['name', 'vlan_id', 'vlan_link'])
     def handle_vlan(self, command):
@@ -520,13 +563,15 @@ class NetworkStateInterpreter(object):
             if not type(addrs) == list:
                 addrs = [addrs]
             for addr in addrs:
-                dns['nameservers'].append(addr)
+                if addr not in dns['nameservers']:
+                    dns['nameservers'].append(addr)
         if 'search' in command:
             paths = command['search']
             if not isinstance(paths, list):
                 paths = [paths]
             for path in paths:
-                dns['search'].append(path)
+                if path not in dns['search']:
+                    dns['search'].append(path)
 
     @ensure_command_keys(['destination'])
     def handle_route(self, command):
@@ -689,18 +734,45 @@ class NetworkStateInterpreter(object):
         LOG.warning('Wifi configuration is only available to distros with'
                     'netplan rendering support.')
 
-    def _v2_common(self, cfg):
-        LOG.debug('v2_common: handling config:\n%s', cfg)
-        if 'nameservers' in cfg:
-            search = cfg.get('nameservers').get('search', [])
-            dns = cfg.get('nameservers').get('addresses', [])
-            name_cmd = {'type': 'nameserver'}
-            if len(search) > 0:
-                name_cmd.update({'search': search})
-            if len(dns) > 0:
-                name_cmd.update({'addresses': dns})
-            LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd)
-            self.handle_nameserver(name_cmd)
+    def _maybe_promote_v2_common(self, cfg):
+        """Possibly promote v2 common/global services from specific devices.
+
+        Since network v2 only supports per-interface DNS config settings, there
+        is no 'global' dns service that can be expressed, unless we set
+        the same dns values on every interface.  If v2 config has the same
+        dns config on every configured interface, it will be assumed that
+        the common dns setting needs to be written to the distribution's
+        'global (read /etc/resolv.conf)' dns config.
+
+        Track reference counts in _global_dns_refs so net/sysconfig renderer
+        can determine whether to use /etc/resolv.conf of not for specific
+        device dns configuration.
+        """
+        LOG.debug('maybe_promote_v2_common: handling config:\n%s', cfg)
+        for if_name, iface_cfg in cfg.items():
+            if 'nameservers' in iface_cfg:
+                search = iface_cfg.get('nameservers').get('search', [])
+                if not search:
+                   search = []
+                elif not isinstance(search, list):
+                    search = [search]
+                dns = iface_cfg.get('nameservers').get('addresses')
+                if not dns:
+                   dns = []
+                elif not isinstance(dns, list):
+                    dns = [dns]
+                name_cmd = {'type': 'nameserver', 'search': [], 'address': []}
+                for sname in search:
+                    if self._global_dns_refs['search'][sname]:
+                        name_cmd['search'].append[sname]
+                    self._global_dns_refs['search'][sname].append(if_name)
+                for dns_ip in dns:
+                    if self._global_dns_refs['nameserver'][dns_ip]:
+                        name_cmd['address'].append[dns_ip]
+                    self._global_dns_refs['nameserver'][dns_ip].append(if_name)
+                if any([name_cmd['search'], name_cmd['address']]):
+                    # promote DNS config seen by multiple interfaces
+                    self.handle_nameserver(name_cmd)
 
     def _handle_bond_bridge(self, command, cmd_type=None):
         """Common handler for bond and bridge types"""
@@ -827,7 +899,7 @@ def _normalize_net_keys(network, address_keys=()):
 
     @returns: A dict containing normalized prefix and matching addr_key.
     """
-    net = dict((k, v) for k, v in network.items() if v)
+    net = dict((k, v) for k, v in network.items() if v is not None)
     addr_key = None
     for key in address_keys:
         if net.get(key):
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index be5dede..0fa0508 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -444,9 +444,9 @@ class Renderer(renderer.Renderer):
 
                 if _is_default_route(route):
                     if (
-                            (subnet.get('ipv4') and
+                            (not is_ipv6 and
                              route_cfg.has_set_default_ipv4) or
-                            (subnet.get('ipv6') and
+                            (is_ipv6 and
                              route_cfg.has_set_default_ipv6)
                     ):
                         raise ValueError("Duplicate declaration of default "
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 8f06911..ceaf868 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -17,6 +17,7 @@ import six
 from cloudinit import ec2_utils
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.net import network_state
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
@@ -82,6 +83,36 @@ KNOWN_PHYSICAL_TYPES = (
     'vif',
 )
 
+LINK_TYPE_TO_NETWORK_V2_KEYS = {
+    'bond': 'bonds',
+    'vlan': 'vlans',
+}
+
+NET_V2_PHYSICAL_TYPES = ['ethernets', 'vlans', 'bonds', 'bridges', 'wifi']
+
+NETWORK_DATA_TO_V2 = {
+    'bond': {
+        'mtu': 'mtu',
+        'bond_links': 'interfaces',
+        'ethernet_mac_address': 'macaddress',
+    },
+    'ethernets': {
+        'mtu': 'mtu',
+        'ethernet_mac_address': 'match.macaddress',
+        'id': 'set-name',
+    },
+    'vlan': {
+        'key_rename': '{link}.{id}',   # override top level key for object
+        'mtu': 'mtu',
+        'vlan_mac_address': 'macaddress',
+        'vlan_id': 'id',
+        'vlan_link': 'link',
+    },
+}
+
+# network_data.json keys for which converted net v2 values should be lowercase
+LOWERCASE_KEY_VALUE_V2 = ('macaddress',)
+
 
 class NonReadable(IOError):
     pass
@@ -496,8 +527,233 @@ class MetadataReader(BaseReader):
                                                retries=self.retries)
 
 
-# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def _find_v2_device_type(name, net_v2):
+    """Return the netv2 physical device type containing matching name."""
+    for device_type in NET_V2_PHYSICAL_TYPES:
+        if name in net_v2.get(device_type, {}):
+            return device_type
+    return None
+
+
+def _convert_network_json_network_to_net_v2(src_json):
+    """Parse a single network item from the networks list in network_data.json
+
+    @param src_json: One network item from network_data.json 'networks' key.
+
+    @return: Tuple of <interface_name>, network v2 configuration dict for the
+         src_json. For example: eth0, {'addresses': [...], 'dhcp4': True}
+    """
+    net_v2 = {'addresses': []}
+    ignored_keys = set()
+
+    # In Liberty spec https://specs.openstack.org/openstack/nova-specs/
+    # specs/liberty/implemented/metadata-service-network-info.html
+    if src_json['type'] == 'ipv4_dhcp':
+        net_v2['dhcp4'] = True
+    elif src_json['type'] == 'ipv6_dhcp':
+        net_v2['dhcp6'] = True
+
+    for service in src_json.get('services', []):
+        if service['type'] != 'dns':
+            ignored_keys.update(['services.type(%s)' % service['type']])
+            continue
+        if 'nameservers' not in net_v2:
+            net_v2['nameservers'] = {'addresses': [], 'search': []}
+        net_v2['nameservers']['addresses'].append(service['address'])
+    # In Rocky spec https://specs.openstack.org/openstack/nova-specs/specs/
+    # rocky/approved/multiple-fixed-ips-network-information.html
+    dns_nameservers = src_json.get('dns_nameservers', [])
+    if dns_nameservers:
+        if 'nameservers' not in net_v2:
+            net_v2['nameservers'] = {'addresses': [], 'search': []}
+        net_v2['nameservers']['addresses'] = copy.copy(dns_nameservers)
+
+    # Parse routes for network, prefix and gateway
+    route_keys = set(['netmask', 'network', 'gateway'])
+    for route in src_json.get('routes', []):
+        ignored_route_keys = (set(route.keys()).difference(route_keys))
+        ignored_keys.update(['route.%s' % key for key in ignored_route_keys])
+        route_cfg = {
+            'to': '{network}/{prefix}'.format(
+                network=route['network'],
+                prefix=net.network_state.mask_to_net_prefix(route['netmask'])),
+            'via': route['gateway']}
+        if route.get('metric'):
+            route_cfg['metric'] = route.get('metric')
+        if 'routes' not in net_v2:
+            net_v2['routes'] = []
+        net_v2['routes'].append(route_cfg)
+
+    # Parse ip addresses on Rocky and Liberty
+    for ip_cfg in src_json.get('ip_addresses', []):
+        if ip_cfg.get('netmask'):
+            prefix = net.network_state.mask_to_net_prefix(ip_cfg['netmask'])
+            cidr_fmt = '{ip}/{prefix}'
+        else:
+            cidr_fmt = '{ip}'
+            prefix = None
+        net_v2['addresses'].append(
+            cidr_fmt.format(ip=ip_cfg['address'], prefix=prefix))
+    liberty_ip = src_json.get('ip_address')
+    if liberty_ip:
+        if src_json.get('netmask'):
+            prefix = net.network_state.mask_to_net_prefix(src_json['netmask'])
+            cidr_fmt = '{ip}/{prefix}'
+        else:
+            cidr_fmt = '{ip}'
+            prefix = None
+        liberty_cidr = cidr_fmt.format(ip=liberty_ip, prefix=prefix)
+        if liberty_cidr not in net_v2['addresses']:
+            net_v2['addresses'].append(liberty_cidr)
+    if not net_v2['addresses']:
+        net_v2.pop('addresses')
+    if ignored_keys:
+        LOG.debug(
+            'Ignoring the network_data.json %s config keys %s',
+            src_json['id'], ', '.join(ignored_keys))
+    return src_json['link'], net_v2
+
+
+def _convert_network_json_to_net_v2(src_json, var_map):
+    """Return network v2 for an element of OpenStack NetworkData json.
+
+    @param src_json: Dict of network_data.json for a single src_json object
+    @param var_map: Dict with a variable name map from network_data.json to
+        network v2
+
+    @return Tuple of the interface name and the converted network v2 for the
+        src_json object. For example: eth0, {'match': {'macaddress': 'AA:BB'}}
+    """
+    net_v2 = {}
+    # Map openstack bond keys to network v2
+    # Copy key values
+    current_keys = set(src_json)
+    for key in current_keys.intersection(set(var_map)):
+        keyparts = var_map[key].split('.')
+        tmp_cfg = net_v2  # allow traversing net_v2 dict
+        while keyparts:
+            keypart = keyparts.pop(0)
+            if keyparts:
+                if keypart not in tmp_cfg:
+                    tmp_cfg[keypart] = {}
+                tmp_cfg = tmp_cfg[keypart]
+            elif isinstance(src_json[key], list):
+                tmp_cfg[keypart] = copy.copy(src_json[key])
+            elif src_json[key]:
+                if keypart in LOWERCASE_KEY_VALUE_V2:
+                    tmp_cfg[keypart] = src_json[key].lower()
+                else:
+                    tmp_cfg[keypart] = src_json[key]
+    if 'key_rename' in var_map:
+        net_v2['key_rename'] = var_map['key_rename'].format(**net_v2)
+    return src_json['id'], net_v2
+
+
 def convert_net_json(network_json=None, known_macs=None):
+    """Parse OpenStack ConfigDrive NetworkData json, returning network cfg v2.
+
+    OpenStack network_data.json provides a 3 element dictionary
+      - "links" (links are network devices, physical or virtual)
+      - "networks" (networks are ip network configurations for one or more
+                    links)
+      -  services (non-ip services, like dns)
+
+    networks and links are combined via network items referencing specific
+    links via a 'link_id' which maps to a links 'id' field.
+    """
+    if network_json is None:
+        return None
+    net_config = {'version': 2}
+    for link in network_json.get('links', []):
+        link_type = link['type']
+        v2_key = LINK_TYPE_TO_NETWORK_V2_KEYS.get(link_type)
+        if not v2_key:
+            v2_key = 'ethernets'
+            if link_type not in KNOWN_PHYSICAL_TYPES:
+                LOG.warning('Unknown network_data link type (%s); treating as'
+                            ' physical ethernet', link_type)
+        if v2_key not in net_config:
+            net_config[v2_key] = {}
+        var_map = copy.deepcopy(NETWORK_DATA_TO_V2.get(v2_key))
+        if not var_map:
+            var_map = copy.deepcopy(NETWORK_DATA_TO_V2.get(link_type))
+
+        # Add v2 config parameters map for this link_type if present
+        if link_type in network_state.NET_CONFIG_TO_V2:
+            var_map.update(dict(
+                (k.replace('-', '_'), 'parameters.{v}'.format(v=v))
+                for k, v in network_state.NET_CONFIG_TO_V2[link_type].items()))
+        intf_id, intf_cfg = _convert_network_json_to_net_v2(link, var_map)
+        if v2_key in ('ethernets', 'bonds') and 'name' not in intf_cfg:
+            if known_macs is None:
+                known_macs = net.get_interfaces_by_mac()
+            lower_known_macs = dict(
+                (k.lower(), v) for k, v in known_macs.items())
+            mac = intf_cfg.get(  # top-level macaddress and match.macaddress
+                'macaddress', intf_cfg.get('match', {}).get('macaddress'))
+            if not mac:
+                raise ValueError("No mac_address or name entry for %s" % d)
+            mac = mac.lower()
+            if mac not in lower_known_macs:
+                raise ValueError(
+                    'Unable to find a system nic for %s' % mac)
+            intf_cfg['key_rename'] = lower_known_macs[mac]
+        net_config[v2_key].update({intf_id: intf_cfg})
+    for network in network_json.get('networks', []):
+        v2_key = _find_v2_device_type(network['link'], net_config)
+        intf_id, network_cfg = _convert_network_json_network_to_net_v2(network)
+        for key, val in network_cfg.items():
+            if isinstance(val, list):
+                if key not in net_config[v2_key][intf_id]:
+                    net_config[v2_key][intf_id][key] = []
+                net_config[v2_key][intf_id][key].extend(val)
+            else:
+                net_config[v2_key][intf_id][key] = val
+
+    # Inject global nameserver values under each all interface which
+    # has addresses and do not already have a DNS configuration
+    ignored_keys = set()
+    global_dns = []
+    for service in network_json.get('services', []):
+        if service['type'] != 'dns':
+            ignored_keys.update('services.type(%s)' % service['type'])
+            continue
+        global_dns.append(service['address'])
+
+    # Handle renames and global_dns
+    for dev_type in NET_V2_PHYSICAL_TYPES:
+        if dev_type not in net_config:
+            continue
+        renames = {}
+        for dev in net_config[dev_type]:
+            renames[dev] = net_config[dev_type][dev].pop('key_rename', None)
+            if not global_dns:
+                continue
+            dev_keys = set(net_config[dev_type][dev].keys())
+            if set(['nameservers', 'dhcp4', 'dhcp6']).intersection(dev_keys):
+                # Do not add nameservers if we already have dns config
+                continue
+            if 'addresses' not in net_config[dev_type][dev]:
+                # No configured address, needs no nameserver
+                continue
+            net_config[dev_type][dev]['nameservers'] = {
+                'addresses': copy.copy(global_dns), 'search': []}
+        for dev, rename in renames.items():
+            if rename:
+                net_config[dev_type][rename] = net_config[dev_type].pop(dev)
+                if 'set-name' in net_config[dev_type][rename]:
+                    net_config[dev_type][rename]['set-name'] = rename
+
+    if ignored_keys:
+        LOG.debug(
+            'Ignoring the network_data.json config keys %s',
+            ', '.join(ignored_keys))
+
+    return net_config
+
+
+# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def convert_net_json1(network_json=None, known_macs=None):
     """Return a dictionary of network_config by parsing provided
        OpenStack ConfigDrive NetworkData json format
 
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 520c50f..f4181f5 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -719,11 +719,11 @@ class TestConvertNetworkData(CiTestCase):
         for link in my_netdata['links']:
             link['ethernet_mac_address'] = link['ethernet_mac_address'].upper()
 
+        import pdb; pdb.set_trace()
         ncfg = openstack.convert_net_json(my_netdata, known_macs=KNOWN_MACS)
         config_name2mac = {}
-        for n in ncfg['config']:
-            if n['type'] == 'physical':
-                config_name2mac[n['name']] = n['mac_address']
+        for name, if_cfg in ncfg['ethernets'].items():
+            config_name2mac[name] = if_cfg['match']['macaddress']
 
         expected = {'nic0': 'fa:16:3e:05:30:fe', 'enp0s1': 'fa:16:3e:69:b0:58',
                     'enp0s2': 'fa:16:3e:d4:57:ad'}