Merge lp:~smoser/cloud-init/trunk.smartos-fabric into lp:~cloud-init-dev/cloud-init/trunk

[Scott Moser]

SmartOS: more improvements for network configuration

This improves smart os network configuration
 - fix the SocketClient which was previously completely broken.
 - adds support for configuring dns servers and dns search (based off the
   sdc:dns_domain).
 - support 'sdc:gateways' information from the datasource for configuring
   default routes.
 - add converted network information to output when module is run as a main

TODO: add support for sdc:routes as described at http://eng.joyent.com/mdata/datadict.html

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:1257
https://server-team-jenkins.canonical.com/job/cloud-init-ci/28/
Executed test runs:
    None: https://server-team-jenkins.canonical.com/job/lp-vote-on-merge/2/console

Click here to trigger a rebuild:
https://server-team-jenkins.canonical.com/job/cloud-init-ci/28/rebuild

review: Approve (continuous-integration)

[lp:~smoser/cloud-init/trunk.smartos-fabric]

1258. By Scott Moser on 2016-07-25

add knowledge of the 'sdc:routes' key (but do not use it)

1259. By Scott Moser on 2016-07-25

add test case covering not all nics have 'gateways' entry.

also for brevity import convert_smartos_network_data as convert_net.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:1259
https://server-team-jenkins.canonical.com/job/cloud-init-ci/1/
Executed test runs:
    SUCCESS: https://server-team-jenkins.canonical.com/job/cloud-init-ci/nodes=amd64/1
    SUCCESS: https://server-team-jenkins.canonical.com/job/cloud-init-ci/nodes=ppc64el/1

Click here to trigger a rebuild:
https://server-team-jenkins.canonical.com/job/cloud-init-ci/1/rebuild

review: Approve (continuous-integration)

[Scott Moser (smoser)]

merged into git at https://git.launchpad.net/cloud-init/commit/?id=c52b8eb9671be454d22283c81057f97b98bb49fe

[Scott Moser (smoser)]

review: Approve

=== modified file 'cloudinit/sources/DataSourceSmartOS.py'
--- cloudinit/sources/DataSourceSmartOS.py	2016-07-13 22:18:46 +0000
+++ cloudinit/sources/DataSourceSmartOS.py	2016-07-25 15:23:44 +0000
@@ -60,11 +60,15 @@
     'availability_zone': ('sdc:datacenter_name', True),
     'vendor-data': ('sdc:vendor-data', False),
     'operator-script': ('sdc:operator-script', False),
+    'hostname': ('sdc:hostname', True),
+    'dns_domain': ('sdc:dns_domain', True),
 }
 
 SMARTOS_ATTRIB_JSON = {
     # Cloud-init Key : (SmartOS Key known JSON)
     'network-data': 'sdc:nics',
+    'dns_servers': 'sdc:resolvers',
+    'routes': 'sdc:routes',
 }
 
 SMARTOS_ENV_LX_BRAND = "lx-brand"
@@ -311,7 +315,10 @@
         if self._network_config is None:
             if self.network_data is not None:
                 self._network_config = (
-                    convert_smartos_network_data(self.network_data))
+                    convert_smartos_network_data(
+                        network_data=self.network_data,
+                        dns_servers=self.metadata['dns_servers'],
+                        dns_domain=self.metadata['dns_domain']))
         return self._network_config
 
 
@@ -445,7 +452,8 @@
 
 
 class JoyentMetadataSocketClient(JoyentMetadataClient):
-    def __init__(self, socketpath):
+    def __init__(self, socketpath, smartos_type=SMARTOS_ENV_LX_BRAND):
+        super(JoyentMetadataSocketClient, self).__init__(smartos_type)
         self.socketpath = socketpath
 
     def open_transport(self):
@@ -461,7 +469,7 @@
 
 
 class JoyentMetadataSerialClient(JoyentMetadataClient):
-    def __init__(self, device, timeout=10, smartos_type=None):
+    def __init__(self, device, timeout=10, smartos_type=SMARTOS_ENV_KVM):
         super(JoyentMetadataSerialClient, self).__init__(smartos_type)
         self.device = device
         self.timeout = timeout
@@ -583,7 +591,8 @@
             device=serial_device, timeout=serial_timeout,
             smartos_type=smartos_type)
     elif smartos_type == SMARTOS_ENV_LX_BRAND:
-        return JoyentMetadataSocketClient(socketpath=metadata_sockfile)
+        return JoyentMetadataSocketClient(socketpath=metadata_sockfile,
+                                          smartos_type=smartos_type)
 
     raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
 
@@ -671,8 +680,9 @@
     return None
 
 
-# Covert SMARTOS 'sdc:nics' data to network_config yaml
-def convert_smartos_network_data(network_data=None):
+# Convert SMARTOS 'sdc:nics' data to network_config yaml
+def convert_smartos_network_data(network_data=None,
+                                 dns_servers=None, dns_domain=None):
     """Return a dictionary of network_config by parsing provided
        SMARTOS sdc:nics configuration data
 
@@ -706,9 +716,7 @@
             'broadcast',
             'dns_nameservers',
             'dns_search',
-            'gateway',
             'metric',
-            'netmask',
             'pointopoint',
             'routes',
             'scope',
@@ -716,6 +724,29 @@
         ],
     }
 
+    if dns_servers:
+        if not isinstance(dns_servers, (list, tuple)):
+            dns_servers = [dns_servers]
+    else:
+        dns_servers = []
+
+    if dns_domain:
+        if not isinstance(dns_domain, (list, tuple)):
+            dns_domain = [dns_domain]
+    else:
+        dns_domain = []
+
+    def is_valid_ipv4(addr):
+        return '.' in addr
+
+    def is_valid_ipv6(addr):
+        return ':' in addr
+
+    pgws = {
+        'ipv4': {'match': is_valid_ipv4, 'gw': None},
+        'ipv6': {'match': is_valid_ipv6, 'gw': None},
+    }
+
     config = []
     for nic in network_data:
         cfg = dict((k, v) for k, v in nic.items()
@@ -727,18 +758,40 @@
             cfg.update({'mac_address': nic['mac']})
 
         subnets = []
-        for ip, gw in zip(nic['ips'], nic['gateways']):
-            subnet = dict((k, v) for k, v in nic.items()
-                          if k in valid_keys['subnet'])
-            subnet.update({
-                'type': 'static',
-                'address': ip,
-                'gateway': gw,
-            })
+        for ip in nic.get('ips', []):
+            if ip == "dhcp":
+                subnet = {'type': 'dhcp4'}
+            else:
+                subnet = dict((k, v) for k, v in nic.items()
+                              if k in valid_keys['subnet'])
+                subnet.update({
+                    'type': 'static',
+                    'address': ip,
+                })
+
+                proto = 'ipv4' if is_valid_ipv4(ip) else 'ipv6'
+                # Only use gateways for 'primary' nics
+                if 'primary' in nic and nic.get('primary', False):
+                    # the ips and gateways list may be N to M, here
+                    # we map the ip index into the gateways list,
+                    # and handle the case that we could have more ips
+                    # than gateways.  we only consume the first gateway
+                    if not pgws[proto]['gw']:
+                        gateways = [gw for gw in nic.get('gateways', [])
+                                    if pgws[proto]['match'](gw)]
+                        if len(gateways):
+                            pgws[proto]['gw'] = gateways[0]
+                            subnet.update({'gateway': pgws[proto]['gw']})
+
             subnets.append(subnet)
         cfg.update({'subnets': subnets})
         config.append(cfg)
 
+    if dns_servers:
+        config.append(
+            {'type': 'nameserver', 'address': dns_servers,
+             'search': dns_domain})
+
     return {'version': 1, 'config': config}
 
 
@@ -761,21 +814,36 @@
         sys.exit(1)
     if len(sys.argv) == 1:
         keys = (list(SMARTOS_ATTRIB_JSON.keys()) +
-                list(SMARTOS_ATTRIB_MAP.keys()))
+                list(SMARTOS_ATTRIB_MAP.keys()) + ['network_config'])
     else:
         keys = sys.argv[1:]
 
-    data = {}
-    for key in keys:
+    def load_key(client, key, data):
+        if key in data:
+            return data[key]
+
         if key in SMARTOS_ATTRIB_JSON:
             keyname = SMARTOS_ATTRIB_JSON[key]
-            data[key] = jmc.get_json(keyname)
+            data[key] = client.get_json(keyname)
+        elif key == "network_config":
+            for depkey in ('network-data', 'dns_servers', 'dns_domain'):
+                load_key(client, depkey, data)
+            data[key] = convert_smartos_network_data(
+                network_data=data['network-data'],
+                dns_servers=data['dns_servers'],
+                dns_domain=data['dns_domain'])
         else:
             if key in SMARTOS_ATTRIB_MAP:
                 keyname, strip = SMARTOS_ATTRIB_MAP[key]
             else:
                 keyname, strip = (key, False)
-            val = jmc.get(keyname, strip=strip)
-            data[key] = jmc.get(keyname, strip=strip)
-
-    print(json.dumps(data, indent=1))
+            data[key] = client.get(keyname, strip=strip)
+
+        return data[key]
+
+    data = {}
+    for key in keys:
+        load_key(client=jmc, key=key, data=data)
+
+    print(json.dumps(data, indent=1, sort_keys=True,
+                     separators=(',', ': ')))
 
=== modified file 'tests/unittests/test_datasource/test_smartos.py'
--- tests/unittests/test_datasource/test_smartos.py	2016-06-10 21:22:17 +0000
+++ tests/unittests/test_datasource/test_smartos.py	2016-07-25 15:23:44 +0000
@@ -36,6 +36,8 @@
 
 from cloudinit import serial
 from cloudinit.sources import DataSourceSmartOS
+from cloudinit.sources.DataSourceSmartOS import (
+    convert_smartos_network_data as convert_net)
 
 import six
 
@@ -86,6 +88,229 @@
 ]
 """)
 
+
+SDC_NICS_ALT = json.loads("""
+[
+    {
+        "interface": "net0",
+        "mac": "90:b8:d0:ae:64:51",
+        "vlan_id": 324,
+        "nic_tag": "external",
+        "gateway": "8.12.42.1",
+        "gateways": [
+          "8.12.42.1"
+        ],
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.51",
+        "ips": [
+          "8.12.42.51/24"
+        ],
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "model": "virtio",
+        "mtu": 1500,
+        "primary": true
+    },
+    {
+        "interface": "net1",
+        "mac": "90:b8:d0:bd:4f:9c",
+        "vlan_id": 600,
+        "nic_tag": "internal",
+        "netmask": "255.255.255.0",
+        "ip": "10.210.1.217",
+        "ips": [
+          "10.210.1.217/24"
+        ],
+        "network_uuid": "98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+        "model": "virtio",
+        "mtu": 1500
+    }
+]
+""")
+
+SDC_NICS_DHCP = json.loads("""
+[
+    {
+        "interface": "net0",
+        "mac": "90:b8:d0:ae:64:51",
+        "vlan_id": 324,
+        "nic_tag": "external",
+        "gateway": "8.12.42.1",
+        "gateways": [
+          "8.12.42.1"
+        ],
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.51",
+        "ips": [
+          "8.12.42.51/24"
+        ],
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "model": "virtio",
+        "mtu": 1500,
+        "primary": true
+    },
+    {
+        "interface": "net1",
+        "mac": "90:b8:d0:bd:4f:9c",
+        "vlan_id": 600,
+        "nic_tag": "internal",
+        "netmask": "255.255.255.0",
+        "ip": "10.210.1.217",
+        "ips": [
+          "dhcp"
+        ],
+        "network_uuid": "98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+        "model": "virtio",
+        "mtu": 1500
+    }
+]
+""")
+
+SDC_NICS_MIP = json.loads("""
+[
+    {
+        "interface": "net0",
+        "mac": "90:b8:d0:ae:64:51",
+        "vlan_id": 324,
+        "nic_tag": "external",
+        "gateway": "8.12.42.1",
+        "gateways": [
+          "8.12.42.1"
+        ],
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.51",
+        "ips": [
+          "8.12.42.51/24",
+          "8.12.42.52/24"
+        ],
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "model": "virtio",
+        "mtu": 1500,
+        "primary": true
+    },
+    {
+        "interface": "net1",
+        "mac": "90:b8:d0:bd:4f:9c",
+        "vlan_id": 600,
+        "nic_tag": "internal",
+        "netmask": "255.255.255.0",
+        "ip": "10.210.1.217",
+        "ips": [
+          "10.210.1.217/24",
+          "10.210.1.151/24"
+        ],
+        "network_uuid": "98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+        "model": "virtio",
+        "mtu": 1500
+    }
+]
+""")
+
+SDC_NICS_MIP_IPV6 = json.loads("""
+[
+    {
+        "interface": "net0",
+        "mac": "90:b8:d0:ae:64:51",
+        "vlan_id": 324,
+        "nic_tag": "external",
+        "gateway": "8.12.42.1",
+        "gateways": [
+          "8.12.42.1"
+        ],
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.51",
+        "ips": [
+          "2001:4800:78ff:1b:be76:4eff:fe06:96b3/64",
+          "8.12.42.51/24"
+        ],
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "model": "virtio",
+        "mtu": 1500,
+        "primary": true
+    },
+    {
+        "interface": "net1",
+        "mac": "90:b8:d0:bd:4f:9c",
+        "vlan_id": 600,
+        "nic_tag": "internal",
+        "netmask": "255.255.255.0",
+        "ip": "10.210.1.217",
+        "ips": [
+          "10.210.1.217/24"
+        ],
+        "network_uuid": "98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+        "model": "virtio",
+        "mtu": 1500
+    }
+]
+""")
+
+SDC_NICS_IPV4_IPV6 = json.loads("""
+[
+    {
+        "interface": "net0",
+        "mac": "90:b8:d0:ae:64:51",
+        "vlan_id": 324,
+        "nic_tag": "external",
+        "gateway": "8.12.42.1",
+        "gateways": ["8.12.42.1", "2001::1", "2001::2"],
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.51",
+        "ips": ["2001::10/64", "8.12.42.51/24", "2001::11/64",
+                "8.12.42.52/32"],
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "model": "virtio",
+        "mtu": 1500,
+        "primary": true
+    },
+    {
+        "interface": "net1",
+        "mac": "90:b8:d0:bd:4f:9c",
+        "vlan_id": 600,
+        "nic_tag": "internal",
+        "netmask": "255.255.255.0",
+        "ip": "10.210.1.217",
+        "ips": ["10.210.1.217/24"],
+        "gateways": ["10.210.1.210"],
+        "network_uuid": "98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+        "model": "virtio",
+        "mtu": 1500
+    }
+]
+""")
+
+SDC_NICS_SINGLE_GATEWAY = json.loads("""
+[
+  {
+    "interface":"net0",
+    "mac":"90:b8:d0:d8:82:b4",
+    "vlan_id":324,
+    "nic_tag":"external",
+    "gateway":"8.12.42.1",
+    "gateways":["8.12.42.1"],
+    "netmask":"255.255.255.0",
+    "ip":"8.12.42.26",
+    "ips":["8.12.42.26/24"],
+    "network_uuid":"992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+    "model":"virtio",
+    "mtu":1500,
+    "primary":true
+  },
+  {
+    "interface":"net1",
+    "mac":"90:b8:d0:0a:51:31",
+    "vlan_id":600,
+    "nic_tag":"internal",
+    "netmask":"255.255.255.0",
+    "ip":"10.210.1.27",
+    "ips":["10.210.1.27/24"],
+    "network_uuid":"98657fdf-11f4-4ee2-88a4-ce7fe73e33a6",
+    "model":"virtio",
+    "mtu":1500
+  }
+]
+""")
+
+
 MOCK_RETURNS = {
     'hostname': 'test-host',
     'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
@@ -524,20 +749,135 @@
 
 
 class TestNetworkConversion(TestCase):
-
     def test_convert_simple(self):
         expected = {
             'version': 1,
             'config': [
                 {'name': 'net0', 'type': 'physical',
                  'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
-                              'netmask': '255.255.255.0',
                               'address': '8.12.42.102/24'}],
                  'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'},
                 {'name': 'net1', 'type': 'physical',
-                 'subnets': [{'type': 'static', 'gateway': '192.168.128.1',
-                              'netmask': '255.255.252.0',
+                 'subnets': [{'type': 'static',
                               'address': '192.168.128.93/22'}],
                  'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]}
-        found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS)
+        found = convert_net(SDC_NICS)
+        self.assertEqual(expected, found)
+
+    def test_convert_simple_alt(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'address': '8.12.42.51/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:ae:64:51'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'static',
+                              'address': '10.210.1.217/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:bd:4f:9c'}]}
+        found = convert_net(SDC_NICS_ALT)
+        self.assertEqual(expected, found)
+
+    def test_convert_simple_dhcp(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'address': '8.12.42.51/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:ae:64:51'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'dhcp4'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:bd:4f:9c'}]}
+        found = convert_net(SDC_NICS_DHCP)
+        self.assertEqual(expected, found)
+
+    def test_convert_simple_multi_ip(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'address': '8.12.42.51/24'},
+                             {'type': 'static',
+                              'address': '8.12.42.52/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:ae:64:51'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'static',
+                              'address': '10.210.1.217/24'},
+                             {'type': 'static',
+                              'address': '10.210.1.151/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:bd:4f:9c'}]}
+        found = convert_net(SDC_NICS_MIP)
+        self.assertEqual(expected, found)
+
+    def test_convert_with_dns(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'address': '8.12.42.51/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:ae:64:51'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'dhcp4'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:bd:4f:9c'},
+                {'type': 'nameserver',
+                 'address': ['8.8.8.8', '8.8.8.1'], 'search': ["local"]}]}
+        found = convert_net(
+            network_data=SDC_NICS_DHCP, dns_servers=['8.8.8.8', '8.8.8.1'],
+            dns_domain="local")
+        self.assertEqual(expected, found)
+
+    def test_convert_simple_multi_ipv6(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'address':
+                              '2001:4800:78ff:1b:be76:4eff:fe06:96b3/64'},
+                             {'type': 'static', 'gateway': '8.12.42.1',
+                              'address': '8.12.42.51/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:ae:64:51'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'static',
+                              'address': '10.210.1.217/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:bd:4f:9c'}]}
+        found = convert_net(SDC_NICS_MIP_IPV6)
+        self.assertEqual(expected, found)
+
+    def test_convert_simple_both_ipv4_ipv6(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'mac_address': '90:b8:d0:ae:64:51', 'mtu': 1500,
+                 'name': 'net0', 'type': 'physical',
+                 'subnets': [{'address': '2001::10/64', 'gateway': '2001::1',
+                              'type': 'static'},
+                             {'address': '8.12.42.51/24',
+                              'gateway': '8.12.42.1',
+                              'type': 'static'},
+                             {'address': '2001::11/64', 'type': 'static'},
+                             {'address': '8.12.42.52/32', 'type': 'static'}]},
+                {'mac_address': '90:b8:d0:bd:4f:9c', 'mtu': 1500,
+                 'name': 'net1', 'type': 'physical',
+                 'subnets': [{'address': '10.210.1.217/24',
+                              'type': 'static'}]}]}
+        found = convert_net(SDC_NICS_IPV4_IPV6)
+        self.assertEqual(expected, found)
+
+    def test_gateways_not_on_all_nics(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'mac_address': '90:b8:d0:d8:82:b4', 'mtu': 1500,
+                 'name': 'net0', 'type': 'physical',
+                 'subnets': [{'address': '8.12.42.26/24',
+                              'gateway': '8.12.42.1', 'type': 'static'}]},
+                {'mac_address': '90:b8:d0:0a:51:31', 'mtu': 1500,
+                 'name': 'net1', 'type': 'physical',
+                 'subnets': [{'address': '10.210.1.27/24',
+                              'type': 'static'}]}]}
+        found = convert_net(SDC_NICS_SINGLE_GATEWAY)
         self.assertEqual(expected, found)