Merge lp:~jtv/maas/generate-node-networking-config-2 into lp:~maas-committers/maas/trunk

[Jeroen T. Vermeulen]

More curtin networking config: generate DNS server entries, and network entries. (This doesn't include the network entries into the config yet; that's for a later branch.) Also, update to a seemingly newer example of the file format.

This builds on a predecessor branch. You may want to have a look at that first.

Jeroen

[Graham Binns (gmb)]

Great work, Jeroen. Thank you for persevering with this!

Just a few nitpicks; nothing major.

[Graham Binns (gmb)]

review: Approve

[MAAS Lander (maas-lander)]

The attempt to merge lp:~jtv/maas/generate-node-networking-config-2 into lp:maas failed. Below is the output from the failed tests.

Ign http://security.ubuntu.com trusty-security InRelease
Hit http://security.ubuntu.com trusty-security Release.gpg
Hit http://security.ubuntu.com trusty-security Release
Ign http://nova.clouds.archive.ubuntu.com trusty InRelease
Ign http://nova.clouds.archive.ubuntu.com trusty-updates InRelease
Hit http://nova.clouds.archive.ubuntu.com trusty Release.gpg
Hit http://nova.clouds.archive.ubuntu.com trusty-updates Release.gpg
Hit http://nova.clouds.archive.ubuntu.com trusty Release
Hit http://nova.clouds.archive.ubuntu.com trusty-updates Release
Hit http://security.ubuntu.com trusty-security/main Sources
Hit http://security.ubuntu.com trusty-security/universe Sources
Hit http://security.ubuntu.com trusty-security/main amd64 Packages
Hit http://security.ubuntu.com trusty-security/universe amd64 Packages
Hit http://security.ubuntu.com trusty-security/main Translation-en
Hit http://security.ubuntu.com trusty-security/universe Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty/main Sources
Hit http://nova.clouds.archive.ubuntu.com trusty/universe Sources
Hit http://nova.clouds.archive.ubuntu.com trusty/main amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty/universe amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty/main Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty/universe Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/main Sources
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/universe Sources
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/main amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/universe amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/main Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/universe Translation-en
Ign http://nova.clouds.archive.ubuntu.com trusty/main Translation-en_US
Ign http://nova.clouds.archive.ubuntu.com trusty/universe Translation-en_US
Reading package lists...
sudo DEBIAN_FRONTEND=noninteractive apt-get -y \
     --no-install-recommends install apache2 authbind bind9 bind9utils build-essential bzr-builddeb curl daemontools debhelper dh-apport distro-info dnsutils firefox freeipmi-tools ipython isc-dhcp-common libjs-raphael libjs-yui3-full libjs-yui3-min libpq-dev make pep8 postgresql pyflakes python-amqplib python-bzrlib python-celery python-convoy python-crochet python-cssselect python-curtin python-dev python-distro-info python-django python-django-piston python-django-south python-djorm-ext-pgarray python-docutils python-extras python-fixtures python-flake8 python-formencode python-hivex python-httplib2 python-jinja2 python-jsonschema python-lockfile python-lxml python-mimeparse python-mock python-netaddr python-netifaces python-nose python-oauth python-oops python-oops-amqp python-oops-datedir-repo python-oops-twisted python-oops-wsgi python-openssl python-paramiko python-pexpect python-pip python-pocket-lint python-psycopg2 python-pyinotify python-seamicroclient python-simplejson python-simplestreams python-...

=== modified file 'src/maasserver/networking_preseed.py'
--- src/maasserver/networking_preseed.py	2014-09-24 02:48:44 +0000
+++ src/maasserver/networking_preseed.py	2014-09-24 03:07:38 +0000
@@ -7,6 +7,10 @@
 
    https://gist.github.com/jayofdoom/b035067523defec7fb53
 
+A different version of the format is documented here:
+
+    http://bit.ly/1uqWvC8
+
 The installer running on a node will use this to set up the node's networking
 according to MAAS's specifications.
 """
@@ -25,7 +29,13 @@
     ]
 
 from lxml import etree
+from maasserver.dns.zonegenerator import get_dns_server_address
+from maasserver.exceptions import UnresolvableHost
 from maasserver.models.nodeprobeddetails import get_probed_details
+from netaddr import (
+    IPAddress,
+    IPNetwork,
+    )
 
 
 def extract_network_interface_data(element):
@@ -90,11 +100,120 @@
     """
     return {
         'id': interface,
-        'type': 'ethernet',
+        'type': 'phy',
         'ethernet_mac_address': mac,
         }
 
 
+def generate_dns_server_entry(dns_address):
+    """Generate the `services` list entry for the given DNS server.
+
+    :param dns_address: IP address for a DNS server, in text form.
+    :return: A dict specifying the DNS server as a network service, with
+        keys `address` and `type` (and possibly more).
+    """
+    return {
+        'type': 'dns',
+        'address': dns_address,
+        }
+
+
+def list_dns_servers(node):
+    """Return DNS servers, IPv4 and IPv6 as appropriate, for use by `node`.
+
+    These are always the MAAS-controlled DNS servers.
+    """
+    cluster = node.nodegroup
+    servers = []
+    if not node.disable_ipv4:
+        try:
+            servers.append(
+                get_dns_server_address(cluster, ipv4=True, ipv6=False))
+        except UnresolvableHost:
+            # No IPv4 DNS server.
+            pass
+    try:
+        servers.append(get_dns_server_address(cluster, ipv4=False, ipv6=True))
+    except UnresolvableHost:
+        # No IPv6 DNS server.
+        pass
+    return [dns_server for dns_server in servers if dns_server is not None]
+
+
+def generate_route_entries(cluster_interface):
+    """Generate `routes` list entries for a cluster interface.
+
+    Actually this returns exactly one route (the default route) if
+    `cluster_interface` has a router set; or none otherwise.
+    """
+    if cluster_interface.router_ip in ('', None):
+        # No routes available.
+        return []
+    elif IPAddress(cluster_interface.ip).version == 4:
+        return [
+            {
+                'network': '0.0.0.0',
+                'netmask': '0.0.0.0',
+                'gateway': unicode(cluster_interface.router_ip),
+            },
+            ]
+    else:
+        return [
+            {
+                'network': '::',
+                'netmask': '::',
+                'gateway': unicode(cluster_interface.router_ip),
+            },
+            ]
+
+
+def generate_network_entry(network_interface, cluster_interface, ip=None):
+    """Generate the `networks` list entry for the given network connection.
+
+    :param network_interface: Name of the network interface (on the node) that
+        connects to this network.
+    :param cluster_interface: The `NodeGroupInterface` responsible for this
+        network.  (Do not confuse its `interface` property, which is a network
+        interface on the cluster controller, with the `network_interface`
+        parameter which is a network interface on the node.)
+    :param ip: Optional IP address.  If not given, use DHCP.
+    """
+    network_types = {
+        4: 'ipv4',
+        6: 'ipv6',
+        }
+    network = cluster_interface.network
+
+    # Still lacking a few entries that we don't have enough information about:
+    # * id -- does this need to match anything anywhere?
+    # * network_id -- what is this, and how do we compose it?
+    #
+    # It's tempting to use cluster_interface.name for the 'id,' but that
+    # could be confusing: it was probably generated based on the name of its
+    # network interface on the cluster.  Which will probably often match the
+    # name of the node cluster interface, but is completely unrelated to it.
+    entry = {
+        'link': network_interface,
+        # The example we have does not show IPv6 netmasks.  Should we pass
+        # width in bits?
+        'type': network_types[network.version],
+        'routes': generate_route_entries(cluster_interface)
+        }
+    if ip is not None:
+        # Set static IP address.
+        # How do we tell the node to request a dynamic IP address over DHCP?
+        # Is just omitting ip_address the appropriate behaviour?
+        if network.version == 4:
+            entry['ip_address'] = ip
+        else:
+            # Include network size directly in IPv6 address.
+            entry['ip_address'] = unicode(
+                IPNetwork("%s/%s" % (ip, network.netmask)))
+    if network.version == 4:
+        entry['netmask'] = unicode(network.netmask)
+    return entry
+
+
 def generate_networking_config(node):
     """Generate a networking preseed for `node`.
 
@@ -108,7 +227,8 @@
         'provider': "MAAS",
         'network_info': {
             'services': [
-                # List DNS servers here.
+                generate_dns_server_entry(dns_server)
+                for dns_server in list_dns_servers(node)
                 ],
             'networks': [
                 # Write network specs here.

=== modified file 'src/maasserver/tests/test_networking_preseed.py'
--- src/maasserver/tests/test_networking_preseed.py	2014-09-24 02:47:58 +0000
+++ src/maasserver/tests/test_networking_preseed.py	2014-09-24 03:07:38 +0000
@@ -15,16 +15,26 @@
 __all__ = [
     ]
 
+from random import randint
+
 from maasserver import networking_preseed
+from maasserver.dns import zonegenerator
+from maasserver.enum import NODEGROUPINTERFACE_MANAGEMENT
+from maasserver.exceptions import UnresolvableHost
 from maasserver.networking_preseed import (
     extract_network_interfaces,
+    generate_dns_server_entry,
     generate_ethernet_link_entry,
+    generate_network_entry,
     generate_networking_config,
+    generate_route_entries,
+    list_dns_servers,
     normalise_mac,
     )
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import MAASServerTestCase
 from maastesting.matchers import MockCalledOnceWith
+from testtools.matchers import HasLength
 
 
 def make_denormalised_mac():
@@ -211,12 +221,210 @@
         self.assertEqual(
             {
                 'id': interface,
-                'type': 'ethernet',
+                'type': 'phy',
                 'ethernet_mac_address': mac,
             },
             generate_ethernet_link_entry(interface, mac))
 
 
+class TestGenerateDNServerEntry(MAASServerTestCase):
+
+    def test__returns_dict(self):
+        address = factory.make_ipv4_address()
+        self.assertEqual(
+            {
+                'type': 'dns',
+                'address': address,
+            },
+            generate_dns_server_entry(address))
+
+
+def patch_dns_servers(testcase, ipv4_dns=None, ipv6_dns=None):
+    """Patch `get_dns_server_address` to return the given addresses.
+
+    The fake will return `ipv4_dns` or `ipv6_dns` as appropriate to the
+    arguments.  For that reason, this patch does not use a `Mock`.
+    """
+
+    def fake_get_maas_facing_server_address(cluster, ipv4=True, ipv6=True):
+        result = None
+        if ipv4:
+            result = ipv4_dns
+        if result is None and ipv6:
+            result = ipv6_dns
+        if result is None:
+            raise UnresolvableHost()
+        return result
+
+    testcase.patch(
+        zonegenerator, 'get_maas_facing_server_address',
+        fake_get_maas_facing_server_address)
+    testcase.patch(zonegenerator, 'warn_loopback')
+
+
+class ListDNSServers(MAASServerTestCase):
+
+    def test__includes_ipv4_and_ipv6_by_default(self):
+        ipv4_dns = factory.make_ipv4_address()
+        ipv6_dns = factory.make_ipv6_address()
+        patch_dns_servers(self, ipv4_dns=ipv4_dns, ipv6_dns=ipv6_dns)
+        node = factory.make_Node(disable_ipv4=False)
+        self.assertItemsEqual([ipv4_dns, ipv6_dns], list_dns_servers(node))
+
+    def test__omits_ipv4_if_disabled_for_node(self):
+        ipv4_dns = factory.make_ipv4_address()
+        ipv6_dns = factory.make_ipv6_address()
+        patch_dns_servers(self, ipv4_dns=ipv4_dns, ipv6_dns=ipv6_dns)
+        node = factory.make_Node(disable_ipv4=True)
+        self.assertItemsEqual([ipv6_dns], list_dns_servers(node))
+
+    def test__omits_ipv4_if_unvailable(self):
+        ipv6_dns = factory.make_ipv6_address()
+        patch_dns_servers(self, ipv6_dns=ipv6_dns)
+        node = factory.make_Node(disable_ipv4=False)
+        self.assertItemsEqual([ipv6_dns], list_dns_servers(node))
+
+    def test__omits_ipv6_if_unavailable(self):
+        ipv4_dns = factory.make_ipv4_address()
+        patch_dns_servers(self, ipv4_dns=ipv4_dns)
+        node = factory.make_Node(disable_ipv4=False)
+        self.assertItemsEqual([ipv4_dns], list_dns_servers(node))
+
+
+def make_cluster_interface(network=None, **kwargs):
+    return factory.make_NodeGroupInterface(
+        factory.make_NodeGroup(), network=network, **kwargs)
+
+
+class TestGenerateRouteEntries(MAASServerTestCase):
+
+    def test__generates_IPv4_default_route_if_available(self):
+        network = factory.make_ipv4_network()
+        router = factory.pick_ip_in_network(network)
+        cluster_interface = make_cluster_interface(network, router_ip=router)
+        self.assertEqual(
+            [
+                {
+                    'network': '0.0.0.0',
+                    'netmask': '0.0.0.0',
+                    'gateway': unicode(router),
+                },
+            ],
+            generate_route_entries(cluster_interface))
+
+    def test__generates_IPv6_default_route_if_available(self):
+        network = factory.make_ipv6_network()
+        router = factory.pick_ip_in_network(network)
+        cluster_interface = make_cluster_interface(network, router_ip=router)
+        self.assertEqual(
+            [
+                {
+                    'network': '::',
+                    'netmask': '::',
+                    'gateway': unicode(router),
+                },
+            ],
+            generate_route_entries(cluster_interface))
+
+    def test__generates_empty_list_if_no_route_available(self):
+        network = factory.make_ipv4_network()
+        cluster_interface = make_cluster_interface(
+            network, management=NODEGROUPINTERFACE_MANAGEMENT.UNMANAGED,
+            router_ip='')
+        self.assertEqual([], generate_route_entries(cluster_interface))
+
+
+class TestGenerateNetworkEntry(MAASServerTestCase):
+
+    def test__generates_IPv4_dict(self):
+        network = factory.make_ipv4_network()
+        network_interface = factory.make_name('eth')
+        cluster_interface = make_cluster_interface(network)
+        ip = factory.pick_ip_in_network(network)
+
+        entry = generate_network_entry(
+            network_interface, cluster_interface, ip=ip)
+
+        del entry['routes']
+        self.assertEqual(
+            {
+                'type': 'ipv4',
+                'link': network_interface,
+                'ip_address': unicode(ip),
+                'netmask': unicode(network.netmask),
+            },
+            entry)
+
+    def test__generates_IPv6_dict(self):
+        slash = randint(48, 64)
+        network = factory.make_ipv6_network(slash=slash)
+        network_interface = factory.make_name('eth')
+        cluster_interface = make_cluster_interface(network)
+        ip = factory.pick_ip_in_network(network)
+
+        entry = generate_network_entry(
+            network_interface, cluster_interface, ip=ip)
+
+        del entry['routes']
+        self.assertEqual(
+            {
+                'type': 'ipv6',
+                'link': network_interface,
+                'ip_address': '%s/%d' % (ip, slash),
+            },
+            entry)
+
+    def test__omits_IP_if_not_given(self):
+        network = factory.make_ipv4_network()
+        network_interface = factory.make_name('eth')
+        cluster_interface = make_cluster_interface(network)
+
+        entry = generate_network_entry(network_interface, cluster_interface)
+
+        del entry['routes']
+        self.assertEqual(
+            {
+                'type': 'ipv4',
+                'link': network_interface,
+                'netmask': unicode(network.netmask),
+            },
+            entry)
+
+    def test__tells_IPv4_from_IPv6_even_without_IP(self):
+        cluster_interface = make_cluster_interface(factory.make_ipv6_network())
+        entry = generate_network_entry(
+            factory.make_name('eth'), cluster_interface)
+        self.assertEqual('ipv6', entry['type'])
+
+    def test__includes_IPv4_routes_on_IPv4_network(self):
+        network = factory.make_ipv4_network()
+        router = factory.pick_ip_in_network(network)
+        cluster_interface = make_cluster_interface(
+            network, management=NODEGROUPINTERFACE_MANAGEMENT.DHCP,
+            router_ip=router)
+
+        entry = generate_network_entry(
+            factory.make_name('eth'), cluster_interface)
+
+        self.assertThat(entry['routes'], HasLength(1))
+        [route] = entry['routes']
+        self.assertEqual(unicode(router), route['gateway'])
+
+    def test__includes_IPv6_routes_on_IPv6_network(self):
+        network = factory.make_ipv6_network()
+        router = factory.pick_ip_in_network(network)
+        cluster_interface = make_cluster_interface(
+            network, management=NODEGROUPINTERFACE_MANAGEMENT.DHCP,
+            router_ip=router)
+
+        entry = generate_network_entry(
+            factory.make_name('eth'), cluster_interface)
+
+        self.assertThat(entry['routes'], HasLength(1))
+        [route] = entry['routes']
+        self.assertEqual(unicode(router), route['gateway'])
+
+
 class TestGenerateNetworkingConfig(MAASServerTestCase):
 
     def patch_interfaces(self, interface_mac_pairs):
@@ -227,11 +435,13 @@
 
     def test__returns_config_dict(self):
         self.patch_interfaces([])
+        patch_dns_servers(self)
         config = generate_networking_config(factory.make_Node())
         self.assertIsInstance(config, dict)
         self.assertEqual("MAAS", config['provider'])
 
     def test__includes_links(self):
+        patch_dns_servers(self)
         node = factory.make_Node()
         interface = factory.make_name('eth')
         mac = factory.make_mac_address()
@@ -244,7 +454,7 @@
             [
                 {
                     'id': interface,
-                    'type': 'ethernet',
+                    'type': 'phy',
                     'ethernet_mac_address': mac,
                 },
             ],
@@ -252,12 +462,22 @@
 
     def test__includes_networks(self):
         # This section is not yet implemented, so expect an empty list.
+        patch_dns_servers(self)
         self.patch_interfaces([])
         config = generate_networking_config(factory.make_Node())
         self.assertEqual([], config['network_info']['networks'])
 
     def test__includes_dns_servers(self):
-        # This section is not yet implemented, so expect an empty list.
+        dns_address = factory.make_ipv4_address()
+        patch_dns_servers(self, dns_address)
         self.patch_interfaces([])
-        config = generate_networking_config(factory.make_Node())
-        self.assertEqual([], config['network_info']['services'])
+        config = generate_networking_config(
+            factory.make_Node(disable_ipv4=False))
+        self.assertEqual(
+            [
+                {
+                    'type': 'dns',
+                    'address': dns_address,
+                },
+            ],
+            config['network_info']['services'])