Merge lp:~raharper/curtin/trunk.test-vlan into lp:~curtin-dev/curtin/trunk

[Ryan Harper]

vmtests: add network vlan testing

- Utilize a real vlan configuration used in openstack deployments which on
  trusty sometimes triggers a bug/issue with ifupdown (lp:1565711)

- curtin/net now writes: network: {config: disabled} into
  /etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg
  as in Xenial cloud-init network config device naming causes issues which
  prevent cloud-init from finding the data source and then bad things
  happen.

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:377
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~raharper/curtin/trunk.test-vlan/+merge/291023/+edit-commit-message

https://server-team-jenkins.canonical.com/job/curtin-ci/201/
Executed test runs:
    None: https://server-team-jenkins.canonical.com/job/generic-update-mp/198/console

Click here to trigger a rebuild:
https://server-team-jenkins.canonical.com/job/curtin-ci/201/rebuild

review: Needs Fixing (continuous-integration)

[lp:~raharper/curtin/trunk.test-vlan]

373. By Ryan Harper on 2016-04-06

vmtests: enhance vlan testing on all releases

Enable vlan testing for all distros
Fix testcase logic for multiple vlan interfaces

[Ryan Harper (raharper)]

On Wed, Apr 6, 2016 at 1:53 PM, Scott Moser <email address hidden> wrote:

>
>
> Diff comments:
>
> > === modified file 'curtin/net/__init__.py'
> > --- curtin/net/__init__.py 2016-03-31 14:27:52 +0000
> > +++ curtin/net/__init__.py 2016-04-06 16:25:28 +0000
> > @@ -444,4 +446,10 @@
> > LOG.info('Writing ' + netrules)
> > f.write(persistent_net_rules)
> >
> > + ccfg = os.path.sep.join((target, cloud_cfg_disable,))
>
> why not util.write_file() ?
>

Probably before I knew about it. Will convert them all.

>
> > + util.ensure_dir(os.path.dirname(ccfg))
> > + with open(ccfg, 'w+') as f:
> > + LOG.info('Writing ' + ccfg)
> > + f.write('network: {config: disabled}\n')
> > +
> > # vi: ts=4 expandtab syntax=python
>
>
> --
> https://code.launchpad.net/~raharper/curtin/trunk.test-vlan/+merge/291023
> You are the owner of lp:~raharper/curtin/trunk.test-vlan.
>

[lp:~raharper/curtin/trunk.test-vlan]

374. By Ryan Harper on 2016-04-07

use util.write_file for writing network configuration files

=== modified file 'curtin/net/__init__.py'
--- curtin/net/__init__.py	2016-03-31 14:27:52 +0000
+++ curtin/net/__init__.py	2016-04-07 14:33:20 +0000
@@ -429,19 +429,18 @@
 def render_network_state(target, network_state):
     eni = 'etc/network/interfaces'
     netrules = 'etc/udev/rules.d/70-persistent-net.rules'
+    cc = 'etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg'
 
     eni = os.path.sep.join((target, eni,))
-    util.ensure_dir(os.path.dirname(eni))
-    with open(eni, 'w+') as f:
-        LOG.info('Writing ' + eni)
-        f.write(render_interfaces(network_state))
+    LOG.info('Writing ' + eni)
+    util.write_file(eni, content=render_interfaces(network_state))
 
     netrules = os.path.sep.join((target, netrules,))
-    util.ensure_dir(os.path.dirname(netrules))
-    persistent_net_rules = render_persistent_net(network_state)
-    if len(persistent_net_rules) > 0:
-        with open(netrules, 'w+') as f:
-            LOG.info('Writing ' + netrules)
-            f.write(persistent_net_rules)
+    LOG.info('Writing ' + netrules)
+    util.write_file(netrules, content=render_persistent_net(network_state))
+
+    cc_disable = os.path.sep.join((target, cc,))
+    LOG.info('Writing ' + cc_disable)
+    util.write_file(cc_disable, content='network: {config: disabled}\n')
 
 # vi: ts=4 expandtab syntax=python

=== added file 'examples/tests/vlan_network.yaml'
--- examples/tests/vlan_network.yaml	1970-01-01 00:00:00 +0000
+++ examples/tests/vlan_network.yaml	2016-04-07 14:33:20 +0000
@@ -0,0 +1,81 @@
+network:
+  config:
+  - id: eth0
+    mac_address: d4:be:d9:a8:49:13
+    mtu: 1500
+    name: eth0
+    subnets:
+    - address: 10.245.168.16/21
+      dns_nameservers:
+      - 10.245.168.2
+      gateway: 10.245.168.1
+      type: static
+    type: physical
+  - id: eth1
+    mac_address: d4:be:d9:a8:49:15
+    mtu: 1500
+    name: eth1
+    subnets:
+    - address: 10.245.188.2/24
+      dns_nameservers: []
+      type: static
+    type: physical
+  - id: eth2
+    mac_address: d4:be:d9:a8:49:17
+    mtu: 1500
+    name: eth2
+    subnets:
+    - type: manual
+    type: physical
+  - id: eth3
+    mac_address: d4:be:d9:a8:49:19
+    mtu: 1500
+    name: eth3
+    subnets:
+    - type: manual
+    type: physical
+  - id: eth1.2667
+    mtu: 1500
+    name: eth1.2667
+    subnets:
+    - address: 10.245.184.2/24
+      dns_nameservers: []
+      type: static
+    type: vlan
+    vlan_id: 2667
+    vlan_link: eth1
+  - id: eth1.2668
+    mtu: 1500
+    name: eth1.2668
+    subnets:
+    - address: 10.245.185.1/24
+      dns_nameservers: []
+      type: static
+    type: vlan
+    vlan_id: 2668
+    vlan_link: eth1
+  - id: eth1.2669
+    mtu: 1500
+    name: eth1.2669
+    subnets:
+    - address: 10.245.186.1/24
+      dns_nameservers: []
+      type: static
+    type: vlan
+    vlan_id: 2669
+    vlan_link: eth1
+  - id: eth1.2670
+    mtu: 1500
+    name: eth1.2670
+    subnets:
+    - address: 10.245.187.2/24
+      dns_nameservers: []
+      type: static
+    type: vlan
+    vlan_id: 2670
+    vlan_link: eth1
+  - address: 10.245.168.2
+    search:
+    - dellstack
+    type: nameserver
+  version: 1

=== modified file 'tests/vmtests/__init__.py'
--- tests/vmtests/__init__.py	2016-04-04 18:20:33 +0000
+++ tests/vmtests/__init__.py	2016-04-07 14:33:20 +0000
@@ -871,6 +871,7 @@
         'password': 'passw0rd',
         'chpasswd': {'expire': False},
         'power_state': {'mode': 'poweroff'},
+        'network': {'config': 'disabled'},
     }
 
     ssh_keys, _err = util.subp(['tools/ssh-keys-list', 'cloud-config'],

=== modified file 'tests/vmtests/test_network.py'
--- tests/vmtests/test_network.py	2016-03-03 11:46:13 +0000
+++ tests/vmtests/test_network.py	2016-04-07 14:33:20 +0000
@@ -9,7 +9,7 @@
 
 
 def iface_extract(input):
-    mo = re.search(r'^(?P<interface>\w+|\w+:\d+)\s+' +
+    mo = re.search(r'^(?P<interface>\w+|\w+:\d+|\w+\.\d+)\s+' +
                    r'Link encap:(?P<link_encap>\S+)\s+' +
                    r'(HWaddr\s+(?P<mac_address>\S+))?' +
                    r'(\s+inet addr:(?P<address>\S+))?' +
@@ -59,6 +59,7 @@
         cp -av /etc/udev/rules.d/70-persistent-net.rules .
         ip -o route show > ip_route_show
         route -n > route_n
+        cp -av /run/network ./run_network
         """)]
 
     def test_output_files_exist(self):
@@ -237,6 +238,48 @@
     conf_file = "examples/tests/basic_network_static.yaml"
 
 
+class TestNetworkVlanAbs(TestNetworkAbs):
+    conf_file = "examples/tests/vlan_network.yaml"
+    collect_scripts = TestNetworkAbs.collect_scripts + [textwrap.dedent("""
+             cd OUTPUT_COLLECT_D
+             dpkg-query -W -f '${Status}' vlan > vlan_installed
+             ip -d link show eth1.2667 > ip_link_show_eth1.2667
+             ip -d link show eth1.2668 > ip_link_show_eth1.2668
+             ip -d link show eth1.2669 > ip_link_show_eth1.2669
+             ip -d link show eth1.2670 > ip_link_show_eth1.2670
+             """)]
+
+    def get_vlans(self):
+        network_state = self.get_network_state()
+        logger.debug('get_vlans ns:\n{}'.format(
+            yaml.dump(network_state, default_flow_style=False, indent=4)))
+        interfaces = network_state.get('interfaces')
+        return [iface for iface in interfaces.values()
+                if iface['type'] == 'vlan']
+
+    def test_output_files_exist_vlan(self):
+        link_files = ["ip_link_show_{}".format(vlan['name'])
+                      for vlan in self.get_vlans()]
+        self.output_files_exist(["vlan_installed"] + link_files)
+
+    def test_vlan_installed(self):
+        with open(os.path.join(self.td.collect, "vlan_installed")) as fp:
+            status = fp.read().strip()
+            logger.debug('vlan installed?: {}'.format(status))
+            self.assertEqual('install ok installed', status)
+
+    def test_vlan_enabled(self):
+
+        # we must have at least one
+        self.assertGreaterEqual(len(self.get_vlans()), 1)
+
+        # did they get configured?
+        for vlan in self.get_vlans():
+            link_file = "ip_link_show_" + vlan['name']
+            vlan_msg = "vlan protocol 802.1Q id " + str(vlan['vlan_id'])
+            self.check_file_regex(link_file, vlan_msg)
+
+
 class PreciseHWETTestNetwork(relbase.precise_hwe_t, TestNetworkAbs):
     # FIXME: off due to hang at test: Starting execute cloud user/final scripts
     __test__ = False
@@ -321,3 +364,35 @@
     #        over the net.ifnames to the installed system via '---' as the net
     #        config should take care of that.
     extra_kern_args = "net.ifnames=0"
+
+
+class PreciseTestNetworkVlan(relbase.precise, TestNetworkVlanAbs):
+    __test__ = True
+
+    # precise ip -d link show output is different (of course)
+    def test_vlan_enabled(self):
+
+        # we must have at least one
+        self.assertGreaterEqual(len(self.get_vlans()), 1)
+
+        # did they get configured?
+        for vlan in self.get_vlans():
+            link_file = "ip_link_show_" + vlan['name']
+            vlan_msg = "vlan id " + str(vlan['vlan_id'])
+            self.check_file_regex(link_file, vlan_msg)
+
+
+class TrustyTestNetworkVlan(relbase.trusty, TestNetworkVlanAbs):
+    __test__ = True
+
+
+class VividTestNetworkVlan(relbase.vivid, TestNetworkVlanAbs):
+    __test__ = True
+
+
+class WilyTestNetworkVlan(relbase.wily, TestNetworkVlanAbs):
+    __test__ = True
+
+
+class XenialTestNetworkVlan(relbase.xenial, TestNetworkVlanAbs):
+    __test__ = True