Merge ~raharper/curtin:feature/enable-ovs into curtin:master

[Ryan Harper]

net/deps.py: detect openvswitch cfg and install openvswitch packages

Scan the provided network-config and if openvswitch keywords are
present then install the require openvswitch packages into the
target system.

- Enable both version detections, top-level openvswitch and as
  renderer
- Also detect renderer values and install the required renderer
- Add vmtest to verify package is installed

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:6814799023abe43bb60a4d2adc8eb95e029c2156
https://jenkins.ubuntu.com/server/job/curtin-ci/3835/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/3835/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/3835/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/3835/
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=torkoal/3835/

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/3835//rebuild

review: Approve (continuous-integration)

[Chad Smith (chad.smith)]

This iteration looks good to me and is understandable.

I think we may have to circle back for a followup branch on whether DPDK type fields imply a different package name (like whether we need to additionally install openvswitch-switch-dpdk).

But, I think we can address that in a followup when we get more information.
+1 for now.

[Chad Smith (chad.smith)]

review: Approve

diff --git a/curtin/net/deps.py b/curtin/net/deps.py
index cbebae9..fd9e3c0 100644
--- a/curtin/net/deps.py
+++ b/curtin/net/deps.py
@@ -27,8 +27,15 @@ def network_config_required_packages(network_config, mapping=None):
                           for device in network_config['config'])
     else:
         # v2 has no config key
-        dev_configs = set(cfgtype for (cfgtype, cfg) in
-                          network_config.items() if cfgtype not in ['version'])
+        dev_configs = set()
+        for cfgtype, cfg in network_config.items():
+            if cfgtype == 'version':
+                continue
+            dev_configs.add(cfgtype)
+            # the renderer type may trigger package adds
+            for entry, entry_cfg in cfg.items():
+                if entry_cfg.get('renderer'):
+                    dev_configs.add(entry_cfg.get('renderer'))
 
     needed_packages = []
     for dev_type in dev_configs:
@@ -50,6 +57,9 @@ def detect_required_packages_mapping(osfamily=DISTROS.debian):
             'bonds': ['ifenslave'],
             'bridge': ['bridge-utils'],
             'bridges': ['bridge-utils'],
+            'openvswitch': ['openvswitch-switch'],
+            'networkd': ['systemd'],
+            'NetworkManager': ['network-manager'],
             'vlan': ['vlan'],
             'vlans': ['vlan']},
         DISTROS.redhat: {
@@ -57,6 +67,7 @@ def detect_required_packages_mapping(osfamily=DISTROS.debian):
             'bonds': [],
             'bridge': [],
             'bridges': [],
+            'openvswitch': ['openvswitch-switch'],
             'vlan': [],
             'vlans': []},
     }
diff --git a/examples/tests/network_v2_ovs.yaml b/examples/tests/network_v2_ovs.yaml
 new file mode 100644
index 0000000..6d1dc3d
--- /dev/null
+++ b/examples/tests/network_v2_ovs.yaml
@@ -0,0 +1,42 @@
+# example netplan config with openvswitch support
+# showtrace: true
+network:
+    version: 2
+    ethernets:
+        eth0:
+            dhcp4: true
+            match:
+                macaddress: '52:54:00:12:34:00'
+            set-name: eth0
+        eth1:
+            match:
+                macaddress: '52:54:00:12:34:02'
+            set-name: eth1
+        eth2:
+            match:
+                macaddress: '52:54:00:12:34:04'
+            set-name: eth2
+    openvswitch:
+      bridges:
+        br-int:
+          fail-mode: secure
+          datapath_type: system
+          stp: false
+          rstp: false
+          mcast-snooping: false
+          controller:
+            addresses:
+              - tcp:127.0.0.1:6653
+          protocols:
+            - OpenFlow10
+            - OpenFlow12
+            - OpenFlow13
+          ports:
+            patch-tun:
+              type: patch
+              options:
+                peer: patch-int
+            eth1:
+              tag: 2
+            eth2:
+              tag: 2
diff --git a/tests/unittests/test_curthooks.py b/tests/unittests/test_curthooks.py
index 3fefd26..ff38240 100644
--- a/tests/unittests/test_curthooks.py
+++ b/tests/unittests/test_curthooks.py
@@ -984,11 +984,24 @@ class TestDetectRequiredPackages(CiTestCase):
                         {'type': 'static', 'address': '192.168.14.2/24'},
                         {'type': 'static', 'address': '2001:1::1/64'}]}},
             2: {
-                'vlan': {
+                'openvswitch': {
+                    'openvswitch': {
+                        'bridges': {
+                            'br-int': {'ports': {'eth15': {'tag': 2}}}}}},
+                'vlans': {
                     'vlans': {
                         'en-intra': {'id': 1, 'link': 'eno1', 'dhcp4': 'yes'},
                         'en-vpn': {'id': 2, 'link': 'eno1'}}},
-                'bridge': {
+                'renderers': {
+                    'bridges': {
+                        'br-ext': {'renderer': 'openvswitch',
+                                   'ports': {'eth7': {'tag': 9}}}},
+                    'wifis': {
+                        'wlps0': {'renderer': 'NetworkManager',
+                                  'dhcp4': True}},
+                    'ethernets': {
+                        'ens7p0': {'renderer': 'networkd', 'dhcp6': True}}},
+                'bridges': {
                     'bridges': {
                         'br0': {
                             'interfaces': ['wlp1s0', 'switchports'],
@@ -1014,6 +1027,8 @@ class TestDetectRequiredPackages(CiTestCase):
     def _test_req_mappings(self, req_mappings):
         for (config_items, expected_reqs) in req_mappings:
             cfg = self._fmt_config(config_items)
+            print('test_config:\n%s' % config.dump_config(cfg))
+            print()
             actual_reqs = curthooks.detect_required_packages(cfg)
             self.assertEqual(set(actual_reqs), set(expected_reqs),
                              'failed for config: {}'.format(config_items))
@@ -1084,27 +1099,52 @@ class TestDetectRequiredPackages(CiTestCase):
              ('e2fsprogs', '^btrfs-(progs|tools)$', 'vlan', 'ifenslave')),
         ))
 
-    def test_network_v2_detect(self):
+    def test_network_v2_detect_bridges(self):
         self._test_req_mappings((
             ({'network': {
                 'version': 2,
-                'items': ('bridge',)}},
+                'items': ('bridges',)}},
              ('bridge-utils', )),
+        ))
+
+    def test_network_v2_detect_vlan(self):
+        self._test_req_mappings((
             ({'network': {
                 'version': 2,
-                'items': ('vlan',)}},
+                'items': ('vlans',)}},
              ('vlan',)),
+        ))
+
+    def test_network_v2_detect_openvswitch(self):
+        self._test_req_mappings((
+            ({'network': {
+                'version': 2,
+                'items': ('openvswitch',)}},
+             ('openvswitch-switch', )),
+        ))
+
+    def test_network_v2_detect_renderers(self):
+        self._test_req_mappings((
+            ({'network': {
+                'version': 2,
+                'items': ('renderers',)}},
+             ('bridge-utils', 'openvswitch-switch',
+              'systemd', 'network-manager', )),
+        ))
+
+    def test_network_v2_detect_all(self):
+        self._test_req_mappings((
             ({'network': {
                 'version': 2,
-                'items': ('vlan', 'bridge')}},
-             ('bridge-utils', 'vlan')),
+                'items': ('vlans', 'bridges', 'openvswitch')}},
+             ('bridge-utils', 'vlan', 'openvswitch-switch')),
         ))
 
     def test_mixed_storage_v1_network_v2_detect(self):
         self._test_req_mappings((
             ({'network': {
                 'version': 2,
-                'items': ('bridge', 'vlan')},
+                'items': ('bridges', 'vlans')},
              'storage': {
                  'version': 1,
                  'items': ('raid', 'bcache', 'ext4')}},
diff --git a/tests/vmtests/test_network_ovs.py b/tests/vmtests/test_network_ovs.py
 new file mode 100644
index 0000000..3e23bd0
--- /dev/null
+++ b/tests/vmtests/test_network_ovs.py
@@ -0,0 +1,45 @@
+# This file is part of curtin. See LICENSE file for copyright and license info.
+
+from .releases import base_vm_classes as relbase
+from .test_network import TestNetworkBaseTestsAbs
+
+
+class TestNetworkOvsAbs(TestNetworkBaseTestsAbs):
+    """ This class only needs to verify that when provided a v2 config
+        that on Bionic+ openvswitch packages are installed. """
+    conf_file = "examples/tests/network_v2_ovs.yaml"
+
+    def test_openvswitch_package_status(self):
+        """openvswitch-switch is expected installed in Ubuntu >= bionic."""
+        rel = self.target_release
+        pkg = "openvswitch-switch"
+        self.assertIn(
+            pkg, self.debian_packages,
+            "%s package expected in %s but not found" % (pkg, rel))
+
+    def test_etc_network_interfaces(self):
+        pass
+
+    def test_ip_output(self):
+        pass
+
+    def test_etc_resolvconf(self):
+        pass
+
+    def test_bridge_params(self):
+        pass
+
+
+class BionicTestNetworkOvs(relbase.bionic, TestNetworkOvsAbs):
+    __test__ = True
+
+
+class EoanTestNetworkOvs(relbase.eoan, TestNetworkOvsAbs):
+    __test__ = True
+
+
+class FocalTestNetworkOvs(relbase.focal, TestNetworkOvsAbs):
+    __test__ = True
+
+
+# vi: ts=4 expandtab syntax=python