Merge lp:~smoser/curtin/trunk.vmtest-remove-bug-timers into lp:~raharper/curtin/trunk.vmtest-remove-bug-timers

[Scott Moser]

[lp:~smoser/curtin/trunk.vmtest-remove-bug-timers]

549. By Scott Moser on 2017-10-05

vertical space

=== modified file 'curtin/commands/curthooks.py'
--- curtin/commands/curthooks.py	2017-10-05 16:34:52 +0000
+++ curtin/commands/curthooks.py	2017-10-05 20:10:19 +0000
@@ -677,8 +677,8 @@
     '''
 
     installed_packages = util.get_installed_packages(target)
-    needed_packages = [pkg for pkg in detect_required_packages(cfg)
-                       if pkg not in installed_packages]
+    needed_packages = set([pkg for pkg in detect_required_packages(cfg)
+                           if pkg not in installed_packages])
 
     arch_packages = {
         's390x': [('s390-tools', 'zipl')],
@@ -687,26 +687,28 @@
     for pkg, cmd in arch_packages.get(platform.machine(), []):
         if not util.which(cmd, target=target):
             if pkg not in needed_packages:
-                needed_packages.append(pkg)
+                needed_packages.add(pkg)
 
-    # do not install ifenslave if target release is artful as it
-    # triggers an install of ifupdown which will break network rendering
-    if 'ifenslave' in needed_packages:
-        codename = util.lsb_release(target=target).get('codename')
-        LOG.debug('Target release codename: %s', codename)
-        if codename == 'artful':
-            LOG.debug("Skipping install of package 'ifenslave' to prevent"
-                      " network configutation errors on release %s", codename)
-            needed_packages.remove('ifenslave')
+    # FIXME: This needs cleaning up.
+    # do not install certain packages on artful as they are no longer needed.
+    # ifenslave specifically causes issuse due to dependency on ifupdown.
+    codename = util.lsb_release(target=target).get('codename')
+    if codename == 'artful':
+        drops = set(['bridge-utils', 'ifenslave', 'vlan'])
+        if needed_packages.union(drops):
+            LOG.debug("Skipping install of %s.  Not needed on artful.",
+                      needed_packages.union(drops))
+            needed_packages = needed_packages.difference(drops)
 
     if needed_packages:
+        to_add = list(sorted(needed_packages))
         state = util.load_command_environment()
         with events.ReportEventStack(
                 name=state.get('report_stack_prefix'),
                 reporting_enabled=True, level="INFO",
                 description="Installing packages on target system: " +
-                str(needed_packages)):
-            util.install_packages(needed_packages, target=target)
+                str(to_add)):
+            util.install_packages(to_add, target=target)
 
 
 def system_upgrade(cfg, target):

=== modified file 'tests/vmtests/__init__.py'
--- tests/vmtests/__init__.py	2017-10-02 15:32:45 +0000
+++ tests/vmtests/__init__.py	2017-10-05 20:10:19 +0000
@@ -336,7 +336,12 @@
     __test__ = False
     arch_skip = []
     boot_timeout = BOOT_TIMEOUT
-    collect_scripts = []
+    collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        dpkg-query --show \
+            --showformat='${db:Status-Abbrev}\t${Package}\t${Version}\n' \
+            > debian-packages.txt 2> debian-packages.txt.err
+    """)]
     conf_file = "examples/tests/basic.yaml"
     nr_cpus = None
     dirty_disks = False
@@ -369,6 +374,8 @@
     target_krel = None
     target_ftype = "vmtest.root-tgz"
 
+    _debian_packages = None
+
     def shortDescription(self):
         return None
 
@@ -1154,6 +1161,18 @@
                 fp.write(json.dumps(data, indent=2, sort_keys=True,
                                     separators=(',', ': ')) + "\n")
 
+    @property
+    def debian_packages(self):
+        if self._debian_packages is None:
+            data = self.load_collect_file("debian-packages.txt")
+            pkgs = {}
+            for line in data.splitlines():
+                # lines are <status>\t<
+                status, pkg, ver = line.split('\t')
+                pkgs[pkg] = {'status': status, 'version': ver}
+            self._debian_packages = pkgs
+        return self._debian_packages
+
 
 class PsuedoVMBaseClass(VMBaseClass):
     # This mimics much of the VMBaseClass just with faster setUpClass

=== modified file 'tests/vmtests/test_network.py'
--- tests/vmtests/test_network.py	2017-10-04 20:44:57 +0000
+++ tests/vmtests/test_network.py	2017-10-05 20:10:19 +0000
@@ -19,7 +19,7 @@
     extra_nics = []
     # XXX: command | tee output is required for Centos under SELinux
     # http://danwalsh.livejournal.com/22860.html
-    collect_scripts = [textwrap.dedent("""
+    collect_scripts = VMBaseClass.collect_scripts + [textwrap.dedent("""
         cd OUTPUT_COLLECT_D
         echo "waiting for ipv6 to settle" && sleep 5
         route -n | tee first_route_n
@@ -39,9 +39,6 @@
         cp -av /var/log/upstart ./upstart ||:
         cp -av /etc/cloud ./etc_cloud
         cp -av /var/log/cloud*.log ./
-        dpkg-query -W -f '${Version}' cloud-init |tee dpkg_cloud-init_version
-        dpkg-query -W -f '${Version}' nplan |tee dpkg_nplan_version
-        dpkg-query -W -f '${Version}' systemd |tee dpkg_systemd_version
         rpm -q --queryformat '%{VERSION}\n' cloud-init |tee rpm_ci_version
         V=/usr/lib/python*/*-packages/cloudinit/version.py;
         grep -c NETWORK_CONFIG_V2 $V |tee cloudinit_passthrough_available

=== modified file 'tests/vmtests/test_network_bonding.py'
--- tests/vmtests/test_network_bonding.py	2017-10-04 20:46:23 +0000
+++ tests/vmtests/test_network_bonding.py	2017-10-05 20:10:19 +0000
@@ -9,19 +9,10 @@
 
 class TestNetworkBondingAbs(TestNetworkBaseTestsAbs):
     conf_file = "examples/tests/bonding_network.yaml"
-    collect_scripts = TestNetworkBaseTestsAbs.collect_scripts + [
-        textwrap.dedent("""
-        cd OUTPUT_COLLECT_D
-        dpkg-query -W -f '${Status}' ifenslave > ifenslave_installed
-        """)]
-
-    def test_output_files_exist_ifenslave(self):
-        self.output_files_exist(["ifenslave_installed"])
 
     def test_ifenslave_installed(self):
-        status = self.load_collect_file("ifenslave_installed")
-        logger.debug('ifenslave installed: {}'.format(status))
-        self.assertEqual('install ok installed', status)
+        self.assertIn("ifenslave", self.debian_packages,
+                      "ifenslave deb not installed")
 
 
 class CentosTestNetworkBondingAbs(TestNetworkBondingAbs):
@@ -50,12 +41,10 @@
 
 class PreciseHWETTestBonding(relbase.precise_hwe_t, TestNetworkBondingAbs):
     __test__ = True
-    # package names on precise are different, need to check on ifenslave-2.6
-    collect_scripts = TestNetworkBondingAbs.collect_scripts + [
-        textwrap.dedent("""
-        cd OUTPUT_COLLECT_D
-        dpkg-query -W -f '${Status}' ifenslave-2.6 > ifenslave_installed
-        """)]
+
+    def test_ifenslave_installed(self):
+        self.assertIn("ifenslave-2.6", self.debian_packages,
+                      "ifenslave deb not installed")
 
 
 class TrustyTestBonding(relbase.trusty, TestNetworkBondingAbs):
@@ -89,16 +78,15 @@
 class ArtfulTestBonding(relbase.artful, TestNetworkBondingAbs):
     __test__ = True
 
-    @unittest.skip("Artful does not install ifenslave")
     def test_ifenslave_installed(self):
+        """Artful should not have ifenslave installed."""
         pass
 
     def test_ifenslave_not_installed(self):
         """Confirm that ifenslave is not installed on artful"""
-        status = self.load_collect_file("ifenslave_installed")
-        logger.debug('ifenslave installed: [{}]'.format(status))
-        self.assertEqual('', status,
-                         "ifenslave installed on Artful when it shouldn't be")
+        self.assertNotIn('ifenslave', self.debian_packages,
+                         "ifenslave is not expected in artful: %s" %
+                         self.debian_packages.get('ifenslave'))
 
 
 class Centos66TestNetworkBonding(centos_relbase.centos66fromxenial,

=== modified file 'tests/vmtests/test_network_bridging.py'
--- tests/vmtests/test_network_bridging.py	2017-10-04 01:11:14 +0000
+++ tests/vmtests/test_network_bridging.py	2017-10-05 20:10:19 +0000
@@ -101,7 +101,6 @@
         grep -r . /sys/class/net/br0 > sysfs_br0
         grep -r . /sys/class/net/br0/brif/eth1 > sysfs_br0_eth1
         grep -r . /sys/class/net/br0/brif/eth2 > sysfs_br0_eth2
-        dpkg-query -W -f '${Status}' bridge-utils 2>&1 > bridge-utils_installed
         """)]
 
     def test_output_files_exist_bridge(self):
@@ -110,10 +109,8 @@
                                  "sysfs_br0_eth2"])
 
     def test_bridge_utils_installed(self):
-        self.output_files_exist(["bridge-utils_installed"])
-        status = self.load_collect_file("bridge-utils_installed").strip()
-        logger.debug('bridge-utils installed: {}'.format(status))
-        self.assertEqual('install ok installed', status)
+        self.assertIn("bridge-utils", self.debian_packages,
+                      "bridge-utilsi deb not installed")
 
     def test_bridge_params(self):
 
@@ -234,3 +231,12 @@
         cls.skip_by_date(cls.__name__, cls.release, "1721157",
                          fixby=(2017, 10, 16), removeby=(2017, 11, 16))
         super().setUpClass()
+
+    def test_bridge_utils_installed(self):
+        """bridge-utils not needed in artful."""
+        pass
+
+    def test_bridge_utils_not_installed(self):
+        self.assertNotIn("bridge-utils" self.debian_packages,
+                         "bridge-utils is not expected in artful: %s" %
+                         self.debian_packages.get('bridge-utils'))

=== modified file 'tests/vmtests/test_network_enisource.py'
--- tests/vmtests/test_network_enisource.py	2017-10-03 16:44:50 +0000
+++ tests/vmtests/test_network_enisource.py	2017-10-05 20:10:19 +0000
@@ -26,6 +26,8 @@
         This testcase then uses curtin.net.deb_parse_config method to
         extract information about what curtin wrote and compare that
         with what was actually configured (which we capture via ifconfig)
+
+        Note: This test is *not* valid for Artful as it has no ENI.
     """
 
     conf_file = "examples/tests/network_source.yaml"
@@ -98,7 +100,3 @@
 
 class ZestyTestNetworkENISource(relbase.zesty, TestNetworkENISource):
     __test__ = True
-
-
-class ArtfulTestNetworkENISource(relbase.artful, TestNetworkENISource):
-    __test__ = False

=== modified file 'tests/vmtests/test_network_vlan.py'
--- tests/vmtests/test_network_vlan.py	2017-10-03 16:44:50 +0000
+++ tests/vmtests/test_network_vlan.py	2017-10-05 20:10:19 +0000
@@ -12,7 +12,6 @@
     collect_scripts = TestNetworkBaseTestsAbs.collect_scripts + [
         textwrap.dedent("""
              cd OUTPUT_COLLECT_D
-             dpkg-query -W -f '${Status}' vlan > vlan_installed
              ip -d link show interface1.2667 |tee ip_link_show_interface1.2667
              ip -d link show interface1.2668 |tee ip_link_show_interface1.2668
              ip -d link show interface1.2669 |tee ip_link_show_interface1.2669
@@ -31,15 +30,11 @@
     def test_output_files_exist_vlan(self):
         link_files = ["ip_link_show_%s" % vlan['name']
                       for vlan in self.get_vlans()]
-        self.output_files_exist(["vlan_installed"] + link_files)
 
     def test_vlan_installed(self):
-        status = self.load_collect_file("vlan_installed").strip()
-        logger.debug('vlan installed?: %s', status)
-        self.assertEqual('install ok installed', status)
+        self.assertIn("vlan", self.debian_packages, "vlan deb not installed")
 
     def test_vlan_enabled(self):
-
         # we must have at least one
         self.assertGreaterEqual(len(self.get_vlans()), 1)