Merge ~raharper/curtin:ubuntu/devel/newupstream-20181010 into curtin:ubuntu/devel

[Ryan Harper]

curtin (18.1-59-g0f993084-0ubuntu1) cosmic; urgency=medium

  * New upstream snapshot.
    - distro: fix system_upgrade command using string instead of function
      (LP: #1796968)
    - Capture stdout when using lxc file push
    - vmtest: boot ephemeral with 'ro' on the kernel command line.

 -- Ryan Harper <email address hidden> Wed, 10 Oct 2018 11:22:30 -0500

diff --git a/curtin/distro.py b/curtin/distro.py
index f2a78ed..9714515 100644
--- a/curtin/distro.py
+++ b/curtin/distro.py
@@ -337,10 +337,10 @@ def system_upgrade(opts=None, target=None, env=None, allow_daemons=False,
     LOG.debug("Upgrading system in %s", target)
 
     distro_cfg = {
-        DISTROS.debian: {'function': 'run_apt_command',
+        DISTROS.debian: {'function': run_apt_command,
                          'subcommands': ('dist-upgrade', 'autoremove')},
-        DISTROS.redhat: {'function': 'run_yum_command',
-                         'subcommands': ('upgrade')},
+        DISTROS.redhat: {'function': run_yum_command,
+                         'subcommands': ('upgrade',)},
     }
     if osfamily not in distro_cfg:
         raise ValueError('Distro "%s" does not have system_upgrade support',
diff --git a/debian/changelog b/debian/changelog
index 4f5237f..89bb20f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+curtin (18.1-59-g0f993084-0ubuntu1) cosmic; urgency=medium
+
+  * New upstream snapshot.
+    - distro: fix system_upgrade command using string instead of function
+      (LP: #1796968)
+    - Capture stdout when using lxc file push
+    - vmtest: boot ephemeral with 'ro' on the kernel command line.
+
+ -- Ryan Harper <[e-mail]>  Wed, 10 Oct 2018 11:22:30 -0500
+
 curtin (18.1-56-g3aafe77d-0ubuntu1) cosmic; urgency=medium
 
   * New upstream snapshot.
diff --git a/tests/unittests/test_distro.py b/tests/unittests/test_distro.py
index d4e5a1e..b079162 100644
--- a/tests/unittests/test_distro.py
+++ b/tests/unittests/test_distro.py
@@ -235,21 +235,92 @@ class TestYumInstall(CiTestCase):
         ]
 
         # call yum_install directly
+        self.assertFalse(m_subp.called)
         distro.yum_install(mode, pkglist, target=target)
         m_subp.assert_has_calls(expected_calls)
 
-        # call yum_install through run_yum_command
-        m_subp.reset()
+        # call yum_install through run_yum_command; expect the same calls
+        # so clear m_subp's call stack.
+        m_subp.reset_mock()
+        self.assertFalse(m_subp.called)
         distro.run_yum_command('install', pkglist, target=target)
         m_subp.assert_has_calls(expected_calls)
 
-        # call yum_install through install_packages
-        m_subp.reset()
+        # call yum_install through install_packages; expect the same calls
+        # so clear m_subp's call stack.
+        m_subp.reset_mock()
+        self.assertFalse(m_subp.called)
         osfamily = distro.DISTROS.redhat
         distro.install_packages(pkglist, osfamily=osfamily, target=target)
         m_subp.assert_has_calls(expected_calls)
 
 
+class TestSystemUpgrade(CiTestCase):
+
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    @mock.patch('curtin.util.subp')
+    def test_system_upgrade_redhat(self, m_subp):
+        """system_upgrade osfamily=redhat calls run_yum_command mode=upgrade"""
+        osfamily = distro.DISTROS.redhat
+        target = 'mytarget'
+        mode = 'upgrade'
+        pkglist = []
+        expected_calls = [
+            mock.call(['yum', '--assumeyes', '--quiet', mode,
+                       '--downloadonly', '--setopt=keepcache=1'] + pkglist,
+                      env=None, retries=[1] * 10,
+                      target=paths.target_path(target)),
+            mock.call(['yum', '--assumeyes', '--quiet', mode,
+                       '--cacheonly'] + pkglist, env=None,
+                      target=paths.target_path(target))
+        ]
+        # call system_upgrade via osfamily; note that we expect the same calls
+        # call system_upgrade via osfamily; note that we expect the same calls
+
+        # call yum_install through run_yum_command
+        distro.run_yum_command(mode, pkglist, target=target)
+        m_subp.assert_has_calls(expected_calls)
+
+        # call system_upgrade via osfamily; note that we expect the same calls
+        # but to prevent a false positive we clear m_subp's call stack.
+        m_subp.reset_mock()
+        self.assertFalse(m_subp.called)
+        distro.system_upgrade(target=target, osfamily=osfamily)
+        m_subp.assert_has_calls(expected_calls)
+
+    @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
+    @mock.patch('curtin.distro.os.environ')
+    @mock.patch('curtin.distro.apt_update')
+    @mock.patch('curtin.distro.which')
+    @mock.patch('curtin.util.subp')
+    def test_system_upgrade_debian(self, m_subp, m_which, m_apt_update, m_env):
+        """system_upgrade osfamily=debian calls run_apt_command mode=upgrade"""
+        osfamily = distro.DISTROS.debian
+        target = 'mytarget'
+        m_env.copy.return_value = {}
+        m_which.return_value = None
+        env = {'DEBIAN_FRONTEND': 'noninteractive'}
+        pkglist = []
+        apt_base = [
+            'apt-get', '--quiet', '--assume-yes',
+            '--option=Dpkg::options::=--force-unsafe-io',
+            '--option=Dpkg::Options::=--force-confold']
+        apt_cmd = apt_base + ['dist-upgrade'] + pkglist
+        auto_remove = apt_base + ['autoremove']
+        expected_calls = [
+            mock.call(apt_cmd, env=env, target=paths.target_path(target)),
+            mock.call(auto_remove, env=env, target=paths.target_path(target)),
+        ]
+        which_calls = [mock.call('eatmydata', target=target)]
+        apt_update_calls = [
+            mock.call(target, env=env, comment=' '.join(apt_cmd))]
+
+        distro.system_upgrade(target=target, osfamily=osfamily)
+        m_which.assert_has_calls(which_calls)
+        m_apt_update.assert_has_calls(apt_update_calls)
+        m_subp.assert_has_calls(expected_calls)
+
+
 class TestHasPkgAvailable(CiTestCase):
 
     def setUp(self):
diff --git a/tests/vmtests/__init__.py b/tests/vmtests/__init__.py
index af75599..3823e39 100644
--- a/tests/vmtests/__init__.py
+++ b/tests/vmtests/__init__.py
@@ -887,6 +887,7 @@ class VMBaseClass(TestCase):
         cmd.extend([
             "--root-arg=root=%s" % root_url,
             "--append=overlayroot=tmpfs",
+            "--append=ro",
         ])
 
         # getting resolvconf configured is only fixed in bionic
diff --git a/tools/curtin-from-container b/tools/curtin-from-container
index ed7d1fc..7addd1a 100755
--- a/tools/curtin-from-container
+++ b/tools/curtin-from-container
@@ -63,8 +63,12 @@ send() {
         return
     }
 
-    lxc file push "$fpath" "$name/$prefix/$afile" || {
-        rerror "failed: lxc file push '$fpath' '$name/$prefix/$afile'"
+    # 'lxc file' command may write progress info to stdout which breaks
+    # this program when called as 'curtin pack' as it needs to write the
+    # payload to stdout. LXD issue #2683
+    local out=""
+    out=$(lxc file push "$fpath" "$name/$prefix/$afile" 2>&1) || {
+        rerror "failed: lxc file push '$fpath' '$name/$prefix/$afile': $out"
         return
     }
     _RET="$afile:$prefix/$afile"