Merge lp:~brian-murray/ubuntu-release-upgrader/remove-aufs into lp:ubuntu-release-upgrader

[Brian Murray]

I spoke with Michael about the --sandbox option failing and he recommended just removing the aufs support so I did.

[Steve Langasek (vorlon)]

LGTM, one minor comment inline.

review: Approve

[Brian Murray (brian-murray)]

Okay, I'll make that change.

[lp:~brian-murray/ubuntu-release-upgrader/remove-aufs]

3054. By Brian Murray on 2017-05-18

Remove error handling for AUFS option in environ.

=== removed file 'DistUpgrade/DistUpgradeAufs.py'
--- DistUpgrade/DistUpgradeAufs.py	2015-01-20 22:44:02 +0000
+++ DistUpgrade/DistUpgradeAufs.py	1970-01-01 00:00:00 +0000
@@ -1,289 +0,0 @@
-# DistUpgradeAufs.py
-#
-#  Copyright (c) 2009-2012 Canonical
-#
-#  Author: Michael Vogt <[e-mail]>
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU General Public License as
-#  published by the Free Software Foundation; either version 2 of the
-#  License, or (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-#  USA
-
-import logging
-import os
-import subprocess
-import tempfile
-
-
-def aufsOptionsAndEnvironmentSetup(options, config):
-    """ setup the environment based on the config and options
-    It will use
-    config("Aufs","Enabled") - to show if its enabled
-    and
-    config("Aufs","RWDir") - for the writable overlay dir
-    """
-    logging.debug("aufsOptionsAndEnvironmentSetup()")
-    # enabled from the commandline (full overlay by default)
-    if options and options.useAufs:
-        logging.debug("enabling full overlay from commandline")
-        config.set("Aufs", "Enabled", "True")
-        config.set("Aufs", "EnableFullOverlay", "True")
-    
-    # setup environment based on config
-    tmprw = tempfile.mkdtemp(prefix="upgrade-rw-")
-    aufs_rw_dir = config.getWithDefault("Aufs", "RWDir", tmprw)
-    logging.debug("using '%s' as aufs_rw_dir" % aufs_rw_dir)
-    os.environ["RELEASE_UPGRADE_AUFS_RWDIR"] = aufs_rw_dir
-    config.set("Aufs", "RWDir", aufs_rw_dir)
-    # now the chroot tmpdir
-    tmpchroot = tempfile.mkdtemp(prefix="upgrade-chroot-")
-    os.chmod(tmpchroot, 0o755)
-    aufs_chroot_dir = config.getWithDefault("Aufs", "ChrootDir", tmpchroot)
-    logging.debug("using '%s' as aufs chroot dir" % aufs_chroot_dir)
-    
-    if config.getWithDefault("Aufs", "EnableFullOverlay", False):
-        logging.debug("enabling aufs full overlay (from config)")
-        config.set("Aufs", "Enabled", "True")
-        os.environ["RELEASE_UPGRADE_USE_AUFS_FULL_OVERLAY"] = "1"
-    if config.getWithDefault("Aufs", "EnableChrootOverlay", False):
-        logging.debug("enabling aufs chroot overlay")
-        config.set("Aufs", "Enabled", "True")        
-        os.environ["RELEASE_UPGRADE_USE_AUFS_CHROOT"] = aufs_chroot_dir
-    if config.getWithDefault("Aufs", "EnableChrootRsync", False):
-        logging.debug("enable aufs chroot rsync back to real system")
-        os.environ["RELEASE_UPGRADE_RSYNC_AUFS_CHROOT"] = "1"
-    
-
-def _bindMount(from_dir, to_dir, rbind=False):
-    " helper that bind mounts a given dir to a new place "
-    if not os.path.exists(to_dir):
-        os.makedirs(to_dir)
-    if rbind:
-        bind = "--rbind"
-    else:
-        bind = "--bind"
-    cmd = ["mount", bind, from_dir, to_dir]
-    logging.debug("cmd: %s" % cmd)
-    res = subprocess.call(cmd)
-    if res != 0:
-        # FIXME: revert already mounted stuff
-        logging.error(
-            "Failed to bind mount from '%s' to '%s'" % (from_dir, to_dir))
-        return False
-    return True
-
-
-def _aufsOverlayMount(target, rw_dir, chroot_dir="/"):
-    """ 
-    helper that takes a target dir and mounts a rw dir over it, e.g.
-    /var , /tmp/upgrade-rw
-    """
-    if not os.path.exists(rw_dir + target):
-        os.makedirs(rw_dir + target)
-    if not os.path.exists(chroot_dir + target):
-        os.makedirs(chroot_dir + target)
-    # FIXME: figure out when to use aufs and when to use overlayfs
-    use_overlayfs = False
-    if use_overlayfs:
-        cmd = ["mount",
-               "-t", "overlayfs",
-               "-o", "upperdir=%s,lowerdir=%s" % (rw_dir + target, target),
-               "none",
-               chroot_dir + target]
-    else:
-        cmd = ["mount",
-               "-t", "aufs",
-               "-o", "br:%s:%s=ro" % (rw_dir + target, target),
-               "none",
-               chroot_dir + target]
-    res = subprocess.call(cmd)
-    if res != 0:
-        # FIXME: revert already mounted stuff
-        logging.error("Failed to mount rw aufs overlay for '%s'" % target)
-        return False
-    logging.debug("cmd '%s' return '%s' " % (cmd, res))
-    return True
-
-
-def is_aufs_mount(dir):
-    " test if the given dir is already mounted with aufs overlay "
-    with open("/proc/mounts") as f:
-        for line in f:
-            (device, mountpoint, fstype, options, a, b) = line.split()
-            if device == "none" and fstype == "aufs" and mountpoint == dir:
-                return True
-    return False
-
-
-def is_submount(mountpoint, systemdirs):
-    " helper: check if the given mountpoint is a submount of a systemdir "
-    logging.debug("is_submount: %s %s" % (mountpoint, systemdirs))
-    for d in systemdirs:
-        if not d.endswith("/"):
-            d += "/"
-        if mountpoint.startswith(d):
-            return True
-    return False
-
-
-def is_real_fs(fs):
-    if fs.startswith("fuse"):
-        return False
-    if fs in ["rootfs", "tmpfs", "proc", "fusectrl", "aufs",
-              "devpts", "binfmt_misc", "sysfs"]:
-        return False
-    return True
-
-
-def doAufsChrootRsync(aufs_chroot_dir):
-    """
-    helper that rsyncs the changes in the aufs chroot back to the
-    real system
-    """
-    from .DistUpgradeMain import SYSTEM_DIRS
-    for d in SYSTEM_DIRS:
-        if not os.path.exists(d):
-            continue
-        # its important to have the "/" at the end of source
-        # and dest so that rsync knows what to do
-        cmd = ["rsync", "-aHAX", "--del", "-v", "--progress",
-               "/%s/%s/" % (aufs_chroot_dir, d),
-               "/%s/" % d]
-        logging.debug("running: '%s'" % cmd)
-        ret = subprocess.call(cmd)
-        logging.debug("rsync back result for %s: %i" % (d, ret))
-    return True
-
-
-def doAufsChroot(aufs_rw_dir, aufs_chroot_dir):
-    " helper that sets the chroot up and does chroot() into it "
-    if not setupAufsChroot(aufs_rw_dir, aufs_chroot_dir):
-        return False
-    os.chroot(aufs_chroot_dir)
-    os.chdir("/")
-    return True
-
-
-def setupAufsChroot(rw_dir, chroot_dir):
-    " setup aufs chroot that is based on / but with a writable overlay "
-    # with the chroot aufs we can just rsync the changes back
-    # from the chroot dir to the real dirs
-    # 
-    # (if we run rsync with --backup --backup-dir we could even
-    # create something vaguely rollbackable
-
-    # get the mount points before the aufs buisiness starts
-    with open("/proc/mounts") as f:
-        mounts = f.read()
-    from .DistUpgradeMain import SYSTEM_DIRS
-    systemdirs = SYSTEM_DIRS
-    
-    # aufs mount or bind mount required dirs
-    for d in os.listdir("/"):
-        d = os.path.join("/", d)
-        if os.path.isdir(d):
-            if d in systemdirs:
-                logging.debug("bind mounting %s" % d)
-                if not _aufsOverlayMount(d, rw_dir, chroot_dir):
-                    return False
-            else:
-                logging.debug("overlay mounting %s" % d)
-                if not _bindMount(d, chroot_dir + d, rbind=True):
-                    return False
-
-    # create binds for the systemdirs
-    #needs_bind_mount = set()
-    for line in mounts.split("\n"):
-        line = line.strip()
-        if not line: 
-            continue
-        (device, mountpoint, fstype, options, a, b) = line.split()
-        if (fstype != "aufs" and
-                not is_real_fs(fstype) and
-                is_submount(mountpoint, systemdirs)):
-            logging.debug("found %s that needs bind mount", mountpoint)
-            if not _bindMount(mountpoint, chroot_dir + mountpoint):
-                return False
-    return True
-
-
-def setupAufs(rw_dir):
-    " setup aufs overlay over the rootfs "
-    #        * we need to find a way to tell all the existing daemon 
-    #          to look into the new namespace. so probably something
-    #          like a reboot is required and some hackery in initramfs-tools
-    #          to ensure that we boot into a overlay ready system
-    #        * this is much less of a issue with the aufsChroot code
-    logging.debug("setupAufs")
-    if not os.path.exists("/proc/mounts"):
-        logging.debug("no /proc/mounts, can not do aufs overlay")
-        return False
-
-    from .DistUpgradeMain import SYSTEM_DIRS
-    systemdirs = SYSTEM_DIRS
-    # verify that there are no submounts of a systemdir and collect
-    # the stuff that needs bind mounting (because a aufs does not
-    # include sub mounts)
-    needs_bind_mount = set()
-    needs_bind_mount.add("/var/cache/apt/archives")
-    with open("/proc/mounts") as f:
-        for line in f:
-            (device, mountpoint, fstype, options, a, b) = line.split()
-            if is_real_fs(fstype) and is_submount(mountpoint, systemdirs):
-                logging.warning("mountpoint %s submount of systemdir" % 
-                                mountpoint)
-                return False
-            if (fstype != "aufs" and
-                    not is_real_fs(fstype) and
-                    is_submount(mountpoint, systemdirs)):
-                logging.debug("found %s that needs bind mount", mountpoint)
-                needs_bind_mount.add(mountpoint)
-
-    # aufs mounts do not support stacked filesystems, so
-    # if we mount /var we will loose the tmpfs stuff
-    # first bind mount varun and varlock into the tmpfs
-    for d in needs_bind_mount:
-        if not _bindMount(d, rw_dir + "/needs_bind_mount/" + d):
-            return False
-    # setup writable overlay into /tmp/upgrade-rw so that all 
-    # changes are written there instead of the real fs
-    for d in systemdirs:
-        if not is_aufs_mount(d):
-            if not _aufsOverlayMount(d, rw_dir):
-                return False
-    # now bind back the tempfs to the original location
-    for d in needs_bind_mount:
-        if not _bindMount(rw_dir + "/needs_bind_mount/" + d, d):
-            return False
-
-    # The below information is only of historical relevance:
-    #        now what we *could* do to apply the changes is to
-    #        mount -o bind / /orig 
-    #        (bind is important, *not* rbind that includes submounts)
-    # 
-    #        This will give us the original "/" without the 
-    #        aufs rw overlay  - *BUT* only if "/" is all on one parition
-    #             
-    #        then apply the diff (including the whiteouts) to /orig
-    #        e.g. by "rsync -av /tmp/upgrade-rw /orig"
-    #                "script that search for whiteouts and removes them"
-    #        (whiteout files start with .wh.$name
-    #         whiteout dirs with .wh..? - check with aufs man page)
-    return True
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    #print(setupAufs("/tmp/upgrade-rw"))
-    print(setupAufsChroot("/tmp/upgrade-chroot-rw",
-                          "/tmp/upgrade-chroot"))

=== modified file 'DistUpgrade/DistUpgradeController.py'
--- DistUpgrade/DistUpgradeController.py	2016-08-10 15:09:46 +0000
+++ DistUpgrade/DistUpgradeController.py	2017-05-18 17:34:09 +0000
@@ -57,7 +57,6 @@
 from .DistUpgradeConfigParser import DistUpgradeConfig
 from .DistUpgradeQuirks import DistUpgradeQuirks
 from .DistUpgradeAptCdrom import AptCdrom
-from .DistUpgradeAufs import setupAufs, aufsOptionsAndEnvironmentSetup
 
 # workaround broken relative import in python-apt (LP: #871007), we
 # want the local version of distinfo.py from oneiric, but because of
@@ -138,9 +137,6 @@
         self.config.add_section("Options")
         self.config.set("Options","withNetwork", str(self.useNetwork))
 
-        # aufs stuff
-        aufsOptionsAndEnvironmentSetup(self.options, self.config)
-
         # some constants here
         self.fromDist = self.config.get("Sources","From")
         self.toDist = self.config.get("Sources","To")
@@ -399,27 +395,6 @@
                                "supported with this tool." % (release, self.toDist)))
             sys.exit(1)
 
-        # setup aufs
-        if self.config.getWithDefault("Aufs", "EnableFullOverlay", False):
-            aufs_rw_dir = self.config.get("Aufs","RWDir")
-            if not setupAufs(aufs_rw_dir):
-                logging.error("aufs setup failed")
-                self._view.error(_("Sandbox setup failed"),
-                                 _("It was not possible to create the sandbox "
-                                   "environment."))
-                return False
-
-            # all good, tell the user about the sandbox mode
-            logging.info("running in aufs overlay mode")
-            self._view.information(_("Sandbox mode"),
-                                   _("This upgrade is running in sandbox "
-                                     "(test) mode. All changes are written "
-                                     "to '%s' and will be lost on the next "
-                                     "reboot.\n\n"
-                                     "*No* changes written to a system directory "
-                                     "from now until the next reboot are "
-                                     "permanent.") % aufs_rw_dir)
-
         # setup backports (if we have them)
         if self.options and self.options.havePrerequists:
             backportsdir = os.getcwd()+"/backports"

=== modified file 'DistUpgrade/DistUpgradeMain.py'
--- DistUpgrade/DistUpgradeMain.py	2017-05-03 10:03:08 +0000
+++ DistUpgrade/DistUpgradeMain.py	2017-05-18 17:34:09 +0000
@@ -33,8 +33,8 @@
 from optparse import OptionParser
 from gettext import gettext as _
 
-# dirs that the packages will touch, this is needed for AUFS/overlayfs
-# and also for the sanity check before the upgrade
+# dirs that the packages will touch, this is needed for the sanity check
+# before the upgrade
 SYSTEM_DIRS = ["/bin",
               "/boot",
               "/etc",
@@ -54,9 +54,6 @@
 def do_commandline():
     " setup option parser and parse the commandline "
     parser = OptionParser()
-    parser.add_option("-s", "--sandbox", dest="useAufs", default=False,
-                      action="store_true",
-                      help=_("Sandbox upgrade using aufs"))
     parser.add_option("-c", "--cdrom", dest="cdromPath", default=None,
                       help=_("Use the given path to search for a cdrom with upgradable packages"))
     parser.add_option("--have-prerequists", dest="havePrerequists",

=== modified file 'DistUpgrade/DistUpgradeView.py'
--- DistUpgrade/DistUpgradeView.py	2016-02-08 22:40:04 +0000
+++ DistUpgrade/DistUpgradeView.py	2017-05-18 17:34:09 +0000
@@ -30,7 +30,6 @@
 import signal
 import select
 
-from .DistUpgradeAufs import doAufsChroot, doAufsChrootRsync
 from .DistUpgradeApport import apport_pkgfailure
 
 
@@ -212,12 +211,6 @@
   def run(self, pm):
     pid = self.fork()
     if pid == 0:
-      # check if we need to setup/enable the aufs chroot stuff
-      if "RELEASE_UPGRADE_USE_AUFS_CHROOT" in os.environ:
-        if not doAufsChroot(os.environ["RELEASE_UPGRADE_AUFS_RWDIR"],
-                            os.environ["RELEASE_UPGRADE_USE_AUFS_CHROOT"]):
-          print("ERROR: failed to setup aufs chroot overlay")
-          os._exit(1)
       # child, ignore sigpipe, there are broken scripts out there
       # like etckeeper (LP: #283642)
       signal.signal(signal.SIGPIPE,signal.SIG_IGN) 
@@ -232,14 +225,6 @@
       os._exit(res)
     self.child_pid = pid
     res = os.WEXITSTATUS(self.wait_child())
-    # check if we want to sync the changes back, *only* do that
-    # if res is positive
-    if (res == 0 and
-        "RELEASE_UPGRADE_RSYNC_AUFS_CHROOT" in os.environ):
-      logging.info("doing rsync commit of the update")
-      if not doAufsChrootRsync(os.environ["RELEASE_UPGRADE_USE_AUFS_CHROOT"]):
-        logging.error("FATAL ERROR: doAufsChrootRsync() returned FALSE")
-        return pm.RESULT_FAILED
     return res
   
   def error(self, pkg, errormsg):

=== modified file 'data/DistUpgrade.cfg'
--- data/DistUpgrade.cfg	2017-04-24 21:15:03 +0000
+++ data/DistUpgrade.cfg	2017-05-18 17:34:09 +0000
@@ -128,17 +128,6 @@
 ;SourcesList-ia64=prerequists-sources.ports.list
 ;SourcesList-hppa=prerequists-sources.ports.list
 
-[Aufs]
-; this is a xor option, either full or chroot overlay
-;EnableFullOverlay=yes
-;EnableChrootOverlay=yes
-; sync changes from the chroot back to the real system
-;EnableChrootRsync=yes
-; what chroot dir to use
-;ChrootDir=/tmp/upgrade-chroot
-; the RW dir to use (either for full overlay or chroot overlay)
-;RWDir=/tmp/upgrade-rw
-
 [Network]
 MaxRetries=3
 

=== modified file 'data/do-release-upgrade.8'
--- data/do-release-upgrade.8	2012-06-13 11:47:22 +0000
+++ data/do-release-upgrade.8	2017-05-18 17:34:09 +0000
@@ -28,8 +28,5 @@
 .TP 
 \fB\-f\fR FRONTEND, \fB\-\-frontend\fR=\fIFRONTEND\fR
 Run the specified frontend
-.TP 
-\fB\-s\fR, \fB\-\-sandbox\fR
-Test upgrade with a sandbox aufs overlay
 .SH "SEE ALSO"
 \fBupdate\-manager\fR(8), \fBapt\-get\fR(8)

=== modified file 'debian/changelog'
--- debian/changelog	2017-05-03 16:12:01 +0000
+++ debian/changelog	2017-05-18 17:34:09 +0000
@@ -1,3 +1,10 @@
+ubuntu-release-upgrader (1:17.10.3) UNRELEASED; urgency=medium
+
+  * Remove --sandbox (aufs support) as it has been broken for some time and
+    was rather unused. (LP: #1605259)
+
+ -- Brian Murray <[e-mail]>  Wed, 17 May 2017 14:38:24 -0700
+
 ubuntu-release-upgrader (1:17.10.2) artful; urgency=medium
 
   [ Balint Reczey ]

=== modified file 'do-release-upgrade'
--- do-release-upgrade	2015-04-16 23:00:33 +0000
+++ do-release-upgrade	2017-05-18 17:34:09 +0000
@@ -96,8 +96,6 @@
   parser.add_option ("-f", "--frontend", default="DistUpgradeViewText",
                      dest="frontend", 
                      help=_("Run the specified frontend"))
-  parser.add_option ("-s","--sandbox", action="store_true", default=False,
-                     help=_("Test upgrade with a sandbox aufs overlay"))
   parser.add_option ("-c", "--check-dist-upgrade-only", action="store_true",
                      default=check_only,
                      help=_("Check only if a new distribution release is "
@@ -158,8 +156,6 @@
   fetcher.run_options += ["--mode=%s" % options.mode,
                           "--frontend=%s" % options.frontend,
                           ]
-  if options.sandbox:
-    fetcher.run_options.append("--sandbox")
   if options.devel_release:
     fetcher.run_options.append("--devel-release")
   fetcher.run()