Merge lp:~mterry/duplicity/manifest-oddities into lp:duplicity/0.6

[Michael Terry]

We may accidentally end up with an oddly inconsistent manifest like so:

Volume 1
Volume 2
Volume 3
Volume 2

As did get reported recently on the mailing list: http://lists.nongnu.org/archive/html/duplicity-talk/2013-11/msg00009.html

One way this can happen (the only way?) is if you back up, then duplicity gets interrupted between writing the manifest and uploading the volume. Then, when restarted, there is no longer enough data to create as many volumes as existed previously.

This situation can cause an exception when trying to restart the backup.

This branch fixes it by deleting any excess volume information encountered when loading in the manifest. We discard volume with higher numbers than the last one read.

[lp:~mterry/duplicity/manifest-oddities]

932. By Michael Terry on 2013-11-16

Fix test comments

=== modified file 'duplicity/manifest.py'
--- duplicity/manifest.py	2011-06-17 06:21:42 +0000
+++ duplicity/manifest.py	2013-11-16 02:23:17 +0000
@@ -177,12 +177,23 @@
         next_vi_string_regexp = re.compile("(^|\\n)(volume\\s.*?)"
                                            "(\\nvolume\\s|$)", re.I | re.S)
         starting_s_index = 0
+        highest_vol = 0
+        latest_vol = 0
         while 1:
             match = next_vi_string_regexp.search(s[starting_s_index:])
             if not match:
                 break
-            self.add_volume_info(VolumeInfo().from_string(match.group(2)))
+            vi = VolumeInfo().from_string(match.group(2))
+            self.add_volume_info(vi)
+            highest_vol = max(highest_vol, vi.volume_number)
+            latest_vol = vi.volume_number
             starting_s_index += match.end(2)
+        # If we restarted after losing some remote volumes, the highest volume
+        # seen may be higher than the last volume recorded.  That is, the
+        # manifest could contain "vol1, vol2, vol3, vol2."  If so, we don't
+        # want to keep vol3's info.
+        for i in range(latest_vol + 1, highest_vol + 1):
+            self.del_volume_info(i)
         return self
 
     def __eq__(self, other):

=== modified file 'testing/tests/restarttest.py'
--- testing/tests/restarttest.py	2013-01-07 16:13:29 +0000
+++ testing/tests/restarttest.py	2013-11-16 02:23:17 +0000
@@ -446,6 +446,31 @@
         assert not os.system("diff %s/file1 testfiles/restore_out/file1" % source)
         assert not os.system("diff %s/z testfiles/restore_out/z" % source)
 
+    def test_changed_source_dangling_manifest_volume(self):
+        """
+        If we restart but find remote volumes missing, we can easily end up
+        with a manifest that lists "vol1, vol2, vol3, vol2", leaving a dangling
+        vol3.  Make sure we can gracefully handle that.  This will only happen
+        if the source data changes to be small enough to not create a vol3 on
+        restart.
+        """
+        source = 'testfiles/largefiles'
+        self.make_largefiles(count=5, size=1)
+        # intentionally interrupt initial backup
+        try:
+            self.backup("full", source, options = ["--vol 1", "--fail 3"])
+            self.fail()
+        except CmdError, e:
+            self.assertEqual(30, e.exit_status)
+        # now delete the last volume on remote end and some source files
+        assert not os.system("rm testfiles/output/duplicity-full*vol3.difftar*")
+        assert not os.system("rm %s/file[2345]" % source)
+        assert not os.system("echo hello > %s/z" % source)
+        # finish backup
+        self.backup("full", source)
+        # and verify we can restore
+        self.restore()
+
 
 # Note that this class duplicates all the tests in RestartTest
 class RestartTestWithoutEncryption(RestartTest):