Merge lp:~aaron-whitehouse/duplicity/bug_884371_asterisks_in_includes into lp:~duplicity-team/duplicity/0.7-series

[Aaron Whitehouse]

Added tests to unit/test_selection.py and funtional/test_selection.py to show the behaviour reported in Bug #884371, i.e. that selection is incorrect when there is a * or ** on an include line of a filelist or commandline --include.

[Aaron Whitehouse (aaron-whitehouse)]

You may have noticed that I've been adding test cases to trunk for some of the filelist-related bugs in the tracker. In addition to helping people fix the bugs, this is so that, if I get a chance to work on tidying up the selection code, I will know if I've fixed any of the bugs as a consequence.

[lp:~aaron-whitehouse/duplicity/bug_884371_asterisks_in_includes]

1068. By Aaron Whitehouse <email address hidden> on 2015-01-29

Added credit to Elifarley Cruz for one of the test cases.

[Kenneth Loafman (kenneth-loafman)]

> You may have noticed that I've been adding test cases to trunk for some of the
> filelist-related bugs in the tracker. In addition to helping people fix the
> bugs, this is so that, if I get a chance to work on tidying up the selection
> code, I will know if I've fixed any of the bugs as a consequence.

I really appreciate the tests and the thoroughness of the tests. Keep it up!

=== modified file 'testing/functional/test_selection.py'
--- testing/functional/test_selection.py	2015-01-29 13:34:43 +0000
+++ testing/functional/test_selection.py	2015-01-29 23:17:01 +0000
@@ -582,6 +582,7 @@
         restored = self.directory_tree_to_list_of_lists(restore_dir)
         self.assertEqual(restored, self.expected_restored_tree)
 
+
 class TestIncludeExcludedForContents(IncludeExcludeFunctionalTest):
     """ Test to check that folders that are excluded are included if they contain includes of higher priority.
      Exhibits the issue reported in Bug #1408411 (https://bugs.launchpad.net/duplicity/+bug/1408411). """
@@ -639,5 +640,92 @@
         self.backup("full", "testfiles/select/1", options=["--exclude-filelist=testfiles/exclude.txt"])
         self.restore_and_check()
 
+
+class TestAsterisks(IncludeExcludeFunctionalTest):
+    """ Test to check that asterisks work as expected
+     Exhibits the issue reported in Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371).
+     See the unit tests for more granularity on the issue."""
+
+    def restore_and_check(self):
+        """Restores the backup and compares to what is expected."""
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, [['2'], ['1']])
+
+    def test_exclude_globbing_filelist_asterisks_none(self):
+        """Basic exclude globbing filelist."""
+        with open("testfiles/filelist.txt", 'w') as f:
+            f.write("+ testfiles/select/1/2/1\n"
+                    "- testfiles/select/1/2\n"
+                    "- testfiles/select/1/1\n"
+                    "- testfiles/select/1/3")
+        self.backup("full", "testfiles/select/1", options=["--exclude-globbing-filelist=testfiles/filelist.txt"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_exclude_globbing_filelist_asterisks_single(self):
+        """Exclude globbing filelist with asterisks replacing folders."""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        with open("testfiles/filelist.txt", 'w') as f:
+            f.write("+ */select/1/2/1\n"
+                    "- */select/1/2\n"
+                    "- testfiles/*/1/1\n"
+                    "- */*/1/3")
+        self.backup("full", "testfiles/select/1", options=["--exclude-globbing-filelist=testfiles/filelist.txt"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_exclude_globbing_filelist_asterisks_double_asterisks(self):
+        """Exclude globbing filelist with double asterisks replacing folders."""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        with open("testfiles/filelist.txt", 'w') as f:
+            f.write("+ **/1/2/1\n"
+                    "- **/1/2\n"
+                    "- **/select/1/1\n"
+                    "- testfiles/select/1/3")
+        self.backup("full", "testfiles/select/1", options=["--exclude-globbing-filelist=testfiles/filelist.txt"])
+        self.restore_and_check()
+
+    def test_commandline_asterisks_single_excludes_only(self):
+        """test_commandline_include_exclude with single asterisks on exclude lines."""
+        self.backup("full", "testfiles/select/1",
+                    options=["--include", "testfiles/select/1/2/1",
+                             "--exclude", "testfiles/*/1/2",
+                             "--exclude", "*/select/1/1",
+                             "--exclude", "*/select/1/3"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_commandline_asterisks_single_both(self):
+        """test_commandline_include_exclude with single asterisks on both exclude and include lines."""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.backup("full", "testfiles/select/1",
+                    options=["--include", "*/select/1/2/1",
+                             "--exclude", "testfiles/*/1/2",
+                             "--exclude", "*/select/1/1",
+                             "--exclude", "*/select/1/3"])
+        self.restore_and_check()
+
+    def test_commandline_asterisks_double_exclude_only(self):
+        """test_commandline_include_exclude with double asterisks on exclude lines."""
+        self.backup("full", "testfiles/select/1",
+                    options=["--include", "testfiles/select/1/2/1",
+                             "--exclude", "**/1/2",
+                             "--exclude", "**/1/1",
+                             "--exclude", "**/1/3"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_commandline_asterisks_double_both(self):
+        """test_commandline_include_exclude with double asterisks on both exclude and include lines."""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.backup("full", "testfiles/select/1",
+                    options=["--include", "**/1/2/1",
+                             "--exclude", "**/1/2",
+                             "--exclude", "**/1/1",
+                             "--exclude", "**/1/3"])
+        self.restore_and_check()
+
 if __name__ == "__main__":
     unittest.main()

=== modified file 'testing/unit/test_selection.py'
--- testing/unit/test_selection.py	2015-01-22 23:38:29 +0000
+++ testing/unit/test_selection.py	2015-01-29 23:17:01 +0000
@@ -682,6 +682,215 @@
                         '- testfiles/select/1\n'
                         '- **'])
 
+    def test_include_globbing_filelist_asterisk(self):
+        """Filelist glob test with * instead of 'testfiles'"""
+        # Thank you to Elifarley Cruz for this test case
+        # (https://bugs.launchpad.net/duplicity/+bug/884371).
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '1'),
+                        ('1', '1', '2'), ('1', '1', '3')],
+                       ["*/select/1/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_asterisk_2(self):
+        """Identical to test_globbing_filelist, but with the exclude 'select' replaced with '*'"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/*/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_asterisk_3(self):
+        """Identical to test_globbing_filelist, but with the auto-include 'select' replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "testfiles/*/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_asterisk_4(self):
+        """Identical to test_globbing_filelist, but with a specific include 'select' replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "+ testfiles/*/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_asterisk_5(self):
+        """Identical to test_globbing_filelist, but with all 'select's replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/*/1/1/1\n"
+                        "+ testfiles/*/1/1\n"
+                        "- testfiles/*/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_asterisk_6(self):
+        """Identical to test_globbing_filelist, but with numerous excluded folders replaced with '*'"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- */*/1/1/1\n"
+                        "+ testfiles/select/1/1\n"
+                        "- */*/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_asterisk_7(self):
+        """Identical to test_globbing_filelist, but with numerous included/excluded folders replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- */*/1/1/1\n"
+                        "+ */*/1/1\n"
+                        "- */*/1\n"
+                        "- **"])
+
+
+    def test_include_globbing_filelist_double_asterisk_1(self):
+        """Identical to test_globbing_filelist, but with the exclude 'select' replaced with '**'"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/**/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_double_asterisk_2(self):
+        """Identical to test_globbing_filelist, but with the include 'select' replaced with '**'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "testfiles/**/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_double_asterisk_3(self):
+        """Identical to test_globbing_filelist, but with the exclude 'testfiles/select' replaced with '**'"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- **/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_double_asterisk_4(self):
+        """Identical to test_globbing_filelist, but with the include 'testfiles/select' replaced with '**'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "**/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_include_globbing_filelist_double_asterisk_5(self):
+        """Identical to test_globbing_filelist, but with all 'testfiles/select's replaced with '**'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- **/1/1/1\n"
+                        "**/1/1\n"
+                        "- **/1\n"
+                        "- **"])
+
+    def test_exclude_globbing_filelist(self):
+        """Exclude version of test_globbing_filelist"""
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["testfiles/select/1/1/1\n"
+                        "+ testfiles/select/1/1\n"
+                        "testfiles/select/1\n"
+                        "- **"])
+
+    def test_exclude_globbing_filelist_asterisk_1(self):
+        """Exclude version of test_include_globbing_filelist_asterisk"""
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '1'),
+                        ('1', '1', '2'), ('1', '1', '3')],
+                       ["+ */select/1/1\n"
+                        "- **"])
+
+    def test_exclude_globbing_filelist_asterisk_2(self):
+        """Identical to test_exclude_globbing_filelist, but with the exclude 'select' replaced with '*'"""
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["testfiles/*/1/1/1\n"
+                        "+ testfiles/select/1/1\n"
+                        "testfiles/select/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_exclude_globbing_filelist_asterisk_3(self):
+        """Identical to test_exclude_globbing_filelist, but with the include 'select' replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["testfiles/select/1/1/1\n"
+                        "+ testfiles/*/1/1\n"
+                        "testfiles/select/1\n"
+                        "- **"])
+
+    def test_exclude_globbing_filelist_asterisk_4(self):
+        """Identical to test_exclude_globbing_filelist, but with numerous excluded folders replaced with '*'"""
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["*/select/1/1/1\n"
+                        "+ testfiles/select/1/1\n"
+                        "*/*/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_exclude_globbing_filelist_asterisk_5(self):
+        """Identical to test_exclude_globbing_filelist, but with numerous included/excluded folders replaced with '*'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["*/select/1/1/1\n"
+                        "+ */*/1/1\n"
+                        "*/*/1\n"
+                        "- **"])
+
+    @unittest.expectedFailure
+    def test_exclude_globbing_filelist_double_asterisk(self):
+        """Identical to test_exclude_globbing_filelist, but with all included/excluded folders replaced with '**'"""
+        # Todo: Bug #884371 (https://bugs.launchpad.net/duplicity/+bug/884371)
+        self.ParseTest([("--exclude-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["**/1/1/1\n"
+                        "+ **/1/1\n"
+                        "**/1\n"
+                        "- **"])
+
     def testGlob(self):
         """Test globbing expression"""
         self.ParseTest([("--exclude", "**[3-5]"),