Merge lp:~aaron-whitehouse/duplicity/process_filelists_for_spaces_etc into lp:~duplicity-team/duplicity/0.7-series

[Aaron Whitehouse]

Process filelists to remove imperfections such as blank lines, comments and leading/trailing whitespace. Also correctly processes quoted folders containing spaces in their names. Extensive unit and functional tests to test these changes (and selection more generally).

The branch does add an additional folder to testfiles.tar.gz called select2. This included a folder with a trailing space, to test the quote test. The subfolders also have clearer names than in the "select" folder (eg "1sub2sub3") which makes it easier to keep track of issues in tests.

=== modified file 'duplicity/selection.py'
--- duplicity/selection.py	2014-12-12 14:39:54 +0000
+++ duplicity/selection.py	2015-01-07 22:00:24 +0000
@@ -2,6 +2,7 @@
 #
 # Copyright 2002 Ben Escoto <[e-mail]>
 # Copyright 2007 Kenneth Loafman <[e-mail]>
+# Copyright 2014 Aaron Whitehouse <[e-mail]>
 #
 # This file is part of duplicity.
 #
@@ -349,30 +350,37 @@
         something_excluded, tuple_list = None, []
         separator = globals.null_separator and "\0" or "\n"
         for line in filelist_fp.read().split(separator):
-            if not line:
-                continue  # skip blanks
             try:
                 tuple = self.filelist_parse_line(line, include)
             except FilePrefixError as exc:
                 incr_warnings(exc)
                 continue
-            tuple_list.append(tuple)
+            if not tuple:
+                # Skip blanks/full-line comments
+                continue
+            else:
+                tuple_list.append(tuple)
             if not tuple[1]:
                 something_excluded = 1
         if filelist_fp not in (sys.stdin,) and filelist_fp.close():
             log.Warn(_("Error closing filelist %s") % filelist_name)
         return (tuple_list, something_excluded)
 
-    def filelist_parse_line(self, line, include):
-        """Parse a single line of a filelist, returning a pair
-
-        pair will be of form (index, include), where index is another
-        tuple, and include is 1 if the line specifies that we are
-        including a file.  The default is given as an argument.
-        prefix is the string that the index is relative to.
-
-        """
+    def filelist_sanitise_line(self, line, include_default):
+        """
+        Sanitises lines of both normal and globbing filelists, returning (line, include) and line=None if blank/comment
+
+        The aim is to parse filelists in a consistent way, prior to the interpretation of globbing statements.
+        The function removes whitespace, comment lines and processes modifiers (leading +/-) and quotes.
+        """
+
         line = line.strip()
+        if not line:  # skip blanks
+            return None, include_default
+        if line[0] == "#":  # skip full-line comments
+            return None, include_default
+
+        include = include_default
         if line[:2] == "+ ":
             # Check for "+ "/"- " syntax
             include = 1
@@ -381,6 +389,27 @@
             include = 0
             line = line[2:]
 
+        if (line[:1] == "'" and line[-1:] == "'") or (line[:1] == '"' and line[-1:] == '"'):
+            line = line[1:-1]
+
+        return line, include
+
+    def filelist_parse_line(self, line, include):
+        """Parse a single line of a filelist, returning a pair or None if the line is blank/a comment
+
+        Pair will be of form (index, include), where index is another
+        tuple, and include is 1 if the line specifies that we are
+        including a file.  The default is given as an argument.
+        prefix is the string that the index is relative to.
+
+        """
+
+        line, include = self.filelist_sanitise_line(line, include)
+
+        if not line:
+            # Skip blanks and comments
+            return None
+
         if not line.startswith(self.prefix):
             raise FilePrefixError(line)
         line = line[len(self.prefix):]  # Discard prefix
@@ -430,16 +459,11 @@
         log.Notice(_("Reading globbing filelist %s") % list_name)
         separator = globals.null_separator and "\0" or "\n"
         for line in filelist_fp.read().split(separator):
-            if not line:  # skip blanks
-                continue
-            if line[0] == "#":  # skip comments
-                continue
-            if line[:2] == "+ ":
-                yield self.glob_get_sf(line[2:], 1)
-            elif line[:2] == "- ":
-                yield self.glob_get_sf(line[2:], 0)
-            else:
-                yield self.glob_get_sf(line, inc_default)
+            line, include = self.filelist_sanitise_line(line, inc_default)
+            if not line:
+                # Skip blanks and comment lines
+                continue
+            yield self.glob_get_sf(line, include)
 
     def other_filesystems_get_sf(self, include):
         """Return selection function matching files on other filesystems"""

=== added file 'testing/functional/test_selection.py'
--- testing/functional/test_selection.py	1970-01-01 00:00:00 +0000
+++ testing/functional/test_selection.py	2015-01-07 22:00:24 +0000
@@ -0,0 +1,606 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2014 Aaron Whitehouse <[e-mail]>
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import os.path
+import unittest
+
+from . import FunctionalTestCase
+
+
+class IncludeExcludeFunctionalTest(FunctionalTestCase):
+    """
+    This contains methods used in the tests below for testing the include, exclude and various filelist features.
+    """
+
+    # These tests assume the following files and logic, with:
+        # "is" meaning that the file is included specifically
+        # "ia" meaning that the file should be included automatically because its parent is included
+        # "ic" meaning that the folder is included because its contents are included
+        # "es" meaning that the file is excluded specifically
+        # "ea" meaning that the file should be excluded automatically because its parent is included
+        # select2 (es)
+        # --- 1.doc (ea)
+        # --- 1.py (is)
+        # --- 1 (is)
+        # ------ 1sub1 (ia)
+        # --------- 1sub1sub1 (ia)
+        # ------------ 1sub1sub1_file.txt (ia)
+        # --------- 1sub1sub2 (es)
+        # ------------ 1sub1sub2_file.txt (ea)
+        # --------- 1sub1sub3 (ia)
+        # ------------ 1sub1sub3_file.txt (es)
+        # ------ 1sub2 (ic)
+        # --------- 1sub2sub1 (is)
+        # --------- 1sub2sub2 (ea)
+        # --------- 1sub2sub3 (es)  # Not necessary as also ea, but to ensure there are no issues doing so
+        # ------ 1sub3 (ia)
+        # --------- 1sub3sub1 (es)
+        # --------- 1sub3sub2 (es)
+        # --------- 1sub3sub3 (ia)
+        # --- 2 (ic)
+        # ------ 2sub1 (is)
+        # --------- 2sub1sub1 (ia)
+        # ------------ 2sub1sub1_file.txt (ia)
+        # --------- 2sub1sub2 (es)
+        # --------- 2sub1sub3 (es)
+        # ------ 2sub2 (ea)
+        # --------- 2sub2sub1 (ea)
+        # --------- 2sub2sub2 (ea)
+        # --------- 2sub2sub3 (ea)
+        # ------ 2sub3 (ea)
+        # --------- 2sub3sub1 (ea)
+        # --------- 2sub3sub3 (ea)
+        # --------- 2sub3sub2 (ea)
+        # --- 3 (is)
+        # ------ 3sub1 (es)
+        # --------- 3sub1sub1 (ea)
+        # --------- 3sub1sub2 (ea)
+        # --------- 3sub1sub3 (ea)
+        # ------ 3sub2 (ia)
+        # --------- 3sub2sub1 (ia)
+        # --------- 3sub2sub2 (ia)
+        # --------- 3sub2sub3 (ia)
+        # ------ 3sub3 (is)  # Not necessary as also ia, but to ensure there are no issues doing so
+        # --------- 3sub3sub1 (ia)
+        # --------- 3sub3sub2 (es, ic)
+        # ------------ 3sub3sub2_file.txt (is)
+        # --------- 3sub3sub3 (ia)
+        # --- trailing_space  (ea)  # Note this is "trailing_space ". Excluded until trailing_space test, when (is)
+        # ------ trailing_space sub1 (ea)  # Excluded until trailing_space test, when (ia)
+        # ------ trailing_space sub2 (ea)  # Excluded until trailing_space test, when (es, ic)
+        # --------- trailing_space sub2_file.txt (ea)  # Excluded until trailing_space test, when (is)
+
+    complete_directory_tree = [['1', '2', '3', 'trailing_space ', '1.doc', '1.py'],
+                              ['1sub1', '1sub2', '1sub3'],
+                              ['1sub1sub1', '1sub1sub2', '1sub1sub3'],
+                              ['1sub1sub1_file.txt'],
+                              ['1sub1sub2_file.txt'],
+                              ['1sub1sub3_file.txt'],
+                              ['1sub2sub1', '1sub2sub2', '1sub2sub3'],
+                              ['1sub3sub1', '1sub3sub2', '1sub3sub3'],
+                              ['2sub1', '2sub2', '2sub3'],
+                              ['2sub1sub1', '2sub1sub2', '2sub1sub3'],
+                              ['2sub1sub1_file.txt'],
+                              ['2sub2sub1', '2sub2sub2', '2sub2sub3'],
+                              ['2sub3sub1', '2sub3sub2', '2sub3sub3'],
+                              ['3sub1', '3sub2', '3sub3'],
+                              ['3sub1sub1', '3sub1sub2', '3sub1sub3'],
+                              ['3sub2sub1', '3sub2sub2', '3sub2sub3'],
+                              ['3sub3sub1', '3sub3sub2', '3sub3sub3'],
+                              ['3sub3sub2_file.txt'],
+                              ['trailing_space sub1', 'trailing_space sub2'],
+                              ['trailing_space sub2_file.txt']]
+
+    expected_restored_tree = [['1', '2', '3', '1.py'],
+                              ['1sub1', '1sub2', '1sub3'],
+                              ['1sub1sub1', '1sub1sub3'],
+                              ['1sub1sub1_file.txt'],
+                              ['1sub2sub1'],
+                              ['1sub3sub3'],
+                              ['2sub1'],
+                              ['2sub1sub1'],
+                              ['2sub1sub1_file.txt'],
+                              ['3sub2', '3sub3'],
+                              ['3sub2sub1', '3sub2sub2', '3sub2sub3'],
+                              ['3sub3sub1', '3sub3sub2', '3sub3sub3'],
+                              ['3sub3sub2_file.txt']]
+
+    expected_restored_tree_with_trailing_space = [['1', '2', '3', 'trailing_space ', '1.py'],
+                                                  ['1sub1', '1sub2', '1sub3'],
+                                                  ['1sub1sub1', '1sub1sub3'],
+                                                  ['1sub1sub1_file.txt'],
+                                                  ['1sub2sub1'],
+                                                  ['1sub3sub3'],
+                                                  ['2sub1'],
+                                                  ['2sub1sub1'],
+                                                  ['2sub1sub1_file.txt'],
+                                                  ['3sub2', '3sub3'],
+                                                  ['3sub2sub1', '3sub2sub2', '3sub2sub3'],
+                                                  ['3sub3sub1', '3sub3sub2', '3sub3sub3'],
+                                                  ['3sub3sub2_file.txt'],
+                                                  ['trailing_space sub1', 'trailing_space sub2'],
+                                                  ['trailing_space sub2_file.txt']]
+
+    def directory_tree_to_list_of_lists(self, parent_directory):
+        """
+        This takes a folder as an input and returns a list with its contents. If the directory has subdirectories, it
+        returns a list of lists with the contents of those subdirectories.
+        """
+        directory_list = []
+        for root, dirs, files in os.walk(parent_directory):
+            to_add = []
+            if dirs:
+                dirs.sort()  # So that we can easily compare to what we expect
+                to_add = dirs
+            if files:
+                files.sort()  # So that we can easily compare to what we expect
+                to_add += files
+            if to_add:
+                directory_list.append(to_add)
+        return directory_list
+
+
+class TestCheckTestFiles(IncludeExcludeFunctionalTest):
+    """ Tests the testfiles required by the exclude/include tests are as expected. """
+
+    def test_files_are_as_expected(self):
+        """Test that the contents of testfiles/select are as expected."""
+        testfiles = self.directory_tree_to_list_of_lists('testfiles/select2')
+        #print(testfiles)
+        self.assertEqual(testfiles, self.complete_directory_tree)
+
+
+class TestIncludeExcludeOptions(IncludeExcludeFunctionalTest):
+    """ This tests the behaviour of the duplicity binary when the include/exclude options are passed directly """
+
+    def test_include_exclude_basic(self):
+        """ Test --include and --exclude work in the basic case """
+        self.backup("full", "testfiles/select2",
+                    options=["--include", "testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt",
+                             "--exclude", "testfiles/select2/3/3sub3/3sub3sub2",
+                             "--include", "testfiles/select2/3/3sub2/3sub2sub2",
+                             "--include", "testfiles/select2/3/3sub3",
+                             "--exclude", "testfiles/select2/3/3sub1",
+                             "--exclude", "testfiles/select2/2/2sub1/2sub1sub3",
+                             "--exclude", "testfiles/select2/2/2sub1/2sub1sub2",
+                             "--include", "testfiles/select2/2/2sub1",
+                             "--exclude", "testfiles/select2/1/1sub3/1sub3sub2",
+                             "--exclude", "testfiles/select2/1/1sub3/1sub3sub1",
+                             "--exclude", "testfiles/select2/1/1sub2/1sub2sub3",
+                             "--include", "testfiles/select2/1/1sub2/1sub2sub1",
+                             "--exclude", "testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt",
+                             "--exclude", "testfiles/select2/1/1sub1/1sub1sub2",
+                             "--exclude", "testfiles/select2/1/1sub2",
+                             "--include", "testfiles/select2/1.py",
+                             "--include", "testfiles/select2/3",
+                             "--include", "testfiles/select2/1",
+                             "--exclude", "testfiles/select2/**"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_include_exclude_trailing_whitespace(self):
+        """Test that folders with trailing whitespace in the names work correctly when passing as include/exclude"""
+        # Note that, because this only passes items in as a list of options, this test does not test whether duplicity
+        # would correctly interpret commandline options with spaces. However, bin/duplicity uses sys.argv[1:], which
+        # should return a list of strings after having correctly processed quotes etc.
+        self.backup("full", "testfiles/select2",
+                    options=["--include",
+                             "testfiles/select2/trailing_space /trailing_space sub2/trailing_space sub2_file.txt",
+                             "--exclude", "testfiles/select2/trailing_space /trailing_space sub2",
+                             "--include", "testfiles/select2/trailing_space ",
+                             "--include", "testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt",
+                             "--exclude", "testfiles/select2/3/3sub3/3sub3sub2",
+                             "--include", "testfiles/select2/3/3sub2/3sub2sub2",
+                             "--include", "testfiles/select2/3/3sub3",
+                             "--exclude", "testfiles/select2/3/3sub1",
+                             "--exclude", "testfiles/select2/2/2sub1/2sub1sub3",
+                             "--exclude", "testfiles/select2/2/2sub1/2sub1sub2",
+                             "--include", "testfiles/select2/2/2sub1",
+                             "--exclude", "testfiles/select2/1/1sub3/1sub3sub2",
+                             "--exclude", "testfiles/select2/1/1sub3/1sub3sub1",
+                             "--exclude", "testfiles/select2/1/1sub2/1sub2sub3",
+                             "--include", "testfiles/select2/1/1sub2/1sub2sub1",
+                             "--exclude", "testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt",
+                             "--exclude", "testfiles/select2/1/1sub1/1sub1sub2",
+                             "--exclude", "testfiles/select2/1/1sub2",
+                             "--include", "testfiles/select2/1.py",
+                             "--include", "testfiles/select2/3",
+                             "--include", "testfiles/select2/1",
+                             "--exclude", "testfiles/select2/**"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree_with_trailing_space)
+
+
+class TestExcludeGlobbingFilelistTest(IncludeExcludeFunctionalTest):
+    """
+    Test --exclude-globbing-filelist using duplicity binary.
+    """
+
+    def test_exclude_globbing_filelist(self):
+        """Test that exclude globbing filelist works in the basic case """
+        # As this is an exclude filelist any lines with no +/- modifier should be treated as if they have a -.
+        # Create a filelist
+        with open('testfiles/exclude.txt', 'w') as f:
+            f.write('+ testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    'testfiles/select2/3/3sub3/3sub3sub2\n'
+                    '+ testfiles/select2/3/3sub2/3sub2sub2\n'
+                    '+ testfiles/select2/3/3sub3\n'
+                    '- testfiles/select2/3/3sub1\n'  # - added to ensure it makes no difference
+                    'testfiles/select2/2/2sub1/2sub1sub3\n'
+                    'testfiles/select2/2/2sub1/2sub1sub2\n'
+                    '+ testfiles/select2/2/2sub1\n'
+                    'testfiles/select2/1/1sub3/1sub3sub2\n'
+                    'testfiles/select2/1/1sub3/1sub3sub1\n'
+                    'testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'
+                    'testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    'testfiles/select2/1/1sub1/1sub1sub2\n'
+                    '- testfiles/select2/1/1sub2\n'  # - added to ensure it makes no difference
+                    '+ testfiles/select2/1.py\n'
+                    '+ testfiles/select2/3\n'
+                    '+ testfiles/select2/1\n'
+                    'testfiles/select2/**')
+        self.backup("full", "testfiles/select2", options=["--exclude-globbing-filelist=testfiles/exclude.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_exclude_globbing_filelist_combined_imperfections(self):
+        """Test that exclude globbing filelist works with imperfections in the input file"""
+        # This is a combined test for speed reasons. The individual imperfections are tested as unittests in
+        # unit/test_selection.
+        # Imperfections tested are;
+        # * Leading space/spaces before the modifier
+        # * Trailing space/spaces after the filename (but before the newline)
+        # * Blank lines (newline character only)
+        # * Line only containing spaces
+        # * Full-line comments with # as the first character and with leading/trailing spaces
+        # * Unnecessarily quoted filenames with/without modifier (both " and ')
+
+        # Create a filelist
+        with open('testfiles/exclude.txt', 'w') as f:
+            f.write('+ testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    'testfiles/select2/3/3sub3/3sub3sub2\n'
+                    '+ testfiles/select2/3/3sub2/3sub2sub2\n'
+                    ' + testfiles/select2/3/3sub3\n'  # Note leading space added here
+                    '- testfiles/select2/3/3sub1\n'
+                    '  testfiles/select2/2/2sub1/2sub1sub3\n'  # Note leading spaces added here
+                    '\n'
+                    'testfiles/select2/2/2sub1/2sub1sub2\n'
+                    ' + testfiles/select2/2/2sub1 \n'  # Note added trailing/leading space here
+                    '- "testfiles/select2/1/1sub3/1sub3sub2"\n'  # Unnecessary quotes
+                    '# Testing a full-line comment\n'
+                    "'testfiles/select2/1/1sub3/1sub3sub1'  \n"  # Note added spaces and quotes here
+                    'testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '    \n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    'testfiles/select2/1/1sub1/1sub1sub2\n'
+                    '     # Testing a full-line comment with leading and trailing spaces     \n'
+                    'testfiles/select2/1/1sub2  \n'  # Note added spaces here
+                    '+ testfiles/select2/1.py\n'
+                    '+ testfiles/select2/3 \n'  # Note added space here
+                    '+ testfiles/select2/1\n'
+                    '- testfiles/select2/**')
+        self.backup("full", "testfiles/select2", options=["--exclude-globbing-filelist=testfiles/exclude.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_exclude_globbing_filelist_trailing_whitespace_folders_work_with_quotes(self):
+        """Test that folders with trailing whitespace in the names work correctly if they are enclosed in quotes"""
+        # Create a filelist
+        with open('testfiles/exclude.txt', 'w') as f:
+            f.write('+ "testfiles/select2/trailing_space /trailing_space sub2/trailing_space sub2_file.txt"\n'  # New
+                    '- "testfiles/select2/trailing_space /trailing_space sub2"\n'  # New
+                    '+ "testfiles/select2/trailing_space "\n'  # New
+                    '+ testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    'testfiles/select2/3/3sub3/3sub3sub2\n'
+                    '+ testfiles/select2/3/3sub2/3sub2sub2\n'
+                    '+ testfiles/select2/3/3sub3\n'
+                    '- testfiles/select2/3/3sub1\n'
+                    'testfiles/select2/2/2sub1/2sub1sub3\n'
+                    'testfiles/select2/2/2sub1/2sub1sub2\n'
+                    '+ testfiles/select2/2/2sub1\n'
+                    'testfiles/select2/1/1sub3/1sub3sub2\n'
+                    'testfiles/select2/1/1sub3/1sub3sub1\n'
+                    'testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'
+                    'testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    'testfiles/select2/1/1sub1/1sub1sub2\n'
+                    '- testfiles/select2/1/1sub2\n'
+                    '+ testfiles/select2/1.py\n'
+                    '+ testfiles/select2/3\n'
+                    '+ testfiles/select2/1\n'
+                    'testfiles/select2/**')
+        self.backup("full", "testfiles/select2", options=["--exclude-globbing-filelist=testfiles/exclude.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree_with_trailing_space)
+
+class TestIncludeGlobbingFilelistTest(IncludeExcludeFunctionalTest):
+    """
+    Test --include-globbing-filelist using duplicity binary.
+    """
+
+    def test_include_globbing_filelist(self):
+        """Test that include globbing filelist works in the basic case"""
+        # See test_exclude_globbing_filelist above for explanation of what is expected. As this is an include filelist
+        # any lines with no +/- modifier should be treated as if they have a +.
+        # Create a filelist
+        with open('testfiles/include.txt', 'w') as f:
+            f.write('testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    '- testfiles/select2/3/3sub3/3sub3sub2\n'
+                    'testfiles/select2/3/3sub2/3sub2sub2\n'
+                    '+ testfiles/select2/3/3sub3\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/3/3sub1\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub3\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub2\n'
+                    'testfiles/select2/2/2sub1\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub2\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub1\n'
+                    '- testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    '- testfiles/select2/1/1sub1/1sub1sub2\n'
+                    '- testfiles/select2/1/1sub2\n'
+                    'testfiles/select2/1.py\n'
+                    'testfiles/select2/3\n'
+                    'testfiles/select2/1\n'
+                    '- testfiles/select2/**')
+        self.backup("full", "testfiles/select2", options=["--include-globbing-filelist=testfiles/include.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_include_globbing_filelist_combined_imperfections(self):
+        """Test that include globbing filelist works with imperfections in the input file"""
+        # This is a combined test for speed reasons. The individual imperfections are tested as unittests in
+        # unit/test_selection.
+        # Imperfections tested are;
+        # * Leading space/spaces before the modifier
+        # * Trailing space/spaces after the filename (but before the newline)
+        # * Blank lines (newline character only)
+        # * Line only containing spaces
+        # * Full-line comments with # as the first character and with leading/trailing spaces
+        # * Unnecessarily quoted filenames with/without modifier  (both " and ')
+        # Create a filelist
+        with open('testfiles/include.txt', 'w') as f:
+            f.write('testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    '- testfiles/select2/3/3sub3/3sub3sub2\n'
+                    '"testfiles/select2/3/3sub2/3sub2sub2"\n'
+                    '  + testfiles/select2/3/3sub3\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/3/3sub1\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub3\n'
+                    ' - "testfiles/select2/2/2sub1/2sub1sub2"\n'
+                    'testfiles/select2/2/2sub1  \n'
+                    '\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub2\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub1 \n'
+                    "- 'testfiles/select2/1/1sub2/1sub2sub3'\n"
+                    '             \n'
+                    ' + testfiles/select2/1/1sub2/1sub2sub1 \n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    '  - testfiles/select2/1/1sub1/1sub1sub2  \n'
+                    '# Testing full-line comment\n'
+                    '- testfiles/select2/1/1sub2\n'
+                    "'testfiles/select2/1.py'\n"
+                    'testfiles/select2/3\n'
+                    '        #  Testing another full-line comment      \n'
+                    'testfiles/select2/1\n'
+                    '- testfiles/select2/**')
+        self.backup("full", "testfiles/select2", options=["--include-globbing-filelist=testfiles/include.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+
+class TestIncludeFilelistTest(IncludeExcludeFunctionalTest):
+    """
+    Test --include-filelist using duplicity binary.
+    """
+    @unittest.expectedFailure
+    def test_include_filelist(self):
+        """Test that include filelist works in the basic case"""
+        # See test_exclude_globbing_filelist above for explanation of what is expected. As this is an include filelist
+        # any lines with no +/- modifier should be treated as if they have a +.
+        # ToDo Currently fails - Bug #1408411 (https://bugs.launchpad.net/duplicity/+bug/1408411)
+        # Create a filelist
+        with open('testfiles/include.txt', 'w') as f:
+            f.write('testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    '- testfiles/select2/3/3sub3/3sub3sub2\n'
+                    'testfiles/select2/3/3sub2/3sub2sub2\n'
+                    '+ testfiles/select2/3/3sub3\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/3/3sub1\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub3\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub2\n'
+                    'testfiles/select2/2/2sub1\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub2\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub1\n'
+                    '- testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    '- testfiles/select2/1/1sub1/1sub1sub2\n'
+                    '- testfiles/select2/1/1sub2\n'
+                    'testfiles/select2/1.py\n'
+                    'testfiles/select2/3\n'
+                    '- testfiles/select2/2\n' # es instead of ea as no globbing - **
+                    'testfiles/select2/1\n'
+                    '- "testfiles/select2/trailing_space "\n'  # es instead of ea as no globbing - **
+                    '- testfiles/select2/1.doc')  # es instead of ea as no globbing - **
+        self.backup("full", "testfiles/select2", options=["--include-filelist=testfiles/include.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_include_filelist_workaround(self):
+        """Test that include filelist works in the basic case"""
+        # This is a modified version of test_include_filelist that passes, despite Bug #1408411
+        # It is still a valid test, it just does not test as many selection features as the above.
+        # Create a filelist
+        with open('testfiles/include.txt', 'w') as f:
+            f.write('testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    #  '- testfiles/select2/3/3sub3/3sub3sub2\n'  # Commented out because of Bug #1408411
+                    'testfiles/select2/3/3sub2/3sub2sub2\n'
+                    '+ testfiles/select2/3/3sub3\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/3/3sub1\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub3\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub2\n'
+                    'testfiles/select2/2/2sub1\n'
+                    '- testfiles/select2/2/2sub3\n'  # Added because of Bug #1408411
+                    '- testfiles/select2/2/2sub2\n'  # Added because of Bug #1408411
+                    '- testfiles/select2/1/1sub3/1sub3sub2\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub1\n'
+                    '- testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '- testfiles/select2/1/1sub2/1sub2sub2\n'  # Added because of Bug #1408411
+                    '+ testfiles/select2/1/1sub2/1sub2sub1\n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    '- testfiles/select2/1/1sub1/1sub1sub2\n'
+                    #  '- testfiles/select2/1/1sub2\n'  # Commented out because of Bug #1408411
+                    'testfiles/select2/1.py\n'
+                    'testfiles/select2/3\n'
+                    #  '- testfiles/select2/2\n' # Commented out because of Bug #1408411
+                    'testfiles/select2/1\n'
+                    '- "testfiles/select2/trailing_space "\n'  # es instead of ea as no globbing - **
+                    '- testfiles/select2/1.doc')  # es instead of ea as no globbing - **
+        self.backup("full", "testfiles/select2", options=["--include-filelist=testfiles/include.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+    def test_include_filelist_workaround_combined_imperfections(self):
+        """Test that include filelist works with imperfections in the input file"""
+        # This is a modified version of test_include_filelist that passes, despite Bug #1408411
+        # It is still a valid test, it just does not test as many selection features as the above.
+        # This is a combined test for speed reasons. The individual imperfections are tested as unittests in
+        # unit/test_selection.
+        # Imperfections tested are;
+        # * Leading space/spaces before the modifier
+        # * Trailing space/spaces after the filename (but before the newline)
+        # * Blank lines (newline character only)
+        # * Line only containing spaces
+        # * Full-line comments with # as the first character and with leading/trailing spaces
+        # * Unnecessarily quoted filenames with/without modifier  (both " and ')
+        # Create a filelist
+        with open('testfiles/include.txt', 'w') as f:
+            f.write('testfiles/select2/3/3sub3/3sub3sub2/3sub3sub2_file.txt\n'
+                    'testfiles/select2/3/3sub2/3sub2sub2 \n'
+                    '  + testfiles/select2/3/3sub3\n'  # + added to ensure it makes no difference
+                    ' - testfiles/select2/3/3sub1  \n'
+                    '- testfiles/select2/2/2sub1/2sub1sub3\n'
+                    '- testfiles/select2/2/2sub1/2sub1sub2\n'
+                    '"testfiles/select2/2/2sub1"\n'
+                    '   - testfiles/select2/2/2sub3 \n'  # Added because of Bug #1408411
+                    '- testfiles/select2/2/2sub2\n'  # Added because of Bug #1408411
+                    "- 'testfiles/select2/1/1sub3/1sub3sub2'\n"
+                    '\n'
+                    '- testfiles/select2/1/1sub3/1sub3sub1\n'
+                    '- testfiles/select2/1/1sub2/1sub2sub3\n'
+                    '- "testfiles/select2/1/1sub2/1sub2sub2"\n'  # Added because of Bug #1408411
+                    '# This is a full-line comment\n'
+                    '+ testfiles/select2/1/1sub2/1sub2sub1  \n'  # + added to ensure it makes no difference
+                    '- testfiles/select2/1/1sub1/1sub1sub3/1sub1sub3_file.txt\n'
+                    '          \n'
+                    '- testfiles/select2/1/1sub1/1sub1sub2\n'
+                    #  '- testfiles/select2/1/1sub2\n'  # Commented out because of Bug #1408411
+                    "'testfiles/select2/1.py'\n"
+                    '       # This is another full-line comment, with spaces     \n'
+                    'testfiles/select2/3\n'
+                    #  '- testfiles/select2/2\n' # Commented out because of Bug #1408411
+                    'testfiles/select2/1\n'
+                    '- "testfiles/select2/trailing_space "\n'  # es instead of ea as no globbing - **
+                    '- testfiles/select2/1.doc')  # es instead of ea as no globbing - **
+        self.backup("full", "testfiles/select2", options=["--include-filelist=testfiles/include.txt"])
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, self.expected_restored_tree)
+
+class TestIncludeExcludedForContents(IncludeExcludeFunctionalTest):
+    """ Test to check that folders that are excluded are included if they contain includes of higher priority.
+     Exhibits the issue reported in Bug #1408411 (https://bugs.launchpad.net/duplicity/+bug/1408411). """
+
+    def write_filelist(self, filelist_name):
+        """Used by the below tests to write the filelist"""
+        assert filelist_name is not None
+        with open(filelist_name, 'w') as f:
+            f.write("+ testfiles/select/1/2/1\n"
+                    "- testfiles/select/1/2\n"
+                    "- testfiles/select/1/1\n"
+                    "- testfiles/select/1/3")
+
+    def restore_and_check(self):
+        """Restores the backup and compares to what was expected (based on the filelist in write_filelist)"""
+        self.restore()
+        restore_dir = 'testfiles/restore_out'
+        restored = self.directory_tree_to_list_of_lists(restore_dir)
+        self.assertEqual(restored, [['2'], ['1']])
+
+    def test_commandline_include_exclude(self):
+        """test an excluded folder is included for included contents when using commandline includes and excludes"""
+        self.backup("full", "testfiles/select/1",
+                    options=["--include", "testfiles/select/1/2/1",
+                             "--exclude", "testfiles/select/1/2",
+                             "--exclude", "testfiles/select/1/1",
+                             "--exclude", "testfiles/select/1/3"])
+        self.restore_and_check()
+
+    def test_include_globbing_filelist(self):
+        """test an excluded folder is included for included contents with an include-globbing-filelist """
+        self.write_filelist("testfiles/include.txt")
+        self.backup("full", "testfiles/select/1", options=["--include-globbing-filelist=testfiles/include.txt"])
+        self.restore_and_check()
+
+    def test_exclude_globbing_filelist(self):
+        """test an excluded folder is included for included contents with an exclude-globbing-filelist """
+        self.write_filelist("testfiles/exclude.txt")
+        self.backup("full", "testfiles/select/1", options=["--exclude-globbing-filelist=testfiles/exclude.txt"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_include_filelist(self):
+        """test an excluded folder is included for included contents with an include-filelist (non-globbing) """
+        # ToDo - currently fails - Bug #1408411 (https://bugs.launchpad.net/duplicity/+bug/1408411)
+        self.write_filelist("testfiles/include.txt")
+        self.backup("full", "testfiles/select/1", options=["--include-filelist=testfiles/include.txt"])
+        self.restore_and_check()
+
+    @unittest.expectedFailure
+    def test_exclude_filelist(self):
+        """test an excluded folder is included for included contents with an exclude-filelist  (non-globbing) """
+        # ToDo - currently fails - Bug #1408411 (https://bugs.launchpad.net/duplicity/+bug/1408411)
+        self.write_filelist("testfiles/exclude.txt")
+        self.backup("full", "testfiles/select/1", options=["--exclude-filelist=testfiles/exclude.txt"])
+        self.restore_and_check()
+
+if __name__ == "__main__":
+    unittest.main()

=== modified file 'testing/testfiles.tar.gz'
Binary files testing/testfiles.tar.gz	2011-11-04 04:07:59 +0000 and testing/testfiles.tar.gz	2015-01-07 22:00:24 +0000 differ
=== modified file 'testing/unit/test_selection.py'
--- testing/unit/test_selection.py	2014-12-12 14:39:54 +0000
+++ testing/unit/test_selection.py	2015-01-07 22:00:24 +0000
@@ -2,6 +2,7 @@
 #
 # Copyright 2002 Ben Escoto <[e-mail]>
 # Copyright 2007 Kenneth Loafman <[e-mail]>
+# Copyright 2014 Aaron Whitehouse <[e-mail]>
 #
 # This file is part of duplicity.
 #
@@ -27,7 +28,6 @@
 from . import UnitTestCase
 
 
-
 class MatchingTest(UnitTestCase):
     """Test matching of file names against various selection functions"""
     def setUp(self):
@@ -117,6 +117,70 @@
         assert sf(self.makeext("3/3")) == 1
         assert sf(self.makeext("3/3/3")) == None
 
+    def test_filelist_include_1_trailing_white_space(self):
+        """Test trailing whitespace is ignored in included filelist (1 space)"""
+        fp = StringIO.StringIO("testfiles/select/1/2\n"
+                               "testfiles/select/1 \n"
+                               "testfiles/select/1/2/3\n"
+                               "testfiles/select/3/3/2")
+        sf = self.Select.filelist_get_sf(fp, 1, "test")
+        assert sf(self.root) == 1
+        assert sf(self.makeext("1")) == 1
+        assert sf(self.makeext("1/1")) == None
+        assert sf(self.makeext("1/2/3")) == 1
+        assert sf(self.makeext("2/2")) == None
+        assert sf(self.makeext("3")) == 1
+        assert sf(self.makeext("3/3")) == 1
+        assert sf(self.makeext("3/3/3")) == None
+
+    def test_filelist_include_2_trailing_white_spaces(self):
+        """Test trailing whitespace is ignored in included filelist (2 space)"""
+        fp = StringIO.StringIO("testfiles/select/1/2\n"
+                               "testfiles/select/1\n"
+                               "testfiles/select/1/2/3  \n"
+                               "testfiles/select/3/3/2")
+        sf = self.Select.filelist_get_sf(fp, 1, "test")
+        assert sf(self.root) == 1
+        assert sf(self.makeext("1")) == 1
+        assert sf(self.makeext("1/1")) == None
+        assert sf(self.makeext("1/2/3")) == 1
+        assert sf(self.makeext("2/2")) == None
+        assert sf(self.makeext("3")) == 1
+        assert sf(self.makeext("3/3")) == 1
+        assert sf(self.makeext("3/3/3")) == None
+
+    def test_filelist_include_1_leading_white_space(self):
+        """Test leading whitespace is ignored in included filelist (1 space)"""
+        fp = StringIO.StringIO(" testfiles/select/1/2\n"
+                               "testfiles/select/1\n"
+                               "testfiles/select/1/2/3\n"
+                               "testfiles/select/3/3/2")
+        sf = self.Select.filelist_get_sf(fp, 1, "test")
+        assert sf(self.root) == 1
+        assert sf(self.makeext("1")) == 1
+        assert sf(self.makeext("1/1")) == None
+        assert sf(self.makeext("1/2/3")) == 1
+        assert sf(self.makeext("2/2")) == None
+        assert sf(self.makeext("3")) == 1
+        assert sf(self.makeext("3/3")) == 1
+        assert sf(self.makeext("3/3/3")) == None
+
+    def test_filelist_include_2_leading_white_spaces(self):
+        """Test leading whitespace is ignored in included filelist (1 space)"""
+        fp = StringIO.StringIO("testfiles/select/1/2\n"
+                               "testfiles/select/1\n"
+                               "testfiles/select/1/2/3\n"
+                               "  testfiles/select/3/3/2")
+        sf = self.Select.filelist_get_sf(fp, 1, "test")
+        assert sf(self.root) == 1
+        assert sf(self.makeext("1")) == 1
+        assert sf(self.makeext("1/1")) == None
+        assert sf(self.makeext("1/2/3")) == 1
+        assert sf(self.makeext("2/2")) == None
+        assert sf(self.makeext("3")) == 1
+        assert sf(self.makeext("3/3")) == 1
+        assert sf(self.makeext("3/3/3")) == None
+
     def testFilelistIncludeNullSep(self):
         """Test included filelist but with null_separator set"""
         fp = StringIO.StringIO("""\0testfiles/select/1/2\0testfiles/select/1\0testfiles/select/1/2/3\0testfiles/select/3/3/2\0testfiles/select/hello\nthere\0""")
@@ -168,6 +232,42 @@
         assert sf(self.makeext("2")) == None
         assert sf(self.makeext("3")) == 0
 
+    def testFilelistInclude3(self):
+        """testFilelistInclude3 - with modifiers to check - works as expected"""
+        fp = StringIO.StringIO("""
+testfiles/select/1/1
+- testfiles/select/1/2
++ testfiles/select/1/3
+testfiles/select/1""")
+        sf = self.Select.filelist_get_sf(fp, 1, "test1")
+        assert sf(self.makeext("1")) == 1
+        assert sf(self.makeext("1/1")) == 1
+        assert sf(self.makeext("1/1/2")) == None
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == None
+
+#     def test_filelist_include_excluded_folder_with_included_contents(self):
+#         """Check that excluded folder is included if subfolder is included at higher priority. """
+#         # ToDo - currently fails. 1/2 should be included (scanned) because 1/2/1 is. Commandline --include/--exclude
+#         # ToDo - and globbing filelists work this way
+#         fp = StringIO.StringIO("""
+# + testfiles/select/1/2/1
+# - testfiles/select/1/2
+# + testfiles/select/1/3
+# testfiles/select/1""")
+#         sf = self.Select.filelist_get_sf(fp, 1, "test1")
+#         assert sf(self.makeext("1")) == 1
+#         assert sf(self.makeext("1/1")) == None
+#         assert sf(self.makeext("1/2/1")) == 1
+#         assert sf(self.makeext("1/2")) == 0  # ToDo - what should this return?
+#         assert sf(self.makeext("1/2/3")) == 0
+#         assert sf(self.makeext("1/3")) == 1
+#         assert sf(self.makeext("2")) == None
+#         assert sf(self.makeext("3")) == None
+
     def testFilelistExclude2(self):
         """testFilelistExclude2 - with modifiers"""
         fp = StringIO.StringIO("""
@@ -188,6 +288,123 @@
         assert sf(self.makeext("2")) == None
         assert sf(self.makeext("3")) == 0
 
+    def test_filelist_exclude_2_with_trailing_white_space(self):
+        """testFilelistExclude2 with modifiers - test trailing whitespace is ignored (1 and 2 spaces)"""
+        fp = StringIO.StringIO("testfiles/select/1/1\n"
+                               "- testfiles/select/1/2 \n"
+                               "+ testfiles/select/1/3  \n"
+                               "- testfiles/select/3")
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
+    def test_filelist_exclude_with_single_quotes(self):
+        """testFilelistExclude2 with modifiers - test unnecessary single quotes are ignored"""
+        fp = StringIO.StringIO("testfiles/select/1/1\n"
+                               "- testfiles/select/1/2\n"
+                               "+ 'testfiles/select/1/3'\n"
+                               "- testfiles/select/3")
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
+    def test_filelist_exclude_with_full_line_comment(self):
+        """testFilelistExclude2 with modifiers - test full-line comment is ignored"""
+        fp = StringIO.StringIO("testfiles/select/1/1\n"
+                               "- testfiles/select/1/2\n"
+                               "# This is a full-line comment\n"
+                               "+ testfiles/select/1/3\n"
+                               "- testfiles/select/3")
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
+    def test_filelist_exclude_with_blank_line(self):
+        """testFilelistExclude2 with modifiers - test blank line is ignored"""
+        fp = StringIO.StringIO("testfiles/select/1/1\n"
+                               "- testfiles/select/1/2\n"
+                               "\n"
+                               "+ testfiles/select/1/3\n"
+                               "- testfiles/select/3")
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
+    def test_filelist_exclude_with_blank_line_and_whitespace(self):
+        """testFilelistExclude2 with modifiers - test blank line with whitespace is ignored"""
+        fp = StringIO.StringIO("testfiles/select/1/1\n"
+                               "- testfiles/select/1/2\n"
+                               " \n"
+                               "+ testfiles/select/1/3\n"
+                               "- testfiles/select/3")
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
+    def test_filelist_exclude_with_double_quotes(self):
+        """testFilelistExclude2 with modifiers - test unnecessary double quotes are ignored"""
+        fp = StringIO.StringIO('testfiles/select/1/1\n'
+                               '- testfiles/select/1/2\n'
+                               '+ "testfiles/select/1/3"\n'
+                               '- testfiles/select/3')
+        sf = self.Select.filelist_get_sf(fp, 0, "test1")
+        sf_val1 = sf(self.root)
+        assert sf_val1 == 1 or sf_val1 == None  # either is OK
+        sf_val2 = sf(self.makeext("1"))
+        assert sf_val2 == 1 or sf_val2 == None
+        assert sf(self.makeext("1/1")) == 0
+        assert sf(self.makeext("1/1/2")) == 0
+        assert sf(self.makeext("1/2")) == 0
+        assert sf(self.makeext("1/2/3")) == 0
+        assert sf(self.makeext("1/3")) == 1
+        assert sf(self.makeext("2")) == None
+        assert sf(self.makeext("3")) == 0
+
     def testGlobRE(self):
         """testGlobRE - test translation of shell pattern to regular exp"""
         assert self.Select.glob_to_re("hello") == "hello"
@@ -316,12 +533,154 @@
         self.ParseTest([("--include-globbing-filelist", "file")],
                        [(), ('1',), ('1', '1'), ('1', '1', '2'),
                         ('1', '1', '3')],
-                       ["""
-- testfiles/select/1/1/1
-testfiles/select/1/1
-- testfiles/select/1
-- **
-"""])
+                       ["- testfiles/select/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_1_trailing_whitespace(self):
+        """Filelist glob test similar to globbing filelist, but with 1 trailing whitespace on include"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "testfiles/select/1/1 \n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_2_trailing_whitespaces(self):
+        """Filelist glob test similar to globbing filelist, but with 2 trailing whitespaces on include"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "testfiles/select/1/1  \n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_1_leading_whitespace(self):
+        """Filelist glob test similar to globbing filelist, but with 1 leading whitespace on include"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        " testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_2_leading_whitespaces(self):
+        """Filelist glob test similar to globbing filelist, but with 2 leading whitespaces on include"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1\n"
+                        "  testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_1_trailing_whitespace_exclude(self):
+        """Filelist glob test similar to globbing filelist, but with 1 trailing whitespace on exclude"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1 \n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_2_trailing_whitespace_exclude(self):
+        """Filelist glob test similar to globbing filelist, but with 2 trailing whitespaces on exclude"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- testfiles/select/1/1/1  \n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_1_leading_whitespace_exclude(self):
+        """Filelist glob test similar to globbing filelist, but with 1 leading whitespace on exclude"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       [" - testfiles/select/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_2_leading_whitespaces_exclude(self):
+        """Filelist glob test similar to globbing filelist, but with 2 leading whitespaces on exclude"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["  - testfiles/select/1/1/1\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_check_excluded_folder_included_for_contents(self):
+        """Filelist glob test to check excluded folder is included if contents are"""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3'), ('1', '2'), ('1', '2', '1'), ('1', '3'), ('1', '3', '1'), ('1', '3', '2'),
+                        ('1', '3', '3')],
+                       ["+ testfiles/select/1/2/1\n"
+                        "- testfiles/select/1/2\n"
+                        "testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_with_unnecessary_quotes(self):
+        """Filelist glob test similar to globbing filelist, but with quotes around one of the paths."""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ["- 'testfiles/select/1/1/1'\n"
+                        "testfiles/select/1/1\n"
+                        "- testfiles/select/1\n"
+                        "- **"])
+
+    def test_include_globbing_filelist_with_unnecessary_double_quotes(self):
+        """Filelist glob test similar to globbing filelist, but with double quotes around one of the paths."""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ['- "testfiles/select/1/1/1"\n'
+                        'testfiles/select/1/1\n'
+                        '- testfiles/select/1\n'
+                        '- **'])
+
+    def test_include_globbing_filelist_with_full_line_comment(self):
+        """Filelist glob test similar to globbing filelist, but with a full-line comment."""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ['- testfiles/select/1/1/1\n'
+                        '# This is a test\n'
+                        'testfiles/select/1/1\n'
+                        '- testfiles/select/1\n'
+                        '- **'])
+
+    def test_include_globbing_filelist_with_blank_line(self):
+        """Filelist glob test similar to globbing filelist, but with a blank line."""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ['- testfiles/select/1/1/1\n'
+                        '\n'
+                        'testfiles/select/1/1\n'
+                        '- testfiles/select/1\n'
+                        '- **'])
+
+    def test_include_globbing_filelist_with_blank_line_and_whitespace(self):
+        """Filelist glob test similar to globbing filelist, but with a blank line and whitespace."""
+        self.ParseTest([("--include-globbing-filelist", "file")],
+                       [(), ('1',), ('1', '1'), ('1', '1', '2'),
+                        ('1', '1', '3')],
+                       ['- testfiles/select/1/1/1\n'
+                        '  \n'
+                        'testfiles/select/1/1\n'
+                        '- testfiles/select/1\n'
+                        '- **'])
 
     def testGlob(self):
         """Test globbing expression"""