Merge lp:~jelmer/brz/osutils-fstype into lp:brz

[Jelmer Vernooij]

Use basis tree executability status if the filesystem doesn't support executable bit.

Use basis tree executability status if the filesystem doesn't support executable bit.

For now, just support vfat and ntfs as filesystems without the executable bit as well as all filesystems on Windows.

[lp:~jelmer/brz/osutils-fstype]

7128. By Jelmer Vernooij on 2018-09-21

Fix GitMemoryTree tests.

[Martin Packman (gz)]

Good change to make, some inline questions about specifics to address.

review: Needs Fixing

[lp:~jelmer/brz/osutils-fstype]

7129. By Jelmer Vernooij on 2018-09-23

cope with psutil not being available.

7130. By Jelmer Vernooij on 2018-09-23

Move case_sensitive_filename attribute to bzr-specific moduke.

7131. By Jelmer Vernooij on 2018-09-23

Add trust_executable_bit.

7132. By Jelmer Vernooij on 2018-12-18

Merge trunk.

7133. By Jelmer Vernooij on 2019-03-04

Review feedback:

* Cache disk partitions
* Rename supports_executable => fs_supports_executable

7134. By Jelmer Vernooij on 2019-03-04

Add dependency on psutil.

[lp:~jelmer/brz/osutils-fstype]

7135. By Jelmer Vernooij on 2019-03-04

merge trunk.

=== modified file 'breezy/bzr/dirstate.py'
--- breezy/bzr/dirstate.py	2018-11-18 19:48:57 +0000
+++ breezy/bzr/dirstate.py	2019-03-04 02:41:29 +0000
@@ -366,7 +366,8 @@
     HEADER_FORMAT_2 = b'#bazaar dirstate flat format 2\n'
     HEADER_FORMAT_3 = b'#bazaar dirstate flat format 3\n'
 
-    def __init__(self, path, sha1_provider, worth_saving_limit=0):
+    def __init__(self, path, sha1_provider, worth_saving_limit=0,
+                 use_filesystem_for_exec=True):
         """Create a  DirState object.
 
         :param path: The path at which the dirstate file on disk should live.
@@ -375,6 +376,8 @@
             entries is known, only bother saving the dirstate if more than
             this count of entries have changed.
             -1 means never save hash changes, 0 means always save hash changes.
+        :param use_filesystem_for_exec: Whether to trust the filesystem
+            for executable bit information
         """
         # _header_state and _dirblock_state represent the current state
         # of the dirstate metadata and the per-row data respectiely.
@@ -423,6 +426,7 @@
         self._worth_saving_limit = worth_saving_limit
         self._config_stack = config.LocationStack(urlutils.local_path_to_url(
             path))
+        self._use_filesystem_for_exec = use_filesystem_for_exec
 
     def __repr__(self):
         return "%s(%r)" % \
@@ -1949,14 +1953,10 @@
 
     def _is_executable(self, mode, old_executable):
         """Is this file executable?"""
-        return bool(S_IEXEC & mode)
-
-    def _is_executable_win32(self, mode, old_executable):
-        """On win32 the executable bit is stored in the dirstate."""
-        return old_executable
-
-    if sys.platform == 'win32':
-        _is_executable = _is_executable_win32
+        if self._use_filesystem_for_exec:
+            return bool(S_IEXEC & mode)
+        else:
+            return old_executable
 
     def _read_link(self, abspath, old_link):
         """Read the target of a symlink"""
@@ -2403,7 +2403,8 @@
         return len(self._parents) - len(self._ghosts)
 
     @classmethod
-    def on_file(cls, path, sha1_provider=None, worth_saving_limit=0):
+    def on_file(cls, path, sha1_provider=None, worth_saving_limit=0,
+                use_filesystem_for_exec=True):
         """Construct a DirState on the file at path "path".
 
         :param path: The path at which the dirstate file on disk should live.
@@ -2412,12 +2413,15 @@
         :param worth_saving_limit: when the exact number of hash changed
             entries is known, only bother saving the dirstate if more than
             this count of entries have changed. -1 means never save.
+        :param use_filesystem_for_exec: Whether to trust the filesystem
+            for executable bit information
         :return: An unlocked DirState object, associated with the given path.
         """
         if sha1_provider is None:
             sha1_provider = DefaultSHA1Provider()
         result = cls(path, sha1_provider,
-                     worth_saving_limit=worth_saving_limit)
+                     worth_saving_limit=worth_saving_limit,
+                     use_filesystem_for_exec=use_filesystem_for_exec)
         return result
 
     def _read_dirblocks_if_needed(self):
 
=== modified file 'breezy/bzr/workingtree.py'
--- breezy/bzr/workingtree.py	2018-12-11 00:51:46 +0000
+++ breezy/bzr/workingtree.py	2019-03-04 02:41:29 +0000
@@ -126,6 +126,7 @@
 
         self._control_files = _control_files
         self._detect_case_handling()
+        self._detect_trust_executable()
 
         if _inventory is None:
             # This will be acquired on lock_read() or lock_write()
@@ -162,6 +163,12 @@
 
         self._setup_directory_is_tree_reference()
 
+    def _detect_trust_executable(self):
+        config_stack = self.get_config_stack()
+        self.trust_executable_bit = config_stack.get('trust_executable_bit')
+        if self.trust_executable_bit is None:
+            self.trust_executable_bit = osutils.fs_supports_executable(self.basedir)
+
     def _serialize(self, inventory, out_file):
         xml5.serializer_v5.write_inventory(
             self._inventory, out_file, working=True)
@@ -751,7 +758,7 @@
         return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
 
     def is_executable(self, path):
-        if not self._supports_executable():
+        if not self.trust_executable_bit:
             ie = self._path2ie(path)
             return ie.executable
         else:
@@ -759,7 +766,7 @@
             return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
 
     def _is_executable_from_path_and_stat(self, path, stat_result):
-        if not self._supports_executable():
+        if not self.trust_executable_bit:
             return self._is_executable_from_path_and_stat_from_basis(
                 path, stat_result)
         else:
@@ -1753,6 +1760,8 @@
 
     ignore_filename = '.bzrignore'
 
+    case_sensitive_filename = "FoRMaT"
+
     def __init__(self):
         WorkingTreeFormat.__init__(self)
         bzrdir.BzrFormat.__init__(self)
 
=== modified file 'breezy/bzr/workingtree_4.py'
--- breezy/bzr/workingtree_4.py	2019-02-04 19:39:30 +0000
+++ breezy/bzr/workingtree_4.py	2019-03-04 02:41:29 +0000
@@ -121,6 +121,7 @@
         # -------------
         self._setup_directory_is_tree_reference()
         self._detect_case_handling()
+        self._detect_trust_executable()
         self._rules_searcher = None
         self.views = self._make_views()
         # --- allow tests to select the dirstate iter_changes implementation
@@ -255,8 +256,9 @@
             return self._dirstate
         local_path = self.controldir.get_workingtree_transport(
             None).local_abspath('dirstate')
-        self._dirstate = dirstate.DirState.on_file(
-            local_path, self._sha1_provider(), self._worth_saving_limit())
+        self._dirstate = dirstate.DirState.on_file(local_path,
+            self._sha1_provider(), self._worth_saving_limit(),
+            self.trust_executable_bit)
         return self._dirstate
 
     def _sha1_provider(self):
@@ -504,7 +506,7 @@
 
         Note: The caller is expected to take a read-lock before calling this.
         """
-        if not self._supports_executable():
+        if not self.trust_executable_bit:
             entry = self._get_entry(path=path)
             if entry == (None, None):
                 return False
@@ -2244,7 +2246,7 @@
             search_specific_files_utf8.add(path.encode('utf8'))
 
         iter_changes = self.target._iter_changes(
-            include_unchanged, self.target._supports_executable(),
+            include_unchanged, self.target.trust_executable_bit,
             search_specific_files_utf8, state, source_index, target_index,
             want_unversioned, self.target)
         return iter_changes.iter_changes()
 
=== modified file 'breezy/git/memorytree.py'
--- breezy/git/memorytree.py	2018-11-18 00:25:19 +0000
+++ breezy/git/memorytree.py	2019-03-04 02:41:29 +0000
@@ -57,6 +57,7 @@
         self._locks = 0
         self._lock_mode = None
         self._populate_from_branch()
+        self.trust_executable_bit = True
 
     @property
     def controldir(self):
 
=== modified file 'breezy/git/tree.py'
--- breezy/git/tree.py	2018-12-18 20:55:37 +0000
+++ breezy/git/tree.py	2019-03-04 02:41:29 +0000
@@ -1447,6 +1447,7 @@
     # Report dirified directories to commit_tree first, so that they can be
     # replaced with non-empty directories if they have contents.
     dirified = []
+    trust_executable = target.trust_executable_bit
     for path, index_entry in target._recurse_index_entries():
         try:
             live_entry = target._live_entry(path)
@@ -1465,7 +1466,13 @@
             else:
                 raise
         else:
-            blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode))
+            mode = live_entry.mode
+            if not trust_executable:
+                if mode_is_executable(index_entry.mode):
+                    mode |= 0o111
+                else:
+                    mode &= ~0o111
+            blobs[path] = (live_entry.sha, cleanup_mode(mode))
     if want_unversioned:
         for e in target.extras():
             st = target._lstat(e)
 
=== modified file 'breezy/git/workingtree.py'
--- breezy/git/workingtree.py	2019-02-05 04:00:02 +0000
+++ breezy/git/workingtree.py	2019-03-04 02:41:29 +0000
@@ -103,6 +103,7 @@
         self.views = self._make_views()
         self._rules_searcher = None
         self._detect_case_handling()
+        self._detect_trust_executable_bit()
         self._reset_data()
 
     def supports_tree_reference(self):
@@ -209,6 +210,12 @@
         else:
             self.case_sensitive = False
 
+    def _detect_trust_executable_bit(self):
+        config_stack = self.repository._git.get_config_stack()
+        self.trust_executable_bit = config_stack.get_boolean(('core', ), 'filemode', default=None)
+        if self.trust_executable_bit is None:
+            self.trust_executable_bit = osutils.fs_supports_executable(self.basedir)
+
     def merge_modified(self):
         return {}
 
@@ -743,8 +750,7 @@
 
     def is_executable(self, path):
         with self.lock_read():
-            if getattr(self, "_supports_executable",
-                       osutils.supports_executable)():
+            if self.trust_executable_bit:
                 mode = self._lstat(path).st_mode
             else:
                 (index, subpath) = self._lookup_index(path.encode('utf-8'))
@@ -755,8 +761,7 @@
             return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
 
     def _is_executable_from_path_and_stat(self, path, stat_result):
-        if getattr(self, "_supports_executable",
-                   osutils.supports_executable)():
+        if self.trust_executable_bit:
             return self._is_executable_from_path_and_stat_from_stat(
                 path, stat_result)
         else:
@@ -1166,7 +1171,8 @@
             self.store,
             None
             if self.branch.head is None
-            else self.store[self.branch.head].tree)
+            else self.store[self.branch.head].tree,
+            honor_filemode=self.trust_executable_bit)
 
     def reset_state(self, revision_ids=None):
         """Reset the state of the working tree.
 
=== modified file 'breezy/osutils.py'
--- breezy/osutils.py	2019-03-02 23:49:52 +0000
+++ breezy/osutils.py	2019-03-04 02:41:29 +0000
@@ -1662,8 +1662,24 @@
     _terminal_size = _ioctl_terminal_size
 
 
-def supports_executable():
-    return sys.platform != "win32"
+def fs_supports_executable(path):
+    """Return if filesystem at path supports executable bit.
+
+    :param path: Path for which to check the file system
+    :return: boolean indicating whether executable bit can be stored/relied upon
+    """
+    if sys.platform == 'win32':
+        return False
+    try:
+        fs_type = get_fs_type(path)
+    except errors.DependencyNotPresent:
+        # TODO(jelmer): Warn here?
+        pass
+    else:
+        if fs_type in ('vfat', 'ntfs'):
+            # filesystems known to not support executable bit
+            return False
+    return True
 
 
 def supports_posix_readonly():
@@ -2602,6 +2618,31 @@
     return False
 
 
+_disk_partitions = None
+
+
+def get_fs_type(path):
+    """Return the filesystem type for the partition a path is in.
+
+    :param path: Path to search filesystem type for
+    :return: A FS type, as string. E.g. "ext2"
+    """
+    global _disk_partitions
+    # TODO(jelmer): It would be nice to avoid an extra dependency here, but the only
+    # alternative is reading platform-specific files under /proc :(
+    try:
+        import psutil
+    except ImportError as e:
+        raise errors.DependencyNotPresent('psutil', e)
+    if _disk_partitions is None:
+        _disk_partitions = psutil.disk_partitions()
+    for part in sorted(_disk_partitions, key=lambda x: len(x.mountpoint), reverse=True):
+        if is_inside(part.mountpoint, path):
+            return part.fstype
+    # Unable to parse the file? Since otherwise at least the entry for / should match..
+    return None
+
+
 if PY3:
     perf_counter = time.perf_counter
 else:
 
=== modified file 'breezy/tests/per_workingtree/test_executable.py'
--- breezy/tests/per_workingtree/test_executable.py	2018-06-18 02:13:57 +0000
+++ breezy/tests/per_workingtree/test_executable.py	2019-03-04 02:41:29 +0000
@@ -186,7 +186,7 @@
         self.assertFalse(b_executable)
 
     def test_use_exec_from_basis(self):
-        self.wt._supports_executable = lambda: False
+        self.wt.trust_executable_bit = False
         self.addCleanup(self.wt.lock_read().unlock)
         self.assertTrue(self.wt.is_executable('a'))
         self.assertFalse(self.wt.is_executable('b'))
 
=== modified file 'breezy/tests/per_workingtree/test_workingtree.py'
--- breezy/tests/per_workingtree/test_workingtree.py	2019-02-14 03:30:18 +0000
+++ breezy/tests/per_workingtree/test_workingtree.py	2019-03-04 02:41:29 +0000
@@ -1061,17 +1061,14 @@
         tree = tree.controldir.open_workingtree()
         self.assertFalse(tree.case_sensitive)
 
-    def test_supports_executable(self):
+    def test_trust_executable(self):
         self.build_tree(['filename'])
         tree = self.make_branch_and_tree('.')
         tree.add('filename')
-        self.assertIsInstance(tree._supports_executable(), bool)
-        if tree._supports_executable():
-            tree.lock_read()
-            try:
+        self.assertIsInstance(tree.trust_executable_bit, bool)
+        if tree.trust_executable_bit:
+            with tree.lock_read():
                 self.assertFalse(tree.is_executable('filename'))
-            finally:
-                tree.unlock()
             os.chmod('filename', 0o755)
             self.addCleanup(tree.lock_read().unlock)
             self.assertTrue(tree.is_executable('filename'))
@@ -1326,3 +1323,12 @@
         self.assertSubset(
             [self.workingtree_format.supports_store_uncommitted],
             (True, False))
+
+
+class TestTreeAttributes(TestCaseWithWorkingTree):
+
+    def test_trust_executable_bit(self):
+        wt = self.make_branch_and_tree('wt')
+        self.assertSubset(
+            [wt.trust_executable_bit],
+            (True, False))
 
=== modified file 'breezy/tests/test_osutils.py'
--- breezy/tests/test_osutils.py	2018-11-17 18:49:41 +0000
+++ breezy/tests/test_osutils.py	2019-03-04 02:41:29 +0000
@@ -62,6 +62,8 @@
 
 term_ios_feature = features.ModuleAvailableFeature('termios')
 
+psutil_feature = features.ModuleAvailableFeature('psutil')
+
 
 def _already_unicode(s):
     return s
@@ -2340,3 +2342,16 @@
         import pywintypes
         self.assertTrue(osutils.is_environment_error(
             pywintypes.error(errno.EINVAL, "Invalid parameter", "Caller")))
+
+
+class SupportsExecutableTests(tests.TestCaseInTempDir):
+
+    def test_returns_bool(self):
+        self.assertIsInstance(osutils.fs_supports_executable(self.test_dir), bool)
+
+
+class GetFsTypeTests(tests.TestCaseInTempDir):
+
+    def test_returns_string(self):
+        self.requireFeature(psutil_feature)
+        self.assertIsInstance(osutils.get_fs_type(self.test_dir), str)
 
=== modified file 'breezy/transform.py'
--- breezy/transform.py	2019-02-02 15:13:30 +0000
+++ breezy/transform.py	2019-03-04 02:41:29 +0000
@@ -761,7 +761,7 @@
 
     def _set_executability(self, path, trans_id):
         """Set the executability of versioned files """
-        if self._tree._supports_executable():
+        if self._tree.trust_executable_bit:
             new_executability = self._new_executability[trans_id]
             abspath = self._tree.abspath(path)
             current_mode = os.stat(abspath).st_mode
@@ -1263,8 +1263,7 @@
 
     def _limbo_supports_executable(self):
         """Check if the limbo path supports the executable bit."""
-        # FIXME: Check actual file system capabilities of limbodir
-        return osutils.supports_executable()
+        return osutils.fs_supports_executable(self._limbodir)
 
     def _limbo_name(self, trans_id):
         """Generate the limbo name of a file"""
 
=== modified file 'breezy/workingtree.py'
--- breezy/workingtree.py	2019-02-14 22:18:59 +0000
+++ breezy/workingtree.py	2019-03-04 02:41:29 +0000
@@ -152,12 +152,6 @@
         """
         return self._format.supports_merge_modified
 
-    def _supports_executable(self):
-        if sys.platform == 'win32':
-            return False
-        # FIXME: Ideally this should check the file system
-        return True
-
     def break_lock(self):
         """Break a lock if one is present from another instance.
 
@@ -950,7 +944,7 @@
         else:
             mode = stat_value.st_mode
             kind = osutils.file_kind_from_stat_mode(mode)
-            if not self._supports_executable():
+            if not self.trust_executable_bit:
                 executable = entry is not None and entry.executable
             else:
                 executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
@@ -1426,8 +1420,6 @@
 
     requires_normalized_unicode_filenames = False
 
-    case_sensitive_filename = "FoRMaT"
-
     missing_parent_conflicts = False
     """If this format supports missing parent conflicts."""
 
 
=== modified file 'setup.py'
--- setup.py	2019-02-06 05:44:37 +0000
+++ setup.py	2019-03-04 02:41:29 +0000
@@ -61,6 +61,8 @@
         # no way to enable them by default and let users opt out.
         'fastimport>=0.9.8',
         'dulwich>=0.19.11',
+        # Used for getting filesystem information
+        'psutil',
         ],
     'extras_require': {
         'fastimport': [],