Merge lp:~jelmer/brz/osutils-fstype into lp:brz

Proposed by Jelmer Vernooij on 2018-09-21
Status: Merged
Merge reported by: Jelmer Vernooij
Merged at revision: not available
Proposed branch: lp:~jelmer/brz/osutils-fstype
Merge into: lp:brz
Prerequisite: lp:~jelmer/brz/wt-executable
Diff against target: 480 lines (+129/-45)
13 files modified
breezy/bzr/dirstate.py (+15/-11)
breezy/bzr/workingtree.py (+11/-2)
breezy/bzr/workingtree_4.py (+6/-4)
breezy/git/memorytree.py (+1/-0)
breezy/git/tree.py (+8/-1)
breezy/git/workingtree.py (+11/-5)
breezy/osutils.py (+43/-2)
breezy/tests/per_workingtree/test_executable.py (+1/-1)
breezy/tests/per_workingtree/test_workingtree.py (+13/-7)
breezy/tests/test_osutils.py (+15/-0)
breezy/transform.py (+2/-3)
breezy/workingtree.py (+1/-9)
setup.py (+2/-0)
To merge this branch: bzr merge lp:~jelmer/brz/osutils-fstype
Reviewer Review Type Date Requested Status
Martin Packman 2018-09-21 Needs Fixing on 2018-09-21
Review via email: mp+355466@code.launchpad.net

Commit message

Use basis tree executability status if the filesystem doesn't support executable bit.

Description of the change

Use basis tree executability status if the filesystem doesn't support executable bit.

For now, just support vfat and ntfs as filesystems without the executable bit as well as all filesystems on Windows.

To post a comment you must log in.
lp:~jelmer/brz/osutils-fstype updated on 2018-09-21
7128. By Jelmer Vernooij on 2018-09-21

Fix GitMemoryTree tests.

Martin Packman (gz) wrote :

Good change to make, some inline questions about specifics to address.

review: Needs Fixing
lp:~jelmer/brz/osutils-fstype updated on 2019-03-04
7129. By Jelmer Vernooij on 2018-09-23

cope with psutil not being available.

7130. By Jelmer Vernooij on 2018-09-23

Move case_sensitive_filename attribute to bzr-specific moduke.

7131. By Jelmer Vernooij on 2018-09-23

Add trust_executable_bit.

7132. By Jelmer Vernooij on 2018-12-18

Merge trunk.

7133. By Jelmer Vernooij on 2019-03-04

Review feedback:

* Cache disk partitions
* Rename supports_executable => fs_supports_executable

7134. By Jelmer Vernooij on 2019-03-04

Add dependency on psutil.

Jelmer Vernooij (jelmer) :
lp:~jelmer/brz/osutils-fstype updated on 2019-03-04
7135. By Jelmer Vernooij on 2019-03-04

merge trunk.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'breezy/bzr/dirstate.py'
2--- breezy/bzr/dirstate.py 2018-11-18 19:48:57 +0000
3+++ breezy/bzr/dirstate.py 2019-03-04 02:41:29 +0000
4@@ -366,7 +366,8 @@
5 HEADER_FORMAT_2 = b'#bazaar dirstate flat format 2\n'
6 HEADER_FORMAT_3 = b'#bazaar dirstate flat format 3\n'
7
8- def __init__(self, path, sha1_provider, worth_saving_limit=0):
9+ def __init__(self, path, sha1_provider, worth_saving_limit=0,
10+ use_filesystem_for_exec=True):
11 """Create a DirState object.
12
13 :param path: The path at which the dirstate file on disk should live.
14@@ -375,6 +376,8 @@
15 entries is known, only bother saving the dirstate if more than
16 this count of entries have changed.
17 -1 means never save hash changes, 0 means always save hash changes.
18+ :param use_filesystem_for_exec: Whether to trust the filesystem
19+ for executable bit information
20 """
21 # _header_state and _dirblock_state represent the current state
22 # of the dirstate metadata and the per-row data respectiely.
23@@ -423,6 +426,7 @@
24 self._worth_saving_limit = worth_saving_limit
25 self._config_stack = config.LocationStack(urlutils.local_path_to_url(
26 path))
27+ self._use_filesystem_for_exec = use_filesystem_for_exec
28
29 def __repr__(self):
30 return "%s(%r)" % \
31@@ -1949,14 +1953,10 @@
32
33 def _is_executable(self, mode, old_executable):
34 """Is this file executable?"""
35- return bool(S_IEXEC & mode)
36-
37- def _is_executable_win32(self, mode, old_executable):
38- """On win32 the executable bit is stored in the dirstate."""
39- return old_executable
40-
41- if sys.platform == 'win32':
42- _is_executable = _is_executable_win32
43+ if self._use_filesystem_for_exec:
44+ return bool(S_IEXEC & mode)
45+ else:
46+ return old_executable
47
48 def _read_link(self, abspath, old_link):
49 """Read the target of a symlink"""
50@@ -2403,7 +2403,8 @@
51 return len(self._parents) - len(self._ghosts)
52
53 @classmethod
54- def on_file(cls, path, sha1_provider=None, worth_saving_limit=0):
55+ def on_file(cls, path, sha1_provider=None, worth_saving_limit=0,
56+ use_filesystem_for_exec=True):
57 """Construct a DirState on the file at path "path".
58
59 :param path: The path at which the dirstate file on disk should live.
60@@ -2412,12 +2413,15 @@
61 :param worth_saving_limit: when the exact number of hash changed
62 entries is known, only bother saving the dirstate if more than
63 this count of entries have changed. -1 means never save.
64+ :param use_filesystem_for_exec: Whether to trust the filesystem
65+ for executable bit information
66 :return: An unlocked DirState object, associated with the given path.
67 """
68 if sha1_provider is None:
69 sha1_provider = DefaultSHA1Provider()
70 result = cls(path, sha1_provider,
71- worth_saving_limit=worth_saving_limit)
72+ worth_saving_limit=worth_saving_limit,
73+ use_filesystem_for_exec=use_filesystem_for_exec)
74 return result
75
76 def _read_dirblocks_if_needed(self):
77
78=== modified file 'breezy/bzr/workingtree.py'
79--- breezy/bzr/workingtree.py 2018-12-11 00:51:46 +0000
80+++ breezy/bzr/workingtree.py 2019-03-04 02:41:29 +0000
81@@ -126,6 +126,7 @@
82
83 self._control_files = _control_files
84 self._detect_case_handling()
85+ self._detect_trust_executable()
86
87 if _inventory is None:
88 # This will be acquired on lock_read() or lock_write()
89@@ -162,6 +163,12 @@
90
91 self._setup_directory_is_tree_reference()
92
93+ def _detect_trust_executable(self):
94+ config_stack = self.get_config_stack()
95+ self.trust_executable_bit = config_stack.get('trust_executable_bit')
96+ if self.trust_executable_bit is None:
97+ self.trust_executable_bit = osutils.fs_supports_executable(self.basedir)
98+
99 def _serialize(self, inventory, out_file):
100 xml5.serializer_v5.write_inventory(
101 self._inventory, out_file, working=True)
102@@ -751,7 +758,7 @@
103 return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
104
105 def is_executable(self, path):
106- if not self._supports_executable():
107+ if not self.trust_executable_bit:
108 ie = self._path2ie(path)
109 return ie.executable
110 else:
111@@ -759,7 +766,7 @@
112 return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
113
114 def _is_executable_from_path_and_stat(self, path, stat_result):
115- if not self._supports_executable():
116+ if not self.trust_executable_bit:
117 return self._is_executable_from_path_and_stat_from_basis(
118 path, stat_result)
119 else:
120@@ -1753,6 +1760,8 @@
121
122 ignore_filename = '.bzrignore'
123
124+ case_sensitive_filename = "FoRMaT"
125+
126 def __init__(self):
127 WorkingTreeFormat.__init__(self)
128 bzrdir.BzrFormat.__init__(self)
129
130=== modified file 'breezy/bzr/workingtree_4.py'
131--- breezy/bzr/workingtree_4.py 2019-02-04 19:39:30 +0000
132+++ breezy/bzr/workingtree_4.py 2019-03-04 02:41:29 +0000
133@@ -121,6 +121,7 @@
134 # -------------
135 self._setup_directory_is_tree_reference()
136 self._detect_case_handling()
137+ self._detect_trust_executable()
138 self._rules_searcher = None
139 self.views = self._make_views()
140 # --- allow tests to select the dirstate iter_changes implementation
141@@ -255,8 +256,9 @@
142 return self._dirstate
143 local_path = self.controldir.get_workingtree_transport(
144 None).local_abspath('dirstate')
145- self._dirstate = dirstate.DirState.on_file(
146- local_path, self._sha1_provider(), self._worth_saving_limit())
147+ self._dirstate = dirstate.DirState.on_file(local_path,
148+ self._sha1_provider(), self._worth_saving_limit(),
149+ self.trust_executable_bit)
150 return self._dirstate
151
152 def _sha1_provider(self):
153@@ -504,7 +506,7 @@
154
155 Note: The caller is expected to take a read-lock before calling this.
156 """
157- if not self._supports_executable():
158+ if not self.trust_executable_bit:
159 entry = self._get_entry(path=path)
160 if entry == (None, None):
161 return False
162@@ -2244,7 +2246,7 @@
163 search_specific_files_utf8.add(path.encode('utf8'))
164
165 iter_changes = self.target._iter_changes(
166- include_unchanged, self.target._supports_executable(),
167+ include_unchanged, self.target.trust_executable_bit,
168 search_specific_files_utf8, state, source_index, target_index,
169 want_unversioned, self.target)
170 return iter_changes.iter_changes()
171
172=== modified file 'breezy/git/memorytree.py'
173--- breezy/git/memorytree.py 2018-11-18 00:25:19 +0000
174+++ breezy/git/memorytree.py 2019-03-04 02:41:29 +0000
175@@ -57,6 +57,7 @@
176 self._locks = 0
177 self._lock_mode = None
178 self._populate_from_branch()
179+ self.trust_executable_bit = True
180
181 @property
182 def controldir(self):
183
184=== modified file 'breezy/git/tree.py'
185--- breezy/git/tree.py 2018-12-18 20:55:37 +0000
186+++ breezy/git/tree.py 2019-03-04 02:41:29 +0000
187@@ -1447,6 +1447,7 @@
188 # Report dirified directories to commit_tree first, so that they can be
189 # replaced with non-empty directories if they have contents.
190 dirified = []
191+ trust_executable = target.trust_executable_bit
192 for path, index_entry in target._recurse_index_entries():
193 try:
194 live_entry = target._live_entry(path)
195@@ -1465,7 +1466,13 @@
196 else:
197 raise
198 else:
199- blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode))
200+ mode = live_entry.mode
201+ if not trust_executable:
202+ if mode_is_executable(index_entry.mode):
203+ mode |= 0o111
204+ else:
205+ mode &= ~0o111
206+ blobs[path] = (live_entry.sha, cleanup_mode(mode))
207 if want_unversioned:
208 for e in target.extras():
209 st = target._lstat(e)
210
211=== modified file 'breezy/git/workingtree.py'
212--- breezy/git/workingtree.py 2019-02-05 04:00:02 +0000
213+++ breezy/git/workingtree.py 2019-03-04 02:41:29 +0000
214@@ -103,6 +103,7 @@
215 self.views = self._make_views()
216 self._rules_searcher = None
217 self._detect_case_handling()
218+ self._detect_trust_executable_bit()
219 self._reset_data()
220
221 def supports_tree_reference(self):
222@@ -209,6 +210,12 @@
223 else:
224 self.case_sensitive = False
225
226+ def _detect_trust_executable_bit(self):
227+ config_stack = self.repository._git.get_config_stack()
228+ self.trust_executable_bit = config_stack.get_boolean(('core', ), 'filemode', default=None)
229+ if self.trust_executable_bit is None:
230+ self.trust_executable_bit = osutils.fs_supports_executable(self.basedir)
231+
232 def merge_modified(self):
233 return {}
234
235@@ -743,8 +750,7 @@
236
237 def is_executable(self, path):
238 with self.lock_read():
239- if getattr(self, "_supports_executable",
240- osutils.supports_executable)():
241+ if self.trust_executable_bit:
242 mode = self._lstat(path).st_mode
243 else:
244 (index, subpath) = self._lookup_index(path.encode('utf-8'))
245@@ -755,8 +761,7 @@
246 return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
247
248 def _is_executable_from_path_and_stat(self, path, stat_result):
249- if getattr(self, "_supports_executable",
250- osutils.supports_executable)():
251+ if self.trust_executable_bit:
252 return self._is_executable_from_path_and_stat_from_stat(
253 path, stat_result)
254 else:
255@@ -1166,7 +1171,8 @@
256 self.store,
257 None
258 if self.branch.head is None
259- else self.store[self.branch.head].tree)
260+ else self.store[self.branch.head].tree,
261+ honor_filemode=self.trust_executable_bit)
262
263 def reset_state(self, revision_ids=None):
264 """Reset the state of the working tree.
265
266=== modified file 'breezy/osutils.py'
267--- breezy/osutils.py 2019-03-02 23:49:52 +0000
268+++ breezy/osutils.py 2019-03-04 02:41:29 +0000
269@@ -1662,8 +1662,24 @@
270 _terminal_size = _ioctl_terminal_size
271
272
273-def supports_executable():
274- return sys.platform != "win32"
275+def fs_supports_executable(path):
276+ """Return if filesystem at path supports executable bit.
277+
278+ :param path: Path for which to check the file system
279+ :return: boolean indicating whether executable bit can be stored/relied upon
280+ """
281+ if sys.platform == 'win32':
282+ return False
283+ try:
284+ fs_type = get_fs_type(path)
285+ except errors.DependencyNotPresent:
286+ # TODO(jelmer): Warn here?
287+ pass
288+ else:
289+ if fs_type in ('vfat', 'ntfs'):
290+ # filesystems known to not support executable bit
291+ return False
292+ return True
293
294
295 def supports_posix_readonly():
296@@ -2602,6 +2618,31 @@
297 return False
298
299
300+_disk_partitions = None
301+
302+
303+def get_fs_type(path):
304+ """Return the filesystem type for the partition a path is in.
305+
306+ :param path: Path to search filesystem type for
307+ :return: A FS type, as string. E.g. "ext2"
308+ """
309+ global _disk_partitions
310+ # TODO(jelmer): It would be nice to avoid an extra dependency here, but the only
311+ # alternative is reading platform-specific files under /proc :(
312+ try:
313+ import psutil
314+ except ImportError as e:
315+ raise errors.DependencyNotPresent('psutil', e)
316+ if _disk_partitions is None:
317+ _disk_partitions = psutil.disk_partitions()
318+ for part in sorted(_disk_partitions, key=lambda x: len(x.mountpoint), reverse=True):
319+ if is_inside(part.mountpoint, path):
320+ return part.fstype
321+ # Unable to parse the file? Since otherwise at least the entry for / should match..
322+ return None
323+
324+
325 if PY3:
326 perf_counter = time.perf_counter
327 else:
328
329=== modified file 'breezy/tests/per_workingtree/test_executable.py'
330--- breezy/tests/per_workingtree/test_executable.py 2018-06-18 02:13:57 +0000
331+++ breezy/tests/per_workingtree/test_executable.py 2019-03-04 02:41:29 +0000
332@@ -186,7 +186,7 @@
333 self.assertFalse(b_executable)
334
335 def test_use_exec_from_basis(self):
336- self.wt._supports_executable = lambda: False
337+ self.wt.trust_executable_bit = False
338 self.addCleanup(self.wt.lock_read().unlock)
339 self.assertTrue(self.wt.is_executable('a'))
340 self.assertFalse(self.wt.is_executable('b'))
341
342=== modified file 'breezy/tests/per_workingtree/test_workingtree.py'
343--- breezy/tests/per_workingtree/test_workingtree.py 2019-02-14 03:30:18 +0000
344+++ breezy/tests/per_workingtree/test_workingtree.py 2019-03-04 02:41:29 +0000
345@@ -1061,17 +1061,14 @@
346 tree = tree.controldir.open_workingtree()
347 self.assertFalse(tree.case_sensitive)
348
349- def test_supports_executable(self):
350+ def test_trust_executable(self):
351 self.build_tree(['filename'])
352 tree = self.make_branch_and_tree('.')
353 tree.add('filename')
354- self.assertIsInstance(tree._supports_executable(), bool)
355- if tree._supports_executable():
356- tree.lock_read()
357- try:
358+ self.assertIsInstance(tree.trust_executable_bit, bool)
359+ if tree.trust_executable_bit:
360+ with tree.lock_read():
361 self.assertFalse(tree.is_executable('filename'))
362- finally:
363- tree.unlock()
364 os.chmod('filename', 0o755)
365 self.addCleanup(tree.lock_read().unlock)
366 self.assertTrue(tree.is_executable('filename'))
367@@ -1326,3 +1323,12 @@
368 self.assertSubset(
369 [self.workingtree_format.supports_store_uncommitted],
370 (True, False))
371+
372+
373+class TestTreeAttributes(TestCaseWithWorkingTree):
374+
375+ def test_trust_executable_bit(self):
376+ wt = self.make_branch_and_tree('wt')
377+ self.assertSubset(
378+ [wt.trust_executable_bit],
379+ (True, False))
380
381=== modified file 'breezy/tests/test_osutils.py'
382--- breezy/tests/test_osutils.py 2018-11-17 18:49:41 +0000
383+++ breezy/tests/test_osutils.py 2019-03-04 02:41:29 +0000
384@@ -62,6 +62,8 @@
385
386 term_ios_feature = features.ModuleAvailableFeature('termios')
387
388+psutil_feature = features.ModuleAvailableFeature('psutil')
389+
390
391 def _already_unicode(s):
392 return s
393@@ -2340,3 +2342,16 @@
394 import pywintypes
395 self.assertTrue(osutils.is_environment_error(
396 pywintypes.error(errno.EINVAL, "Invalid parameter", "Caller")))
397+
398+
399+class SupportsExecutableTests(tests.TestCaseInTempDir):
400+
401+ def test_returns_bool(self):
402+ self.assertIsInstance(osutils.fs_supports_executable(self.test_dir), bool)
403+
404+
405+class GetFsTypeTests(tests.TestCaseInTempDir):
406+
407+ def test_returns_string(self):
408+ self.requireFeature(psutil_feature)
409+ self.assertIsInstance(osutils.get_fs_type(self.test_dir), str)
410
411=== modified file 'breezy/transform.py'
412--- breezy/transform.py 2019-02-02 15:13:30 +0000
413+++ breezy/transform.py 2019-03-04 02:41:29 +0000
414@@ -761,7 +761,7 @@
415
416 def _set_executability(self, path, trans_id):
417 """Set the executability of versioned files """
418- if self._tree._supports_executable():
419+ if self._tree.trust_executable_bit:
420 new_executability = self._new_executability[trans_id]
421 abspath = self._tree.abspath(path)
422 current_mode = os.stat(abspath).st_mode
423@@ -1263,8 +1263,7 @@
424
425 def _limbo_supports_executable(self):
426 """Check if the limbo path supports the executable bit."""
427- # FIXME: Check actual file system capabilities of limbodir
428- return osutils.supports_executable()
429+ return osutils.fs_supports_executable(self._limbodir)
430
431 def _limbo_name(self, trans_id):
432 """Generate the limbo name of a file"""
433
434=== modified file 'breezy/workingtree.py'
435--- breezy/workingtree.py 2019-02-14 22:18:59 +0000
436+++ breezy/workingtree.py 2019-03-04 02:41:29 +0000
437@@ -152,12 +152,6 @@
438 """
439 return self._format.supports_merge_modified
440
441- def _supports_executable(self):
442- if sys.platform == 'win32':
443- return False
444- # FIXME: Ideally this should check the file system
445- return True
446-
447 def break_lock(self):
448 """Break a lock if one is present from another instance.
449
450@@ -950,7 +944,7 @@
451 else:
452 mode = stat_value.st_mode
453 kind = osutils.file_kind_from_stat_mode(mode)
454- if not self._supports_executable():
455+ if not self.trust_executable_bit:
456 executable = entry is not None and entry.executable
457 else:
458 executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
459@@ -1426,8 +1420,6 @@
460
461 requires_normalized_unicode_filenames = False
462
463- case_sensitive_filename = "FoRMaT"
464-
465 missing_parent_conflicts = False
466 """If this format supports missing parent conflicts."""
467
468
469=== modified file 'setup.py'
470--- setup.py 2019-02-06 05:44:37 +0000
471+++ setup.py 2019-03-04 02:41:29 +0000
472@@ -61,6 +61,8 @@
473 # no way to enable them by default and let users opt out.
474 'fastimport>=0.9.8',
475 'dulwich>=0.19.11',
476+ # Used for getting filesystem information
477+ 'psutil',
478 ],
479 'extras_require': {
480 'fastimport': [],

Subscribers

People subscribed via source and target branches