Merge lp:~jelmer/brz/osutils-fstype into lp:brz
- osutils-fstype
- Merge into trunk
Proposed by
Jelmer Vernooij
Status: | Merged |
---|---|
Merge reported by: | Jelmer Vernooij |
Merged at revision: | not available |
Proposed branch: | lp:~jelmer/brz/osutils-fstype |
Merge into: | lp:brz |
Prerequisite: | lp:~jelmer/brz/wt-executable |
Diff against target: |
480 lines (+129/-45) 13 files modified
breezy/bzr/dirstate.py (+15/-11) breezy/bzr/workingtree.py (+11/-2) breezy/bzr/workingtree_4.py (+6/-4) breezy/git/memorytree.py (+1/-0) breezy/git/tree.py (+8/-1) breezy/git/workingtree.py (+11/-5) breezy/osutils.py (+43/-2) breezy/tests/per_workingtree/test_executable.py (+1/-1) breezy/tests/per_workingtree/test_workingtree.py (+13/-7) breezy/tests/test_osutils.py (+15/-0) breezy/transform.py (+2/-3) breezy/workingtree.py (+1/-9) setup.py (+2/-0) |
To merge this branch: | bzr merge lp:~jelmer/brz/osutils-fstype |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Martin Packman | Needs Fixing | ||
Review via email: mp+355466@code.launchpad.net |
Commit message
Use basis tree executability status if the filesystem doesn't support executable bit.
Description of the change
Use basis tree executability status if the filesystem doesn't support executable bit.
For now, just support vfat and ntfs as filesystems without the executable bit as well as all filesystems on Windows.
To post a comment you must log in.
lp:~jelmer/brz/osutils-fstype
updated
- 7128. By Jelmer Vernooij
-
Fix GitMemoryTree tests.
lp:~jelmer/brz/osutils-fstype
updated
- 7129. By Jelmer Vernooij
-
cope with psutil not being available.
- 7130. By Jelmer Vernooij
-
Move case_sensitive_
filename attribute to bzr-specific moduke. - 7131. By Jelmer Vernooij
-
Add trust_executabl
e_bit. - 7132. By Jelmer Vernooij
-
Merge trunk.
- 7133. By Jelmer Vernooij
-
Review feedback:
* Cache disk partitions
* Rename supports_executable => fs_supports_executable - 7134. By Jelmer Vernooij
-
Add dependency on psutil.
Revision history for this message
Jelmer Vernooij (jelmer) : | # |
lp:~jelmer/brz/osutils-fstype
updated
- 7135. By Jelmer Vernooij
-
merge trunk.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'breezy/bzr/dirstate.py' |
2 | --- breezy/bzr/dirstate.py 2018-11-18 19:48:57 +0000 |
3 | +++ breezy/bzr/dirstate.py 2019-03-04 02:41:29 +0000 |
4 | @@ -366,7 +366,8 @@ |
5 | HEADER_FORMAT_2 = b'#bazaar dirstate flat format 2\n' |
6 | HEADER_FORMAT_3 = b'#bazaar dirstate flat format 3\n' |
7 | |
8 | - def __init__(self, path, sha1_provider, worth_saving_limit=0): |
9 | + def __init__(self, path, sha1_provider, worth_saving_limit=0, |
10 | + use_filesystem_for_exec=True): |
11 | """Create a DirState object. |
12 | |
13 | :param path: The path at which the dirstate file on disk should live. |
14 | @@ -375,6 +376,8 @@ |
15 | entries is known, only bother saving the dirstate if more than |
16 | this count of entries have changed. |
17 | -1 means never save hash changes, 0 means always save hash changes. |
18 | + :param use_filesystem_for_exec: Whether to trust the filesystem |
19 | + for executable bit information |
20 | """ |
21 | # _header_state and _dirblock_state represent the current state |
22 | # of the dirstate metadata and the per-row data respectiely. |
23 | @@ -423,6 +426,7 @@ |
24 | self._worth_saving_limit = worth_saving_limit |
25 | self._config_stack = config.LocationStack(urlutils.local_path_to_url( |
26 | path)) |
27 | + self._use_filesystem_for_exec = use_filesystem_for_exec |
28 | |
29 | def __repr__(self): |
30 | return "%s(%r)" % \ |
31 | @@ -1949,14 +1953,10 @@ |
32 | |
33 | def _is_executable(self, mode, old_executable): |
34 | """Is this file executable?""" |
35 | - return bool(S_IEXEC & mode) |
36 | - |
37 | - def _is_executable_win32(self, mode, old_executable): |
38 | - """On win32 the executable bit is stored in the dirstate.""" |
39 | - return old_executable |
40 | - |
41 | - if sys.platform == 'win32': |
42 | - _is_executable = _is_executable_win32 |
43 | + if self._use_filesystem_for_exec: |
44 | + return bool(S_IEXEC & mode) |
45 | + else: |
46 | + return old_executable |
47 | |
48 | def _read_link(self, abspath, old_link): |
49 | """Read the target of a symlink""" |
50 | @@ -2403,7 +2403,8 @@ |
51 | return len(self._parents) - len(self._ghosts) |
52 | |
53 | @classmethod |
54 | - def on_file(cls, path, sha1_provider=None, worth_saving_limit=0): |
55 | + def on_file(cls, path, sha1_provider=None, worth_saving_limit=0, |
56 | + use_filesystem_for_exec=True): |
57 | """Construct a DirState on the file at path "path". |
58 | |
59 | :param path: The path at which the dirstate file on disk should live. |
60 | @@ -2412,12 +2413,15 @@ |
61 | :param worth_saving_limit: when the exact number of hash changed |
62 | entries is known, only bother saving the dirstate if more than |
63 | this count of entries have changed. -1 means never save. |
64 | + :param use_filesystem_for_exec: Whether to trust the filesystem |
65 | + for executable bit information |
66 | :return: An unlocked DirState object, associated with the given path. |
67 | """ |
68 | if sha1_provider is None: |
69 | sha1_provider = DefaultSHA1Provider() |
70 | result = cls(path, sha1_provider, |
71 | - worth_saving_limit=worth_saving_limit) |
72 | + worth_saving_limit=worth_saving_limit, |
73 | + use_filesystem_for_exec=use_filesystem_for_exec) |
74 | return result |
75 | |
76 | def _read_dirblocks_if_needed(self): |
77 | |
78 | === modified file 'breezy/bzr/workingtree.py' |
79 | --- breezy/bzr/workingtree.py 2018-12-11 00:51:46 +0000 |
80 | +++ breezy/bzr/workingtree.py 2019-03-04 02:41:29 +0000 |
81 | @@ -126,6 +126,7 @@ |
82 | |
83 | self._control_files = _control_files |
84 | self._detect_case_handling() |
85 | + self._detect_trust_executable() |
86 | |
87 | if _inventory is None: |
88 | # This will be acquired on lock_read() or lock_write() |
89 | @@ -162,6 +163,12 @@ |
90 | |
91 | self._setup_directory_is_tree_reference() |
92 | |
93 | + def _detect_trust_executable(self): |
94 | + config_stack = self.get_config_stack() |
95 | + self.trust_executable_bit = config_stack.get('trust_executable_bit') |
96 | + if self.trust_executable_bit is None: |
97 | + self.trust_executable_bit = osutils.fs_supports_executable(self.basedir) |
98 | + |
99 | def _serialize(self, inventory, out_file): |
100 | xml5.serializer_v5.write_inventory( |
101 | self._inventory, out_file, working=True) |
102 | @@ -751,7 +758,7 @@ |
103 | return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode) |
104 | |
105 | def is_executable(self, path): |
106 | - if not self._supports_executable(): |
107 | + if not self.trust_executable_bit: |
108 | ie = self._path2ie(path) |
109 | return ie.executable |
110 | else: |
111 | @@ -759,7 +766,7 @@ |
112 | return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode) |
113 | |
114 | def _is_executable_from_path_and_stat(self, path, stat_result): |
115 | - if not self._supports_executable(): |
116 | + if not self.trust_executable_bit: |
117 | return self._is_executable_from_path_and_stat_from_basis( |
118 | path, stat_result) |
119 | else: |
120 | @@ -1753,6 +1760,8 @@ |
121 | |
122 | ignore_filename = '.bzrignore' |
123 | |
124 | + case_sensitive_filename = "FoRMaT" |
125 | + |
126 | def __init__(self): |
127 | WorkingTreeFormat.__init__(self) |
128 | bzrdir.BzrFormat.__init__(self) |
129 | |
130 | === modified file 'breezy/bzr/workingtree_4.py' |
131 | --- breezy/bzr/workingtree_4.py 2019-02-04 19:39:30 +0000 |
132 | +++ breezy/bzr/workingtree_4.py 2019-03-04 02:41:29 +0000 |
133 | @@ -121,6 +121,7 @@ |
134 | # ------------- |
135 | self._setup_directory_is_tree_reference() |
136 | self._detect_case_handling() |
137 | + self._detect_trust_executable() |
138 | self._rules_searcher = None |
139 | self.views = self._make_views() |
140 | # --- allow tests to select the dirstate iter_changes implementation |
141 | @@ -255,8 +256,9 @@ |
142 | return self._dirstate |
143 | local_path = self.controldir.get_workingtree_transport( |
144 | None).local_abspath('dirstate') |
145 | - self._dirstate = dirstate.DirState.on_file( |
146 | - local_path, self._sha1_provider(), self._worth_saving_limit()) |
147 | + self._dirstate = dirstate.DirState.on_file(local_path, |
148 | + self._sha1_provider(), self._worth_saving_limit(), |
149 | + self.trust_executable_bit) |
150 | return self._dirstate |
151 | |
152 | def _sha1_provider(self): |
153 | @@ -504,7 +506,7 @@ |
154 | |
155 | Note: The caller is expected to take a read-lock before calling this. |
156 | """ |
157 | - if not self._supports_executable(): |
158 | + if not self.trust_executable_bit: |
159 | entry = self._get_entry(path=path) |
160 | if entry == (None, None): |
161 | return False |
162 | @@ -2244,7 +2246,7 @@ |
163 | search_specific_files_utf8.add(path.encode('utf8')) |
164 | |
165 | iter_changes = self.target._iter_changes( |
166 | - include_unchanged, self.target._supports_executable(), |
167 | + include_unchanged, self.target.trust_executable_bit, |
168 | search_specific_files_utf8, state, source_index, target_index, |
169 | want_unversioned, self.target) |
170 | return iter_changes.iter_changes() |
171 | |
172 | === modified file 'breezy/git/memorytree.py' |
173 | --- breezy/git/memorytree.py 2018-11-18 00:25:19 +0000 |
174 | +++ breezy/git/memorytree.py 2019-03-04 02:41:29 +0000 |
175 | @@ -57,6 +57,7 @@ |
176 | self._locks = 0 |
177 | self._lock_mode = None |
178 | self._populate_from_branch() |
179 | + self.trust_executable_bit = True |
180 | |
181 | @property |
182 | def controldir(self): |
183 | |
184 | === modified file 'breezy/git/tree.py' |
185 | --- breezy/git/tree.py 2018-12-18 20:55:37 +0000 |
186 | +++ breezy/git/tree.py 2019-03-04 02:41:29 +0000 |
187 | @@ -1447,6 +1447,7 @@ |
188 | # Report dirified directories to commit_tree first, so that they can be |
189 | # replaced with non-empty directories if they have contents. |
190 | dirified = [] |
191 | + trust_executable = target.trust_executable_bit |
192 | for path, index_entry in target._recurse_index_entries(): |
193 | try: |
194 | live_entry = target._live_entry(path) |
195 | @@ -1465,7 +1466,13 @@ |
196 | else: |
197 | raise |
198 | else: |
199 | - blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode)) |
200 | + mode = live_entry.mode |
201 | + if not trust_executable: |
202 | + if mode_is_executable(index_entry.mode): |
203 | + mode |= 0o111 |
204 | + else: |
205 | + mode &= ~0o111 |
206 | + blobs[path] = (live_entry.sha, cleanup_mode(mode)) |
207 | if want_unversioned: |
208 | for e in target.extras(): |
209 | st = target._lstat(e) |
210 | |
211 | === modified file 'breezy/git/workingtree.py' |
212 | --- breezy/git/workingtree.py 2019-02-05 04:00:02 +0000 |
213 | +++ breezy/git/workingtree.py 2019-03-04 02:41:29 +0000 |
214 | @@ -103,6 +103,7 @@ |
215 | self.views = self._make_views() |
216 | self._rules_searcher = None |
217 | self._detect_case_handling() |
218 | + self._detect_trust_executable_bit() |
219 | self._reset_data() |
220 | |
221 | def supports_tree_reference(self): |
222 | @@ -209,6 +210,12 @@ |
223 | else: |
224 | self.case_sensitive = False |
225 | |
226 | + def _detect_trust_executable_bit(self): |
227 | + config_stack = self.repository._git.get_config_stack() |
228 | + self.trust_executable_bit = config_stack.get_boolean(('core', ), 'filemode', default=None) |
229 | + if self.trust_executable_bit is None: |
230 | + self.trust_executable_bit = osutils.fs_supports_executable(self.basedir) |
231 | + |
232 | def merge_modified(self): |
233 | return {} |
234 | |
235 | @@ -743,8 +750,7 @@ |
236 | |
237 | def is_executable(self, path): |
238 | with self.lock_read(): |
239 | - if getattr(self, "_supports_executable", |
240 | - osutils.supports_executable)(): |
241 | + if self.trust_executable_bit: |
242 | mode = self._lstat(path).st_mode |
243 | else: |
244 | (index, subpath) = self._lookup_index(path.encode('utf-8')) |
245 | @@ -755,8 +761,7 @@ |
246 | return bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode) |
247 | |
248 | def _is_executable_from_path_and_stat(self, path, stat_result): |
249 | - if getattr(self, "_supports_executable", |
250 | - osutils.supports_executable)(): |
251 | + if self.trust_executable_bit: |
252 | return self._is_executable_from_path_and_stat_from_stat( |
253 | path, stat_result) |
254 | else: |
255 | @@ -1166,7 +1171,8 @@ |
256 | self.store, |
257 | None |
258 | if self.branch.head is None |
259 | - else self.store[self.branch.head].tree) |
260 | + else self.store[self.branch.head].tree, |
261 | + honor_filemode=self.trust_executable_bit) |
262 | |
263 | def reset_state(self, revision_ids=None): |
264 | """Reset the state of the working tree. |
265 | |
266 | === modified file 'breezy/osutils.py' |
267 | --- breezy/osutils.py 2019-03-02 23:49:52 +0000 |
268 | +++ breezy/osutils.py 2019-03-04 02:41:29 +0000 |
269 | @@ -1662,8 +1662,24 @@ |
270 | _terminal_size = _ioctl_terminal_size |
271 | |
272 | |
273 | -def supports_executable(): |
274 | - return sys.platform != "win32" |
275 | +def fs_supports_executable(path): |
276 | + """Return if filesystem at path supports executable bit. |
277 | + |
278 | + :param path: Path for which to check the file system |
279 | + :return: boolean indicating whether executable bit can be stored/relied upon |
280 | + """ |
281 | + if sys.platform == 'win32': |
282 | + return False |
283 | + try: |
284 | + fs_type = get_fs_type(path) |
285 | + except errors.DependencyNotPresent: |
286 | + # TODO(jelmer): Warn here? |
287 | + pass |
288 | + else: |
289 | + if fs_type in ('vfat', 'ntfs'): |
290 | + # filesystems known to not support executable bit |
291 | + return False |
292 | + return True |
293 | |
294 | |
295 | def supports_posix_readonly(): |
296 | @@ -2602,6 +2618,31 @@ |
297 | return False |
298 | |
299 | |
300 | +_disk_partitions = None |
301 | + |
302 | + |
303 | +def get_fs_type(path): |
304 | + """Return the filesystem type for the partition a path is in. |
305 | + |
306 | + :param path: Path to search filesystem type for |
307 | + :return: A FS type, as string. E.g. "ext2" |
308 | + """ |
309 | + global _disk_partitions |
310 | + # TODO(jelmer): It would be nice to avoid an extra dependency here, but the only |
311 | + # alternative is reading platform-specific files under /proc :( |
312 | + try: |
313 | + import psutil |
314 | + except ImportError as e: |
315 | + raise errors.DependencyNotPresent('psutil', e) |
316 | + if _disk_partitions is None: |
317 | + _disk_partitions = psutil.disk_partitions() |
318 | + for part in sorted(_disk_partitions, key=lambda x: len(x.mountpoint), reverse=True): |
319 | + if is_inside(part.mountpoint, path): |
320 | + return part.fstype |
321 | + # Unable to parse the file? Since otherwise at least the entry for / should match.. |
322 | + return None |
323 | + |
324 | + |
325 | if PY3: |
326 | perf_counter = time.perf_counter |
327 | else: |
328 | |
329 | === modified file 'breezy/tests/per_workingtree/test_executable.py' |
330 | --- breezy/tests/per_workingtree/test_executable.py 2018-06-18 02:13:57 +0000 |
331 | +++ breezy/tests/per_workingtree/test_executable.py 2019-03-04 02:41:29 +0000 |
332 | @@ -186,7 +186,7 @@ |
333 | self.assertFalse(b_executable) |
334 | |
335 | def test_use_exec_from_basis(self): |
336 | - self.wt._supports_executable = lambda: False |
337 | + self.wt.trust_executable_bit = False |
338 | self.addCleanup(self.wt.lock_read().unlock) |
339 | self.assertTrue(self.wt.is_executable('a')) |
340 | self.assertFalse(self.wt.is_executable('b')) |
341 | |
342 | === modified file 'breezy/tests/per_workingtree/test_workingtree.py' |
343 | --- breezy/tests/per_workingtree/test_workingtree.py 2019-02-14 03:30:18 +0000 |
344 | +++ breezy/tests/per_workingtree/test_workingtree.py 2019-03-04 02:41:29 +0000 |
345 | @@ -1061,17 +1061,14 @@ |
346 | tree = tree.controldir.open_workingtree() |
347 | self.assertFalse(tree.case_sensitive) |
348 | |
349 | - def test_supports_executable(self): |
350 | + def test_trust_executable(self): |
351 | self.build_tree(['filename']) |
352 | tree = self.make_branch_and_tree('.') |
353 | tree.add('filename') |
354 | - self.assertIsInstance(tree._supports_executable(), bool) |
355 | - if tree._supports_executable(): |
356 | - tree.lock_read() |
357 | - try: |
358 | + self.assertIsInstance(tree.trust_executable_bit, bool) |
359 | + if tree.trust_executable_bit: |
360 | + with tree.lock_read(): |
361 | self.assertFalse(tree.is_executable('filename')) |
362 | - finally: |
363 | - tree.unlock() |
364 | os.chmod('filename', 0o755) |
365 | self.addCleanup(tree.lock_read().unlock) |
366 | self.assertTrue(tree.is_executable('filename')) |
367 | @@ -1326,3 +1323,12 @@ |
368 | self.assertSubset( |
369 | [self.workingtree_format.supports_store_uncommitted], |
370 | (True, False)) |
371 | + |
372 | + |
373 | +class TestTreeAttributes(TestCaseWithWorkingTree): |
374 | + |
375 | + def test_trust_executable_bit(self): |
376 | + wt = self.make_branch_and_tree('wt') |
377 | + self.assertSubset( |
378 | + [wt.trust_executable_bit], |
379 | + (True, False)) |
380 | |
381 | === modified file 'breezy/tests/test_osutils.py' |
382 | --- breezy/tests/test_osutils.py 2018-11-17 18:49:41 +0000 |
383 | +++ breezy/tests/test_osutils.py 2019-03-04 02:41:29 +0000 |
384 | @@ -62,6 +62,8 @@ |
385 | |
386 | term_ios_feature = features.ModuleAvailableFeature('termios') |
387 | |
388 | +psutil_feature = features.ModuleAvailableFeature('psutil') |
389 | + |
390 | |
391 | def _already_unicode(s): |
392 | return s |
393 | @@ -2340,3 +2342,16 @@ |
394 | import pywintypes |
395 | self.assertTrue(osutils.is_environment_error( |
396 | pywintypes.error(errno.EINVAL, "Invalid parameter", "Caller"))) |
397 | + |
398 | + |
399 | +class SupportsExecutableTests(tests.TestCaseInTempDir): |
400 | + |
401 | + def test_returns_bool(self): |
402 | + self.assertIsInstance(osutils.fs_supports_executable(self.test_dir), bool) |
403 | + |
404 | + |
405 | +class GetFsTypeTests(tests.TestCaseInTempDir): |
406 | + |
407 | + def test_returns_string(self): |
408 | + self.requireFeature(psutil_feature) |
409 | + self.assertIsInstance(osutils.get_fs_type(self.test_dir), str) |
410 | |
411 | === modified file 'breezy/transform.py' |
412 | --- breezy/transform.py 2019-02-02 15:13:30 +0000 |
413 | +++ breezy/transform.py 2019-03-04 02:41:29 +0000 |
414 | @@ -761,7 +761,7 @@ |
415 | |
416 | def _set_executability(self, path, trans_id): |
417 | """Set the executability of versioned files """ |
418 | - if self._tree._supports_executable(): |
419 | + if self._tree.trust_executable_bit: |
420 | new_executability = self._new_executability[trans_id] |
421 | abspath = self._tree.abspath(path) |
422 | current_mode = os.stat(abspath).st_mode |
423 | @@ -1263,8 +1263,7 @@ |
424 | |
425 | def _limbo_supports_executable(self): |
426 | """Check if the limbo path supports the executable bit.""" |
427 | - # FIXME: Check actual file system capabilities of limbodir |
428 | - return osutils.supports_executable() |
429 | + return osutils.fs_supports_executable(self._limbodir) |
430 | |
431 | def _limbo_name(self, trans_id): |
432 | """Generate the limbo name of a file""" |
433 | |
434 | === modified file 'breezy/workingtree.py' |
435 | --- breezy/workingtree.py 2019-02-14 22:18:59 +0000 |
436 | +++ breezy/workingtree.py 2019-03-04 02:41:29 +0000 |
437 | @@ -152,12 +152,6 @@ |
438 | """ |
439 | return self._format.supports_merge_modified |
440 | |
441 | - def _supports_executable(self): |
442 | - if sys.platform == 'win32': |
443 | - return False |
444 | - # FIXME: Ideally this should check the file system |
445 | - return True |
446 | - |
447 | def break_lock(self): |
448 | """Break a lock if one is present from another instance. |
449 | |
450 | @@ -950,7 +944,7 @@ |
451 | else: |
452 | mode = stat_value.st_mode |
453 | kind = osutils.file_kind_from_stat_mode(mode) |
454 | - if not self._supports_executable(): |
455 | + if not self.trust_executable_bit: |
456 | executable = entry is not None and entry.executable |
457 | else: |
458 | executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode) |
459 | @@ -1426,8 +1420,6 @@ |
460 | |
461 | requires_normalized_unicode_filenames = False |
462 | |
463 | - case_sensitive_filename = "FoRMaT" |
464 | - |
465 | missing_parent_conflicts = False |
466 | """If this format supports missing parent conflicts.""" |
467 | |
468 | |
469 | === modified file 'setup.py' |
470 | --- setup.py 2019-02-06 05:44:37 +0000 |
471 | +++ setup.py 2019-03-04 02:41:29 +0000 |
472 | @@ -61,6 +61,8 @@ |
473 | # no way to enable them by default and let users opt out. |
474 | 'fastimport>=0.9.8', |
475 | 'dulwich>=0.19.11', |
476 | + # Used for getting filesystem information |
477 | + 'psutil', |
478 | ], |
479 | 'extras_require': { |
480 | 'fastimport': [], |
Good change to make, some inline questions about specifics to address.