Merge lp:~jameinel/bzr/integration into lp:bzr

[John A Meinel]

Merge bzr/2.5 into trunk.

Merge bzr/2.5 into trunk.

[John A Meinel (jameinel)]

sent to pqm by email

[Richard Wilbur (richard-wilbur)]

I like the changes. It seems that, at least for trunk, we might consider moving the trace.mutter from
bzrlib/osutils.py:66

66 + trace.mutter("ignoring error calling fdatasync: %s" % (e,))
67 + if getattr(e, 'errno', None) not in _fdatasync_ignored:
68 + raise

into an 'else' of the test on line 67, since only in that case are we actually ignoring the error. Something like the following:

66 + if getattr(e, 'errno', None) not in _fdatasync_ignored:
67 + raise
68 + else:
69 + trace.mutter("ignoring error calling fdatasync: %s" % (e,))

review: Needs Fixing

=== modified file 'bzrlib/dirstate.py'
--- bzrlib/dirstate.py	2012-01-24 01:35:56 +0000
+++ bzrlib/dirstate.py	2013-05-23 10:07:40 +0000
@@ -2566,13 +2566,6 @@
         self.update_minimal(('', '', new_id), 'd',
             path_utf8='', packed_stat=entry[1][0][4])
         self._mark_modified()
-        # XXX: This was added by Ian, we need to make sure there
-        #      are tests for it, because it isn't in bzr.dev TRUNK
-        #      It looks like the only place it is called is in setting the root
-        #      id of the tree. So probably we never had an _id_index when we
-        #      don't even have a root yet.
-        if self._id_index is not None:
-            self._add_to_id_index(self._id_index, entry[0])
 
     def set_parent_trees(self, trees, ghosts):
         """Set the parent trees for the dirstate.
@@ -3285,10 +3278,20 @@
         if self._id_index is not None:
             for file_id, entry_keys in self._id_index.iteritems():
                 for entry_key in entry_keys:
+                    # Check that the entry in the map is pointing to the same
+                    # file_id
                     if entry_key[2] != file_id:
                         raise AssertionError(
                             'file_id %r did not match entry key %s'
                             % (file_id, entry_key))
+                    # And that from this entry key, we can look up the original
+                    # record
+                    block_index, present = self._find_block_index_from_key(entry_key)
+                    if not present:
+                        raise AssertionError('missing block for entry key: %r', entry_key)
+                    entry_index, present = self._find_entry_index(entry_key, self._dirblocks[block_index][1])
+                    if not present:
+                        raise AssertionError('missing entry for key: %r', entry_key)
                 if len(entry_keys) != len(set(entry_keys)):
                     raise AssertionError(
                         'id_index contained non-unique data for %s'
 
=== modified file 'bzrlib/osutils.py'
--- bzrlib/osutils.py	2012-09-19 07:58:27 +0000
+++ bzrlib/osutils.py	2013-05-23 10:07:40 +0000
@@ -2554,6 +2554,10 @@
 else:
     is_local_pid_dead = _posix_is_local_pid_dead
 
+_maybe_ignored = ['EAGAIN', 'EINTR', 'ENOTSUP', 'EOPNOTSUPP', 'EACCES']
+_fdatasync_ignored = [getattr(errno, name) for name in _maybe_ignored
+                      if getattr(errno, name, None) is not None]
+
 
 def fdatasync(fileno):
     """Flush file contents to disk if possible.
@@ -2563,7 +2567,16 @@
     """
     fn = getattr(os, 'fdatasync', getattr(os, 'fsync', None))
     if fn is not None:
-        fn(fileno)
+        try:
+            fn(fileno)
+        except IOError, e:
+            # See bug #1075108, on some platforms fdatasync exists, but can
+            # raise ENOTSUP. However, we are calling fdatasync to be helpful
+            # and reduce the chance of corruption-on-powerloss situations. It
+            # is not a mandatory call, so it is ok to suppress failures.
+            trace.mutter("ignoring error calling fdatasync: %s" % (e,))
+            if getattr(e, 'errno', None) not in _fdatasync_ignored:
+                raise
 
 
 def ensure_empty_directory_exists(path, exception_class):
 
=== modified file 'bzrlib/tests/__init__.py'
--- bzrlib/tests/__init__.py	2012-08-03 14:23:05 +0000
+++ bzrlib/tests/__init__.py	2013-05-23 10:07:40 +0000
@@ -1780,9 +1780,15 @@
 
         :returns: The actual attr value.
         """
-        value = getattr(obj, attr_name)
         # The actual value is captured by the call below
-        self.addCleanup(setattr, obj, attr_name, value)
+        value = getattr(obj, attr_name, _unitialized_attr)
+        if value is _unitialized_attr:
+            # When the test completes, the attribute should not exist, but if
+            # we aren't setting a value, we don't need to do anything.
+            if new is not _unitialized_attr:
+                self.addCleanup(delattr, obj, attr_name)
+        else:
+            self.addCleanup(setattr, obj, attr_name, value)
         if new is not _unitialized_attr:
             setattr(obj, attr_name, new)
         return value
 
=== modified file 'bzrlib/tests/test_osutils.py'
--- bzrlib/tests/test_osutils.py	2012-09-19 07:58:27 +0000
+++ bzrlib/tests/test_osutils.py	2013-05-23 10:07:40 +0000
@@ -23,6 +23,7 @@
 import select
 import socket
 import sys
+import tempfile
 import time
 
 from bzrlib import (
@@ -436,6 +437,49 @@
         self.assertTrue(-eighteen_hours < offset < eighteen_hours)
 
 
+class TestFdatasync(tests.TestCaseInTempDir):
+
+    def do_fdatasync(self):
+        f = tempfile.NamedTemporaryFile()
+        osutils.fdatasync(f.fileno())
+        f.close()
+
+    @staticmethod
+    def raise_eopnotsupp(*args, **kwargs):
+        raise IOError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
+
+    @staticmethod
+    def raise_enotsup(*args, **kwargs):
+        raise IOError(errno.ENOTSUP, os.strerror(errno.ENOTSUP))
+
+    def test_fdatasync_handles_system_function(self):
+        self.overrideAttr(os, "fdatasync")
+        self.do_fdatasync()
+
+    def test_fdatasync_handles_no_fdatasync_no_fsync(self):
+        self.overrideAttr(os, "fdatasync")
+        self.overrideAttr(os, "fsync")
+        self.do_fdatasync()
+
+    def test_fdatasync_handles_no_EOPNOTSUPP(self):
+        self.overrideAttr(errno, "EOPNOTSUPP")
+        self.do_fdatasync()
+
+    def test_fdatasync_catches_ENOTSUP(self):
+        enotsup = getattr(errno, "ENOTSUP", None)
+        if enotsup is None:
+            raise tests.TestNotApplicable("No ENOTSUP on this platform")
+        self.overrideAttr(os, "fdatasync", self.raise_enotsup)
+        self.do_fdatasync()
+
+    def test_fdatasync_catches_EOPNOTSUPP(self):
+        enotsup = getattr(errno, "EOPNOTSUPP", None)
+        if enotsup is None:
+            raise tests.TestNotApplicable("No EOPNOTSUPP on this platform")
+        self.overrideAttr(os, "fdatasync", self.raise_eopnotsupp)
+        self.do_fdatasync()
+
+
 class TestLinks(tests.TestCaseInTempDir):
 
     def test_dereference_path(self):
 
=== modified file 'bzrlib/tests/test_selftest.py'
--- bzrlib/tests/test_selftest.py	2012-09-19 07:58:27 +0000
+++ bzrlib/tests/test_selftest.py	2013-05-23 10:07:40 +0000
@@ -1654,6 +1654,12 @@
         self.assertRaises(AssertionError,
             self.assertListRaises, _TestException, success_generator)
 
+    def _run_successful_test(self, test):
+        result = testtools.TestResult()
+        test.run(result)
+        self.assertTrue(result.wasSuccessful())
+        return result
+
     def test_overrideAttr_without_value(self):
         self.test_attr = 'original' # Define a test attribute
         obj = self # Make 'obj' visible to the embedded test
@@ -1669,8 +1675,7 @@
                 obj.test_attr = 'modified'
                 self.assertEqual('modified', obj.test_attr)
 
-        test = Test('test_value')
-        test.run(unittest.TestResult())
+        self._run_successful_test(Test('test_value'))
         self.assertEqual('original', obj.test_attr)
 
     def test_overrideAttr_with_value(self):
@@ -1686,10 +1691,41 @@
                 self.assertEqual('original', self.orig)
                 self.assertEqual('modified', obj.test_attr)
 
-        test = Test('test_value')
-        test.run(unittest.TestResult())
+        self._run_successful_test(Test('test_value'))
         self.assertEqual('original', obj.test_attr)
 
+    def test_overrideAttr_with_no_existing_value_and_value(self):
+        # Do not define the test_attribute
+        obj = self # Make 'obj' visible to the embedded test
+        class Test(tests.TestCase):
+
+            def setUp(self):
+                tests.TestCase.setUp(self)
+                self.orig = self.overrideAttr(obj, 'test_attr', new='modified')
+
+            def test_value(self):
+                self.assertEqual(tests._unitialized_attr, self.orig)
+                self.assertEqual('modified', obj.test_attr)
+
+        self._run_successful_test(Test('test_value'))
+        self.assertRaises(AttributeError, getattr, obj, 'test_attr')
+
+    def test_overrideAttr_with_no_existing_value_and_no_value(self):
+        # Do not define the test_attribute
+        obj = self # Make 'obj' visible to the embedded test
+        class Test(tests.TestCase):
+
+            def setUp(self):
+                tests.TestCase.setUp(self)
+                self.orig = self.overrideAttr(obj, 'test_attr')
+
+            def test_value(self):
+                self.assertEqual(tests._unitialized_attr, self.orig)
+                self.assertRaises(AttributeError, getattr, obj, 'test_attr')
+
+        self._run_successful_test(Test('test_value'))
+        self.assertRaises(AttributeError, getattr, obj, 'test_attr')
+
     def test_recordCalls(self):
         from bzrlib.tests import test_selftest
         calls = self.recordCalls(
 
=== modified file 'doc/en/release-notes/bzr-2.4.txt'
--- doc/en/release-notes/bzr-2.4.txt	2012-09-05 20:22:17 +0000
+++ doc/en/release-notes/bzr-2.4.txt	2013-05-23 10:07:40 +0000
@@ -35,6 +35,10 @@
 * Cope with Unix filesystems, such as smbfs, where chmod gives 'permission
   denied'.  (Martin Pool, #606537)
 
+* Fix a traceback when trying to checkout a tree that also has an entry
+  with file-id `TREE_ROOT` somewhere other than at the root directory.
+  (John Arbash Meinel, #830947)
+
 * When the ``limbo`` or ``pending-deletion`` directories exist, typically
   because of an interrupted tree update, but are empty, bzr no longer
   errors out, because there is nothing for the user to clean up.  Also,
@@ -55,6 +59,10 @@
 * Prevent a traceback being printed to stderr when logging has problems and
   accept utf-8 byte string without breaking. (Martin Packman, #714449)
 
+* Some filesystems give ``EOPNOTSUPP`` when trying to call ``fdatasync``.
+  This shouldn't be treated as a fatal error.
+  (John Arbash Meinel, #1075108)
+
 * Use ``encoding_type='exact'`` for ``bzr testament`` so that on Windows
   the sha hash of the long testament matches the sha hash in the short
   form. (John Arbash Meinel, #1010339)
 
=== modified file 'doc/en/release-notes/bzr-2.5.txt'
--- doc/en/release-notes/bzr-2.5.txt	2012-09-19 07:58:27 +0000
+++ doc/en/release-notes/bzr-2.5.txt	2013-05-23 10:07:40 +0000
@@ -35,6 +35,10 @@
 * ``bzr config`` properly handles aliases and references in the
   ``--directory`` parameter (Vincent Ladeuil, Wouter van Heyst, #947049)
 
+* Fix a traceback when trying to checkout a tree that also has an entry
+  with file-id `TREE_ROOT` somewhere other than at the root directory.
+  (John Arbash Meinel, #830947)
+
 * Lightweight checkouts of remote repositories had a bug with how they
   extracted texts from the repository. (Just an ordering constraint on how
   they consumed the stream.) (John Arbash Meinel, #1046284)
@@ -48,6 +52,10 @@
 
 * Revert use of --no-tty when gpg signing commits. (Jelmer Vernooij, #1014570)
 
+* Some filesystems give ``EOPNOTSUPP`` when trying to call ``fdatasync``.
+  This shouldn't be treated as a fatal error.
+  (John Arbash Meinel, #1075108)
+
 * Some small bug fixes wrt lightweight checkouts and remote repositories.
   A test permutation was added that runs all working tree tests against a
   lightweight checkout. (John Arbash Meinel, #1046697)
 
=== modified file 'doc/en/release-notes/bzr-2.6.txt'
--- doc/en/release-notes/bzr-2.6.txt	2012-12-04 16:36:55 +0000
+++ doc/en/release-notes/bzr-2.6.txt	2013-05-23 10:07:40 +0000
@@ -60,6 +60,13 @@
   ``BZR_PROGRESS_BAR``. This can be set in ``bazaar.conf`` or specified from
   the command line with ``-Oprogress_bar=text``. (Vincent Ladeuil, #388275)
 
+* ``Authentication.Config`` now always returns unicode user names and passwords.
+  (Vincent Ladeuil, #514301)
+
+* Fix a traceback when trying to checkout a tree that also has an entry
+  with file-id `TREE_ROOT` somewhere other than at the root directory.
+  (John Arbash Meinel, #830947)
+
 * Fixed a bug where the entire contents of ``/etc/mailname`` is read in.
   We only want to read in the first line so that comments could be added
   and would be ignored.
@@ -69,8 +76,9 @@
   causes breakage with docutils 0.9.1.
   (Vincent Ladeuil, Jelmer Vernooij, #1066307)
 
-* ``Authentication.Config`` now always returns unicode user names and passwords.
-  (Vincent Ladeuil, #514301)
+* Some filesystems give ``EOPNOTSUPP`` when trying to call ``fdatasync``.
+  This shouldn't be treated as a fatal error.
+  (John Arbash Meinel, #1075108)
 
 * Warn when ``--show-base`` is used for ``pull`` in a treeless branch
   instead of failing. It's useless but harmless. (Vincent Ladeuil, #1022160)