Merge lp:~jelmer/bzr/hpss-add-text into lp:bzr

[Jelmer Vernooij]

Use sinks to store texts when committing.

Implement text addition in the commit builder using sinks.

This brings us a step closer towards a pure-HPSS commit builder.

[Vincent Ladeuil (vila)]

I'm going to trust you there but if you could mention that some tests failed while you refactored I will be even more confident ;)

review: Approve

[Jelmer Vernooij (jelmer)]

I didn't find any failing tests while refactoring this - we're just using a slightly different code path, but one which with more refactoring should reduce the number of roundtrips for commits over HPSS.

[Jelmer Vernooij (jelmer)]

sent to pqm by email

[Jelmer Vernooij (jelmer)]

sent to pqm by email

[Vincent Ladeuil (vila)]

> I didn't find any failing tests while refactoring this - we're just using a
> slightly different code path, but one which with more refactoring should
> reduce the number of roundtrips for commits over HPSS.

Ha ok. Great !

pqm seems to be a bit unhappy though so may *it* found some failing tests ;)

[Vincent Ladeuil (vila)]

Is this stale or are you still working on it ? What happened ?

[Martin Packman (gz)]

There are a lot of failing tests with this branch, all seemingly related to assertions on repository state:

bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_merged_utf8(RevisionTree)
bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_merged_utf8(DirStateRevisionTree,WT4)
bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_merged_utf8(DirStateRevisionTree,WT5)
bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_utf8(RevisionTree)
bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_utf8(DirStateRevisionTree,WT4)
bt.per_tree.test_test_trees.TestTreeShapes.test_tree_with_utf8(DirStateRevisionTree,WT5)

Traceback (most recent call last):
  File ".\bzrlib\tests\per_tree\test_test_trees.py", line 250, in test_tree_with_utf8
    self.assertEqual(revision_id, ie.revision)
AssertionError: not equal:
a = 'r\xc3\xa9v-1'
b = '<email address hidden>'

bt.per_repository.test_commit_builder.TestCommitBuilder [lots]

Traceback (most recent call last):
  File ".\bzrlib\tests\per_repository\test_commit_builder.py", line 639, in test_last_modified_revision_after_commit_file_unchanged
    self._add_commit_check_unchanged(tree, 'file')
  File ".\bzrlib\tests\per_repository\test_commit_builder.py", line 590, in _add_commit_check_unchanged
    mini_commit=mini_commit)
  File ".\bzrlib\tests\per_repository\test_commit_builder.py", line 596, in _commit_check_unchanged
    rev2 = mini_commit(tree, name, name, False, False)
  File ".\bzrlib\tests\per_repository\test_commit_builder.py", line 903, in mini_commit
    self.assertFalse(version_recorded)
AssertionError: True is not false

bt.per_repository_chk.test_supported.TestCommitWriteGroupIntegrityCheck.test_missing_chk_root_for_unchanged_inventory(RepositoryFormat2aSubtree)

Traceback (most recent call last):
  File ".\bzrlib\tests\per_repository_chk\test_supported.py", line 228, in test_missing_chk_root_for_unchanged_inventory
    self.assertEqual(inv_b.id_to_entry.key(), inv_c.id_to_entry.key())
AssertionError: not equal:
a = StaticTuple('sha1:16c15cbde78efa708c7ded82ff3d3d401d22ad88',)
b = StaticTuple('sha1:8396048593fe7662cef670458de4c8698760cc49',)

bt.per_repository_vf.test_fileid_involved.TestFileIdInvolved [lots]

Traceback (most recent call last):
  File ".\bzrlib\tests\per_repository_vf\test_fileid_involved.py", line 243, in test_fileids_altered_between_two_revs
    self.branch.repository.fileids_altered_by_revision_ids(["rev-J","rev-K"]))
AssertionError: not equal:
a = {'b-file-id-2006-01-01-defg': set(['rev-J']),
 'c-funky<file-id>quiji%bo': set(['rev-K'])}
b = {'a-file-id-2006-01-01-abcd': set(['rev-J', 'rev-K']),
 'b-file-id-2006-01-01-defg': set(['rev-J', 'rev-K']),
 'c-funky<file-id>quiji%bo': set(['rev-J', 'rev-K'])}

bt.test_branchbuilder.TestBranchBuilderBuildSnapshot.test_set_merge_parent_and_contents

Traceback (most recent call last):
  File ".\bzrlib\tests\test_branchbuilder.py", line 348, in test_set_merge_parent_and_contents
    self.assertEqual('C-id', d_tree.get_file_revision('c-id'))
AssertionError: not equal:
a = 'C-id'
b = 'D-id'

bt.test_bundle.V4_2aBundleTester.test_multiple_inventories_as_xml
bt.test_bundle.V4_2aBundle...

=== modified file 'bzrlib/vf_repository.py'
--- bzrlib/vf_repository.py	2011-12-19 13:23:58 +0000
+++ bzrlib/vf_repository.py	2012-01-04 14:42:24 +0000
@@ -130,6 +130,7 @@
         # valid. Callers that will call record_delete() should call
         # .will_record_deletes() to indicate that.
         self._recording_deletes = False
+        self._sink = repository._get_sink()
 
     def will_record_deletes(self):
         """Tell the commit builder that deletes are being notified.
@@ -177,9 +178,8 @@
         while missing_keys and fallback_repos:
             fallback_repo = fallback_repos.pop()
             source = fallback_repo._get_source(self.repository._format)
-            sink = self.repository._get_sink()
             stream = source.get_stream_for_missing_keys(missing_keys)
-            missing_keys = sink.insert_stream_without_locking(stream,
+            missing_keys = self._sink.insert_stream_without_locking(stream,
                 self.repository._format)
         if missing_keys:
             raise errors.BzrError('Unable to fill in parent inventories for a'
@@ -828,9 +828,14 @@
 
     def _add_text_to_weave(self, file_id, new_text, parents, nostore_sha):
         parent_keys = tuple([(file_id, parent) for parent in parents])
-        return self.repository.texts._add_text(
-            (file_id, self._new_revision_id), parent_keys, new_text,
-            nostore_sha=nostore_sha, random_id=self.random_revid)[0:2]
+        sha1 = osutils.sha_string(new_text)
+        stream = [("texts", [
+            versionedfile.FulltextContentFactory(
+                    (file_id, self._new_revision_id),
+                    parent_keys, sha1, new_text)])]
+        missing_keys = self._sink.insert_stream_without_locking(stream,
+            self.repository._format)
+        return sha1, len(new_text)
 
 
 class VersionedFileRootCommitBuilder(VersionedFileCommitBuilder):