Merge lp:~mwhudson/launchpad/incremental-save-all-revs into lp:launchpad

[Michael Hudson-Doyle]

Preserve all revisions in an import branch's repository between runs, needed to make incremental imports work reliably.

[Michael Hudson-Doyle (mwhudson)]

The attempt at importing the kernel incrementally failed because some revisions imported in the first step weren't preserved for the second. This branch changes the BazaarBranchStore to push/pull _all_ revisions in the branch's repository, not just those in the ancestry of the branch tip.

[Tim Penhey (thumper)]

Seems reasonable.

review: Approve

=== modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py'
--- lib/lp/codehosting/codeimport/tests/test_worker.py	2010-02-22 05:37:36 +0000
+++ lib/lp/codehosting/codeimport/tests/test_worker.py	2010-02-23 23:27:16 +0000
@@ -254,6 +254,31 @@
             store._getMirrorURL(self.arbitrary_branch_id),
             sftp_prefix_noslash + '/' + '%08x' % self.arbitrary_branch_id)
 
+    def test_all_revisions_saved(self):
+        # All revisions in the branch's repo are transferred, not just those
+        # in the ancestry of the tip.
+        # Consider a branch with two heads in its repo:
+        #            revid
+        #           /     \
+        #       revid1   revid2 <- branch tip
+        # A naive push/pull would just store 'revid' and 'revid2' in the
+        # branch store -- we need to make sure all three revisions are stored
+        # and retrieved.
+        builder = self.make_branch_builder('tree')
+        revid = builder.build_snapshot(
+            None, None, [('add', ('', 'root-id', 'directory', ''))])
+        revid1 = builder.build_snapshot(None, [revid], [])
+        revid2 = builder.build_snapshot(None, [revid], [])
+        branch = builder.get_branch()
+        source_tree = branch.bzrdir.create_workingtree()
+        store = self.makeBranchStore()
+        store.push(self.arbitrary_branch_id, source_tree, default_format)
+        retrieved_tree = store.pull(
+            self.arbitrary_branch_id, 'pulled', default_format)
+        self.assertEqual(
+            set([revid, revid1, revid2]),
+            set(retrieved_tree.branch.repository.all_revision_ids()))
+
 
 class TestImportDataStore(WorkerTest):
     """Tests for `ImportDataStore`."""

=== modified file 'lib/lp/codehosting/codeimport/worker.py'
--- lib/lp/codehosting/codeimport/worker.py	2010-02-19 03:32:39 +0000
+++ lib/lp/codehosting/codeimport/worker.py	2010-02-23 23:27:16 +0000
@@ -71,20 +71,26 @@
         """
         remote_url = self._getMirrorURL(db_branch_id)
         try:
-            bzr_dir = BzrDir.open(remote_url)
+            remote_bzr_dir = BzrDir.open(remote_url)
         except NotBranchError:
             return BzrDir.create_standalone_workingtree(
                 target_path, required_format)
         # XXX Tim Penhey 2009-09-18 bug 432217 Automatic upgrade of import
         # branches disabled.  Need an orderly upgrade process.
-        if False and bzr_dir.needs_format_conversion(format=required_format):
+        if False and remote_bzr_dir.needs_format_conversion(
+            format=required_format):
             try:
-                bzr_dir.root_transport.delete_tree('backup.bzr')
+                remote_bzr_dir.root_transport.delete_tree('backup.bzr')
             except NoSuchFile:
                 pass
             upgrade(remote_url, required_format)
-        bzr_dir.sprout(target_path)
-        return BzrDir.open(target_path).open_workingtree()
+        local_bzr_dir = remote_bzr_dir.sprout(target_path)
+        # Because of the way we do incremental imports, there may be revisions
+        # in the branch's repo that are not in the ancestry of the branch tip.
+        # We need to transfer them too.
+        local_bzr_dir.open_repository().fetch(
+            remote_bzr_dir.open_repository())
+        return local_bzr_dir.open_workingtree()
 
     def push(self, db_branch_id, bzr_tree, required_format):
         """Push up `bzr_tree` as the Bazaar branch for `code_import`.
@@ -101,6 +107,10 @@
             branch_to = BzrDir.create_branch_and_repo(
                 target_url, format=required_format)
         pull_result = branch_to.pull(branch_from, overwrite=True)
+        # Because of the way we do incremental imports, there may be revisions
+        # in the branch's repo that are not in the ancestry of the branch tip.
+        # We need to transfer them too.
+        branch_to.repository.fetch(branch_from.repository)
         return pull_result.old_revid != pull_result.new_revid