Merge lp:~mwhudson/launchpad/more-git-caches into lp:launchpad

[Michael Hudson-Doyle]

Update to the new, massively faster bzr-git and a few Launchpad changes to cope with the new cache structure.

I haven't landed the bzr-git and dulwich updates yet, so sourcedeps.conf needs to change but otherwise this branch adapts us to the new bzr-git hotness.

[Tim Penhey (thumper)]

You have git_revisions_import_limit mentioned twice in the schema-lazr.conf.

# GitImportWorker.pushBazaarWorkingTree saves the git file from the

perhaps should say "git files"

# legacy:
and
# new hotness

As much as I like these, they don't fit our coding standards of full
sentences.

But other than that, great!

review: Approve

=== modified file 'lib/canonical/config/schema-lazr.conf'
--- lib/canonical/config/schema-lazr.conf	2010-04-07 12:45:11 +0000
+++ lib/canonical/config/schema-lazr.conf	2010-04-09 06:17:35 +0000
@@ -440,9 +440,11 @@
 # in a row.
 consecutive_failure_limit: 5
 
-# Import only this many revisions at once.
-# Only applies to git imports for now.
-revisions_import_limit: 1000
+# Import only this many revisions from git at once.
+git_revisions_import_limit: 10000
+
+# Import only this many revisions from svn (via bzr-svn) at once.
+svn_revisions_import_limit: 1000
 
 
 [codeimportdispatcher]
 
=== modified file 'lib/lp/codehosting/codeimport/tests/test_worker.py'
--- lib/lp/codehosting/codeimport/tests/test_worker.py	2010-03-12 02:30:36 +0000
+++ lib/lp/codehosting/codeimport/tests/test_worker.py	2010-04-09 06:17:35 +0000
@@ -28,6 +28,7 @@
 from canonical.testing import BaseLayer
 
 from lp.codehosting import load_optional_plugin
+from lp.codehosting.codeimport.tarball import create_tarball, extract_tarball
 from lp.codehosting.codeimport.worker import (
     BazaarBranchStore, BzrSvnImportWorker, CodeImportWorkerExitCode,
     ForeignTreeStore, GitImportWorker, HgImportWorker, ImportDataStore,
@@ -590,20 +591,23 @@
             source_details, self.get_transport('import_data'),
             self.makeBazaarBranchStore(), logging.getLogger("silent"))
 
-    def test_pushBazaarWorkingTree_saves_git_db(self):
-        # GitImportWorker.pushBazaarWorkingTree saves the git.db file from the
-        # tree's repository in the worker's ImportDataStore.
+    def test_pushBazaarWorkingTree_saves_git_cache(self):
+        # GitImportWorker.pushBazaarWorkingTree saves a tarball of the git
+        # cache from the tree's repository in the worker's ImportDataStore.
         content = self.factory.getUniqueString()
         tree = self.make_branch_and_tree('.')
-        tree.branch.repository._transport.put_bytes('git.db', content)
+        tree.branch.repository._transport.mkdir('git')
+        tree.branch.repository._transport.put_bytes('git/cache', content)
         import_worker = self.makeImportWorker()
         import_worker.pushBazaarWorkingTree(tree)
-        import_worker.import_data_store.fetch('git.db')
-        self.assertEqual(content, open('git.db').read())
+        import_worker.import_data_store.fetch('git-cache.tar.gz')
+        extract_tarball('git-cache.tar.gz', '.')
+        self.assertEqual(content, open('cache').read())
 
-    def test_getBazaarWorkingTree_fetches_git_db(self):
-        # GitImportWorker.getBazaarWorkingTree fetches the git.db file from
-        # the worker's ImportDataStore into the tree's repository.
+    def test_getBazaarWorkingTree_fetches_legacy_git_db(self):
+        # GitImportWorker.getBazaarWorkingTree fetches the legacy git.db file,
+        # if present, from the worker's ImportDataStore into the tree's
+        # repository.
         import_worker = self.makeImportWorker()
         # Store the git.db file in the store.
         content = self.factory.getUniqueString()
@@ -617,6 +621,25 @@
         self.assertEqual(
             content, tree.branch.repository._transport.get('git.db').read())
 
+    def test_getBazaarWorkingTree_fetches_git_cache(self):
+        # GitImportWorker.getBazaarWorkingTree fetches the tarball of the git
+        # cache from the worker's ImportDataStore and expands it into the
+        # tree's repository.
+        import_worker = self.makeImportWorker()
+        # Store a tarred-up cache in the store.x
+        content = self.factory.getUniqueString()
+        os.mkdir('cache')
+        open('cache/git-cache', 'w').write(content)
+        create_tarball('cache', 'git-cache.tar.gz')
+        import_worker.import_data_store.put('git-cache.tar.gz')
+        # Make sure there's a Bazaar branch in the branch store.
+        tree = self.make_branch_and_tree('tree')
+        ImportWorker.pushBazaarWorkingTree(import_worker, tree)
+        # Finally, fetching the tree gets the git.db file too.
+        tree = import_worker.getBazaarWorkingTree()
+        self.assertEqual(
+            content, tree.branch.repository._transport.get('git/git-cache').read())
+
 
 def clean_up_default_stores_for_import(source_details):
     """Clean up the default branch and foreign tree stores for an import.
@@ -913,7 +936,10 @@
         self.makeForeignCommit(worker.source_details)
         self.assertTrue(self.foreign_commit_count > 1)
         self.pushConfig(
-            'codeimport', revisions_import_limit=self.foreign_commit_count-1)
+            'codeimport',
+            git_revisions_import_limit=self.foreign_commit_count-1,
+            svn_revisions_import_limit=self.foreign_commit_count-1,
+            )
         self.assertEqual(
             CodeImportWorkerExitCode.SUCCESS_PARTIAL, worker.run())
         self.assertEqual(
 
=== modified file 'lib/lp/codehosting/codeimport/worker.py'
--- lib/lp/codehosting/codeimport/worker.py	2010-03-15 23:23:29 +0000
+++ lib/lp/codehosting/codeimport/worker.py	2010-04-09 06:17:35 +0000
@@ -26,7 +26,7 @@
 from bzrlib.transport import get_transport
 from bzrlib.errors import NoSuchFile, NotBranchError
 import bzrlib.ui
-from bzrlib.urlutils import join as urljoin
+from bzrlib.urlutils import join as urljoin, local_path_from_url
 from bzrlib.upgrade import upgrade
 
 from canonical.cachedproperty import cachedproperty
@@ -581,7 +581,7 @@
 
     def getExtraPullArgs(self):
         """See `PullingImportWorker.getExtraPullArgs`."""
-        return {'limit': config.codeimport.revisions_import_limit}
+        return {'limit': config.codeimport.git_revisions_import_limit}
 
     def getBazaarWorkingTree(self):
         """See `ImportWorker.getBazaarWorkingTree`.
@@ -591,8 +591,17 @@
         it in the Bazaar tree, that is at '.bzr/repository/git.db'.
         """
         tree = PullingImportWorker.getBazaarWorkingTree(self)
+        # Fetch the legacy cache from the store, if present.
         self.import_data_store.fetch(
             'git.db', tree.branch.repository._transport)
+        # The cache dir from newer bzr-gits is stored as a tarball.
+        local_name = 'git-cache.tar.gz'
+        if self.import_data_store.fetch(local_name):
+            repo_transport = tree.branch.repository._transport
+            repo_transport.mkdir('git')
+            git_db_dir = os.path.join(
+                local_path_from_url(repo_transport.base), 'git')
+            extract_tarball(local_name, git_db_dir)
         return tree
 
     def pushBazaarWorkingTree(self, bazaar_tree):
@@ -604,8 +613,12 @@
         """
         non_trivial = PullingImportWorker.pushBazaarWorkingTree(
             self, bazaar_tree)
-        self.import_data_store.put(
-            'git.db', bazaar_tree.branch.repository._transport)
+        repo_transport = bazaar_tree.branch.repository._transport
+        git_db_dir = os.path.join(
+            local_path_from_url(repo_transport.base), 'git')
+        local_name = 'git-cache.tar.gz'
+        create_tarball(git_db_dir, local_name)
+        self.import_data_store.put(local_name)
         return non_trivial
 
 
@@ -655,7 +668,7 @@
 
     def getExtraPullArgs(self):
         """See `PullingImportWorker.getExtraPullArgs`."""
-        return {'limit': config.codeimport.revisions_import_limit}
+        return {'limit': config.codeimport.svn_revisions_import_limit}
 
     @property
     def format_classes(self):
 
=== modified file 'utilities/sourcedeps.conf'
--- utilities/sourcedeps.conf	2010-04-07 12:27:00 +0000
+++ utilities/sourcedeps.conf	2010-04-09 06:17:35 +0000
@@ -1,10 +1,10 @@
 bzr-builder lp:~launchpad-pqm/bzr-builder/trunk;revno=63
-bzr-git lp:~launchpad-pqm/bzr-git/devel;revno=251
+bzr-git lp:~launchpad-pqm/bzr-git/devel;revno=253
 bzr-hg lp:~launchpad-pqm/bzr-hg/devel;revno=281
 bzr-loom lp:~launchpad-pqm/bzr-loom/trunk;revno=47
 bzr-svn lp:~launchpad-pqm/bzr-svn/devel;revno=2708
 cscvs lp:~launchpad-pqm/launchpad-cscvs/devel;revno=430
-dulwich lp:~launchpad-pqm/dulwich/devel;revno=417
+dulwich lp:~launchpad-pqm/dulwich/devel;revno=418
 launchpad-loggerhead lp:~launchpad-pqm/launchpad-loggerhead/devel;revno=54
 loggerhead lp:~launchpad-pqm/loggerhead/devel;revno=174
 lpreview lp:~launchpad-pqm/bzr-lpreview/devel;revno=23