Merge lp:~wgrant/bzr-git/lp-fixes-caching-hackery into lp:bzr-git

[William Grant]

Unmerged revisions

1634. By William Grant on 2015-05-08

Hack in _tree_to_objects to force-cache blobs that somehow skipped the cache. Big performance win.

1633. By William Grant on 2015-05-08

Correctness/crash fixes for lp:launchpad bzr -> git conversion, apparently fixing bug #818318.

=== modified file 'object_store.py'
--- object_store.py	2015-05-09 00:38:40 +0000
+++ object_store.py	2016-10-02 12:12:04 +0000
@@ -163,7 +163,7 @@
 
 
 def _tree_to_objects(tree, parent_trees, idmap, unusual_modes,
-                     dummy_file_name=None):
+                     dummy_file_name=None, updater=None):
     """Iterate over the objects that were introduced in a revision.
 
     :param idmap: id map
@@ -228,7 +228,15 @@
                     find_unchanged_parent_ie(file_id, kind[1], target, other_parent_trees)
                 except KeyError:
                     yield path[1], blob, (file_id, tree.get_file_revision(file_id, path[1]))
+<<<<<<< TREE
         elif kind[1] not in (None, "directory"):
+=======
+            new_trees[posixpath.dirname(path[1])] = parent[1]
+        elif kind[1] == "directory":
+            if parent[1] is not None:
+                new_trees[posixpath.dirname(path[1])] = parent[1]
+        elif kind[1] != None:
+>>>>>>> MERGE-SOURCE
             raise AssertionError(kind[1])
         for p in parent:
             if p and tree.has_id(p) and tree.kind(p) == "directory":
@@ -254,6 +262,7 @@
         inv = tree.inventory
 
     trees = {}
+<<<<<<< TREE
     while dirty_dirs:
         new_dirs = set()
         for file_id in dirty_dirs:
@@ -264,6 +273,21 @@
             if ie.parent_id is not None:
                 new_dirs.add(ie.parent_id)
         dirty_dirs = new_dirs
+=======
+    while new_trees:
+        items = new_trees.items()
+        new_trees = {}
+        for path, file_id in items:
+            if path != "":
+                parent_path = urlutils.dirname(path)
+                parent_id = tree.path2id(parent_path)
+                # XXX wgrant 2015-05-06: Do we need to do something with
+                # non-existent parents?
+                if parent_id is None:
+                    continue
+                new_trees[parent_path] = parent_id
+            trees[path] = file_id
+>>>>>>> MERGE-SOURCE
 
     def ie_to_hexsha(ie):
         try:
@@ -277,6 +301,15 @@
                     # no-change merge ?
                     blob = Blob()
                     blob.data = tree.get_file_text(ie.file_id)
+                    if ie.kind == 'symlink':
+                        blob = symlink_to_blob(ie.symlink_target)
+                    # XXX: Sometimes objects don't get cached (the
+                    # InventoryFile gets a new revision without us
+                    # seeing a content_changed). To prevent that from
+                    # slowing down every subsequent revision, force it
+                    # into the cache now.
+                    if updater is not None:
+                        updater.add_object(blob, (ie.file_id, ie.revision), None)
                     return blob.id
             elif ie.kind == "directory":
                 # Not all cache backends store the tree information, 
@@ -433,7 +466,8 @@
             [p for p in rev.parent_ids if p in present_parents])
         root_tree = None
         for path, obj, bzr_key_data in _tree_to_objects(tree, parent_trees,
-                self._cache.idmap, unusual_modes, self.mapping.BZR_DUMMY_FILE):
+                self._cache.idmap, unusual_modes, self.mapping.BZR_DUMMY_FILE,
+                updater=self._get_updater(rev)):
             if path == "":
                 root_tree = obj
                 root_key_data = bzr_key_data