Merge lp:~jelmer/brz-git/fix-thin-packs into lp:brz-git

[Jelmer Vernooij]

Fix thin packs.

Fix thin packs.

[Jelmer Vernooij (jelmer)]

Rubberstamp! Proposer approves of own proposal.

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz-git-dev/131/

[lp:~jelmer/brz-git/fix-thin-packs]

1902. By Jelmer Vernooij on 2018-03-28

Fix tests.

=== modified file 'fetch.py'
--- fetch.py	2018-03-26 22:28:24 +0000
+++ fetch.py	2018-03-28 21:16:28 +0000
@@ -33,10 +33,12 @@
     ObjectStoreGraphWalker,
     tree_lookup_path,
     )
+from dulwich.protocol import CAPABILITY_THIN_PACK
 from dulwich.walk import Walker
 from itertools import (
     imap,
     )
+from io import BytesIO
 import posixpath
 import re
 import stat
@@ -847,7 +849,20 @@
               isinstance(self.target, LocalGitRepository)):
             pb = ui.ui_factory.nested_progress_bar()
             try:
-                f, commit, abort = self.target._git.object_store.add_pack()
+                if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:
+                    # TODO(jelmer): Avoid reading entire file into memory and
+                    # only processing it after the whole file has been fetched.
+                    f = BytesIO()
+
+                    def commit():
+                        if f.tell():
+                            f.seek(0)
+                            self.target._git.object_store.move_in_thin_pack(f)
+
+                    def abort():
+                        pass
+                else:
+                    f, commit, abort = self.target._git.object_store.add_pack()
                 try:
                     refs = self.source.controldir.fetch_pack(
                         determine_wants, graphwalker, f.write,

=== modified file 'transportgit.py'
--- transportgit.py	2018-03-28 02:09:45 +0000
+++ transportgit.py	2018-03-28 21:16:28 +0000
@@ -41,7 +41,7 @@
     Pack,
     iter_sha1,
     load_pack_index_file,
-    write_pack_data,
+    write_pack_objects,
     write_pack_index_v2,
     )
 from dulwich.repo import (
@@ -649,21 +649,6 @@
         self._add_known_pack(basename, final_pack)
         return final_pack
 
-    def add_thin_pack(self):
-        """Add a new thin pack to this object store.
-
-        Thin packs are packs that contain deltas with parents that exist
-        in a different pack.
-        """
-        from cStringIO import StringIO
-        f = StringIO()
-        def commit():
-            if len(f.getvalue()) > 0:
-                return self.move_in_thin_pack(f)
-            else:
-                return None
-        return f, commit
-
     def move_in_thin_pack(self, f):
         """Move a specific file containing a pack into the pack directory.
 
@@ -673,29 +658,26 @@
         :param path: Path to the pack file.
         """
         f.seek(0)
-        data = PackData.from_file(self.get_raw, f, len(f.getvalue()))
-        idx = MemoryPackIndex(data.sorted_entries(), data.get_stored_checksum())
-        p = Pack.from_objects(data, idx)
+        p = Pack('', resolve_ext_ref=self.get_raw)
+        p._data = PackData.from_file(f, len(f.getvalue()))
+        p._data.pack = p
+        p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(), p.data.get_stored_checksum())
 
-        pack_sha = idx.objects_sha1()
+        pack_sha = p.index.objects_sha1()
 
         datafile = self.pack_transport.open_write_stream(
                 "pack-%s.pack" % pack_sha)
         try:
-            entries, data_sum = write_pack_data(datafile, p.pack_tuples())
+            entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
         finally:
             datafile.close()
-        entries.sort()
+        entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
         idxfile = self.pack_transport.open_write_stream(
             "pack-%s.idx" % pack_sha)
         try:
-            write_pack_index_v2(idxfile, data.sorted_entries(), data_sum)
+            write_pack_index_v2(idxfile, entries, data_sum)
         finally:
             idxfile.close()
-        basename = "pack-%s" % pack_sha
-        final_pack = Pack(basename)
-        self._add_known_pack(basename, final_pack)
-        return final_pack
 
     def add_pack(self):
         """Add a new pack to this object store.