Merge lp:~jelmer/brz/cf-size into lp:brz

[Jelmer Vernooij]

Add a size attribute to ContentFactory.

Add a size attribute to ContentFactory.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/655/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/657/

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'breezy/bzr/_groupcompress_py.py'
--- breezy/bzr/_groupcompress_py.py	2020-01-25 22:05:40 +0000
+++ breezy/bzr/_groupcompress_py.py	2020-01-26 04:33:55 +0000
@@ -228,8 +228,7 @@
                 if block[-1] < min_match_bytes:
                     # This block may be a 'short' block, check
                     old_start, new_start, range_len = block
-                    matched_bytes = sum(map(len,
-                                            lines[new_start:new_start + range_len]))
+                    matched_bytes = sum(map(len, lines[new_start:new_start + range_len]))
                     if matched_bytes < min_match_bytes:
                         block = None
             if block is not None:
 
=== modified file 'breezy/bzr/groupcompress.py'
--- breezy/bzr/groupcompress.py	2020-01-26 04:33:55 +0000
+++ breezy/bzr/groupcompress.py	2020-01-26 04:33:55 +0000
@@ -441,6 +441,7 @@
         self.key = key
         self.parents = parents
         self.sha1 = None
+        self.size = None
         # Note: This attribute coupled with Manager._factories creates a
         #       reference cycle. Perhaps we would rather use a weakref(), or
         #       find an appropriate time to release the ref. After the first
@@ -569,7 +570,9 @@
         end_point = 0
         for factory in self._factories:
             chunks = factory.get_bytes_as('chunked')
-            chunks_len = sum(map(len, chunks))
+            chunks_len = factory.size
+            if chunks_len is None:
+                chunks_len = sum(map(len, chunks))
             (found_sha1, start_point, end_point,
              type) = compressor.compress(
                  factory.key, chunks, chunks_len, factory.sha1)
@@ -1349,9 +1352,9 @@
         # chunks in memory.
         chunks = list(chunk_iter)
         record = ChunkedContentFactory(key, parents, None, chunks)
-        sha1 = list(self._insert_record_stream(
+        sha1, size = list(self._insert_record_stream(
             [record], random_id=random_id, nostore_sha=nostore_sha))[0]
-        return sha1, sum(map(len, chunks)), None
+        return sha1, size, None
 
     def add_fallback_versioned_files(self, a_versioned_files):
         """Add a source of texts for texts not present in this knit.
@@ -1832,7 +1835,9 @@
                 adapter_key = record.storage_kind, 'chunked'
                 adapter = get_adapter(adapter_key)
                 chunks = adapter.get_bytes(record, 'chunked')
-            chunks_len = sum(map(len, chunks))
+            chunks_len = record.size
+            if chunks_len is None:
+                chunks_len = sum(map(len, chunks))
             if len(record.key) > 1:
                 prefix = record.key[0]
                 soft = (prefix == last_prefix)
@@ -1873,7 +1878,7 @@
             else:
                 key = record.key
             self._unadded_refs[key] = record.parents
-            yield found_sha1
+            yield found_sha1, chunks_len
             as_st = static_tuple.StaticTuple.from_sequence
             if record.parents is not None:
                 parents = as_st([as_st(p) for p in record.parents])
 
=== modified file 'breezy/bzr/knit.py'
--- breezy/bzr/knit.py	2020-01-25 14:39:08 +0000
+++ breezy/bzr/knit.py	2020-01-26 04:33:55 +0000
@@ -442,6 +442,7 @@
         self.key = key
         self.parents = parents
         self.sha1 = None
+        self.size = None
         self._generator = generator
         self.storage_kind = "knit-delta-closure"
         if not first:
 
=== modified file 'breezy/bzr/versionedfile.py'
--- breezy/bzr/versionedfile.py	2020-01-25 15:55:41 +0000
+++ breezy/bzr/versionedfile.py	2020-01-26 04:33:55 +0000
@@ -75,6 +75,7 @@
     """Abstract interface for insertion and retrieval from a VersionedFile.
 
     :ivar sha1: None, or the sha1 of the content fulltext.
+    :ivar size: None, or the size of the content fulltext.
     :ivar storage_kind: The native storage kind of this factory. One of
         'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
         'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
@@ -89,6 +90,7 @@
     def __init__(self):
         """Create a ContentFactory."""
         self.sha1 = None
+        self.size = None
         self.storage_kind = None
         self.key = None
         self.parents = None
@@ -102,6 +104,7 @@
     satisfies this, as does a list of lines.
 
     :ivar sha1: None, or the sha1 of the content fulltext.
+    :ivar size: None, or the size of the content fulltext.
     :ivar storage_kind: The native storage kind of this factory. Always
         'chunked'
     :ivar key: The key of this content. Each key is a tuple with a single
@@ -115,6 +118,7 @@
     def __init__(self, key, parents, sha1, chunks, chunks_are_lines=None):
         """Create a ContentFactory."""
         self.sha1 = sha1
+        self.size = sum(map(len, chunks))
         self.storage_kind = 'chunked'
         self.key = key
         self.parents = parents
@@ -153,6 +157,7 @@
     def __init__(self, key, parents, sha1, text):
         """Create a ContentFactory."""
         self.sha1 = sha1
+        self.size = len(text)
         self.storage_kind = 'fulltext'
         self.key = key
         self.parents = parents
@@ -181,6 +186,7 @@
         self.file = fileobj
         self.storage_kind = 'file'
         self._sha1 = None
+        self._size = None
 
     @property
     def sha1(self):
@@ -214,6 +220,7 @@
     def __init__(self, key):
         """Create a ContentFactory."""
         self.sha1 = None
+        self.size = None
         self.storage_kind = 'absent'
         self.key = key
         self.parents = None
 
=== modified file 'breezy/git/annotate.py'
--- breezy/git/annotate.py	2020-01-25 14:07:41 +0000
+++ breezy/git/annotate.py	2020-01-26 04:33:55 +0000
@@ -56,6 +56,7 @@
         self.storage_kind = 'git-blob'
         self.parents = None
         self.blob_id = blob_id
+        self.size = None
 
     def get_bytes_as(self, storage_kind):
         if storage_kind == 'fulltext':
@@ -87,6 +88,7 @@
         self.key = (path, revision)
         self.storage_kind = 'absent'
         self.parents = None
+        self.size = None
 
     def get_bytes_as(self, storage_kind):
         raise ValueError