Merge lp:~jelmer/brz/file-content-factory into lp:brz

[Jelmer Vernooij]

Add a 'lines' storage kind.

Add a 'lines' storage kind.

Avoid converting back and forth between fulltext and lines in various places.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/650/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/651/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/652/

=== modified file 'breezy/_annotator_py.py'
--- breezy/_annotator_py.py	2019-03-02 21:46:18 +0000
+++ breezy/_annotator_py.py	2020-01-25 16:26:49 +0000
@@ -127,7 +127,7 @@
             if record.storage_kind == 'absent':
                 raise errors.RevisionNotPresent(record.key, self._vf)
             this_key = record.key
-            lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
+            lines = record.get_bytes_as('lines')
             num_lines = len(lines)
             self._text_cache[this_key] = lines
             yield this_key, lines, num_lines

=== modified file 'breezy/bzr/groupcompress.py'
--- breezy/bzr/groupcompress.py	2020-01-25 14:46:10 +0000
+++ breezy/bzr/groupcompress.py	2020-01-25 16:26:49 +0000
@@ -466,7 +466,7 @@
                 return self._manager._wire_bytes()
             else:
                 return b''
-        if storage_kind in ('fulltext', 'chunked'):
+        if storage_kind in ('fulltext', 'chunked', 'lines'):
             if self._bytes is None:
                 # Grab and cache the raw bytes for this entry
                 # and break the ref-cycle with _manager since we don't need it
@@ -482,8 +482,10 @@
                 # refcycle here, but instead in manager.get_record_stream()
             if storage_kind == 'fulltext':
                 return self._bytes
-            else:
+            elif storage_kind == 'chunked':
                 return [self._bytes]
+            else:
+                return osutils.split_lines(self._bytes)
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
@@ -566,9 +568,9 @@
         old_length = self._block._content_length
         end_point = 0
         for factory in self._factories:
-            bytes = factory.get_bytes_as('fulltext')
+            chunks = factory.get_bytes_as('chunked')
             (found_sha1, start_point, end_point,
-             type) = compressor.compress(factory.key, [bytes], factory.sha1)
+             type) = compressor.compress(factory.key, chunks, factory.sha1)
             # Now update this factory with the new offsets, etc
             factory.sha1 = found_sha1
             factory._start = start_point
@@ -1369,7 +1371,7 @@
         if keys is None:
             keys = self.keys()
             for record in self.get_record_stream(keys, 'unordered', True):
-                record.get_bytes_as('fulltext')
+                record.get_bytes_as('chunked')
         else:
             return self.get_record_stream(keys, 'unordered', True)
 
@@ -1670,8 +1672,8 @@
                 result[record.key] = record.sha1
             else:
                 if record.storage_kind != 'absent':
-                    result[record.key] = osutils.sha_string(
-                        record.get_bytes_as('fulltext'))
+                    result[record.key] = osutils.sha_strings(
+                        record.get_bytes_as('chunked'))
         return result
 
     def insert_record_stream(self, stream):
@@ -1823,25 +1825,26 @@
                     self._index.add_records(nodes, random_id=random_id)
                     continue
             try:
-                bytes = record.get_bytes_as('fulltext')
+                chunks = record.get_bytes_as('chunked')
             except errors.UnavailableRepresentation:
-                adapter_key = record.storage_kind, 'fulltext'
+                adapter_key = record.storage_kind, 'chunked'
                 adapter = get_adapter(adapter_key)
-                bytes = adapter.get_bytes(record)
+                chunks = adapter.get_bytes(record, 'chunked')
+            chunks_len = sum(map(len, chunks))
             if len(record.key) > 1:
                 prefix = record.key[0]
                 soft = (prefix == last_prefix)
             else:
                 prefix = None
                 soft = False
-            if max_fulltext_len < len(bytes):
-                max_fulltext_len = len(bytes)
+            if max_fulltext_len < chunks_len:
+                max_fulltext_len = chunks_len
                 max_fulltext_prefix = prefix
             (found_sha1, start_point, end_point,
              type) = self._compressor.compress(record.key,
-                                               [bytes], record.sha1, soft=soft,
+                                               chunks, record.sha1, soft=soft,
                                                nostore_sha=nostore_sha)
-            # delta_ratio = float(len(bytes)) / (end_point - start_point)
+            # delta_ratio = float(chunks_len) / (end_point - start_point)
             # Check if we want to continue to include that text
             if (prefix == max_fulltext_prefix
                     and end_point < 2 * max_fulltext_len):
@@ -1859,10 +1862,10 @@
             if start_new_block:
                 self._compressor.pop_last()
                 flush()
-                max_fulltext_len = len(bytes)
+                max_fulltext_len = chunks_len
                 (found_sha1, start_point, end_point,
                  type) = self._compressor.compress(
-                     record.key, [bytes], record.sha1)
+                     record.key, chunks, record.sha1)
             if record.key[-1] is None:
                 key = record.key[:-1] + (b'sha1:' + found_sha1,)
             else:
@@ -1915,7 +1918,7 @@
                 pb.update('Walking content', key_idx, total)
             if record.storage_kind == 'absent':
                 raise errors.RevisionNotPresent(key, self)
-            lines = osutils.split_lines(record.get_bytes_as('fulltext'))
+            lines = record.get_bytes_as('lines')
             for line in lines:
                 yield line, key
         if pb is not None:

=== modified file 'breezy/bzr/groupcompress_repo.py'
--- breezy/bzr/groupcompress_repo.py	2019-10-19 22:53:54 +0000
+++ breezy/bzr/groupcompress_repo.py	2020-01-25 16:26:49 +0000
@@ -698,9 +698,9 @@
                         % (chk_inv.revision_id, p_id_map.key()[0],
                            canon_p_id_map.key()[0]))
                     self._data_changed = True
-                yield versionedfile.ChunkedContentFactory(record.key,
-                                                          record.parents, record.sha1,
-                                                          canonical_inv.to_lines())
+                yield versionedfile.ChunkedContentFactory(
+                    record.key, record.parents, record.sha1, canonical_inv.to_lines(),
+                    chunks_are_lines=True)
             # We have finished processing all of the inventory records, we
             # don't need these sets anymore
         return _filtered_inv_stream()

=== modified file 'breezy/bzr/knit.py'
--- breezy/bzr/knit.py	2020-01-25 04:20:44 +0000
+++ breezy/bzr/knit.py	2020-01-25 16:26:49 +0000
@@ -223,7 +223,10 @@
 class FTAnnotatedToUnannotated(KnitAdapter):
     """An adapter from FT annotated knits to unannotated ones."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
+        if target_storage_kind != 'knit-ft-gz':
+            raise errors.UnavailableRepresentation(
+                factory.key, target_storage_kind, factory.storage_kind)
         annotated_compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(annotated_compressed_bytes)
@@ -236,7 +239,10 @@
 class DeltaAnnotatedToUnannotated(KnitAdapter):
     """An adapter for deltas from annotated to unannotated."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
+        if target_storage_kind != 'knit-delta-gz':
+            raise errors.UnavailableRepresentation(
+                factory.key, target_storage_kind, factory.storage_kind)
         annotated_compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(annotated_compressed_bytes)
@@ -250,19 +256,24 @@
 class FTAnnotatedToFullText(KnitAdapter):
     """An adapter from FT annotated knits to unannotated ones."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
         annotated_compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(annotated_compressed_bytes)
         content, delta = self._annotate_factory.parse_record(factory.key[-1],
                                                              contents, factory._build_details, None)
-        return b''.join(content.text())
+        if target_storage_kind == 'fulltext':
+            return b''.join(content.text())
+        elif target_storage_kind in ('chunked', 'lines'):
+            return content.text()
+        raise errors.UnavailableRepresentation(
+            factory.key, target_storage_kind, factory.storage_kind)
 
 
 class DeltaAnnotatedToFullText(KnitAdapter):
     """An adapter for deltas from annotated to unannotated."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
         annotated_compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(annotated_compressed_bytes)
@@ -273,32 +284,42 @@
             [compression_parent], 'unordered', True))
         if basis_entry.storage_kind == 'absent':
             raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
-        basis_chunks = basis_entry.get_bytes_as('chunked')
-        basis_lines = osutils.chunks_to_lines(basis_chunks)
+        basis_lines = basis_entry.get_bytes_as('lines')
         # Manually apply the delta because we have one annotated content and
         # one plain.
         basis_content = PlainKnitContent(basis_lines, compression_parent)
         basis_content.apply_delta(delta, rec[1])
         basis_content._should_strip_eol = factory._build_details[1]
-        return b''.join(basis_content.text())
+
+        if target_storage_kind == 'fulltext':
+            return b''.join(basis_content.text())
+        elif target_storage_kind in ('chunked', 'lines'):
+            return basis_content.text()
+        raise errors.UnavailableRepresentation(
+            factory.key, target_storage_kind, factory.storage_kind)
 
 
 class FTPlainToFullText(KnitAdapter):
     """An adapter from FT plain knits to unannotated ones."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
         compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(compressed_bytes)
         content, delta = self._plain_factory.parse_record(factory.key[-1],
                                                           contents, factory._build_details, None)
-        return b''.join(content.text())
+        if target_storage_kind == 'fulltext':
+            return b''.join(content.text())
+        elif target_storage_kind in ('chunked', 'lines'):
+            return content.text()
+        raise errors.UnavailableRepresentation(
+            factory.key, target_storage_kind, factory.storage_kind)
 
 
 class DeltaPlainToFullText(KnitAdapter):
     """An adapter for deltas from annotated to unannotated."""
 
-    def get_bytes(self, factory):
+    def get_bytes(self, factory, target_storage_kind):
         compressed_bytes = factory._raw_record
         rec, contents = \
             self._data._parse_record_unchecked(compressed_bytes)
@@ -309,14 +330,18 @@
             [compression_parent], 'unordered', True))
         if basis_entry.storage_kind == 'absent':
             raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
-        basis_chunks = basis_entry.get_bytes_as('chunked')
-        basis_lines = osutils.chunks_to_lines(basis_chunks)
+        basis_lines = basis_entry.get_bytes_as('lines')
         basis_content = PlainKnitContent(basis_lines, compression_parent)
         # Manually apply the delta because we have one annotated content and
         # one plain.
         content, _ = self._plain_factory.parse_record(rec[1], contents,
                                                       factory._build_details, basis_content)
-        return b''.join(content.text())
+        if target_storage_kind == 'fulltext':
+            return b''.join(content.text())
+        elif target_storage_kind in ('chunked', 'lines'):
+            return content.text()
+        raise errors.UnavailableRepresentation(
+            factory.key, target_storage_kind, factory.storage_kind)
 
 
 class KnitContentFactory(ContentFactory):
@@ -381,19 +406,15 @@
                 self._create_network_bytes()
             return self._network_bytes
         if ('-ft-' in self.storage_kind
-                and storage_kind in ('chunked', 'fulltext')):
-            adapter_key = (self.storage_kind, 'fulltext')
+                and storage_kind in ('chunked', 'fulltext', 'lines')):
+            adapter_key = (self.storage_kind, storage_kind)
             adapter_factory = adapter_registry.get(adapter_key)
             adapter = adapter_factory(None)
-            bytes = adapter.get_bytes(self)
-            if storage_kind == 'chunked':
-                return [bytes]
-            else:
-                return bytes
+            return adapter.get_bytes(self, storage_kind)
         if self._knit is not None:
             # Not redundant with direct conversion above - that only handles
             # fulltext cases.
-            if storage_kind == 'chunked':
+            if storage_kind in ('chunked', 'lines'):
                 return self._knit.get_lines(self.key[0])
             elif storage_kind == 'fulltext':
                 return self._knit.get_text(self.key[0])
@@ -435,9 +456,9 @@
                 # all the keys etc are contained in the bytes returned in the
                 # first record.
                 return b''
-        if storage_kind in ('chunked', 'fulltext'):
+        if storage_kind in ('chunked', 'fulltext', 'lines'):
             chunks = self._generator._get_one_work(self.key).text()
-            if storage_kind == 'chunked':
+            if storage_kind in ('chunked', 'lines'):
                 return chunks
             else:
                 return b''.join(chunks)
@@ -1708,7 +1729,7 @@
                     except KeyError:
                         adapter_key = (record.storage_kind, "knit-ft-gz")
                         adapter = get_adapter(adapter_key)
-                    bytes = adapter.get_bytes(record)
+                    bytes = adapter.get_bytes(record, adapter_key[1])
                 else:
                     # It's a knit record, it has a _raw_record field (even if
                     # it was reconstituted from a network stream).
@@ -1745,9 +1766,8 @@
                         buffered = True
                 if not buffered:
                     self._index.add_records([index_entry])
-            elif record.storage_kind == 'chunked':
-                self.add_lines(record.key, parents,
-                               osutils.chunks_to_lines(record.get_bytes_as('chunked')))
+            elif record.storage_kind in ('chunked', 'file'):
+                self.add_lines(record.key, parents, record.get_bytes_as('lines'))
             else:
                 # Not suitable for direct insertion as a
                 # delta, either because it's not the right format, or this
@@ -1757,12 +1777,11 @@
                 self._access.flush()
                 try:
                     # Try getting a fulltext directly from the record.
-                    bytes = record.get_bytes_as('fulltext')
+                    lines = record.get_bytes_as('lines')
                 except errors.UnavailableRepresentation:
-                    adapter_key = record.storage_kind, 'fulltext'
+                    adapter_key = record.storage_kind, 'lines'
                     adapter = get_adapter(adapter_key)
-                    bytes = adapter.get_bytes(record)
-                lines = split_lines(bytes)
+                    lines = adapter.get_bytes(record, 'lines')
                 try:
                     self.add_lines(record.key, parents, lines)
                 except errors.RevisionAlreadyPresent:
@@ -2107,7 +2126,7 @@
         if key in self.nonlocal_keys:
             record = next(self.get_record_stream())
             # Create a content object on the fly
-            lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
+            lines = record.get_bytes_as('lines')
             return PlainKnitContent(lines, record.key)
         else:
             # local keys we can ask for directly
@@ -2198,9 +2217,9 @@
                 if component_id in self._contents_map:
                     content = self._contents_map[component_id]
                 else:
-                    content, delta = self._factory.parse_record(key[-1],
-                                                                record, record_details, content,
-                                                                copy_base_content=multiple_versions)
+                    content, delta = self._factory.parse_record(
+                        key[-1], record, record_details, content,
+                        copy_base_content=multiple_versions)
                     if multiple_versions:
                         self._contents_map[component_id] = content
 

=== modified file 'breezy/bzr/knitpack_repo.py'
--- breezy/bzr/knitpack_repo.py	2020-01-11 17:50:28 +0000
+++ breezy/bzr/knitpack_repo.py	2020-01-25 16:26:49 +0000
@@ -1092,8 +1092,8 @@
                     raise errors.BzrError('Mismatched key parent %r:%r' %
                                           (key, parent_keys))
                 parents.append(parent_key[1])
-            text_lines = osutils.split_lines(next(repo.texts.get_record_stream(
-                [key], 'unordered', True)).get_bytes_as('fulltext'))
+            text_lines = next(repo.texts.get_record_stream(
+                [key], 'unordered', True)).get_bytes_as('lines')
             output_texts.add_lines(key, parent_keys, text_lines,
                                    random_id=True, check_content=False)
         # 5) check that nothing inserted has a reference outside the keyspace.

=== modified file 'breezy/bzr/reconcile.py'
--- breezy/bzr/reconcile.py	2018-11-30 12:39:04 +0000
+++ breezy/bzr/reconcile.py	2020-01-25 16:26:49 +0000
@@ -35,7 +35,7 @@
 from ..i18n import gettext
 from ..trace import mutter
 from ..tsort import topo_sort
-from .versionedfile import AdapterFactory, FulltextContentFactory
+from .versionedfile import AdapterFactory, ChunkedContentFactory
 
 
 class VersionedFileRepoReconciler(object):
@@ -152,8 +152,8 @@
                 # The check for the left most parent only handles knit
                 # compressors, but this code only applies to knit and weave
                 # repositories anyway.
-                bytes = record.get_bytes_as('fulltext')
-                yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
+                chunks = record.get_bytes_as('chunked')
+                yield ChunkedContentFactory(record.key, wanted_parents, record.sha1, chunks)
             else:
                 adapted_record = AdapterFactory(
                     record.key, wanted_parents, record)
@@ -344,10 +344,10 @@
 
         def fix_parents(stream):
             for record in stream:
-                bytes = record.get_bytes_as('fulltext')
+                chunks = record.get_bytes_as('chunked')
                 new_key = (new_file_id, record.key[-1])
                 parents = new_parents[new_key]
-                yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
+                yield ChunkedContentFactory(new_key, parents, record.sha1, chunks)
         stream = self.repo.texts.get_record_stream(
             needed_keys, 'topological', True)
         self.repo._remove_file_id(new_file_id)

=== modified file 'breezy/bzr/smart/repository.py'
--- breezy/bzr/smart/repository.py	2019-02-15 18:57:38 +0000
+++ breezy/bzr/smart/repository.py	2020-01-25 16:26:49 +0000
@@ -1270,7 +1270,9 @@
                 inv_delta = inv._make_delta(prev_inv)
                 lines = serializer.delta_to_lines(
                     prev_inv.revision_id, inv.revision_id, inv_delta)
-                yield ChunkedContentFactory(inv.revision_id, None, None, lines)
+                yield ChunkedContentFactory(
+                    inv.revision_id, None, None, lines,
+                    chunks_are_lines=True)
                 prev_inv = inv
 
     def body_stream(self, repository, ordering, revids):

=== modified file 'breezy/bzr/versionedfile.py'
--- breezy/bzr/versionedfile.py	2020-01-11 17:50:28 +0000
+++ breezy/bzr/versionedfile.py	2020-01-25 16:26:49 +0000
@@ -56,20 +56,19 @@
 
 
 adapter_registry = Registry()
-adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'breezy.bzr.knit',
-                               'DeltaPlainToFullText')
-adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'breezy.bzr.knit',
-                               'FTPlainToFullText')
 adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
                                'breezy.bzr.knit', 'DeltaAnnotatedToUnannotated')
-adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
-                               'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
 adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
                                'breezy.bzr.knit', 'FTAnnotatedToUnannotated')
-adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
-                               'breezy.bzr.knit', 'FTAnnotatedToFullText')
-# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
-#     'breezy.bzr.knit', 'FTAnnotatedToChunked')
+for target_storage_kind in ('fulltext', 'chunked', 'lines'):
+    adapter_registry.register_lazy(('knit-delta-gz', target_storage_kind), 'breezy.bzr.knit',
+                                   'DeltaPlainToFullText')
+    adapter_registry.register_lazy(('knit-ft-gz', target_storage_kind), 'breezy.bzr.knit',
+                                   'FTPlainToFullText')
+    adapter_registry.register_lazy(('knit-annotated-ft-gz', target_storage_kind),
+                                   'breezy.bzr.knit', 'FTAnnotatedToFullText')
+    adapter_registry.register_lazy(('knit-annotated-delta-gz', target_storage_kind),
+                                   'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
 
 
 class ContentFactory(object):
@@ -110,21 +109,27 @@
     :ivar parents: A tuple of parent keys for self.key. If the object has
         no parent information, None (as opposed to () for an empty list of
         parents).
+    :ivar chunks_are_lines: Whether chunks are lines.
      """
 
-    def __init__(self, key, parents, sha1, chunks):
+    def __init__(self, key, parents, sha1, chunks, chunks_are_lines=None):
         """Create a ContentFactory."""
         self.sha1 = sha1
         self.storage_kind = 'chunked'
         self.key = key
         self.parents = parents
         self._chunks = chunks
+        self._chunks_are_lines = chunks_are_lines
 
     def get_bytes_as(self, storage_kind):
         if storage_kind == 'chunked':
             return self._chunks
         elif storage_kind == 'fulltext':
             return b''.join(self._chunks)
+        elif storage_kind == 'lines':
+            if self._chunks_are_lines:
+                return self._chunks
+            return list(osutils.chunks_to_lines(self._chunks))
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
@@ -160,6 +165,38 @@
             return self._text
         elif storage_kind == 'chunked':
             return [self._text]
+        elif storage_kind == 'lines':
+            return osutils.split_lines(self._text)
+        raise errors.UnavailableRepresentation(self.key, storage_kind,
+                                               self.storage_kind)
+
+
+class FileContentFactory(ContentFactory):
+    """File-based content factory.
+    """
+
+    def __init__(self, key, parents, fileobj):
+        self.key = key
+        self.parents = parents
+        self.file = fileobj
+        self.storage_kind = 'file'
+        self._sha1 = None
+
+    @property
+    def sha1(self):
+        if self._sha1 is None:
+            self.file.seek(0)
+            self._size, self._sha1 = osutils.size_sha_file(self.file)
+        return self._sha1
+
+    def get_bytes_as(self, storage_kind):
+        self.file.seek(0)
+        if storage_kind == 'fulltext':
+            return self.file.read()
+        elif storage_kind == 'chunked':
+            return list(osutils.file_iterator(self.file))
+        elif storage_kind == 'lines':
+            return self.file.readlines()
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
@@ -1030,13 +1067,11 @@
                                   if not mpvf.has_version(p))
         # It seems likely that adding all the present parents as fulltexts can
         # easily exhaust memory.
-        chunks_to_lines = osutils.chunks_to_lines
         for record in self.get_record_stream(needed_parents, 'unordered',
                                              True):
             if record.storage_kind == 'absent':
                 continue
-            mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
-                             record.key, [])
+            mpvf.add_version(record.get_bytes_as('lines'), record.key, [])
         for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
                 records, mpvf.get_line_list(versions)):
             if len(parent_keys) == 1:
@@ -1546,7 +1581,9 @@
                 lines = self._lines[key]
                 parents = self._parents[key]
                 pending.remove(key)
-                yield ChunkedContentFactory(key, parents, None, lines)
+                yield ChunkedContentFactory(
+                    key, parents, None, lines,
+                    chunks_are_lines=True)
         for versionedfile in self.fallback_versionedfiles:
             for record in versionedfile.get_record_stream(
                     pending, 'unordered', True):
@@ -1775,9 +1812,9 @@
             if lines is not None:
                 if not isinstance(lines, list):
                     raise AssertionError
-                yield ChunkedContentFactory((k,), None,
-                                            sha1=osutils.sha_strings(lines),
-                                            chunks=lines)
+                yield ChunkedContentFactory(
+                    (k,), None, sha1=osutils.sha_strings(lines),
+                    chunks=lines, chunks_are_lines=True)
             else:
                 yield AbsentContentFactory((k,))
 

=== modified file 'breezy/bzr/vf_repository.py'
--- breezy/bzr/vf_repository.py	2020-01-20 00:03:54 +0000
+++ breezy/bzr/vf_repository.py	2020-01-25 16:26:49 +0000
@@ -853,8 +853,8 @@
         """Check a single text from this repository."""
         if kind == 'inventories':
             rev_id = record.key[0]
-            inv = self._deserialise_inventory(rev_id,
-                                              record.get_bytes_as('fulltext'))
+            inv = self._deserialise_inventory(
+                rev_id, record.get_bytes_as('fulltext'))
             if last_object is not None:
                 delta = inv._make_delta(last_object)
                 for old_path, path, file_id, ie in delta:
@@ -880,14 +880,9 @@
         """Check a single text."""
         # Check it is extractable.
         # TODO: check length.
-        if record.storage_kind == 'chunked':
-            chunks = record.get_bytes_as(record.storage_kind)
-            sha1 = osutils.sha_strings(chunks)
-            length = sum(map(len, chunks))
-        else:
-            content = record.get_bytes_as('fulltext')
-            sha1 = osutils.sha_string(content)
-            length = len(content)
+        chunks = record.get_bytes_as('chunked')
+        sha1 = osutils.sha_strings(chunks)
+        length = sum(map(len, chunks))
         if item_data and sha1 != item_data[1]:
             checker._report_items.append(
                 'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
@@ -1197,9 +1192,8 @@
         stream = self.inventories.get_record_stream(keys, 'unordered', True)
         for record in stream:
             if record.storage_kind != 'absent':
-                chunks = record.get_bytes_as('chunked')
+                lines = record.get_bytes_as('lines')
                 revid = record.key[-1]
-                lines = osutils.chunks_to_lines(chunks)
                 for line in lines:
                     yield line, revid
 

=== modified file 'breezy/bzr/weave.py'
--- breezy/bzr/weave.py	2019-03-02 21:46:18 +0000
+++ breezy/bzr/weave.py	2020-01-25 16:26:49 +0000
@@ -179,7 +179,7 @@
     def get_bytes_as(self, storage_kind):
         if storage_kind == 'fulltext':
             return self._weave.get_text(self.key[-1])
-        elif storage_kind == 'chunked':
+        elif storage_kind in ('chunked', 'lines'):
             return self._weave.get_lines(self.key[-1])
         else:
             raise UnavailableRepresentation(self.key, storage_kind, 'fulltext')
@@ -422,20 +422,19 @@
                 raise RevisionNotPresent([record.key[0]], self)
             # adapt to non-tuple interface
             parents = [parent[0] for parent in record.parents]
-            if (record.storage_kind == 'fulltext' or
-                    record.storage_kind == 'chunked'):
+            if record.storage_kind in ('fulltext', 'chunked', 'lines'):
                 self.add_lines(
                     record.key[0], parents,
-                    osutils.chunks_to_lines(record.get_bytes_as('chunked')))
+                    record.get_bytes_as('lines'))
             else:
-                adapter_key = record.storage_kind, 'fulltext'
+                adapter_key = record.storage_kind, 'lines'
                 try:
                     adapter = adapters[adapter_key]
                 except KeyError:
                     adapter_factory = adapter_registry.get(adapter_key)
                     adapter = adapter_factory(self)
                     adapters[adapter_key] = adapter
-                lines = split_lines(adapter.get_bytes(record))
+                lines = adapter.get_bytes(record, 'lines')
                 try:
                     self.add_lines(record.key[0], parents, lines)
                 except RevisionAlreadyPresent:

=== modified file 'breezy/git/annotate.py'
--- breezy/git/annotate.py	2019-10-20 23:46:25 +0000
+++ breezy/git/annotate.py	2020-01-25 16:26:49 +0000
@@ -22,6 +22,7 @@
     tree_lookup_path,
     )
 
+from .. import osutils
 from ..errors import (
     NoSuchRevision,
     UnavailableRepresentation,
@@ -32,7 +33,7 @@
     )
 
 
-class GitFulltextContentFactory(object):
+class GitBlobContentFactory(object):
     """Static data content factory.
 
     This takes a fulltext when created and just returns that during
@@ -52,17 +53,19 @@
         """Create a ContentFactory."""
         self.store = store
         self.key = (path, revision)
-        self.storage_kind = 'fulltext'
+        self.storage_kind = 'git-blob'
         self.parents = None
         self.blob_id = blob_id
 
     def get_bytes_as(self, storage_kind):
         if storage_kind == 'fulltext':
             return self.store[self.blob_id].as_raw_string()
+        elif storage_kind == 'lines':
+            return osutils.chunks_to_lines(self.store[self.blob_id].as_raw_chunks())
         elif storage_kind == 'chunked':
             return self.store[self.blob_id].as_raw_chunks()
         raise UnavailableRepresentation(self.key, storage_kind,
-                                        'fulltext')
+                                        self.storage_kind)
 
 
 class GitAbsentContentFactory(object):
@@ -153,5 +156,5 @@
             except KeyError:
                 yield GitAbsentContentFactory(store, path, text_revision)
             else:
-                yield GitFulltextContentFactory(
+                yield GitBlobContentFactory(
                     store, path, text_revision, blob_sha)

=== modified file 'breezy/merge.py'
--- breezy/merge.py	2020-01-18 16:14:28 +0000
+++ breezy/merge.py	2020-01-25 16:26:49 +0000
@@ -2007,8 +2007,7 @@
         for record in self.vf.get_record_stream(keys, 'unordered', True):
             if record.storage_kind == 'absent':
                 raise errors.RevisionNotPresent(record.key, self.vf)
-            result[record.key[-1]] = osutils.chunks_to_lines(
-                record.get_bytes_as('chunked'))
+            result[record.key[-1]] = record.get_bytes_as('lines')
         return result
 
     def plan_merge(self):

=== modified file 'breezy/plugins/weave_fmt/repository.py'
--- breezy/plugins/weave_fmt/repository.py	2018-11-12 01:41:38 +0000
+++ breezy/plugins/weave_fmt/repository.py	2020-01-25 16:26:49 +0000
@@ -628,19 +628,19 @@
             if record.storage_kind == 'absent':
                 raise errors.RevisionNotPresent([record.key[0]], self)
             # adapt to non-tuple interface
-            if record.storage_kind == 'fulltext':
+            if record.storage_kind in ('fulltext', 'chunks', 'lines'):
                 self.add_lines(record.key, None,
-                               osutils.split_lines(record.get_bytes_as('fulltext')))
+                               record.get_bytes_as('lines'))
             else:
-                adapter_key = record.storage_kind, 'fulltext'
+                adapter_key = record.storage_kind, 'lines'
                 try:
                     adapter = adapters[adapter_key]
                 except KeyError:
                     adapter_factory = adapter_registry.get(adapter_key)
                     adapter = adapter_factory(self)
                     adapters[adapter_key] = adapter
-                lines = osutils.split_lines(adapter.get_bytes(
-                    record, record.get_bytes_as(record.storage_kind)))
+                lines = adapter.get_bytes(
+                    record, record.get_bytes_as(record.storage_kind))
                 try:
                     self.add_lines(record.key, None, lines)
                 except errors.RevisionAlreadyPresent:

=== modified file 'breezy/tests/per_versionedfile.py'
--- breezy/tests/per_versionedfile.py	2019-06-16 15:54:33 +0000
+++ breezy/tests/per_versionedfile.py	2020-01-25 16:26:49 +0000
@@ -1214,7 +1214,11 @@
         # Each is source_kind, requested_kind, adapter class
         scenarios = [
             ('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
+            ('knit-delta-gz', 'lines', _mod_knit.DeltaPlainToFullText),
+            ('knit-delta-gz', 'chunked', _mod_knit.DeltaPlainToFullText),
             ('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
+            ('knit-ft-gz', 'lines', _mod_knit.FTPlainToFullText),
+            ('knit-ft-gz', 'chunked', _mod_knit.FTPlainToFullText),
             ('knit-annotated-delta-gz', 'knit-delta-gz',
                 _mod_knit.DeltaAnnotatedToUnannotated),
             ('knit-annotated-delta-gz', 'fulltext',
@@ -1223,6 +1227,10 @@
                 _mod_knit.FTAnnotatedToUnannotated),
             ('knit-annotated-ft-gz', 'fulltext',
                 _mod_knit.FTAnnotatedToFullText),
+            ('knit-annotated-ft-gz', 'lines',
+                _mod_knit.FTAnnotatedToFullText),
+            ('knit-annotated-ft-gz', 'chunked',
+                _mod_knit.FTAnnotatedToFullText),
             ]
         for source, requested, klass in scenarios:
             adapter_factory = versionedfile.adapter_registry.get(
@@ -1235,16 +1243,16 @@
         transport = self.get_transport()
         return make_file_factory(annotated, mapper)(transport)
 
-    def helpGetBytes(self, f, ft_adapter, delta_adapter):
+    def helpGetBytes(self, f, ft_name, ft_adapter, delta_name, delta_adapter):
         """Grab the interested adapted texts for tests."""
         # origin is a fulltext
         entries = f.get_record_stream([(b'origin',)], 'unordered', False)
         base = next(entries)
-        ft_data = ft_adapter.get_bytes(base)
+        ft_data = ft_adapter.get_bytes(base, ft_name)
         # merged is both a delta and multiple parents.
         entries = f.get_record_stream([(b'merged',)], 'unordered', False)
         merged = next(entries)
-        delta_data = delta_adapter.get_bytes(merged)
+        delta_data = delta_adapter.get_bytes(merged, delta_name)
         return ft_data, delta_data
 
     def test_deannotation_noeol(self):
@@ -1252,10 +1260,9 @@
         # we need a full text, and a delta
         f = self.get_knit()
         get_diamond_files(f, 1, trailing_eol=False)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTAnnotatedToUnannotated(
-                                                    None),
-                                                _mod_knit.DeltaAnnotatedToUnannotated(None))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'knit-ft-gz', _mod_knit.FTAnnotatedToUnannotated(None),
+            'knit-delta-gz', _mod_knit.DeltaAnnotatedToUnannotated(None))
         self.assertEqual(
             b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
             b'origin\n'
@@ -1271,10 +1278,9 @@
         # we need a full text, and a delta
         f = self.get_knit()
         get_diamond_files(f, 1)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTAnnotatedToUnannotated(
-                                                    None),
-                                                _mod_knit.DeltaAnnotatedToUnannotated(None))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'knit-ft-gz', _mod_knit.FTAnnotatedToUnannotated(None),
+            'knit-delta-gz', _mod_knit.DeltaAnnotatedToUnannotated(None))
         self.assertEqual(
             b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
             b'origin\n'
@@ -1293,10 +1299,9 @@
         # Reconstructing a full text requires a backing versioned file, and it
         # must have the base lines requested from it.
         logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTAnnotatedToFullText(
-                                                    None),
-                                                _mod_knit.DeltaAnnotatedToFullText(logged_vf))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'fulltext', _mod_knit.FTAnnotatedToFullText(None),
+            'fulltext', _mod_knit.DeltaAnnotatedToFullText(logged_vf))
         self.assertEqual(b'origin', ft_data)
         self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
         self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
@@ -1310,10 +1315,9 @@
         # Reconstructing a full text requires a backing versioned file, and it
         # must have the base lines requested from it.
         logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTAnnotatedToFullText(
-                                                    None),
-                                                _mod_knit.DeltaAnnotatedToFullText(logged_vf))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'fulltext', _mod_knit.FTAnnotatedToFullText(None),
+            'fulltext', _mod_knit.DeltaAnnotatedToFullText(logged_vf))
         self.assertEqual(b'origin\n', ft_data)
         self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
         self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
@@ -1330,10 +1334,9 @@
         # Reconstructing a full text requires a backing versioned file, and it
         # must have the base lines requested from it.
         logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTPlainToFullText(
-                                                    None),
-                                                _mod_knit.DeltaPlainToFullText(logged_vf))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'fulltext', _mod_knit.FTPlainToFullText(None),
+            'fulltext', _mod_knit.DeltaPlainToFullText(logged_vf))
         self.assertEqual(b'origin\n', ft_data)
         self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
         self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
@@ -1350,10 +1353,9 @@
         # Reconstructing a full text requires a backing versioned file, and it
         # must have the base lines requested from it.
         logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
-        ft_data, delta_data = self.helpGetBytes(f,
-                                                _mod_knit.FTPlainToFullText(
-                                                    None),
-                                                _mod_knit.DeltaPlainToFullText(logged_vf))
+        ft_data, delta_data = self.helpGetBytes(
+            f, 'fulltext', _mod_knit.FTPlainToFullText(None),
+            'fulltext', _mod_knit.DeltaPlainToFullText(logged_vf))
         self.assertEqual(b'origin', ft_data)
         self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
         self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',