Merge lp:~jelmer/brz/chk-inventory-iter into lp:brz

[Jelmer Vernooij]

parse_text_bytes now takes a list of lines.

parse_text_bytes now takes a list of lines.

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'breezy/bzr/groupcompress.py'
--- breezy/bzr/groupcompress.py	2020-01-31 19:21:09 +0000
+++ breezy/bzr/groupcompress.py	2020-02-09 01:38:14 +0000
@@ -499,7 +499,7 @@
         if storage_kind == 'chunked':
             return iter([self._bytes])
         elif storage_kind == 'lines':
-            return osutils.split_lines(self._bytes)
+            return iter(osutils.split_lines(self._bytes))
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
 
=== modified file 'breezy/bzr/inventory_delta.py'
--- breezy/bzr/inventory_delta.py	2019-02-02 15:13:30 +0000
+++ breezy/bzr/inventory_delta.py	2020-02-09 01:38:14 +0000
@@ -277,23 +277,25 @@
         else:
             raise InventoryDeltaError("value %(val)r is not a bool", val=value)
 
-    def parse_text_bytes(self, bytes):
+    def parse_text_bytes(self, lines):
         """Parse the text bytes of a serialized inventory delta.
 
         If versioned_root and/or tree_references flags were set via
         require_flags, then the parsed flags must match or a BzrError will be
         raised.
 
-        :param bytes: The bytes to parse. This can be obtained by calling
-            delta_to_lines and then doing ''.join(delta_lines).
+        :param lines: The lines to parse. This can be obtained by calling
+            delta_to_lines.
         :return: (parent_id, new_id, versioned_root, tree_references,
             inventory_delta)
         """
-        if not bytes.endswith(b'\n'):
-            last_line = bytes.rsplit(b'\n', 1)[-1]
-            raise InventoryDeltaError(
-                'last line not empty: %(line)r', line=last_line)
-        lines = bytes.split(b'\n')[:-1]  # discard the last empty line
+        if not lines:
+            raise InventoryDeltaError(
+                'inventory delta is empty')
+        if not lines[-1].endswith(b'\n'):
+            raise InventoryDeltaError(
+                'last line not empty: %(line)r', line=lines[-1])
+        lines = [line.rstrip(b'\n') for line in lines]  # discard the last empty line
         if not lines or lines[0] != b'format: %s' % FORMAT_1:
             raise InventoryDeltaError(
                 'unknown format %(line)r', line=lines[0:1])
 
=== modified file 'breezy/bzr/remote.py'
--- breezy/bzr/remote.py	2020-01-31 10:39:02 +0000
+++ breezy/bzr/remote.py	2020-02-09 01:38:14 +0000
@@ -2048,7 +2048,7 @@
                 "Unexpected stream %r received" % substream_kind)
         for record in substream:
             (parent_id, new_id, versioned_root, tree_references, invdelta) = (
-                deserializer.parse_text_bytes(record.get_bytes_as("fulltext")))
+                deserializer.parse_text_bytes(record.get_bytes_as("lines")))
             if parent_id != prev_inv.revision_id:
                 raise AssertionError("invalid base %r != %r" % (parent_id,
                                                                 prev_inv.revision_id))
 
=== modified file 'breezy/bzr/versionedfile.py'
--- breezy/bzr/versionedfile.py	2020-01-30 18:07:32 +0000
+++ breezy/bzr/versionedfile.py	2020-02-09 01:38:14 +0000
@@ -143,7 +143,7 @@
         elif storage_kind == 'lines':
             if self._chunks_are_lines:
                 return iter(self._chunks)
-            return osutils.chunks_to_lines(self._chunks)
+            return iter(osutils.chunks_to_lines(self._chunks))
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
 
=== modified file 'breezy/bzr/vf_repository.py'
--- breezy/bzr/vf_repository.py	2020-01-31 10:39:02 +0000
+++ breezy/bzr/vf_repository.py	2020-02-09 01:38:14 +0000
@@ -1937,7 +1937,7 @@
         target_tree_refs = self.target_repo._format.supports_tree_reference
         for record in substream:
             # Insert the delta directly
-            inventory_delta_bytes = record.get_bytes_as('fulltext')
+            inventory_delta_bytes = record.get_bytes_as('lines')
             deserialiser = inventory_delta.InventoryDeltaDeserializer()
             try:
                 parse_result = deserialiser.parse_text_bytes(
 
=== modified file 'breezy/git/annotate.py'
--- breezy/git/annotate.py	2020-01-30 18:07:32 +0000
+++ breezy/git/annotate.py	2020-02-09 01:38:14 +0000
@@ -70,7 +70,7 @@
 
     def iter_bytes_as(self, storage_kind):
         if storage_kind == 'lines':
-            return osutils.chunks_to_lines(self.store[self.blob_id].as_raw_chunks())
+            return iter(osutils.chunks_to_lines(self.store[self.blob_id].as_raw_chunks()))
         elif storage_kind == 'chunked':
             return iter(self.store[self.blob_id].as_raw_chunks())
         raise UnavailableRepresentation(self.key, storage_kind,
 
=== modified file 'breezy/tests/test_inventory_delta.py'
--- breezy/tests/test_inventory_delta.py	2018-11-11 04:08:32 +0000
+++ breezy/tests/test_inventory_delta.py	2020-02-09 01:38:14 +0000
@@ -19,6 +19,7 @@
 See doc/developer/inventory.txt for more information.
 """
 
+from .. import osutils
 from ..bzr import (
     inventory,
     inventory_delta,
@@ -96,28 +97,28 @@
     def test_parse_no_bytes(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(
-            InventoryDeltaError, deserializer.parse_text_bytes, b'')
-        self.assertContainsRe(str(err), 'last line not empty')
+            InventoryDeltaError, deserializer.parse_text_bytes, [])
+        self.assertContainsRe(str(err), 'inventory delta is empty')
 
     def test_parse_bad_format(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, b'format: foo\n')
+                                deserializer.parse_text_bytes, [b'format: foo\n'])
         self.assertContainsRe(str(err), 'unknown format')
 
     def test_parse_no_parent(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
                                 deserializer.parse_text_bytes,
-                                b'format: bzr inventory delta v1 (bzr 1.14)\n')
+                                [b'format: bzr inventory delta v1 (bzr 1.14)\n'])
         self.assertContainsRe(str(err), 'missing parent: marker')
 
     def test_parse_no_version(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
                                 deserializer.parse_text_bytes,
-                                b'format: bzr inventory delta v1 (bzr 1.14)\n'
-                                b'parent: null:\n')
+                                [b'format: bzr inventory delta v1 (bzr 1.14)\n',
+                                 b'parent: null:\n'])
         self.assertContainsRe(str(err), 'missing version: marker')
 
     def test_parse_duplicate_key_errors(self):
@@ -132,12 +133,12 @@
 None\x00/\x00an-id\x00\x00a@e\xc3\xa5ample.com--2004\x00dir\x00\x00
 """
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, double_root_lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(double_root_lines))
         self.assertContainsRe(str(err), 'duplicate file id')
 
     def test_parse_versioned_root_only(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
-        parse_result = deserializer.parse_text_bytes(root_only_lines)
+        parse_result = deserializer.parse_text_bytes(osutils.split_lines(root_only_lines))
         expected_entry = inventory.make_entry(
             'directory', u'', None, b'an-id')
         expected_entry.revision = b'a@e\xc3\xa5ample.com--2004'
@@ -155,8 +156,9 @@
 tree_references: true
 None\x00/\x00TREE_ROOT\x00\x00null:\x00dir\x00\x00
 """
-        err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, root_only_lines)
+        err = self.assertRaises(
+            InventoryDeltaError, deserializer.parse_text_bytes,
+            osutils.split_lines(root_only_lines))
         self.assertContainsRe(str(err), 'special revisionid found')
 
     def test_parse_versioned_root_versioned_disabled(self):
@@ -168,8 +170,9 @@
 tree_references: true
 None\x00/\x00TREE_ROOT\x00\x00a@e\xc3\xa5ample.com--2004\x00dir\x00\x00
 """
-        err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, root_only_lines)
+        err = self.assertRaises(
+            InventoryDeltaError, deserializer.parse_text_bytes,
+            osutils.split_lines(root_only_lines))
         self.assertContainsRe(str(err), 'Versioned root found')
 
     def test_parse_unique_root_id_root_versioned_disabled(self):
@@ -182,12 +185,13 @@
 None\x00/\x00an-id\x00\x00parent-id\x00dir\x00\x00
 """
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, root_only_lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(root_only_lines))
         self.assertContainsRe(str(err), 'Versioned root found')
 
     def test_parse_unversioned_root_versioning_enabled(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer()
-        parse_result = deserializer.parse_text_bytes(root_only_unversioned)
+        parse_result = deserializer.parse_text_bytes(
+            osutils.split_lines(root_only_unversioned))
         expected_entry = inventory.make_entry(
             'directory', u'', None, b'TREE_ROOT')
         expected_entry.revision = b'entry-version'
@@ -200,14 +204,14 @@
         deserializer = inventory_delta.InventoryDeltaDeserializer(
             allow_versioned_root=False)
         err = self.assertRaises(inventory_delta.IncompatibleInventoryDelta,
-                                deserializer.parse_text_bytes, root_only_lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(root_only_lines))
         self.assertEqual("versioned_root not allowed", str(err))
 
     def test_parse_tree_when_disabled(self):
         deserializer = inventory_delta.InventoryDeltaDeserializer(
             allow_tree_references=False)
         err = self.assertRaises(inventory_delta.IncompatibleInventoryDelta,
-                                deserializer.parse_text_bytes, reference_lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(reference_lines))
         self.assertEqual("Tree reference not allowed", str(err))
 
     def test_parse_tree_when_header_disallows(self):
@@ -223,7 +227,7 @@
 None\x00/foo\x00id\x00TREE_ROOT\x00changed\x00tree\x00subtree-version
 """
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(lines))
         self.assertContainsRe(str(err), 'Tree reference found')
 
     def test_parse_versioned_root_when_header_disallows(self):
@@ -239,7 +243,7 @@
 None\x00/\x00TREE_ROOT\x00\x00a@e\xc3\xa5ample.com--2004\x00dir
 """
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(lines))
         self.assertContainsRe(str(err), 'Versioned root found')
 
     def test_parse_last_line_not_empty(self):
@@ -248,7 +252,7 @@
         lines = root_only_lines[:-1]
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(lines))
         self.assertContainsRe(str(err), 'last line not empty')
 
     def test_parse_invalid_newpath(self):
@@ -257,7 +261,7 @@
         lines += b"None\x00bad\x00TREE_ROOT\x00\x00version\x00dir\n"
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(lines))
         self.assertContainsRe(str(err), 'newpath invalid')
 
     def test_parse_invalid_oldpath(self):
@@ -266,7 +270,7 @@
         lines += b"bad\x00/new\x00file-id\x00\x00version\x00dir\n"
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         err = self.assertRaises(InventoryDeltaError,
-                                deserializer.parse_text_bytes, lines)
+                                deserializer.parse_text_bytes, osutils.split_lines(lines))
         self.assertContainsRe(str(err), 'oldpath invalid')
 
     def test_parse_new_file(self):
@@ -277,7 +281,7 @@
             b"None\x00/new\x00file-id\x00an-id\x00version\x00file\x00123\x00" +
             b"\x00" + fake_sha + b"\n")
         deserializer = inventory_delta.InventoryDeltaDeserializer()
-        parse_result = deserializer.parse_text_bytes(lines)
+        parse_result = deserializer.parse_text_bytes(osutils.split_lines(lines))
         expected_entry = inventory.make_entry(
             'file', u'new', b'an-id', b'file-id')
         expected_entry.revision = b'version'
@@ -292,7 +296,7 @@
         lines += (
             b"/old-file\x00None\x00deleted-id\x00\x00null:\x00deleted\x00\x00\n")
         deserializer = inventory_delta.InventoryDeltaDeserializer()
-        parse_result = deserializer.parse_text_bytes(lines)
+        parse_result = deserializer.parse_text_bytes(osutils.split_lines(lines))
         delta = parse_result[4]
         self.assertEqual(
             (u'old-file', None, b'deleted-id', None), delta[-1])
@@ -339,7 +343,7 @@
         deserializer = inventory_delta.InventoryDeltaDeserializer()
         self.assertEqual(
             (NULL_REVISION, b'entry-version', False, False, delta),
-            deserializer.parse_text_bytes(b''.join(serialized_lines)))
+            deserializer.parse_text_bytes(serialized_lines))
 
     def test_unversioned_non_root_errors(self):
         old_inv = Inventory(None)