Merge lp:~jelmer/brz/extract into lp:brz

[Jelmer Vernooij]

More chunkification of groupcompress APIs.

More chunkification of groupcompress APIs.

[Jelmer Vernooij (jelmer)]

review: Approve

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/696/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/699/

=== modified file 'breezy/bzr/bundle/serializer/v4.py'
--- breezy/bzr/bundle/serializer/v4.py	2020-01-30 16:13:31 +0000
+++ breezy/bzr/bundle/serializer/v4.py	2020-02-09 13:41:06 +0000
@@ -181,9 +181,9 @@
         """
         name = self.encode_name(repo_kind, revision_id, file_id)
         encoded_metadata = bencode.bencode(metadata)
-        self._container.add_bytes_record(encoded_metadata, [(name, )])
+        self._container.add_bytes_record([encoded_metadata], len(encoded_metadata), [(name, )])
         if metadata[b'storage_kind'] != b'header':
-            self._container.add_bytes_record(bytes, [])
+            self._container.add_bytes_record([bytes], len(bytes), [])
 
 
 class BundleReader(object):

=== modified file 'breezy/bzr/groupcompress.py'
--- breezy/bzr/groupcompress.py	2020-02-09 01:27:02 +0000
+++ breezy/bzr/groupcompress.py	2020-02-09 13:41:06 +0000
@@ -276,7 +276,7 @@
         :return: The bytes for the content
         """
         if start == end == 0:
-            return b''
+            return []
         self._ensure_content(end)
         # The bytes are 'f' or 'd' for the type, then a variable-length
         # base128 integer for the content size, then the actual content
@@ -297,9 +297,9 @@
             raise ValueError('end != len according to field header'
                              ' %s != %s' % (end, content_start + content_len))
         if c == b'f':
-            return self._content[content_start:end]
+            return [self._content[content_start:end]]
         # Must be type delta as checked above
-        return apply_delta_to_source(self._content, content_start, end)
+        return [apply_delta_to_source(self._content, content_start, end)]
 
     def set_chunked_content(self, content_chunks, length):
         """Set the content of this block to the given chunks."""
@@ -448,7 +448,7 @@
         #       get_bytes_as call? After Manager.get_record_stream() returns
         #       the object?
         self._manager = manager
-        self._bytes = None
+        self._chunks = None
         self.storage_kind = 'groupcompress-block'
         if not first:
             self.storage_kind = 'groupcompress-block-ref'
@@ -469,7 +469,7 @@
         except zlib.error as value:
             raise DecompressCorruption("zlib: " + str(value))
         block = self._manager._block
-        self._bytes = block.extract(self.key, self._start, self._end)
+        self._chunks = block.extract(self.key, self._start, self._end)
         # There are code paths that first extract as fulltext, and then
         # extract as storage_kind (smart fetch). So we don't break the
         # refcycle here, but instead in manager.get_record_stream()
@@ -482,24 +482,24 @@
             else:
                 return b''
         if storage_kind in ('fulltext', 'chunked', 'lines'):
-            if self._bytes is None:
+            if self._chunks is None:
                 self._extract_bytes()
             if storage_kind == 'fulltext':
-                return self._bytes
+                return b''.join(self._chunks)
             elif storage_kind == 'chunked':
-                return [self._bytes]
+                return self._chunks
             else:
-                return osutils.split_lines(self._bytes)
+                return osutils.chunks_to_lines(self._chunks)
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
     def iter_bytes_as(self, storage_kind):
-        if self._bytes is None:
+        if self._chunks is None:
             self._extract_bytes()
         if storage_kind == 'chunked':
-            return iter([self._bytes])
+            return iter(self._chunks)
         elif storage_kind == 'lines':
-            return iter(osutils.split_lines(self._bytes))
+            return iter(osutils.chunks_to_lines(self._chunks))
         raise errors.UnavailableRepresentation(self.key, storage_kind,
                                                self.storage_kind)
 
@@ -912,7 +912,7 @@
         """Extract a key previously added to the compressor.
 
         :param key: The key to extract.
-        :return: An iterable over bytes and the sha1.
+        :return: An iterable over chunks and the sha1.
         """
         (start_byte, start_chunk, end_byte,
          end_chunk) = self.labels_deltas[key]
@@ -926,7 +926,7 @@
                 raise ValueError('Index claimed fulltext len, but stored bytes'
                                  ' claim %s != %s'
                                  % (len(stored_bytes), data_len))
-            data = stored_bytes[offset + 1:]
+            data = [stored_bytes[offset + 1:]]
         else:
             if kind != b'd':
                 raise ValueError('Unknown content kind, bytes claim %s' % kind)
@@ -938,8 +938,8 @@
                 raise ValueError('Index claimed delta len, but stored bytes'
                                  ' claim %s != %s'
                                  % (len(stored_bytes), data_len))
-            data = apply_delta(source, stored_bytes[offset + 1:])
-        data_sha1 = osutils.sha_string(data)
+            data = [apply_delta(source, stored_bytes[offset + 1:])]
+        data_sha1 = osutils.sha_strings(data)
         return data, data_sha1
 
     def flush(self):
@@ -1662,9 +1662,9 @@
                         # self._compressor.
                         for factory in batcher.yield_factories(full_flush=True):
                             yield factory
-                        bytes, sha1 = self._compressor.extract(key)
+                        chunks, sha1 = self._compressor.extract(key)
                         parents = self._unadded_refs[key]
-                        yield FulltextContentFactory(key, parents, sha1, bytes)
+                        yield ChunkedContentFactory(key, parents, sha1, chunks)
                         continue
                     if batcher.add_key(key) > BATCH_SIZE:
                         # Ok, this batch is big enough.  Yield some results.

=== modified file 'breezy/bzr/knitpack_repo.py'
--- breezy/bzr/knitpack_repo.py	2020-01-25 14:07:41 +0000
+++ breezy/bzr/knitpack_repo.py	2020-02-09 13:41:06 +0000
@@ -687,7 +687,7 @@
                 else:
                     df, _ = knit._parse_record_header(key, raw_data)
                     df.close()
-                pos, size = writer.add_bytes_record(raw_data, names)
+                pos, size = writer.add_bytes_record([raw_data], len(raw_data), names)
                 write_index.add_node(key, eol_flag + b"%d %d" % (pos, size))
                 pb.update("Copied record", record_index)
                 record_index += 1
@@ -739,7 +739,7 @@
                     # check the header only
                     df, _ = knit._parse_record_header(key, raw_data)
                     df.close()
-                pos, size = writer.add_bytes_record(raw_data, names)
+                pos, size = writer.add_bytes_record([raw_data], len(raw_data), names)
                 write_index.add_node(key, eol_flag + b"%d %d" %
                                      (pos, size), references)
                 pb.update("Copied record", record_index)

=== modified file 'breezy/bzr/pack.py'
--- breezy/bzr/pack.py	2018-11-11 04:08:32 +0000
+++ breezy/bzr/pack.py	2020-02-09 13:41:06 +0000
@@ -141,10 +141,11 @@
         """Finish writing a container."""
         self.write_func(self._serialiser.end())
 
-    def add_bytes_record(self, bytes, names):
+    def add_bytes_record(self, chunks, length, names):
         """Add a Bytes record with the given names.
 
-        :param bytes: The bytes to insert.
+        :param bytes: The chunks to insert.
+        :param length: Total length of bytes in chunks
         :param names: The names to give the inserted bytes. Each name is
             a tuple of bytestrings. The bytestrings may not contain
             whitespace.
@@ -156,13 +157,13 @@
             and thus are only suitable for use by a ContainerReader.
         """
         current_offset = self.current_offset
-        length = len(bytes)
         if length < self._JOIN_WRITES_THRESHOLD:
             self.write_func(self._serialiser.bytes_header(length, names)
-                            + bytes)
+                            + b''.join(chunks))
         else:
             self.write_func(self._serialiser.bytes_header(length, names))
-            self.write_func(bytes)
+            for chunk in chunks:
+                self.write_func(chunk)
         self.records_written += 1
         # return a memo of where we wrote data to allow random access.
         return current_offset, self.current_offset - current_offset

=== modified file 'breezy/bzr/pack_repo.py'
--- breezy/bzr/pack_repo.py	2020-01-26 13:58:01 +0000
+++ breezy/bzr/pack_repo.py	2020-02-09 13:41:06 +0000
@@ -1978,7 +1978,7 @@
             object supplied to the PackAccess object.
         """
         p_offset, p_length = self._container_writer.add_bytes_record(
-            b''.join(raw_data), [])
+            raw_data, size, [])
         return (self._write_index, p_offset, p_length)
 
     def add_raw_records(self, key_sizes, raw_data):

=== modified file 'breezy/tests/test_bundle.py'
--- breezy/tests/test_bundle.py	2020-01-30 16:13:31 +0000
+++ breezy/tests/test_bundle.py	2020-02-09 13:41:06 +0000
@@ -1796,7 +1796,7 @@
         writer = v4.BundleWriter(fileobj)
         writer.begin()
         writer.add_info_record({b'foo': b'bar'})
-        writer._container.add_bytes_record(b'blah', [(b'two', ), (b'names', )])
+        writer._container.add_bytes_record([b'blah'], len(b'blah'), [(b'two', ), (b'names', )])
         writer.end()
         fileobj.seek(0)
         record_iter = v4.BundleReader(fileobj).iter_records()

=== modified file 'breezy/tests/test_groupcompress.py'
--- breezy/tests/test_groupcompress.py	2020-01-26 04:30:31 +0000
+++ breezy/tests/test_groupcompress.py	2020-02-09 13:41:06 +0000
@@ -124,12 +124,12 @@
         sha1_2, _, end_point, _ = compressor.compress(
             ('newlabel',), [text], len(text), None)
         # get the first out
-        self.assertEqual((b'strange\ncommon long line\n'
-                          b'that needs a 16 byte match\n', sha1_1),
+        self.assertEqual(([b'strange\ncommon long line\n'
+                           b'that needs a 16 byte match\n'], sha1_1),
                          compressor.extract(('label',)))
         # and the second
-        self.assertEqual((b'common long line\nthat needs a 16 byte match\n'
-                          b'different\n', sha1_2),
+        self.assertEqual(([b'common long line\nthat needs a 16 byte match\n'
+                           b'different\n'], sha1_2),
                          compressor.extract(('newlabel',)))
 
     def test_pop_last(self):

=== modified file 'breezy/tests/test_pack.py'
--- breezy/tests/test_pack.py	2018-11-12 01:41:38 +0000
+++ breezy/tests/test_pack.py	2020-02-09 13:41:06 +0000
@@ -119,14 +119,14 @@
     def test_non_empty_end_does_not_add_a_record_to_records_written(self):
         """The end() method does not count towards the records written."""
         self.writer.begin()
-        self.writer.add_bytes_record(b'foo', names=[])
+        self.writer.add_bytes_record([b'foo'], len(b'foo'), names=[])
         self.writer.end()
         self.assertEqual(1, self.writer.records_written)
 
     def test_add_bytes_record_no_name(self):
         """Add a bytes record with no name."""
         self.writer.begin()
-        offset, length = self.writer.add_bytes_record(b'abc', names=[])
+        offset, length = self.writer.add_bytes_record([b'abc'], len(b'abc'), names=[])
         self.assertEqual((42, 7), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\nB3\n\nabc')
@@ -136,7 +136,7 @@
         self.writer.begin()
 
         offset, length = self.writer.add_bytes_record(
-            b'abc', names=[(b'name1', )])
+            [b'abc'], len(b'abc'), names=[(b'name1', )])
         self.assertEqual((42, 13), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -157,7 +157,7 @@
 
         self.writer.begin()
         offset, length = self.writer.add_bytes_record(
-            b'abcabc', names=[(b'name1', )])
+            [b'abcabc'], len(b'abcabc'), names=[(b'name1', )])
         self.assertEqual((42, 16), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -173,7 +173,7 @@
         """Add a bytes record with two names."""
         self.writer.begin()
         offset, length = self.writer.add_bytes_record(
-            b'abc', names=[(b'name1', ), (b'name2', )])
+            [b'abc'], len(b'abc'), names=[(b'name1', ), (b'name2', )])
         self.assertEqual((42, 19), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -183,7 +183,7 @@
         """Add a bytes record with two names."""
         self.writer.begin()
         offset, length = self.writer.add_bytes_record(
-            b'abc', names=[(b'name1', ), (b'name2', )])
+            [b'abc'], len(b'abc'), names=[(b'name1', ), (b'name2', )])
         self.assertEqual((42, 19), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -193,7 +193,7 @@
         """Add a bytes record with a two-element name."""
         self.writer.begin()
         offset, length = self.writer.add_bytes_record(
-            b'abc', names=[(b'name1', b'name2')])
+            [b'abc'], len(b'abc'), names=[(b'name1', b'name2')])
         self.assertEqual((42, 19), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -201,8 +201,8 @@
 
     def test_add_second_bytes_record_gets_higher_offset(self):
         self.writer.begin()
-        self.writer.add_bytes_record(b'abc', names=[])
-        offset, length = self.writer.add_bytes_record(b'abc', names=[])
+        self.writer.add_bytes_record([b'a', b'bc'], len(b'abc'), names=[])
+        offset, length = self.writer.add_bytes_record([b'abc'], len(b'abc'), names=[])
         self.assertEqual((49, 7), (offset, length))
         self.assertOutput(
             b'Bazaar pack format 1 (introduced in 0.18)\n'
@@ -216,14 +216,14 @@
         self.writer.begin()
         self.assertRaises(
             errors.InvalidRecordError,
-            self.writer.add_bytes_record, b'abc', names=[(b'bad name', )])
+            self.writer.add_bytes_record, [b'abc'], len(b'abc'), names=[(b'bad name', )])
 
     def test_add_bytes_records_add_to_records_written(self):
         """Adding a Bytes record increments the records_written counter."""
         self.writer.begin()
-        self.writer.add_bytes_record(b'foo', names=[])
+        self.writer.add_bytes_record([b'foo'], len(b'foo'), names=[])
         self.assertEqual(1, self.writer.records_written)
-        self.writer.add_bytes_record(b'foo', names=[])
+        self.writer.add_bytes_record([b'foo'], len(b'foo'), names=[])
         self.assertEqual(2, self.writer.records_written)
 
 
@@ -539,10 +539,10 @@
         writer = pack.ContainerWriter(pack_data.write)
         writer.begin()
         memos = []
-        memos.append(writer.add_bytes_record(b'abc', names=[]))
-        memos.append(writer.add_bytes_record(b'def', names=[(b'name1', )]))
-        memos.append(writer.add_bytes_record(b'ghi', names=[(b'name2', )]))
-        memos.append(writer.add_bytes_record(b'jkl', names=[]))
+        memos.append(writer.add_bytes_record([b'abc'], 3, names=[]))
+        memos.append(writer.add_bytes_record([b'def'], 3, names=[(b'name1', )]))
+        memos.append(writer.add_bytes_record([b'ghi'], 3, names=[(b'name2', )]))
+        memos.append(writer.add_bytes_record([b'jkl'], 3, names=[]))
         writer.end()
         transport = self.get_transport()
         transport.put_bytes('mypack', pack_data.getvalue())