Merge lp:~jelmer/brz/delta-refactor into lp:brz

[Jelmer Vernooij]

Use reader pattern

Use reader pattern

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'Cargo.lock'
--- Cargo.lock	2023-11-11 15:04:09 +0000
+++ Cargo.lock	2023-11-12 22:03:08 +0000
@@ -97,6 +97,7 @@
  "chrono",
  "crc32fast",
  "fancy-regex",
+ "flate2",
  "lazy-regex",
  "lazy_static",
  "log",
@@ -108,6 +109,7 @@
  "sha1",
  "tempfile",
  "xmltree",
+ "xz2",
 ]
 
 [[package]]
@@ -793,6 +795,17 @@
 ]
 
 [[package]]
+name = "lzma-sys"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
 name = "malloc_buf"
 version = "0.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -980,6 +993,12 @@
 checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
+name = "pkg-config"
+version = "0.3.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
+
+[[package]]
 name = "ppv-lite86"
 version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1752,6 +1771,15 @@
 ]
 
 [[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
+[[package]]
 name = "zerocopy"
 version = "0.7.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"

=== modified file 'crates/bazaar/Cargo.toml'
--- crates/bazaar/Cargo.toml	2023-11-11 15:04:09 +0000
+++ crates/bazaar/Cargo.toml	2023-11-12 22:03:08 +0000
@@ -28,6 +28,8 @@
 lazy-regex = "3.0.0"
 byteorder = "1.5.0"
 lru = "0.12.0"
+flate2 = "1.0.28"
+xz2 = "0.1.7"
 
 [target.'cfg(unix)'.dependencies]
 nix = { workspace = true }

=== modified file 'crates/bazaar/src/groupcompress/mod.rs'
--- crates/bazaar/src/groupcompress/mod.rs	2023-11-12 01:37:05 +0000
+++ crates/bazaar/src/groupcompress/mod.rs	2023-11-12 22:03:08 +0000
@@ -1,6 +1,8 @@
 pub mod delta;
 pub mod line_delta;
+use byteorder::ReadBytesExt;
 use sha1::{Digest as _, Sha1};
+use std::io::Read;
 
 lazy_static::lazy_static! {
     pub static ref NULL_SHA1: Vec<u8> = format!("{:x}", Sha1::new().finalize()).as_bytes().to_vec();
@@ -16,83 +18,173 @@
     data
 }
 
-pub fn decode_base128_int(data: &[u8]) -> (u128, usize) {
-    let mut offset = 0;
+pub fn read_base128_int<R: Read>(reader: &mut R) -> Result<u128, std::io::Error> {
     let mut val: u128 = 0;
     let mut shift = 0;
-    let mut bval = data[offset];
-    while bval >= 0x80 {
-        val |= ((bval & 0x7F) as u128) << shift;
+    let mut bval = [0];
+    reader.read_exact(&mut bval)?;
+    while bval[0] >= 0x80 {
+        val |= ((bval[0] & 0x7F) as u128) << shift;
+        reader.read_exact(&mut bval)?;
         shift += 7;
-        offset += 1;
-        bval = data[offset];
-    }
-    val |= (bval as u128) << shift;
-    offset += 1;
-    (val, offset)
-}
-
-pub type CopyInstruction = (usize, usize, usize);
-
+    }
+
+    val |= (bval[0] as u128) << shift;
+    Ok(val)
+}
+
+#[cfg(test)]
+mod test_base128_int {
+    #[test]
+    fn test_decode_base128_int() {
+        assert_eq!(super::decode_base128_int(&[0x00]), (0, 1));
+        assert_eq!(super::decode_base128_int(&[0x01]), (1, 1));
+        assert_eq!(super::decode_base128_int(&[0x7F]), (127, 1));
+        assert_eq!(super::decode_base128_int(&[0x80, 0x01]), (128, 2));
+        assert_eq!(super::decode_base128_int(&[0xFF, 0x01]), (255, 2));
+        assert_eq!(super::decode_base128_int(&[0x80, 0x02]), (256, 2));
+        assert_eq!(super::decode_base128_int(&[0x81, 0x02]), (257, 2));
+        assert_eq!(super::decode_base128_int(&[0x82, 0x02]), (258, 2));
+        assert_eq!(super::decode_base128_int(&[0xFF, 0x7F]), (16383, 2));
+        assert_eq!(super::decode_base128_int(&[0x80, 0x80, 0x01]), (16384, 3));
+        assert_eq!(super::decode_base128_int(&[0xFF, 0xFF, 0x7F]), (2097151, 3));
+        assert_eq!(
+            super::decode_base128_int(&[0x80, 0x80, 0x80, 0x01]),
+            (2097152, 4)
+        );
+        assert_eq!(
+            super::decode_base128_int(&[0xFF, 0xFF, 0xFF, 0x7F]),
+            (268435455, 4)
+        );
+        assert_eq!(
+            super::decode_base128_int(&[0x80, 0x80, 0x80, 0x80, 0x01]),
+            (268435456, 5)
+        );
+        assert_eq!(
+            super::decode_base128_int(&[0xFF, 0xFF, 0xFF, 0xFF, 0x7F]),
+            (34359738367, 5)
+        );
+        assert_eq!(
+            super::decode_base128_int(&[0x80, 0x80, 0x80, 0x80, 0x80, 0x01]),
+            (34359738368, 6)
+        );
+    }
+
+    #[test]
+    fn test_encode_base128_int() {
+        assert_eq!(super::encode_base128_int(0), [0x00]);
+        assert_eq!(super::encode_base128_int(1), [0x01]);
+        assert_eq!(super::encode_base128_int(127), [0x7F]);
+        assert_eq!(super::encode_base128_int(128), [0x80, 0x01]);
+        assert_eq!(super::encode_base128_int(255), [0xFF, 0x01]);
+        assert_eq!(super::encode_base128_int(256), [0x80, 0x02]);
+        assert_eq!(super::encode_base128_int(257), [0x81, 0x02]);
+        assert_eq!(super::encode_base128_int(258), [0x82, 0x02]);
+        assert_eq!(super::encode_base128_int(16383), [0xFF, 0x7F]);
+        assert_eq!(super::encode_base128_int(16384), [0x80, 0x80, 0x01]);
+        assert_eq!(super::encode_base128_int(2097151), [0xFF, 0xFF, 0x7F]);
+        assert_eq!(super::encode_base128_int(2097152), [0x80, 0x80, 0x80, 0x01]);
+        assert_eq!(
+            super::encode_base128_int(268435455),
+            [0xFF, 0xFF, 0xFF, 0x7F]
+        );
+        assert_eq!(
+            super::encode_base128_int(268435456),
+            [0x80, 0x80, 0x80, 0x80, 0x01]
+        );
+        assert_eq!(
+            super::encode_base128_int(34359738367),
+            [0xFF, 0xFF, 0xFF, 0xFF, 0x7F]
+        );
+        assert_eq!(
+            super::encode_base128_int(34359738368),
+            [0x80, 0x80, 0x80, 0x80, 0x80, 0x01]
+        );
+        assert_eq!(
+            super::encode_base128_int(4398046511103),
+            [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F]
+        );
+        assert_eq!(
+            super::encode_base128_int(4398046511104),
+            [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01]
+        );
+    }
+}
+
+#[deprecated]
+pub fn decode_base128_int(data: &[u8]) -> (u128, usize) {
+    let mut cursor = std::io::Cursor::new(data);
+    let val = read_base128_int(&mut cursor).unwrap();
+    (val, cursor.position() as usize)
+}
+
+#[deprecated]
 pub fn decode_copy_instruction(
     data: &[u8],
     cmd: u8,
     pos: usize,
-) -> Result<CopyInstruction, String> {
+) -> Result<(usize, usize, usize), String> {
+    let mut c = std::io::Cursor::new(&data[pos..]);
+
+    let (offset, length) = read_copy_instruction(&mut c, cmd).unwrap();
+
+    Ok((offset, length, pos + c.position() as usize))
+}
+
+pub type CopyInstruction = (usize, usize);
+
+pub fn read_copy_instruction<R: Read>(
+    reader: &mut R,
+    cmd: u8,
+) -> Result<CopyInstruction, std::io::Error> {
     if cmd & 0x80 != 0x80 {
-        return Err("copy instructions must have bit 0x80 set".to_string());
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "copy instructions must have bit 0x80 set".to_string(),
+        ));
     }
     let mut offset = 0;
     let mut length = 0;
-    let mut new_pos = pos;
 
     if cmd & 0x01 != 0 {
-        offset = data[new_pos] as usize;
-        new_pos += 1;
+        offset = reader.read_u8()? as usize;
     }
     if cmd & 0x02 != 0 {
-        offset |= (data[new_pos] as usize) << 8;
-        new_pos += 1;
+        offset |= (reader.read_u8()? as usize) << 8;
     }
     if cmd & 0x04 != 0 {
-        offset |= (data[new_pos] as usize) << 16;
-        new_pos += 1;
+        offset |= (reader.read_u8()? as usize) << 16;
     }
     if cmd & 0x08 != 0 {
-        offset |= (data[new_pos] as usize) << 24;
-        new_pos += 1;
+        offset |= (reader.read_u8()? as usize) << 24;
     }
     if cmd & 0x10 != 0 {
-        length = data[new_pos] as usize;
-        new_pos += 1;
+        length = reader.read_u8()? as usize;
     }
     if cmd & 0x20 != 0 {
-        length |= (data[new_pos] as usize) << 8;
-        new_pos += 1;
+        length |= (reader.read_u8()? as usize) << 8;
     }
     if cmd & 0x40 != 0 {
-        length |= (data[new_pos] as usize) << 16;
-        new_pos += 1;
+        length |= (reader.read_u8()? as usize) << 16;
     }
     if length == 0 {
         length = 65536;
     }
 
-    Ok((offset, length, new_pos))
+    Ok((offset, length))
 }
 
 pub fn apply_delta(basis: &[u8], delta: &[u8]) -> Result<Vec<u8>, String> {
-    let (target_length, mut pos) = decode_base128_int(delta);
+    let mut delta = &delta[..];
+    let target_length = read_base128_int(&mut delta).map_err(|e| e.to_string())?;
     let mut lines = Vec::new();
-    let len_delta = delta.len();
 
-    while pos < len_delta {
-        let cmd = delta[pos];
-        pos += 1;
+    while !delta.is_empty() {
+        let cmd = delta.read_u8().map_err(|e| e.to_string())?;
 
         if cmd & 0x80 != 0 {
-            let (offset, length, new_pos) = decode_copy_instruction(delta, cmd, pos)?;
-            pos = new_pos;
+            let (offset, length) =
+                read_copy_instruction(&mut delta, cmd).map_err(|e| e.to_string())?;
             let last = offset + length;
             if last > basis.len() {
                 return Err("data would copy bytes past the end of source".to_string());
@@ -102,8 +194,8 @@
             if cmd == 0 {
                 return Err("Command == 0 not supported yet".to_string());
             }
-            lines.extend_from_slice(&delta[pos..pos + cmd as usize]);
-            pos += cmd as usize;
+            lines.extend_from_slice(&delta[..cmd as usize]);
+            delta = &delta[cmd as usize..];
         }
     }
 
@@ -118,6 +210,32 @@
     Ok(lines)
 }
 
+#[cfg(test)]
+mod test_apply_delta {
+    const TEXT1: &[u8] = b"This is a bit
+of source text
+which is meant to be matched
+against other text
+";
+
+    const TEXT2: &[u8] = b"This is a bit
+of source text
+which is meant to differ from
+against other text
+";
+
+    #[test]
+    fn test_apply_delta() {
+        let target =
+            super::apply_delta(TEXT1, b"N\x90/\x1fdiffer from\nagainst other text\n").unwrap();
+        assert_eq!(target, TEXT2);
+        let target =
+            super::apply_delta(TEXT2, b"M\x90/\x1ebe matched\nagainst other text\n").unwrap();
+        assert_eq!(target, TEXT1);
+    }
+}
+
+#[deprecated]
 pub fn apply_delta_to_source(
     source: &[u8],
     delta_start: usize,

=== modified file 'crates/osutils/src/chunkreader.rs'
--- crates/osutils/src/chunkreader.rs	2023-11-12 17:50:30 +0000
+++ crates/osutils/src/chunkreader.rs	2023-11-12 22:03:08 +0000
@@ -1,23 +1,23 @@
 use std::borrow::Borrow;
 use std::io::Read;
 
-pub struct ChunksReader<'a, T: Borrow<[u8]>> {
-    chunks: Box<dyn Iterator<Item = T> + 'a>,
+pub struct ChunksReader<T: Borrow<[u8]>> {
+    chunks: Box<dyn Iterator<Item = T>>,
     current_chunk: Option<T>,
     position: usize,
 }
 
-impl<'a, T: Borrow<[u8]>> ChunksReader<'a, T> {
-    pub fn new(chunks: impl Iterator<Item = T> + 'a) -> Self {
+impl<T: Borrow<[u8]>> ChunksReader<T> {
+    pub fn new(chunks: Box<dyn Iterator<Item = T>>) -> Self {
         ChunksReader {
-            chunks: Box::new(chunks),
+            chunks,
             position: 0,
             current_chunk: None,
         }
     }
 }
 
-impl<'a, T: Borrow<[u8]>> Read for ChunksReader<'a, T> {
+impl<T: Borrow<[u8]>> Read for ChunksReader<T> {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
         let mut bytes_read = 0;
 
@@ -47,7 +47,7 @@
 #[test]
 fn test_chunks_reader_vec() {
     let chunks = vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
-    let mut reader = ChunksReader::new(Box::new(chunks.iter().map(|v| v.as_slice())));
+    let mut reader = ChunksReader::new(Box::new(chunks.into_iter()));
 
     let mut buf = [0; 4];
     assert_eq!(reader.read(&mut buf).unwrap(), 4);
@@ -65,7 +65,7 @@
 #[test]
 fn test_chunks_reader_slice() {
     let chunks = [[1, 2, 3], [4, 5, 6], [7, 8, 9]];
-    let mut reader = ChunksReader::new(Box::new(chunks.iter().map(|v| v.as_slice())));
+    let mut reader = ChunksReader::new(Box::new(chunks.into_iter()));
 
     let mut buf = [0; 4];
     assert_eq!(reader.read(&mut buf).unwrap(), 4);