diff --git a/Cargo.toml b/Cargo.toml index e03a33aa..af5eefd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cramjam" -version = "2.3.2" +version = "2.4.0" authors = ["Miles Granger "] edition = "2018" license = "MIT License" diff --git a/src/gzip.rs b/src/gzip.rs index ebf416fb..ded656a6 100644 --- a/src/gzip.rs +++ b/src/gzip.rs @@ -54,15 +54,17 @@ pub fn decompress_into(input: BytesType, mut output: BytesType) -> PyResult(input: R, output: &mut W) -> Result { - let mut decoder = GzDecoder::new(input); - let n_bytes = std::io::copy(&mut decoder, output)?; + let mut decoder = MultiGzDecoder::new(input); + let mut out = vec![]; + let n_bytes = decoder.read_to_end(&mut out)?; + std::io::copy(&mut Cursor::new(out.as_slice()), output)?; Ok(n_bytes as usize) } @@ -73,4 +75,21 @@ pub(crate) mod internal { let n_bytes = std::io::copy(&mut encoder, output)?; Ok(n_bytes as usize) } + + #[cfg(test)] + mod tests { + + #[test] + fn test_gzip_multiple_streams() { + let mut out1 = vec![]; + let mut out2 = vec![]; + super::compress(b"foo".to_vec().as_slice(), &mut out1, None).unwrap(); + super::compress(b"bar".to_vec().as_slice(), &mut out2, None).unwrap(); + + let mut out3 = vec![]; + out1.extend_from_slice(&out2); + super::decompress(out1.as_slice(), &mut out3).unwrap(); + assert_eq!(out3, b"foobar".to_vec()); + } + } } diff --git a/src/lib.rs b/src/lib.rs index 6e74c69f..44851785 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -89,7 +89,7 @@ pub enum BytesType<'a> { RustyBuffer(&'a PyCell), /// `numpy.array` with `dtype=np.uint8` #[pyo3(transparent, annotation = "numpy")] - NumpyArray(RustyNumpyArray<'a>) + NumpyArray(RustyNumpyArray<'a>), } impl<'a> AsBytes for BytesType<'a> { diff --git a/tests/test_variants.py b/tests/test_variants.py index 85940fa4..87bedada 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -1,3 +1,4 @@ +import gzip import pytest import numpy as np import cramjam @@ -221,3 +222,20 @@ def test_lz4_block(compress_kwargs): output_len=len(data) if not compress_kwargs["store_size"] else None, ) assert bytes(out) == data + + +def test_gzip_multiple_streams(): + + out1 = gzip.compress(b"foo") + out2 = gzip.compress(b"bar") + assert gzip.decompress(out1 + out2) == b"foobar" + + # works with data compressed by std gzip lib + out = bytes(cramjam.gzip.decompress(out1 + out2)) + assert out == b"foobar" + + # works with data compressed by cramjam + o1 = bytes(cramjam.gzip.compress(b"foo")) + o2 = bytes(cramjam.gzip.compress(b"bar")) + out = bytes(cramjam.gzip.decompress(o1 + o2)) + assert out == b"foobar"