Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion docs/lz4.block.rst
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,37 @@ did not take this precaution, the code, if ppassed invalid compressed
data would continuously try to allocate a larger and larger buffer for
decompression until the system ran out of memory.

Compressing into a fixed-size block
----------------------------------

``compress_dest_size`` wraps ``LZ4_compress_destSize``. It compresses as much of
its input as fits in ``target_size`` bytes and returns both the compressed block
and the number of source bytes consumed. Unlike ``compress`` with its default
settings, it does not prepend an uncompressed-size header.

.. doctest::

>>> source = bytes(range(256)) * 4
>>> block, consumed = lz4.block.compress_dest_size(source, target_size=64)
>>> len(block) <= 64 and 0 < consumed < len(source)
True
>>> lz4.block.decompress(block, uncompressed_size=consumed) == source[:consumed]
True

To compress the rest, call it again with ``source[consumed:]``. Each result is an
independent block; keep its consumed count for decompression. A very small target
can produce an empty block while consuming zero input bytes. Check for this
before advancing through input, and increase the target size if necessary.
``return_bytearray=True`` selects a mutable compressed result.

This function requires LZ4 1.9.2 or newer and raises ``RuntimeError`` with older
libraries. Earlier LZ4 versions have a known ``LZ4_compress_destSize`` issue that
can require an extra byte in the decompression buffer. The existing compression
and decompression APIs retain their library-version requirements.

Contents
----------------

.. automodule:: lz4.block
:members: compress, decompress
:members: compress, compress_dest_size, decompress

2 changes: 1 addition & 1 deletion lz4/block/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from ._block import compress, decompress, LZ4BlockError # noqa: F401
from ._block import compress, compress_dest_size, decompress, LZ4BlockError # noqa: F401
97 changes: 97 additions & 0 deletions lz4/block/_block.c
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,82 @@ compress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs)
return py_dest;
}

static PyObject *
compress_dest_size (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs)
{
Py_buffer source;
int target_size, source_size, dest_size, output_size;
int return_bytearray = 0;
char *dest;
PyObject *py_dest;
static char *argnames[] = {"source", "target_size", "return_bytearray", NULL};

if (!PyArg_ParseTupleAndKeywords (args, kwargs, "y*i|p", argnames,
&source, &target_size, &return_bytearray))
{
return NULL;
}

if (LZ4_versionNumber () < 10902)
{
PyBuffer_Release (&source);
PyErr_SetString (PyExc_RuntimeError,
"compress_dest_size requires LZ4 1.9.2 or newer");
return NULL;
}
if (target_size < 1)
{
PyBuffer_Release (&source);
PyErr_SetString (PyExc_ValueError, "target_size must be positive");
return NULL;
}
if (source.len > LZ4_MAX_INPUT_SIZE)
{
PyBuffer_Release (&source);
PyErr_SetString (PyExc_OverflowError, "Input too large for LZ4 API");
return NULL;
}

source_size = (int) source.len;
dest_size = LZ4_compressBound (source_size);
if (dest_size > target_size)
{
dest_size = target_size;
}
dest = PyMem_Malloc ((size_t) dest_size);
if (dest == NULL)
{
PyBuffer_Release (&source);
return PyErr_NoMemory ();
}

Py_BEGIN_ALLOW_THREADS
output_size = LZ4_compress_destSize (source.buf, dest, &source_size, dest_size);
Py_END_ALLOW_THREADS
PyBuffer_Release (&source);

if (output_size <= 0)
{
PyMem_Free (dest);
PyErr_SetString (LZ4BlockError, "Compression failed");
return NULL;
}
if (return_bytearray)
{
py_dest = PyByteArray_FromStringAndSize (dest, output_size);
}
else
{
py_dest = PyBytes_FromStringAndSize (dest, output_size);
}
PyMem_Free (dest);
if (py_dest == NULL)
{
return NULL;
}
return Py_BuildValue ("Ni", py_dest, source_size);
}

static PyObject *
decompress (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwargs)
{
Expand Down Expand Up @@ -432,6 +508,21 @@ PyDoc_STRVAR(compress__doc,
"Returns:\n" \
" bytes or bytearray: Compressed data.\n");

PyDoc_STRVAR(compress_dest_size__doc,
"compress_dest_size(source, target_size, return_bytearray=False)\n\n"
"Compress as much of source as fits in target_size bytes.\n\n"
"Args:\n"
" source (bytes or buffer-compatible object): Data to compress.\n"
" target_size (int): Positive maximum compressed block size.\n\n"
"Keyword Args:\n"
" return_bytearray (bool): Return a bytearray instead of bytes.\n\n"
"Returns:\n"
" tuple: (compressed block, number of source bytes consumed).\n"
" No size prefix is stored. Pass the consumed count as\n"
" uncompressed_size to decompress(). A very small target may\n"
" consume zero bytes; increase target_size before retrying.\n\n"
"Requires LZ4 1.9.2 or newer. Older versions raise RuntimeError.\n");

PyDoc_STRVAR(decompress__doc,
"decompress(source, uncompressed_size=-1, return_bytearray=False)\n\n" \
"Decompress source, returning the uncompressed data as a string.\n" \
Expand Down Expand Up @@ -473,6 +564,12 @@ static PyMethodDef module_methods[] = {
METH_VARARGS | METH_KEYWORDS,
compress__doc
},
{
"compress_dest_size",
(PyCFunction) compress_dest_size,
METH_VARARGS | METH_KEYWORDS,
compress_dest_size__doc
},
{
"decompress",
(PyCFunction) decompress,
Expand Down
83 changes: 83 additions & 0 deletions tests/block/test_compress_dest_size.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import os
from concurrent.futures import ThreadPoolExecutor

import pytest

import lz4.block
import lz4


pytestmark = pytest.mark.skipif(
lz4.library_version_number() < 10902,
reason='compress_dest_size requires LZ4 1.9.2 or newer',
)


@pytest.mark.parametrize('factory', [bytes, bytearray, memoryview])
@pytest.mark.parametrize('target_size', [1, 2, 16, 64, 4096])
@pytest.mark.parametrize('data', [b'', b'abcd' * 256, bytes(range(256)) * 4])
def test_roundtrip(factory, target_size, data):
source = factory(data)
compressed, consumed = lz4.block.compress_dest_size(source, target_size)
assert isinstance(compressed, bytes)
assert 0 <= consumed <= len(data)
assert len(compressed) <= target_size
assert lz4.block.decompress(compressed, uncompressed_size=consumed) == data[:consumed]
if target_size == 4096:
assert consumed == len(data)
if isinstance(source, bytearray):
source.extend(b'x') # No exported buffer may survive the call.


def test_compress_all_input_in_bounded_blocks():
source = os.urandom(16384)
offset = 0
blocks = []
while offset < len(source):
block, consumed = lz4.block.compress_dest_size(
memoryview(source)[offset:], target_size=128, return_bytearray=True,
)
assert isinstance(block, bytearray)
assert 0 < consumed <= len(source) - offset
assert len(block) <= 128
blocks.append(lz4.block.decompress(block, uncompressed_size=consumed))
offset += consumed
assert b''.join(blocks) == source


@pytest.mark.parametrize('size,error', [
(0, ValueError), (-1, ValueError), (2 ** 31, OverflowError),
('64', TypeError), (1.5, TypeError),
])
def test_invalid_target_releases_source(size, error):
source = bytearray(b'data')
with pytest.raises(error):
lz4.block.compress_dest_size(source, size)
source.extend(b'x')


def test_rejects_noncontiguous_input():
with pytest.raises(BufferError):
lz4.block.compress_dest_size(memoryview(b'abcdef')[::2], 64)


def test_parallel_calls():
def roundtrip(data):
block, consumed = lz4.block.compress_dest_size(data, 64)
return lz4.block.decompress(block, uncompressed_size=consumed) == data[:consumed]
with ThreadPoolExecutor(max_workers=4) as executor:
assert all(executor.map(roundtrip, [os.urandom(1024) for _ in range(32)]))


def test_input_above_lz4_limit_releases_buffer():
import mmap
# Anonymous mapping reserves address space without materializing a 2 GB bytes object.
with mmap.mmap(-1, 0x7E000001) as source:
with pytest.raises(OverflowError, match='Input too large'):
lz4.block.compress_dest_size(source, 64)


def test_large_target_does_not_require_large_allocation():
block, consumed = lz4.block.compress_dest_size(b'data', 2 ** 31 - 1)
assert consumed == 4
assert lz4.block.decompress(block, uncompressed_size=consumed) == b'data'