Skip to content

gh-160310: Decompress a single stream directly in _decompress_all() #161737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maikschulze wants to merge 1 commit into
base: main
Choose a base branch
from
+19 −12
Open

gh-160310: Decompress a single stream directly in _decompress_all() #161737

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions Lib/compression/_common/_streams.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ def tell(self):
return self._pos


_TRUNCATED = ("Compressed data ended before the "
"end-of-stream marker was reached")


def _decompress_all(data, decomp_factory, trailing_error=(), *,
stream_magic=(), truncated_error, **decomp_args):
"""Decompress all of the concatenated streams in *data*.
Expand All @@ -181,18 +185,27 @@ def _decompress_all(data, decomp_factory, trailing_error=(), *,
data, unless that stream starts with *stream_magic*. A stream without
an end-of-stream marker raises *truncated_error*.
"""
results = []
# There is usually a single stream: decompress all of the data in one
# go, without the views and lists needed for more streams.
decomp = decomp_factory(**decomp_args)
first = decomp.decompress(data)
if not decomp.eof:
raise truncated_error(_TRUNCATED)
unused = len(decomp.unused_data)
if not unused:
return first
results = [first]
# The views are released on exit, so that the caller can resize a
# bytearray argument again, also while handling an exception.
with memoryview(data) as view, view.cast("B") as data:
pos = 0
# There is usually a single stream: the first chunk is all of the
# data, to decompress it in one go.
chunk_size = len(data)
pos = len(data) - unused
while True:
decomp = decomp_factory(**decomp_args)
start = pos
res = []
# Later streams start small: a larger first chunk reads further
# past the end of a short stream, a smaller one needs more calls.
chunk_size = 1024
try:
# Feed the stream in chunks of increasing size, so that what
# is read past its end (and copied into unused_data) is
Expand All @@ -204,8 +217,6 @@ def _decompress_all(data, decomp_factory, trailing_error=(), *,
pos += len(chunk)
chunk_size *= 2
except trailing_error:
if not results:
raise # Error in the first stream; bail out.
if stream_magic:
if isinstance(stream_magic, bytes):
stream_magic = (stream_magic,)
Expand All @@ -217,12 +228,8 @@ def _decompress_all(data, decomp_factory, trailing_error=(), *,
break
results.extend(res)
if not decomp.eof:
raise truncated_error("Compressed data ended before the "
"end-of-stream marker was reached")
raise truncated_error(_TRUNCATED)
pos -= len(decomp.unused_data)
if pos == len(data):
break
# Later streams start small: a larger first chunk reads further
# past the end of a short stream, a smaller one needs more calls.
chunk_size = 1024
return b"".join(results)
Loading