Repository navigation
Expand file tree
/
Copy pathLazyBlocksReader.py
More file actions
299 lines (230 loc) · 9.39 KB
/
Copy pathLazyBlocksReader.py
File metadata and controls
299 lines (230 loc) · 9.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
"""
Lazy-loading reader for UnityFS bundle blocks.
Implements on-demand decompression of bundle blocks with LRU caching,
reducing memory usage for large asset bundles while maintaining
compatibility with the EndianBinaryReader interface.
"""
class LazyBlocksReader:
"""
Lazy-loading reader that decompresses blocks on-demand.
Fully compatible with EndianBinaryReader interface - just reads data lazily.
This reader maintains an LRU cache of decompressed blocks to balance
memory usage and decompression overhead. When reading data, it determines
which block contains the requested position, decompresses only that block
(if not already cached), and returns the requested data.
Attributes:
bundle: BundleFile instance for decompression
blocks_info: List of BlockInfo tuples describing compression metadata
source_reader: Original file reader for accessing compressed data
blocks_start_offset: File offset where compressed blocks begin
BaseOffset: Base offset for position calculations
Length: Total uncompressed size of all blocks
max_cached_blocks: Maximum number of blocks to keep in cache
"""
def __init__(self, bundle, blocks_info, source_reader, blocks_start_offset, base_offset, max_cached_blocks=5):
self.bundle = bundle # BundleFile instance for decompression
self.blocks_info = blocks_info
self.source_reader = source_reader # Original file reader
self.blocks_start_offset = blocks_start_offset
self.BaseOffset = base_offset
# Calculate total uncompressed size
self.Length = sum(block.uncompressedSize for block in blocks_info)
# Track position
self._position = 0
# Cache for decompressed blocks (LRU-style)
self._block_cache = {}
self._cache_order = [] # Track access order for LRU
self._max_cached_blocks = max_cached_blocks
@property
def Position(self):
return self._position
@Position.setter
def Position(self, value):
if value < 0:
raise ValueError("Position cannot be negative")
self._position = value
@property
def endian(self):
return self.source_reader.endian
@endian.setter
def endian(self, value):
self.source_reader.endian = value
def real_offset(self):
"""Returns offset in the underlying file."""
return self.BaseOffset + self._position
def read(self, size=None):
"""Read bytes from the virtual decompressed stream."""
if size is None or size == -1:
size = self.Length - self._position
if size == 0:
return b""
result = b""
remaining = size
while remaining > 0 and self._position < self.Length:
# Find which block contains current position
block_idx, offset_in_block = self._find_block_for_position(self._position)
if block_idx is None:
break
# Get the decompressed block
block_data = self._get_block(block_idx)
# Read from the block
available = len(block_data) - offset_in_block
to_read = min(remaining, available)
result += block_data[offset_in_block:offset_in_block + to_read]
self._position += to_read
remaining -= to_read
return result
def _find_block_for_position(self, position):
"""Find which block contains the given position."""
current_offset = 0
for idx, block_info in enumerate(self.blocks_info):
block_end = current_offset + block_info.uncompressedSize
if current_offset <= position < block_end:
return idx, position - current_offset
current_offset = block_end
return None, None
def _get_block(self, block_idx):
"""Get a decompressed block, using LRU cache."""
# Check cache
if block_idx in self._block_cache:
# Move to end (most recently used)
self._cache_order.remove(block_idx)
self._cache_order.append(block_idx)
return self._block_cache[block_idx]
block_info = self.blocks_info[block_idx]
# Calculate file offset for this block
file_offset = self.blocks_start_offset
for i in range(block_idx):
file_offset += self.blocks_info[i].compressedSize
# Read compressed data from source
original_pos = self.source_reader.Position
self.source_reader.Position = file_offset
compressed_data = self.source_reader.read_bytes(block_info.compressedSize)
self.source_reader.Position = original_pos
# Decompress
decompressed = self.bundle.decompress_data(
compressed_data,
block_info.uncompressedSize,
block_info.flags,
block_idx
)
# Add to cache with LRU eviction
self._block_cache[block_idx] = decompressed
self._cache_order.append(block_idx)
# Evict oldest if cache is full
while len(self._block_cache) > self._max_cached_blocks:
oldest = self._cache_order.pop(0)
del self._block_cache[oldest]
return decompressed
def create_view(self, offset, size):
"""
Create a restricted view of this reader for a specific range.
This allows passing the LazyBlocksReader directly to SerializedFile
without reading the entire file into memory first.
Args:
offset: Start offset within the bundle
size: Size of the view
Returns:
LazyBlocksReaderView that reads only from the specified range
"""
return LazyBlocksReaderView(self, offset, size)
class LazyBlocksReaderView:
"""
A streaming view of a LazyBlocksReader for a specific file range.
Acts as a stream interface, forwarding read operations to the parent
LazyBlocksReader with offset adjustments. The parent handles block
decompression and caching.
This is a TRUE streaming proxy - it does NOT load all data upfront.
Only the data actually read() is decompressed from blocks.
"""
def __init__(self, parent_reader, start_offset, size):
"""
Create a view of the parent reader.
Args:
parent_reader: The LazyBlocksReader to read from
start_offset: Starting offset in parent reader (absolute)
size: Size of this view
"""
self.parent_reader = parent_reader
self.start_offset = start_offset
self.size = size
self._position = 0
# Stream interface methods (for EndianBinaryReader_Streamable)
def read(self, size=-1):
"""
Read bytes from the view, forwarding to parent reader.
Args:
size: Number of bytes to read. -1 means read to end of view.
Returns:
Bytes read from parent reader
"""
if size == -1:
size = self.size - self._position
if size <= 0:
return b''
# Don't read past view bounds
size = min(size, self.size - self._position)
# Calculate absolute position in parent
abs_position = self.start_offset + self._position
# Save parent position
saved_pos = self.parent_reader.Position
# Read from parent at absolute position
self.parent_reader.Position = abs_position
data = self.parent_reader.read(size)
# Restore parent position
self.parent_reader.Position = saved_pos
# Update view position
self._position += len(data)
return data
def seek(self, offset, whence=0):
"""
Seek to a position in the view.
Args:
offset: Offset to seek to
whence: 0=absolute, 1=relative, 2=from end
"""
if whence == 0: # Absolute
new_pos = offset
elif whence == 1: # Relative
new_pos = self._position + offset
elif whence == 2: # From end
new_pos = self.size + offset
else:
raise ValueError(f"Invalid whence: {whence}")
if new_pos < 0 or new_pos > self.size:
raise ValueError(f"Seek position {new_pos} out of range [0, {self.size}]")
self._position = new_pos
return self._position
def tell(self):
"""Return current position in the view."""
return self._position
# Properties for compatibility with EndianBinaryReader
@property
def Position(self):
"""Current position in the view (0-based)."""
return self._position
@Position.setter
def Position(self, value):
"""Set position in the view."""
if value < 0 or value > self.size:
raise ValueError(f"Position {value} out of range [0, {self.size}]")
self._position = value
@property
def Length(self):
"""Length of the view."""
return self.size
@property
def endian(self):
"""Get endian from parent reader."""
return getattr(self.parent_reader, 'endian', '>')
@endian.setter
def endian(self, value):
"""Endian setting (no-op, parent handles this)."""
pass
def real_offset(self):
"""Return the real absolute offset in the parent reader."""
return self.start_offset + self._position
@property
def BaseOffset(self):
"""Base offset of this view in the parent."""
return self.start_offset