Skip to content

Commit ce87733

Browse files
committed
improve EndianBinaryReader performance
1 parent e4d8345 commit ce87733

1 file changed

Lines changed: 258 additions & 24 deletions

File tree

‎UnityPy/streams/EndianBinaryReader.py‎

Lines changed: 258 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,39 @@
11
import io
2-
from struct import unpack
2+
import sys
3+
from struct import Struct, unpack
4+
import re
5+
6+
reNot0 = re.compile(b"(.*?)\x00")
7+
8+
SYS_ENDIAN = "<" if sys.byteorder == "little" else ">"
39

410
from ..math import Color, Matrix4x4, Quaternion, Vector2, Vector3, Vector4, Rectangle
511

12+
# generate unpack and unpack_from functions
13+
TYPE_PARAM_SIZE_LIST = [
14+
("short", "h", 2),
15+
("u_short", "H", 2),
16+
("int", "i", 4),
17+
("u_int", "I", 4),
18+
("long", "q", 8),
19+
("u_long", "Q", 8),
20+
("half", "e", 2),
21+
("float", "f", 4),
22+
("double", "d", 8),
23+
("vec2", "2f", 8),
24+
("vec3", "3f", 12),
25+
("vec4", "4f", 16),
26+
]
27+
28+
LOCALS = locals()
29+
for endian_s, endian_l in (("<", "little"), (">", "big")):
30+
for typ, param, _ in TYPE_PARAM_SIZE_LIST:
31+
LOCALS[f"unpack_{endian_l}_{typ}"] = Struct(f"{endian_s}{param}").unpack
32+
LOCALS[f"unpack_{endian_l}_{typ}_from"] = Struct(
33+
f"{endian_s}{param}"
34+
).unpack_from
35+
unpack_little_vec2 = Struct(f"<ff").unpack
36+
637

738
class EndianBinaryReader:
839
endian: str
@@ -12,8 +43,7 @@ class EndianBinaryReader:
1243

1344
def __new__(cls, item, endian=">", offset=0):
1445
if isinstance(item, (bytes, bytearray, memoryview)):
15-
obj = super(EndianBinaryReader, cls).__new__(
16-
EndianBinaryReader_Memoryview)
46+
obj = super(EndianBinaryReader, cls).__new__(EndianBinaryReader_Memoryview)
1747
else:
1848
obj = super(EndianBinaryReader, cls).__new__(
1949
EndianBinaryReader_Streamable
@@ -138,25 +168,34 @@ def read_matrix(self):
138168
def read_array(self, command, length: int):
139169
return [command() for _ in range(length)]
140170

141-
def read_boolean_array(self):
142-
return self.read_array(self.read_boolean, self.read_int())
171+
def read_array_struct(self, param: str, length: int = None):
172+
if length is None:
173+
length = self.read_int()
174+
struct = Struct(f"{self.endian}{length}{param}")
175+
return struct.unpack(self.read(struct.size))
176+
177+
def read_boolean_array(self, length: int = None):
178+
return self.read_array_struct("?", length)
143179

144-
def read_u_short_array(self):
145-
return self.read_array(self.read_u_short, self.read_int())
180+
def read_u_short_array(self, length: int = None):
181+
return self.read_array_struct("h", length)
146182

147-
def read_int_array(self, length=0):
148-
return self.read_array(self.read_int, length if length else self.read_int())
183+
def read_short_array(self, length: int = None):
184+
return self.read_array_struct("H", length)
149185

150-
def read_u_int_array(self, length=0):
151-
return self.read_array(self.read_u_int, length if length else self.read_int())
186+
def read_int_array(self, length: int = None):
187+
return self.read_array_struct("i", length)
152188

153-
def read_u_int_array_array(self, length=0):
189+
def read_u_int_array(self, length: int = None):
190+
return self.read_array_struct("I", length)
191+
192+
def read_u_int_array_array(self, length: int = None):
154193
return self.read_array(
155-
self.read_u_int_array, length if length else self.read_int()
194+
self.read_u_int_array, length if length is not None else self.read_int()
156195
)
157196

158-
def read_float_array(self, length=0):
159-
return self.read_array(self.read_float, length if length else self.read_int())
197+
def read_float_array(self, length: int = None):
198+
return self.read_array_struct("f", length)
160199

161200
def read_string_array(self):
162201
return self.read_array(self.read_aligned_string, self.read_int())
@@ -171,24 +210,44 @@ def read_matrix_array(self):
171210
return self.read_array(self.read_matrix, self.read_int())
172211

173212
def real_offset(self):
174-
""" Returns offset in the underlying file.
175-
(Not working with unpacked streams.)
213+
"""Returns offset in the underlying file.
214+
(Not working with unpacked streams.)
176215
"""
177216
return self.BaseOffset + self.Position
178217

179218
def read_the_rest(self, obj_start, obj_size):
180-
""" Returns the rest of the current reader bytes."""
219+
"""Returns the rest of the current reader bytes."""
181220
return self.read_bytes(obj_size - (self.Position - obj_start))
182221

183222

184223
class EndianBinaryReader_Memoryview(EndianBinaryReader):
224+
__slots__ = ("view", "_endian", "BaseOffset", "Position", "Length")
185225
view: memoryview
186226

187227
def __init__(self, view, endian=">", offset=0):
228+
self._endian = ""
188229
super().__init__(view, endian=endian, offset=offset)
189230
self.view = memoryview(view)
190231
self.Length = len(view)
191232

233+
@property
234+
def endian(self):
235+
return self._endian
236+
237+
@endian.setter
238+
def endian(self, value):
239+
if value not in ("<", ">"):
240+
raise ValueError("Invalid endian")
241+
if value != self._endian:
242+
setattr(
243+
self,
244+
"__class__",
245+
EndianBinaryReader_Memoryview_LittleEndian
246+
if value == "<"
247+
else EndianBinaryReader_Memoryview_BigEndian,
248+
)
249+
self._endian = value
250+
192251
@property
193252
def bytes(self):
194253
return self.view
@@ -199,7 +258,7 @@ def dispose(self):
199258
def read(self, length: int):
200259
if not length:
201260
return b""
202-
ret = self.view[self.Position: self.Position + length]
261+
ret = self.view[self.Position : self.Position + length]
203262
self.Position += length
204263
return ret
205264

@@ -212,19 +271,142 @@ def read_aligned_string(self):
212271
return result
213272
return ""
214273

274+
def read_string_to_null(self, max_length=32767) -> str:
275+
match = reNot0.search(self.view, self.Position, self.Position + max_length)
276+
if not match:
277+
raise ValueError("Unterminated string")
278+
ret = match[1].decode("utf8", "surrogateescape")
279+
self.Position = match.end()
280+
return ret
281+
282+
283+
class EndianBinaryReader_Memoryview_LittleEndian(EndianBinaryReader_Memoryview):
284+
def read_u_short(self):
285+
(ret,) = unpack_little_u_short_from(self.view, self.Position)
286+
self.Position += 2
287+
return ret
288+
289+
def read_short(self):
290+
(ret,) = unpack_little_short_from(self.view, self.Position)
291+
self.Position += 2
292+
return ret
293+
294+
def read_int(self):
295+
(ret,) = unpack_little_int_from(self.view, self.Position)
296+
self.Position += 4
297+
return ret
298+
299+
def read_u_int(self):
300+
(ret,) = unpack_little_u_int_from(self.view, self.Position)
301+
self.Position += 4
302+
return ret
303+
304+
def read_long(self):
305+
(ret,) = unpack_little_long_from(self.view, self.Position)
306+
self.Position += 8
307+
return ret
308+
309+
def read_u_long(self):
310+
(ret,) = unpack_little_u_long_from(self.view, self.Position)
311+
self.Position += 8
312+
return ret
313+
314+
def read_half(self):
315+
(ret,) = unpack_little_half_from(self.view, self.Position)
316+
self.Position += 2
317+
return ret
318+
319+
def read_float(self):
320+
(ret,) = unpack_little_float_from(self.view, self.Position)
321+
self.Position += 4
322+
return ret
323+
324+
def read_double(self):
325+
(ret,) = unpack_little_double_from(self.view, self.Position)
326+
self.Position += 8
327+
return ret
328+
329+
330+
class EndianBinaryReader_Memoryview_BigEndian(EndianBinaryReader_Memoryview):
331+
def read_u_short(self):
332+
(ret,) = unpack_big_u_short_from(self.view, self.Position)
333+
self.Position += 2
334+
return ret
335+
336+
def read_short(self):
337+
(ret,) = unpack_big_short_from(self.view, self.Position)
338+
self.Position += 2
339+
return ret
340+
341+
def read_int(self):
342+
(ret,) = unpack_big_int_from(self.view, self.Position)
343+
self.Position += 4
344+
return ret
345+
346+
def read_u_int(self):
347+
(ret,) = unpack_big_u_int_from(self.view, self.Position)
348+
self.Position += 4
349+
return ret
350+
351+
def read_long(self):
352+
(ret,) = unpack_big_long_from(self.view, self.Position)
353+
self.Position += 8
354+
return ret
355+
356+
def read_u_long(self):
357+
(ret,) = unpack_big_u_long_from(self.view, self.Position)
358+
self.Position += 8
359+
return ret
360+
361+
def read_half(self):
362+
(ret,) = unpack_big_half_from(self.view, self.Position)
363+
self.Position += 2
364+
return ret
365+
366+
def read_float(self):
367+
(ret,) = unpack_big_float_from(self.view, self.Position)
368+
self.Position += 4
369+
return ret
370+
371+
def read_double(self):
372+
(ret,) = unpack_big_double_from(self.view, self.Position)
373+
self.Position += 8
374+
return ret
375+
215376

216377
class EndianBinaryReader_Streamable(EndianBinaryReader):
378+
__slots__ = ("stream", "_endian", "BaseOffset")
217379
stream: io.BufferedReader
218380

219381
def __init__(self, stream, endian=">", offset=0):
382+
self._endian = ""
220383
self.stream = stream
221384
super().__init__(stream, endian=endian, offset=offset)
385+
self.read = self.stream.read
222386

223387
def get_position(self):
224388
return self.stream.tell()
225389

226390
def set_position(self, value):
227-
self.stream.seek(value+self.BaseOffset)
391+
self.stream.seek(value + self.BaseOffset)
392+
393+
@property
394+
def endian(self):
395+
return self._endian
396+
397+
@endian.setter
398+
def endian(self, value):
399+
if value not in ("<", ">"):
400+
raise ValueError("Invalid endian")
401+
if value != self._endian:
402+
setattr(
403+
self,
404+
"__class__",
405+
EndianBinaryReader_Streamable_LittleEndian
406+
if value == "<"
407+
else EndianBinaryReader_Streamable_BigEndian,
408+
)
409+
self._endian = value
228410

229411
@property
230412
def Length(self):
@@ -247,8 +429,60 @@ def dispose(self):
247429
self.stream.close()
248430
pass
249431

250-
def read(self, length: int):
251-
if not length:
252-
return b""
253-
return self.stream.read(length)
254432

433+
class EndianBinaryReader_Streamable_LittleEndian(EndianBinaryReader_Streamable):
434+
def read_u_short(self):
435+
return unpack_little_u_short(self.read(2))[0]
436+
437+
def read_short(self):
438+
return unpack_little_short(self.read(2))[0]
439+
440+
def read_int(self):
441+
return unpack_little_int(self.read(4))[0]
442+
443+
def read_u_int(self):
444+
return unpack_little_u_int(self.read(4))[0]
445+
446+
def read_long(self):
447+
return unpack_little_long(self.read(8))[0]
448+
449+
def read_u_long(self):
450+
return unpack_little_u_long(self.read(8))[0]
451+
452+
def read_half(self):
453+
return unpack_little_half(self.read(2))[0]
454+
455+
def read_float(self):
456+
return unpack_little_float(self.read(4))[0]
457+
458+
def read_double(self):
459+
return unpack_little_double(self.read(8))[0]
460+
461+
462+
class EndianBinaryReader_Streamable_BigEndian(EndianBinaryReader_Streamable):
463+
def read_u_short(self):
464+
return unpack_big_u_short(self.read(2))[0]
465+
466+
def read_short(self):
467+
return unpack_big_short(self.read(2))[0]
468+
469+
def read_int(self):
470+
return unpack_big_int(self.read(4))[0]
471+
472+
def read_u_int(self):
473+
return unpack_big_u_int(self.read(4))[0]
474+
475+
def read_long(self):
476+
return unpack_big_long(self.read(8))[0]
477+
478+
def read_u_long(self):
479+
return unpack_big_u_long(self.read(8))[0]
480+
481+
def read_half(self):
482+
return unpack_big_half(self.read(2))[0]
483+
484+
def read_float(self):
485+
return unpack_big_float(self.read(4))[0]
486+
487+
def read_double(self):
488+
return unpack_big_double(self.read(8))[0]

0 commit comments

Comments
 (0)