-
Notifications
You must be signed in to change notification settings - Fork 5
Expand file tree
/
Copy pathcompression_helper.py
More file actions
executable file
·105 lines (83 loc) · 2.64 KB
/
Copy pathcompression_helper.py
File metadata and controls
executable file
·105 lines (83 loc) · 2.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import pickle
import zlib
import lz4.block
def dummy_compressor(data):
r"""
Overview:
Return input data.
"""
return data
def zlib_data_compressor(data):
r"""
Overview:
Takes the input compressed data and return the compressed original data (zlib compressor) in binary format.
Examples:
>>> zlib_data_compressor("Hello")
b'x\x9ck`\x99\xca\xc9\x00\x01=\xac\x1e\xa999\xf9S\xf4\x00%L\x04j'
"""
return zlib.compress(pickle.dumps(data))
def lz4_data_compressor(data):
r"""
Overview:
Return the compressed original data (lz4 compressor).The compressor outputs in binary format.
Examples:
>>> lz4.block.compress(pickle.dumps("Hello"))
b'\x14\x00\x00\x00R\x80\x04\x95\t\x00\x01\x00\x90\x8c\x05Hello\x94.'
"""
return lz4.block.compress(pickle.dumps(data))
_COMPRESSORS_MAP = {
'lz4': lz4_data_compressor,
'zlib': zlib_data_compressor,
'none': dummy_compressor,
}
def get_data_compressor(name: str):
r"""
Overview:
Get the data compressor according to the input name
Arguments:
- name(:obj:`str`): Name of the compressor, support ``['lz4', 'zlib', 'none']``
Return:
- (:obj:`Callable`): Corresponding data_compressor, taking input data returning compressed data.
Example:
>>> compress_fn = get_data_compressor('lz4')
>>> compressed_data = compressed(input_data)
"""
return _COMPRESSORS_MAP[name]
def dummy_decompressor(data):
"""
Overview:
Return input data.
"""
return data
def lz4_data_decompressor(compressed_data):
r"""
Overview:
Return the decompressed original data (lz4 compressor).
"""
return pickle.loads(lz4.block.decompress(compressed_data))
def zlib_data_decompressor(compressed_data):
r"""
Overview:
Return the decompressed original data (zlib compressor).
"""
return pickle.loads(zlib.decompress(compressed_data))
_DECOMPRESSORS_MAP = {
'lz4': lz4_data_decompressor,
'zlib': zlib_data_decompressor,
'none': dummy_decompressor,
}
def get_data_decompressor(name: str):
r"""
Overview:
Get the data decompressor according to the input name
Arguments:
- name(:obj:`str`): Name of the decompressor, support ``['lz4', 'zlib', 'none']``
.. note::
For all the decompressors, the input of a bytes-like object is required.
Returns:
- (:obj:`Callable`): Corresponding data_decompressor.
Examples:
>>> decompress_fn = get_data_decompressor('lz4')
>>> origin_data = compressed(compressed_data)
"""
return _DECOMPRESSORS_MAP[name]