forked from Blosc/python-blosc2
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_schunk_update.py
More file actions
111 lines (96 loc) · 3.35 KB
/
test_schunk_update.py
File metadata and controls
111 lines (96 loc) · 3.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#######################################################################
# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
# All rights reserved.
#
# This source code is licensed under a BSD-style license (found in the
# LICENSE file in the root directory of this source tree)
#######################################################################
import random
import numpy as np
import pytest
import blosc2
@pytest.mark.parametrize("gil", [True, False])
@pytest.mark.parametrize("contiguous", [True, False])
@pytest.mark.parametrize("urlpath", [None, "b2frame"])
@pytest.mark.parametrize(
("nchunks", "nupdates"),
[
(0, 0),
(1, 1),
(7, 3),
],
)
@pytest.mark.parametrize("copy", [True, False])
@pytest.mark.parametrize("create_chunk", [True, False])
def test_schunk_update_numpy(contiguous, urlpath, nchunks, nupdates, copy, create_chunk, gil):
blosc2.set_releasegil(gil)
kwargs = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **kwargs)
for i in range(nchunks):
buffer = i * np.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
for _ in range(nupdates):
pos = random.randint(0, nchunks - 1)
buffer = pos * np.arange(200 * 1000, dtype="int32")
if create_chunk:
chunk = blosc2.compress2(buffer)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, buffer, copy)
chunk_ = schunk.decompress_chunk(pos)
bytes_obj = buffer.tobytes()
assert chunk_ == bytes_obj
dest = np.empty(buffer.shape, buffer.dtype)
schunk.decompress_chunk(pos, dest)
assert np.array_equal(buffer, dest)
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
@pytest.mark.parametrize("gil", [True, False])
@pytest.mark.parametrize("contiguous", [True, False])
@pytest.mark.parametrize("urlpath", [None, "b2frame"])
@pytest.mark.parametrize(
("nchunks", "nupdates"),
[
(0, 0),
(1, 1),
(7, 3),
],
)
@pytest.mark.parametrize("copy", [True, False])
@pytest.mark.parametrize("create_chunk", [True, False])
def test_update(contiguous, urlpath, nchunks, nupdates, copy, create_chunk, gil):
blosc2.set_releasegil(gil)
kwargs = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
nbytes = 23401
schunk = blosc2.SChunk(chunksize=nbytes * 2, **kwargs)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
for _ in range(nupdates):
pos = random.randint(0, nchunks - 1)
bytes_obj = b"i " * nbytes
if create_chunk:
chunk = blosc2.compress2(bytes_obj, typesize=1)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, bytes_obj, copy)
res = schunk.decompress_chunk(pos)
assert res == bytes_obj
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)