-
-
Notifications
You must be signed in to change notification settings - Fork 39
Expand file tree
/
Copy pathpostfilter2.py
More file actions
47 lines (37 loc) · 1.52 KB
/
postfilter2.py
File metadata and controls
47 lines (37 loc) · 1.52 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#######################################################################
# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
#######################################################################
import numpy as np
import blosc2
nchunks = 10
input_dtype = np.dtype("M8[D]")
output_dtype = np.int64 # output dtype has to be of the same size as input
# Set the compression and decompression parameters
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4, typesize=input_dtype.itemsize)
dparams = blosc2.DParams(nthreads=1)
contiguous = True
urlpath = "filename"
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode="a")
# Remove previous SChunk
blosc2.remove_urlpath(urlpath)
# Create and set data
chunkshape = 200 * 1000
data = np.arange(0, chunkshape * nchunks, dtype=input_dtype)
schunk = blosc2.SChunk(
chunksize=chunkshape * input_dtype.itemsize, data=data, cparams=cparams, dparams=dparams, storage=storage
)
out1 = np.empty(chunkshape * nchunks, dtype=input_dtype)
schunk.get_slice(0, chunkshape * nchunks, out=out1)
# Set postfilter with decorator
@schunk.postfilter(input_dtype, output_dtype)
def postfilter(input, output, offset):
output[:] = input <= np.datetime64("1997-12-31")
out2 = np.empty(chunkshape * nchunks, dtype=output_dtype)
schunk.get_slice(0, chunkshape * nchunks, out=out2)
res = np.empty(out1.shape, dtype=output_dtype)
postfilter(data, res, None)
# Check postfilter is applied
assert np.array_equal(res, out2)