forked from ClickHouse/ClickHouse
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathDataTypesBinaryEncoding.h
More file actions
205 lines (190 loc) · 34.6 KB
/
Copy pathDataTypesBinaryEncoding.h
File metadata and controls
205 lines (190 loc) · 34.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
#pragma once
#include <DataTypes/IDataType.h>
namespace DB
{
enum class BinaryTypeIndex : uint8_t
{
Nothing = 0x00,
UInt8 = 0x01,
UInt16 = 0x02,
UInt32 = 0x03,
UInt64 = 0x04,
UInt128 = 0x05,
UInt256 = 0x06,
Int8 = 0x07,
Int16 = 0x08,
Int32 = 0x09,
Int64 = 0x0A,
Int128 = 0x0B,
Int256 = 0x0C,
Float32 = 0x0D,
Float64 = 0x0E,
Date = 0x0F,
Date32 = 0x10,
DateTimeUTC = 0x11,
DateTimeWithTimezone = 0x12,
DateTime64UTC = 0x13,
DateTime64WithTimezone = 0x14,
String = 0x15,
FixedString = 0x16,
Enum8 = 0x17,
Enum16 = 0x18,
Decimal32 = 0x19,
Decimal64 = 0x1A,
Decimal128 = 0x1B,
Decimal256 = 0x1C,
UUID = 0x1D,
Array = 0x1E,
UnnamedTuple = 0x1F,
NamedTuple = 0x20,
Set = 0x21,
Interval = 0x22,
Nullable = 0x23,
Function = 0x24,
AggregateFunction = 0x25,
LowCardinality = 0x26,
Map = 0x27,
IPv4 = 0x28,
IPv6 = 0x29,
Variant = 0x2A,
Dynamic = 0x2B,
Custom = 0x2C,
Bool = 0x2D,
SimpleAggregateFunction = 0x2E,
Nested = 0x2F,
JSON = 0x30,
BFloat16 = 0x31,
Time = 0x32,
/* The reason behind putting Time64 to 0x34 instead of 0x33 is following:
Originally, there were Time and Time64 with (and without) timezones, which were making the following indexing:
TimeUTC = 0x32
TimeWithTimezone = 0x33
Time64UTC = 0x34
Time64WithTimezone = 0x35
After that timezones became forbidden for Time[64] types, so we removed those types from here.
But we need to make the indexing consistent to ensure backwards compatibility.
Please don't use 0x33 and 0x35, because older client might try to serialise data as TimeWithTimezone/Time64WithTimezone, and newer server would deserialise them as incorrect types. */
Time64 = 0x34,
/// reserved = 0x35
QBit = 0x36,
/// QBit with an explicit stride parameter (stride != dimension). Non-strided QBit keeps using 0x36 for backward compatibility.
QBitWithStride = 0x37
};
/// Maximum value of BinaryTypeIndex + 1, used for sizing the index array in SimpleDataTypesCache.
inline constexpr size_t BINARY_TYPE_INDEX_SIZE = 0x38;
/**
Binary encoding for ClickHouse data types:
|---------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| ClickHouse data type | Binary encoding |
|---------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Nothing | 0x00 |
| UInt8 | 0x01 |
| UInt16 | 0x02 |
| UInt32 | 0x03 |
| UInt64 | 0x04 |
| UInt128 | 0x05 |
| UInt256 | 0x06 |
| Int8 | 0x07 |
| Int16 | 0x08 |
| Int32 | 0x09 |
| Int64 | 0x0A |
| Int128 | 0x0B |
| Int256 | 0x0C |
| Float32 | 0x0D |
| Float64 | 0x0E |
| Date | 0x0F |
| Date32 | 0x10 |
| DateTime | 0x11 |
| DateTime(time_zone) | 0x12<var_uint_time_zone_name_size><time_zone_name_data> |
| DateTime64(P) | 0x13<uint8_precision> |
| DateTime64(P, time_zone) | 0x14<uint8_precision><var_uint_time_zone_name_size><time_zone_name_data> |
| String | 0x15 |
| FixedString(N) | 0x16<var_uint_size> |
| Enum8 | 0x17<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int8_value_1>...<var_uint_name_size_N><name_data_N><int8_value_N> |
| Enum16 | 0x18<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int16_little_endian_value_1>...><var_uint_name_size_N><name_data_N><int16_little_endian_value_N> |
| Decimal32(P, S) | 0x19<uint8_precision><uint8_scale> |
| Decimal64(P, S) | 0x1A<uint8_precision><uint8_scale> |
| Decimal128(P, S) | 0x1B<uint8_precision><uint8_scale> |
| Decimal256(P, S) | 0x1C<uint8_precision><uint8_scale> |
| UUID | 0x1D |
| Array(T) | 0x1E<nested_type_encoding> |
| Tuple(T1, ..., TN) | 0x1F<var_uint_number_of_elements><nested_type_encoding_1>...<nested_type_encoding_N> |
| Tuple(name1 T1, ..., nameN TN) | 0x20<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N> |
| Set | 0x21 |
| Interval | 0x22<interval_kind> |
| Nullable(T) | 0x23<nested_type_encoding> |
| Function | 0x24<var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N><return_type_encoding> |
| AggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN) | 0x25<var_uint_version><var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N> |
| LowCardinality(T) | 0x26<nested_type_encoding> |
| Map(K, V) | 0x27<key_type_encoding><value_type_encoding> |
| IPv4 | 0x28 |
| IPv6 | 0x29 |
| Variant(T1, ..., TN) | 0x2A<var_uint_number_of_variants><variant_type_encoding_1>...<variant_type_encoding_N> |
| Dynamic(max_types=N) | 0x2B<uint8_max_types> |
| Custom type (Ring, Polygon, etc) | 0x2C<var_uint_type_name_size><type_name_data> |
| Bool | 0x2D |
| SimpleAggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN) | 0x2E<var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N> |
| Nested(name1 T1, ..., nameN TN) | 0x2F<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N> |
| JSON(max_dynamic_paths=N, max_dynamic_types=M, path Type, SKIP skip_path, SKIP REGEXP skip_path_regexp) | 0x30<uint8_serialization_version><var_int_max_dynamic_paths><uint8_max_dynamic_types><var_uint_number_of_typed_paths><var_uint_path_name_size_1><path_name_data_1><encoded_type_1>...<var_uint_number_of_skip_paths><var_uint_skip_path_size_1><skip_path_data_1>...<var_uint_number_of_skip_path_regexps><var_uint_skip_path_regexp_size_1><skip_path_data_regexp_1>... |
| BFloat16 | 0x31 |
| Time | 0x32 |
| Time64(P) | 0x34<uint8_precision> |
| QBit(T, N) | 0x36<element_type_encoding><var_uint_dimension> |
| QBit(T, N, stride) | 0x37<element_type_encoding><var_uint_dimension><var_uint_stride> |
|---------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
For type QBit the 0x37 encoding is used only when stride differs from the dimension N. When stride == N (including when the third argument is written explicitly, as in QBit(Float32, 4, 4)) the type is canonicalized to the two-argument QBit(T, N) form and encoded as 0x36 above, so its binary encoding stays byte-identical to a non-strided QBit.
Interval kind binary encoding:
|---------------|-----------------|
| Interval kind | Binary encoding |
|---------------|-----------------|
| Nanosecond | 0x00 |
| Microsecond | 0x01 |
| Millisecond | 0x02 |
| Second | 0x03 |
| Minute | 0x04 |
| Hour | 0x05 |
| Day | 0x06 |
| Week | 0x07 |
| Month | 0x08 |
| Quarter | 0x09 |
| Year | 0x0A |
|---------------|-----------------|
Aggregate function parameter binary encoding (binary encoding of a Field, see src/Common/FieldBinaryEncoding.h):
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
| Parameter type | Binary encoding |
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
| Null | 0x00 |
| UInt64 | 0x01<var_uint_value> |
| Int64 | 0x02<var_int_value> |
| UInt128 | 0x03<uint128_little_endian_value> |
| Int128 | 0x04<int128_little_endian_value> |
| UInt256 | 0x05<uint256_little_endian_value> |
| Int256 | 0x06<int256_little_endian_value> |
| Float64 | 0x07<float64_little_endian_value> |
| Decimal32 | 0x08<var_uint_scale><int32_little_endian_value> |
| Decimal64 | 0x09<var_uint_scale><int64_little_endian_value> |
| Decimal128 | 0x0A<var_uint_scale><int128_little_endian_value> |
| Decimal256 | 0x0B<var_uint_scale><int256_little_endian_value> |
| String | 0x0C<var_uint_size><data> |
| Array | 0x0D<var_uint_size><value_encoding_1>...<value_encoding_N> |
| Tuple | 0x0E<var_uint_size><value_encoding_1>...<value_encoding_N> |
| Map | 0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N> |
| IPv4 | 0x10<uint32_little_endian_value> |
| IPv6 | 0x11<uint128_little_endian_value> |
| UUID | 0x12<uuid_value> |
| Bool | 0x13<bool_value> |
| Object | 0x14<var_uint_size><var_uint_key_size_1><key_data_1><value_encoding_1>...<var_uint_key_size_N><key_data_N><value_encoding_N> |
| AggregateFunctionState | 0x15<var_uint_name_size><name_data><var_uint_data_size><data> |
| Negative infinity | 0xFE |
| Positive infinity | 0xFF |
|------------------------|------------------------------------------------------------------------------------------------------------------------------|
*/
String encodeDataType(const DataTypePtr & type);
void encodeDataType(const DataTypePtr & type, WriteBuffer & buf);
/// Special version of type encoding that is used for generic hash calculation.
/// It can skip serializing some data types parameters.
void encodeDataTypeForHashCalculation(const DataTypePtr & type, WriteBuffer & buf);
DataTypePtr decodeDataType(const String & data);
DataTypePtr decodeDataType(std::string_view data);
DataTypePtr decodeDataType(ReadBuffer & buf);
}