-
Notifications
You must be signed in to change notification settings - Fork 203
Expand file tree
/
Copy pathmod.rs
More file actions
221 lines (195 loc) · 7.56 KB
/
Copy pathmod.rs
File metadata and controls
221 lines (195 loc) · 7.56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors
//! A type system for Vortex.
//!
//! This crate contains the core logical type system for Vortex, including the definition of data types,
//! and (optionally) logic for their serialization and deserialization.
//!
//! Vortex dtypes are logical domains, not physical layouts. Encodings are tracked separately on
//! arrays, so the same dtype may be backed by canonical buffers, dictionary codes, compressed
//! children, or a lazy expression array.
//!
//! Every non-null logical dtype carries [`Nullability`] directly. This differs from Apache Arrow,
//! where nullability is usually field metadata.
#[cfg(feature = "arbitrary")]
mod arbitrary;
mod bigint;
mod decimal;
mod dtype_impl;
pub mod extension;
mod f16;
mod field;
mod field_mask;
mod field_names;
mod map;
mod native_dtype;
mod nullability;
mod ptype;
pub mod serde;
pub mod session;
mod struct_;
mod union;
use std::sync::Arc;
/// The logical types of elements in Vortex arrays.
///
/// `DType` represents the different logical data types that can be represented in a Vortex array.
///
/// This is different from physical types, which represent the actual layout of data (compressed or
/// uncompressed). The set of physical types/formats (or data layout) is surjective into the set of
/// logical types (or in other words, all physical types map to a single logical type).
///
/// Note that a `DType` represents the logical type of the elements in the `Array`s, **not** the
/// logical type of the `Array` itself.
///
/// For example, an array with [`DType::Primitive`]([`I32`], [`NonNullable`]) could be physically
/// encoded as any of the following:
///
/// - A flat array of `i32` values.
/// - A run-length encoded sequence.
/// - Dictionary encoded values with bitpacked codes.
///
/// All of these physical encodings preserve the same logical [`I32`] type, even if the physical
/// data is different.
///
/// [`I32`]: PType::I32
/// [`NonNullable`]: Nullability::NonNullable
#[allow(
clippy::derived_hash_with_manual_eq,
reason = "manual PartialEq adds Arc::ptr_eq fast path only"
)]
#[derive(Debug, Clone, Eq, Hash)]
pub enum DType {
/// A logical null type.
///
/// `Null` only has a single value, `null`.
Null,
/// A logical boolean type.
///
/// `Bool` can be `true` or `false` if non-nullable. It can be `true`, `false`, or `null` if
/// nullable.
Bool(Nullability),
/// A logical fixed-width numeric type.
///
/// This can be unsigned, signed, or floating point. See [`PType`] for more information.
Primitive(PType, Nullability),
/// Logical real numbers with fixed precision and scale.
///
/// See [`DecimalDType`] for more information.
Decimal(DecimalDType, Nullability),
/// Logical UTF-8 strings.
Utf8(Nullability),
/// Logical binary data.
Binary(Nullability),
/// A logical variable-length list type.
///
/// This is parameterized by a single `DType` that represents the element type of the inner
/// lists.
List(Arc<DType>, Nullability),
/// A logical fixed-size list type.
///
/// This is parameterized by a `DType` that represents the element type of the inner lists, as
/// well as a `u32` size that determines the fixed length of each `FixedSizeList` scalar.
FixedSizeList(Arc<DType>, u32, Nullability),
/// A logical map type.
///
/// Map keys are non-nullable, values may be nullable, and [`MapDType`] stores Arrow's
/// `keys_sorted` assertion.
Map(MapDType, Nullability),
/// A logical struct type.
///
/// A `Struct` type is composed of an ordered list of fields, each with a corresponding name and
/// `DType`. See [`StructFields`] for more information.
Struct(StructFields, Nullability),
/// A logical union (sum) type.
///
/// A `Union` is composed of one or more **variants**, each with a name and a `DType`. A per-row
/// `u8` tag selects which variant is "live" at that row.
///
/// A union has independent outer nullability in addition to the nullability of each variant.
/// This distinguishes a null union from a non-null union whose selected child is null.
///
/// See [`UnionVariants`] for the type-tag conventions and accessors.
Union(UnionVariants, Nullability),
/// Dynamically typed values stored as Vortex scalars.
///
/// A variant value preserves its full logical value in variant storage and may also carry a
/// typed, row-aligned shredded representation for selected paths when materialized as a
/// [`VariantArray`](crate::arrays::VariantArray).
Variant(Nullability),
/// A user-defined extension type.
///
/// See [`ExtDTypeRef`] for more information.
Extension(ExtDTypeRef),
}
impl PartialEq for DType {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Null, Self::Null) => true,
(Self::Bool(a), Self::Bool(b)) => a == b,
(Self::Primitive(pa, na), Self::Primitive(pb, nb)) => pa == pb && na == nb,
(Self::Decimal(da, na), Self::Decimal(db, nb)) => da == db && na == nb,
(Self::Utf8(a), Self::Utf8(b)) => a == b,
(Self::Binary(a), Self::Binary(b)) => a == b,
(Self::List(da, na), Self::List(db, nb)) => {
na == nb && (Arc::ptr_eq(da, db) || da == db)
}
(Self::FixedSizeList(da, sa, na), Self::FixedSizeList(db, sb, nb)) => {
sa == sb && na == nb && (Arc::ptr_eq(da, db) || da == db)
}
(Self::Map(ma, na), Self::Map(mb, nb)) => na == nb && ma == mb,
// StructFields handles its own Arc::ptr_eq in its PartialEq impl.
(Self::Struct(a, na), Self::Struct(b, nb)) => na == nb && a == b,
// UnionVariants handles its own Arc::ptr_eq in its PartialEq impl.
(Self::Union(a, na), Self::Union(b, nb)) => na == nb && a == b,
(Self::Variant(a), Self::Variant(b)) => a == b,
(Self::Extension(a), Self::Extension(b)) => a == b,
// Every variant is listed in the first position so that adding a new
// variant produces a non-exhaustive match compile error.
(Self::Null, _)
| (Self::Bool(_), _)
| (Self::Primitive(..), _)
| (Self::Decimal(..), _)
| (Self::Utf8(_), _)
| (Self::Binary(_), _)
| (Self::List(..), _)
| (Self::FixedSizeList(..), _)
| (Self::Map(..), _)
| (Self::Struct(..), _)
| (Self::Union(..), _)
| (Self::Variant(_), _)
| (Self::Extension(_), _) => false,
}
}
}
pub use bigint::*;
pub use decimal::*;
pub use dtype_impl::NativeDType;
pub use f16::*;
pub use field::*;
pub use field_mask::*;
pub use field_names::*;
pub use half;
pub use map::*;
pub use nullability::*;
pub use ptype::*;
pub use struct_::*;
pub use union::*;
use crate::dtype::extension::ExtDTypeRef;
pub mod proto {
//! Protocol buffer representations for DTypes
//!
//! This module contains the code to serialize and deserialize DTypes to and from protocol buffers.
pub use vortex_proto::dtype;
}
pub mod flatbuffers {
//! Flatbuffer representations for DTypes
//!
//! This module contains the code to serialize and deserialize DTypes to and from flatbuffers.
pub use vortex_flatbuffers::dtype::*;
}
#[cfg(test)]
mod test {
use std::sync::LazyLock;
use vortex_session::VortexSession;
pub(crate) static SESSION: LazyLock<VortexSession> = LazyLock::new(crate::array_session);
}