forked from apache/arrow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path_dataset.pxd
More file actions
181 lines (114 loc) · 4.69 KB
/
Copy path_dataset.pxd
File metadata and controls
181 lines (114 loc) · 4.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# cython: language_level = 3
"""Dataset is currently unstable. APIs subject to change without notice."""
from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow_dataset cimport *
from pyarrow.lib cimport *
from pyarrow._fs cimport FileSystem
cdef CFileSource _make_file_source(object file, FileSystem filesystem=*)
cdef class DatasetFactory(_Weakrefable):
cdef:
shared_ptr[CDatasetFactory] wrapped
CDatasetFactory* factory
cdef init(self, const shared_ptr[CDatasetFactory]& sp)
@staticmethod
cdef wrap(const shared_ptr[CDatasetFactory]& sp)
cdef inline shared_ptr[CDatasetFactory] unwrap(self) nogil
cdef class Dataset(_Weakrefable):
cdef:
shared_ptr[CDataset] wrapped
CDataset* dataset
public dict _scan_options
cdef void init(self, const shared_ptr[CDataset]& sp)
@staticmethod
cdef wrap(const shared_ptr[CDataset]& sp)
cdef shared_ptr[CDataset] unwrap(self) nogil
cdef class Scanner(_Weakrefable):
cdef:
shared_ptr[CScanner] wrapped
CScanner* scanner
cdef void init(self, const shared_ptr[CScanner]& sp)
@staticmethod
cdef wrap(const shared_ptr[CScanner]& sp)
cdef shared_ptr[CScanner] unwrap(self)
@staticmethod
cdef shared_ptr[CScanOptions] _make_scan_options(Dataset dataset, dict py_scanoptions) except *
cdef class FragmentScanOptions(_Weakrefable):
cdef:
shared_ptr[CFragmentScanOptions] wrapped
cdef void init(self, const shared_ptr[CFragmentScanOptions]& sp)
@staticmethod
cdef wrap(const shared_ptr[CFragmentScanOptions]& sp)
cdef class FileFormat(_Weakrefable):
cdef:
shared_ptr[CFileFormat] wrapped
CFileFormat* format
cdef void init(self, const shared_ptr[CFileFormat]& sp)
@staticmethod
cdef wrap(const shared_ptr[CFileFormat]& sp)
cdef inline shared_ptr[CFileFormat] unwrap(self)
cdef _set_default_fragment_scan_options(self, FragmentScanOptions options)
# Return a WrittenFile after a file was written.
# May be overridden by subclasses, e.g. to add metadata.
cdef WrittenFile _finish_write(self, path, base_dir,
CFileWriter* file_writer)
cdef class FileWriteOptions(_Weakrefable):
cdef:
shared_ptr[CFileWriteOptions] wrapped
CFileWriteOptions* c_options
cdef void init(self, const shared_ptr[CFileWriteOptions]& sp)
@staticmethod
cdef wrap(const shared_ptr[CFileWriteOptions]& sp)
cdef inline shared_ptr[CFileWriteOptions] unwrap(self)
cdef class Fragment(_Weakrefable):
cdef:
shared_ptr[CFragment] wrapped
CFragment* fragment
cdef void init(self, const shared_ptr[CFragment]& sp)
@staticmethod
cdef wrap(const shared_ptr[CFragment]& sp)
cdef inline shared_ptr[CFragment] unwrap(self)
cdef class FileFragment(Fragment):
cdef:
CFileFragment* file_fragment
cdef void init(self, const shared_ptr[CFragment]& sp)
cdef class Partitioning(_Weakrefable):
cdef:
shared_ptr[CPartitioning] wrapped
CPartitioning* partitioning
cdef init(self, const shared_ptr[CPartitioning]& sp)
@staticmethod
cdef wrap(const shared_ptr[CPartitioning]& sp)
cdef inline shared_ptr[CPartitioning] unwrap(self)
cdef class PartitioningFactory(_Weakrefable):
cdef:
shared_ptr[CPartitioningFactory] wrapped
CPartitioningFactory* factory
cdef init(self, const shared_ptr[CPartitioningFactory]& sp)
@staticmethod
cdef wrap(const shared_ptr[CPartitioningFactory]& sp)
cdef inline shared_ptr[CPartitioningFactory] unwrap(self)
cdef class WrittenFile(_Weakrefable):
# The full path to the created file
cdef public str path
# Optional Parquet metadata
# This metadata will have the file path attribute set to the path of
# the written file.
cdef public object metadata
# The size of the file in bytes
cdef public int64_t size