cdef _sequence_to_array(object sequence, object mask, object size,

DataType type, CMemoryPool* pool, c_bool from_pandas):

cdef int64_t c_size

cdef PyConversionOptions options

if type is not None:

options.type = type.sp_type

if size is not None:

options.size = size

options.pool = pool

options.from_pandas = from_pandas

cdef shared_ptr[CChunkedArray] out

with nogil:

check_status(ConvertPySequence(sequence, mask, options, &out))

if out.get().num_chunks() == 1:

return pyarrow_wrap_array(out.get().chunk(0))

else:

return pyarrow_wrap_chunked_array(out)

cdef inline _is_array_like(obj):

if isinstance(obj, np.ndarray):

return True

return pandas_api._have_pandas_internal() and pandas_api.is_array_like(obj)

def _ndarray_to_arrow_type(object values, DataType type):

return pyarrow_wrap_data_type(_ndarray_to_type(values, type))

cdef shared_ptr[CDataType] _ndarray_to_type(object values,

DataType type) except *:

cdef shared_ptr[CDataType] c_type

dtype = values.dtype

if type is None and dtype != object:

with nogil:

check_status(NumPyDtypeToArrow(dtype, &c_type))

if type is not None:

c_type = type.sp_type

return c_type

cdef _ndarray_to_array(object values, object mask, DataType type,

c_bool from_pandas, c_bool safe, CMemoryPool* pool):

cdef:

shared_ptr[CChunkedArray] chunked_out

shared_ptr[CDataType] c_type = _ndarray_to_type(values, type)

CCastOptions cast_options = CCastOptions(safe)

with nogil:

check_status(NdarrayToArrow(pool, values, mask, from_pandas,

c_type, cast_options, &chunked_out))

if chunked_out.get().num_chunks() > 1:

return pyarrow_wrap_chunked_array(chunked_out)

else:

return pyarrow_wrap_array(chunked_out.get().chunk(0))

def _handle_arrow_array_protocol(obj, type, mask, size):

if mask is not None or size is not None:

raise ValueError(

"Cannot specify a mask or a size when passing an object that is "

"converted with the __arrow_array__ protocol.")

res = obj.__arrow_array__(type=type)

if not isinstance(res, Array):

raise TypeError("The object's __arrow_array__ method does not "

"return a pyarrow Array.")

return res

def array(object obj, type=None, mask=None, size=None, from_pandas=None,

bint safe=True, MemoryPool memory_pool=None):

"""

cdef:

CMemoryPool* pool = maybe_unbox_memory_pool(memory_pool)

bint is_pandas_object = False

bint c_from_pandas

type = ensure_type(type, allow_none=True)

if from_pandas is None:

c_from_pandas = False

else:

c_from_pandas = from_pandas

if hasattr(obj, '__arrow_array__'):

return _handle_arrow_array_protocol(obj, type, mask, size)

elif _is_array_like(obj):

if mask is not None:

# out argument unused

mask = get_series_values(mask, &is_pandas_object)

values = get_series_values(obj, &is_pandas_object)

if is_pandas_object and from_pandas is None:

c_from_pandas = True

if isinstance(values, np.ma.MaskedArray):

if mask is not None:

raise ValueError("Cannot pass a numpy masked array and "

"specify a mask at the same time")

else:

mask = values.mask

values = values.data

if hasattr(values, '__arrow_array__'):

return _handle_arrow_array_protocol(values, type, mask, size)

elif pandas_api.is_categorical(values):

return DictionaryArray.from_arrays(

values.codes, values.categories.values,

mask=mask, ordered=values.ordered,

from_pandas=True, safe=safe,

memory_pool=memory_pool)

else:

if pandas_api.have_pandas:

values, type = pandas_api.compat.get_datetimetz_type(

values, obj.dtype, type)

return _ndarray_to_array(values, mask, type, c_from_pandas, safe,

pool)

else:

# ConvertPySequence does strict conversion if type is explicitly passed

return _sequence_to_array(obj, mask, size, type, pool, c_from_pandas)

def asarray(values, type=None):

"""

if isinstance(values, (Array, ChunkedArray)):

if type is not None and not values.type.equals(type):

values = values.cast(type)

return values

else:

return array(values, type=type)

def infer_type(values, mask=None, from_pandas=False):

"""

cdef:

shared_ptr[CDataType] out

c_bool use_pandas_sentinels = from_pandas

if mask is not None and not isinstance(mask, np.ndarray):

mask = np.array(mask, dtype=bool)

check_status(InferArrowType(values, mask, use_pandas_sentinels, &out))

return pyarrow_wrap_data_type(out)

def _normalize_slice(object arrow_obj, slice key):

cdef:

Py_ssize_t start, stop, step

Py_ssize_t n = len(arrow_obj)

start = key.start or 0

if start < 0:

start += n

if start < 0:

start = 0

elif start >= n:

start = n

stop = key.stop if key.stop is not None else n

if stop < 0:

stop += n

if stop < 0:

stop = 0

elif stop >= n:

stop = n

step = key.step or 1

if step != 1:

raise IndexError('only slices with step 1 supported')

else:

return arrow_obj.slice(start, stop - start)

cdef Py_ssize_t _normalize_index(Py_ssize_t index,

Py_ssize_t length) except -1:

if index < 0:

index += length

if index < 0:

raise IndexError("index out of bounds")

elif index >= length:

raise IndexError("index out of bounds")

return index

cdef class _FunctionContext:

self.ctx.reset(new CFunctionContext(c_default_memory_pool()))

cdef _FunctionContext _global_ctx = _FunctionContext()

cdef CFunctionContext* _context() nogil:

return _global_ctx.ctx.get()

cdef wrap_datum(const CDatum& datum):

if datum.kind() == DatumType_ARRAY:

return pyarrow_wrap_array(MakeArray(datum.array()))

elif datum.kind() == DatumType_CHUNKED_ARRAY:

return pyarrow_wrap_chunked_array(datum.chunked_array())

elif datum.kind() == DatumType_SCALAR:

return pyarrow_wrap_scalar(datum.scalar())

else:

raise ValueError("Unable to wrap Datum in a Python object")

cdef _append_array_buffers(const CArrayData* ad, list res):

"""

cdef size_t i, n

assert ad != NULL

n = ad.buffers.size()

for i in range(n):

buf = ad.buffers[i]

res.append(pyarrow_wrap_buffer(buf)

if buf.get() != NULL else None)

n = ad.child_data.size()

for i in range(n):

_append_array_buffers(ad.child_data[i].get(), res)

cdef _reduce_array_data(const CArrayData* ad):

"""

cdef size_t i, n

assert ad != NULL

n = ad.buffers.size()

buffers = []

for i in range(n):

buf = ad.buffers[i]

buffers.append(pyarrow_wrap_buffer(buf)

if buf.get() != NULL else None)

children = []

n = ad.child_data.size()

for i in range(n):

children.append(_reduce_array_data(ad.child_data[i].get()))

return pyarrow_wrap_data_type(ad.type), ad.length, ad.null_count, \

ad.offset, buffers, children

cdef shared_ptr[CArrayData] _reconstruct_array_data(data):

"""

cdef:

int64_t length, null_count, offset, i

DataType dtype

Buffer buf

vector[shared_ptr[CBuffer]] c_buffers

vector[shared_ptr[CArrayData]] c_children

dtype, length, null_count, offset, buffers, children = data

for i in range(len(buffers)):

buf = buffers[i]

if buf is None:

c_buffers.push_back(shared_ptr[CBuffer]())

else:

c_buffers.push_back(buf.buffer)

for i in range(len(children)):

c_children.push_back(_reconstruct_array_data(children[i]))

return CArrayData.MakeWithChildren(

dtype.sp_type,

length,

c_buffers,

c_children,

null_count,

offset)

def _restore_array(data):

"""

cdef shared_ptr[CArrayData] ad = _reconstruct_array_data(data)

return pyarrow_wrap_array(MakeArray(ad))

cdef class _PandasConvertible:

ignore_metadata=ignore_metadata)

cdef PandasOptions _convert_pandas_options(dict options):

cdef PandasOptions result

result.pool = maybe_unbox_memory_pool(options['pool'])

result.strings_to_categorical = options['strings_to_categorical']

result.zero_copy_only = options['zero_copy_only']

result.integer_object_nulls = options['integer_object_nulls']

result.date_as_object = options['date_as_object']

result.use_threads = options['use_threads']

result.deduplicate_objects = options['deduplicate_objects']

return result

cdef class Array(_PandasConvertible):

self.type = pyarrow_wrap_data_type(self.sp_array.get().type())

def __eq__(self, other):

raise NotImplementedError('Comparisons with pyarrow.Array are not '

'implemented')

def _debug_print(self):

with nogil:

check_status(DebugPrint(deref(self.ap), 0))

def diff(self, Array other):

return frombytes(result)

def cast(self, object target_type, bint safe=True):

return pyarrow_wrap_array(result)

def view(self, object target_type):

return pyarrow_wrap_array(result)

def sum(self):

return wrap_datum(out)

def unique(self):

return pyarrow_wrap_array(result)

def dictionary_encode(self):

return wrap_datum(out)

@staticmethod

def from_pandas(obj, mask=None, type=None, bint safe=True,

memory_pool=memory_pool)

def __reduce__(self):

return _restore_array, \

(_reduce_array_data(self.sp_array.get().data().get()),)

@staticmethod

def from_buffers(DataType type, length, buffers, null_count=-1, offset=0,

return result

@property

def null_count(self):

return self.sp_array.get().null_count()

def __iter__(self):

for i in range(len(self)):

yield self.getitem(i)

def __repr__(self):

type_format = object.__repr__(self)

return '{0}\n{1}'.format(type_format, str(self))

def format(self, int indent=0, int window=10):

return frombytes(result)

def __str__(self):

return self.format()

def equals(Array self, Array other):

return self.ap.Equals(deref(other.ap))

def __len__(self):

return self.length()

cdef int64_t length(self):

if self.sp_array.get():

return self.sp_array.get().length()

else:

return 0

def isnull(self):

raise NotImplemented

def __getitem__(self, index):

"""

if PySlice_Check(index):

return _normalize_slice(self, index)

return self.getitem(_normalize_index(index, self.length()))

cdef getitem(self, int64_t i):

return box_scalar(self.type, self.sp_array, i)

def slice(self, offset=0, length=None):

"""

cdef:

shared_ptr[CArray] result

if offset < 0:

raise IndexError('Offset must be non-negative')

if length is None:

result = self.ap.Slice(offset)

else:

result = self.ap.Slice(offset, length)

return pyarrow_wrap_array(result)

def take(self, Array indices):

"""

cdef:

cdef CTakeOptions options

cdef CDatum out

with nogil:

check_status(Take(_context(), CDatum(self.sp_array),

CDatum(indices.sp_array), options, &out))

return wrap_datum(out)

def filter(self, Array mask):

"""

cdef:

cdef CDatum out

with nogil:

check_status(FilterKernel(_context(), CDatum(self.sp_array),

CDatum(mask.sp_array), &out))

return wrap_datum(out)

def _to_pandas(self, options, **kwargs):

cdef:

PyObject* out

PandasOptions c_options = _convert_pandas_options(options)

Array array

if self.type.id == _Type_TIMESTAMP and self.type.unit != 'ns':

# pandas only stores ns data - casting here is faster

array = self.cast(timestamp('ns'))

else:

array = self

with nogil:

check_status(ConvertArrayToPandas(c_options, array.sp_array,

array, &out))

result = pandas_api.series(wrap_array_output(out), name=self._name)

if isinstance(self.type, TimestampType) and self.type.tz is not None:

from pyarrow.pandas_compat import make_tz_aware

result = make_tz_aware(result, self.type.tz)

return result

def __array__(self, dtype=None):

cdef:

PyObject* out

PandasOptions c_options

object values

with nogil:

check_status(ConvertArrayToPandas(c_options, self.sp_array,

self, &out))

# wrap_array_output uses pandas to convert to Categorical, here

# always convert to numpy array

values = PyObject_to_object(out)

if isinstance(values, dict):

values = np.take(values['dictionary'], values['indices'])

if dtype is None:

return values

return values.astype(dtype)

def to_numpy(self):

"""

if self.null_count:

raise NotImplementedError('NumPy array view is only supported '

'for arrays without nulls.')

if not is_primitive(self.type.id) or self.type.id == _Type_BOOL:

raise NotImplementedError('NumPy array view is only supported '

'for primitive types.')

buflist = self.buffers()

assert len(buflist) == 2

return np.frombuffer(buflist[-1], dtype=self.type.to_pandas_dtype())[

self.offset:self.offset + len(self)]

def to_pylist(self):

"""

return [x.as_py() for x in self]

def validate(self):

"""

with nogil:

check_status(self.ap.Validate())

@property

def offset(self):

return self.sp_array.get().offset()

def buffers(self):