"""Statistics for a single column in a single row group."""

def __cinit__(self):

pass

def __repr__(self):

return """{}

converted_type (legacy): {}""".format(object.__repr__(self),

self.has_min_max,

self.min,

self.max,

self.null_count,

self.distinct_count,

self.num_values,

self.physical_type,

str(self.logical_type),

self.converted_type)

def to_dict(self):

"""

d = dict(

has_min_max=self.has_min_max,

min=self.min,

max=self.max,

null_count=self.null_count,

distinct_count=self.distinct_count,

num_values=self.num_values,

physical_type=self.physical_type

)

return d

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def equals(self, Statistics other):

"""

return self.statistics.get().Equals(deref(other.statistics.get()))

@property

def has_min_max(self):

"""Whether min and max are present (bool)."""

return self.statistics.get().HasMinMax()

@property

def has_null_count(self):

"""Whether null count is present (bool)."""

return self.statistics.get().HasNullCount()

@property

def has_distinct_count(self):

"""Whether distinct count is preset (bool)."""

return self.statistics.get().HasDistinctCount()

@property

def min_raw(self):

"""Min value as physical type (bool, int, float, or bytes)."""

if self.has_min_max:

return _cast_statistic_raw_min(self.statistics.get())

else:

return None

@property

def max_raw(self):

"""Max value as physical type (bool, int, float, or bytes)."""

if self.has_min_max:

return _cast_statistic_raw_max(self.statistics.get())

else:

return None

@property

def min(self):

"""

if self.has_min_max:

min_scalar, _ = _cast_statistics(self.statistics.get())

return min_scalar.as_py()

else:

return None

@property

def max(self):

"""

if self.has_min_max:

_, max_scalar = _cast_statistics(self.statistics.get())

return max_scalar.as_py()

else:

return None

@property

def null_count(self):

"""Number of null values in chunk (int)."""

return self.statistics.get().null_count()

@property

def distinct_count(self):

"""

# This seems to be zero if not set. See: ARROW-11793

return self.statistics.get().distinct_count()

@property

def num_values(self):

"""Number of non-null values (int)."""

return self.statistics.get().num_values()

@property

def physical_type(self):

"""Physical type of column (str)."""

raw_physical_type = self.statistics.get().physical_type()

return physical_type_name_from_enum(raw_physical_type)

@property

def logical_type(self):

"""Logical type of column (:class:`ParquetLogicalType`)."""

return wrap_logical_type(self.statistics.get().descr().logical_type())

@property

def converted_type(self):

"""Legacy converted type (str or None)."""

raw_converted_type = self.statistics.get().descr().converted_type()

"""Logical type of parquet type."""

cdef:

shared_ptr[const CParquetLogicalType] type

def __cinit__(self):

pass

cdef init(self, const shared_ptr[const CParquetLogicalType]& type):

"""

"""Column metadata for a single row group."""

def __cinit__(self):

pass

def __repr__(self):

statistics = indent(repr(self.statistics), 4 * ' ')

return """{0}

total_uncompressed_size: {14}""".format(object.__repr__(self),

self.file_offset,

self.file_path,

self.physical_type,

self.num_values,

self.path_in_schema,

self.is_stats_set,

statistics,

self.compression,

self.encodings,

self.has_dictionary_page,

self.dictionary_page_offset,

self.data_page_offset,

self.total_compressed_size,

self.total_uncompressed_size)

def to_dict(self):

"""

statistics = self.statistics.to_dict() if self.is_stats_set else None

d = dict(

file_offset=self.file_offset,

file_path=self.file_path,

physical_type=self.physical_type,

num_values=self.num_values,

path_in_schema=self.path_in_schema,

is_stats_set=self.is_stats_set,

statistics=statistics,

compression=self.compression,

encodings=self.encodings,

has_dictionary_page=self.has_dictionary_page,

dictionary_page_offset=self.dictionary_page_offset,

data_page_offset=self.data_page_offset,

total_compressed_size=self.total_compressed_size,

total_uncompressed_size=self.total_uncompressed_size

)

return d

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def equals(self, ColumnChunkMetaData other):

"""

return self.metadata.Equals(deref(other.metadata))

@property

def file_offset(self):

"""Offset into file where column chunk is located (int)."""

return self.metadata.file_offset()

@property

def file_path(self):

"""Optional file path if set (str or None)."""

return frombytes(self.metadata.file_path())

@property

def physical_type(self):

"""Physical type of column (str)."""

return physical_type_name_from_enum(self.metadata.type())

@property

def num_values(self):

"""Total number of values (int)."""

return self.metadata.num_values()

@property

def path_in_schema(self):

"""Nested path to field, separated by periods (str)."""

path = self.metadata.path_in_schema().get().ToDotString()

return frombytes(path)

@property

def is_stats_set(self):

"""Whether or not statistics are present in metadata (bool)."""

return self.metadata.is_stats_set()

@property

def statistics(self):

"""Statistics for column chunk (:class:`Statistics`)."""

if not self.metadata.is_stats_set():

return None

statistics = Statistics()

statistics.init(self.metadata.statistics(), self)

return statistics

@property

def compression(self):

"""

return compression_name_from_enum(self.metadata.compression())

@property

def encodings(self):

"""

return tuple(map(encoding_name_from_enum, self.metadata.encodings()))

@property

def has_dictionary_page(self):

"""Whether there is dictionary data present in the column chunk (bool)."""

return bool(self.metadata.has_dictionary_page())

@property

def dictionary_page_offset(self):

"""Offset of dictionary page reglative to column chunk offset (int)."""

if self.has_dictionary_page:

return self.metadata.dictionary_page_offset()

else:

return None

@property

def data_page_offset(self):

"""Offset of data page reglative to column chunk offset (int)."""

return self.metadata.data_page_offset()

@property

def has_index_page(self):

"""Not yet supported."""

raise NotImplementedError('not supported in parquet-cpp')

@property

def index_page_offset(self):

"""Not yet supported."""

raise NotImplementedError("parquet-cpp doesn't return valid values")

@property

def total_compressed_size(self):

"""Compresssed size in bytes (int)."""

return self.metadata.total_compressed_size()

@property

def total_uncompressed_size(self):

"""Uncompressed size in bytes (int)."""

"""Metadata for a single row group."""

def __cinit__(self, FileMetaData parent, int index):

if index < 0 or index >= parent.num_row_groups:

raise IndexError('{0} out of bounds'.format(index))

self.up_metadata = parent._metadata.RowGroup(index)

self.metadata = self.up_metadata.get()

self.parent = parent

self.index = index

def __reduce__(self):

return RowGroupMetaData, (self.parent, self.index)

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def equals(self, RowGroupMetaData other):

"""

return self.metadata.Equals(deref(other.metadata))

def column(self, int i):

"""

if i < 0 or i >= self.num_columns:

raise IndexError('{0} out of bounds'.format(i))

chunk = ColumnChunkMetaData()

chunk.init(self, i)

return chunk

def __repr__(self):

return """{0}

total_byte_size: {3}""".format(object.__repr__(self),

self.num_columns,

self.num_rows,

self.total_byte_size)

def to_dict(self):

"""

columns = []

d = dict(

num_columns=self.num_columns,

num_rows=self.num_rows,

total_byte_size=self.total_byte_size,

columns=columns,

)

for i in range(self.num_columns):

columns.append(self.column(i).to_dict())

return d

@property

def num_columns(self):

"""Number of columns in this row group (int)."""

return self.metadata.num_columns()

@property

def num_rows(self):

"""Number of rows in this row group (int)."""

return self.metadata.num_rows()

@property

def total_byte_size(self):

"""Total byte size of all the uncompressed column data in this row group (int)."""

cdef:

FileMetaData metadata = FileMetaData.__new__(FileMetaData)

CBuffer *buffer = serialized.buffer.get()

uint32_t metadata_len = <uint32_t>buffer.size()

metadata.init(CFileMetaData_Make(buffer.data(), &metadata_len))

"""Parquet metadata for a single file."""

def __cinit__(self):

pass

def __reduce__(self):

cdef:

NativeFile sink = BufferOutputStream()

COutputStream* c_sink = sink.get_output_stream().get()

with nogil:

self._metadata.WriteTo(c_sink)

cdef Buffer buffer = sink.getvalue()

return _reconstruct_filemetadata, (buffer,)

def __repr__(self):

return """{0}

serialized_size: {6}""".format(object.__repr__(self),

self.created_by, self.num_columns,

self.num_rows, self.num_row_groups,

self.format_version,

self.serialized_size)

def to_dict(self):

"""

row_groups = []

d = dict(

created_by=self.created_by,

num_columns=self.num_columns,

num_rows=self.num_rows,

num_row_groups=self.num_row_groups,

row_groups=row_groups,

format_version=self.format_version,

serialized_size=self.serialized_size

)

for i in range(self.num_row_groups):

row_groups.append(self.row_group(i).to_dict())

return d

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def equals(self, FileMetaData other not None):

"""

return self._metadata.Equals(deref(other._metadata))

@property

def schema(self):

"""Schema of the file (:class:`ParquetSchema`)."""

if self._schema is None:

self._schema = ParquetSchema(self)

return self._schema

@property

def serialized_size(self):

"""Size of the original thrift encoded metadata footer (int)."""

return self._metadata.size()

@property

def num_columns(self):

"""Number of columns in file (int)."""

return self._metadata.num_columns()

@property

def num_rows(self):

"""Total number of rows in file (int)."""

return self._metadata.num_rows()

@property

def num_row_groups(self):

"""Number of row groups in file (int)."""

return self._metadata.num_row_groups()

@property

def format_version(self):

"""

cdef ParquetVersion version = self._metadata.version()

if version == ParquetVersion_V1:

return '1.0'

elif version == ParquetVersion_V2_0:

return 'pseudo-2.0'

elif version == ParquetVersion_V2_4:

return '2.4'

elif version == ParquetVersion_V2_6:

return '2.6'

else:

warnings.warn('Unrecognized file version, assuming 2.4: {}'

.format(version))

return '2.4'

@property

def created_by(self):

"""

return frombytes(self._metadata.created_by())

@property

def metadata(self):

"""Additional metadata as key value pairs (dict[bytes, bytes])."""

cdef:

unordered_map[c_string, c_string] metadata

const CKeyValueMetadata* underlying_metadata

underlying_metadata = self._metadata.key_value_metadata().get()

if underlying_metadata != NULL:

underlying_metadata.ToUnorderedMap(&metadata)

return metadata

else:

return None

def row_group(self, int i):

"""

return RowGroupMetaData(self, i)

def set_file_path(self, path):

"""

cdef:

c_string c_path = tobytes(path)

self._metadata.set_file_path(c_path)

def append_row_groups(self, FileMetaData other):

"""

cdef shared_ptr[CFileMetaData] c_metadata

c_metadata = other.sp_metadata

self._metadata.AppendRowGroups(deref(c_metadata))

def write_metadata_file(self, where):

"""

cdef:

shared_ptr[COutputStream] sink

c_string c_where

try:

where = _stringify_path(where)

except TypeError:

get_writer(where, &sink)

else:

c_where = tobytes(where)

with nogil:

sink = GetResultValue(FileOutputStream.Open(c_where))

with nogil:

check_status(

"""A Parquet schema."""

def __cinit__(self, FileMetaData container):

self.parent = container

self.schema = container._metadata.schema()

def __repr__(self):

return "{0}\n{1}".format(

object.__repr__(self),

frombytes(self.schema.ToString(), safe=True))

def __reduce__(self):

return ParquetSchema, (self.parent,)

def __len__(self):

return self.schema.num_columns()

def __getitem__(self, i):

return self.column(i)

@property

def names(self):

"""Name of each field (list of str)."""

return [self[i].name for i in range(len(self))]

def to_arrow_schema(self):

"""

cdef shared_ptr[CSchema] sp_arrow_schema

with nogil:

check_status(FromParquetSchema(

self.schema, default_arrow_reader_properties(),

self.parent._metadata.key_value_metadata(),

&sp_arrow_schema))

return pyarrow_wrap_schema(sp_arrow_schema)

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def equals(self, ParquetSchema other):

"""

return self.schema.Equals(deref(other.schema))

def column(self, i):

"""

if i < 0 or i >= len(self):

raise IndexError('{0} out of bounds'.format(i))

"""Schema for a single column."""

cdef:

int index

ParquetSchema parent

const ColumnDescriptor* descr

def __cinit__(self, ParquetSchema schema, int index):

self.parent = schema

self.index = index # for pickling support

self.descr = schema.schema.Column(index)

def __eq__(self, other):

try:

return self.equals(other)

except TypeError:

return NotImplemented

def __reduce__(self):

return ColumnSchema, (self.parent, self.index)

def equals(self, ColumnSchema other):

"""

return self.descr.Equals(deref(other.descr))

def __repr__(self):

physical_type = self.physical_type

converted_type = self.converted_type

if converted_type == 'DECIMAL':

converted_type = 'DECIMAL({0}, {1})'.format(self.precision,

self.scale)

elif physical_type == 'FIXED_LEN_BYTE_ARRAY':

converted_type = ('FIXED_LEN_BYTE_ARRAY(length={0})'

.format(self.length))

return """<ParquetColumnSchema>

converted_type (legacy): {6}""".format(self.name, self.path,

self.max_definition_level,

self.max_repetition_level,

physical_type,

str(self.logical_type),

converted_type)

@property

def name(self):

"""Name of field (str)."""

return frombytes(self.descr.name())

@property

def path(self):

"""Nested path to field, separated by periods (str)."""

return frombytes(self.descr.path().get().ToDotString())

@property

def max_definition_level(self):

"""Maximum definition level (int)."""

return self.descr.max_definition_level()

@property

def max_repetition_level(self):

"""Maximum repetition level (int)."""

return self.descr.max_repetition_level()

@property

def physical_type(self):

"""Name of physical type (str)."""

return physical_type_name_from_enum(self.descr.physical_type())

@property

def logical_type(self):

"""Logical type of column (:class:`ParquetLogicalType`)."""