switch (chunked.num_chunks()) {

case 0: {

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Array> array,

::arrow::MakeArrayOfNull(chunked.type(), 0));

return array->data();

}

case 1:

return chunked.chunk(0)->data();

default:

// ARROW-3762(wesm): If item reader yields a chunked array, we reject as

// this is not yet implemented

return Status::NotImplemented(

"Nested data conversions not implemented for chunked array outputs");

}

public:

virtual Status GetDefLevels(const int16_t** data, int64_t* length) = 0;

virtual Status GetRepLevels(const int16_t** data, int64_t* length) = 0;

virtual const std::shared_ptr<Field> field() = 0;

::arrow::Status NextBatch(int64_t batch_size,

std::shared_ptr<::arrow::ChunkedArray>* out) final {

RETURN_NOT_OK(LoadBatch(batch_size));

RETURN_NOT_OK(BuildArray(batch_size, out));

for (int x = 0; x < (*out)->num_chunks(); x++) {

RETURN_NOT_OK((*out)->chunk(x)->Validate());

}

return Status::OK();

}

virtual ::arrow::Status LoadBatch(int64_t num_records) = 0;

virtual ::arrow::Status BuildArray(int64_t length_upper_bound,

std::shared_ptr<::arrow::ChunkedArray>* out) = 0;

virtual bool IsOrHasRepeatedChild() const = 0;

const std::vector<int>& values) {

std::shared_ptr<std::unordered_set<int>> result(new std::unordered_set<int>());

result->insert(values.begin(), values.end());

return result;

public:

FileReaderImpl(MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader,

ArrowReaderProperties properties)

: pool_(pool),

reader_(std::move(reader)),

reader_properties_(std::move(properties)) {}

Status Init() {

return SchemaManifest::Make(reader_->metadata()->schema(),

reader_->metadata()->key_value_metadata(),

reader_properties_, &manifest_);

}

FileColumnIteratorFactory SomeRowGroupsFactory(std::vector<int> row_groups) {

return [row_groups](int i, ParquetFileReader* reader) {

return new FileColumnIterator(i, reader, row_groups);

};

}

FileColumnIteratorFactory AllRowGroupsFactory() {

return SomeRowGroupsFactory(Iota(reader_->metadata()->num_row_groups()));

}

Status BoundsCheckColumn(int column) {

if (column < 0 || column >= this->num_columns()) {

return Status::Invalid("Column index out of bounds (got ", column,

", should be "

"between 0 and ",

this->num_columns() - 1, ")");

}

return Status::OK();

}

Status BoundsCheckRowGroup(int row_group) {

// row group indices check

if (row_group < 0 || row_group >= num_row_groups()) {

return Status::Invalid("Some index in row_group_indices is ", row_group,

", which is either < 0 or >= num_row_groups(",

num_row_groups(), ")");

}

return Status::OK();

}

Status BoundsCheck(const std::vector<int>& row_groups,

const std::vector<int>& column_indices) {

for (int i : row_groups) {

RETURN_NOT_OK(BoundsCheckRowGroup(i));

}

for (int i : column_indices) {

RETURN_NOT_OK(BoundsCheckColumn(i));

}

return Status::OK();

}

std::shared_ptr<RowGroupReader> RowGroup(int row_group_index) override;

Status ReadTable(const std::vector<int>& indices,

std::shared_ptr<Table>* out) override {

return ReadRowGroups(Iota(reader_->metadata()->num_row_groups()), indices, out);

}

Status GetFieldReader(int i,

const std::shared_ptr<std::unordered_set<int>>& included_leaves,

const std::vector<int>& row_groups,

std::unique_ptr<ColumnReaderImpl>* out) {

// Should be covered by GetRecordBatchReader checks but

// manifest_.schema_fields is a separate variable so be extra careful.

if (ARROW_PREDICT_FALSE(i < 0 ||

static_cast<size_t>(i) >= manifest_.schema_fields.size())) {

return Status::Invalid("Column index out of bounds (got ", i,

", should be "

"between 0 and ",

manifest_.schema_fields.size(), ")");

}

auto ctx = std::make_shared<ReaderContext>();

ctx->reader = reader_.get();

ctx->pool = pool_;

ctx->iterator_factory = SomeRowGroupsFactory(row_groups);

ctx->filter_leaves = true;

ctx->included_leaves = included_leaves;

return GetReader(manifest_.schema_fields[i], ctx, out);

}

Status GetFieldReaders(const std::vector<int>& column_indices,

const std::vector<int>& row_groups,

std::vector<std::shared_ptr<ColumnReaderImpl>>* out,

std::shared_ptr<::arrow::Schema>* out_schema) {

// We only need to read schema fields which have columns indicated

// in the indices vector

ARROW_ASSIGN_OR_RAISE(std::vector<int> field_indices,

manifest_.GetFieldIndices(column_indices));

auto included_leaves = VectorToSharedSet(column_indices);

out->resize(field_indices.size());

::arrow::FieldVector out_fields(field_indices.size());

for (size_t i = 0; i < out->size(); ++i) {

std::unique_ptr<ColumnReaderImpl> reader;

RETURN_NOT_OK(

GetFieldReader(field_indices[i], included_leaves, row_groups, &reader));

out_fields[i] = reader->field();

out->at(i) = std::move(reader);

}

*out_schema = ::arrow::schema(std::move(out_fields), manifest_.schema_metadata);

return Status::OK();

}

Status GetColumn(int i, FileColumnIteratorFactory iterator_factory,

std::unique_ptr<ColumnReader>* out);

Status GetColumn(int i, std::unique_ptr<ColumnReader>* out) override {

return GetColumn(i, AllRowGroupsFactory(), out);

}

Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override {

return FromParquetSchema(reader_->metadata()->schema(), reader_properties_,

reader_->metadata()->key_value_metadata(), out);

}

Status ReadSchemaField(int i, std::shared_ptr<ChunkedArray>* out) override {

auto included_leaves = VectorToSharedSet(Iota(reader_->metadata()->num_columns()));

std::vector<int> row_groups = Iota(reader_->metadata()->num_row_groups());

std::unique_ptr<ColumnReaderImpl> reader;

RETURN_NOT_OK(GetFieldReader(i, included_leaves, row_groups, &reader));

return ReadColumn(i, row_groups, reader.get(), out);

}

Status ReadColumn(int i, const std::vector<int>& row_groups, ColumnReader* reader,

std::shared_ptr<ChunkedArray>* out) {

BEGIN_PARQUET_CATCH_EXCEPTIONS

// TODO(wesm): This calculation doesn't make much sense when we have repeated

// schema nodes

int64_t records_to_read = 0;

for (auto row_group : row_groups) {

// Can throw exception

records_to_read +=

reader_->metadata()->RowGroup(row_group)->ColumnChunk(i)->num_values();

}

#ifdef ARROW_WITH_OPENTELEMETRY

std::string column_name = reader_->metadata()->schema()->Column(i)->name();

std::string phys_type =

TypeToString(reader_->metadata()->schema()->Column(i)->physical_type());

::arrow::util::tracing::Span span;

START_SPAN(span, "parquet::arrow::read_column",

{{"parquet.arrow.columnindex", i},

{"parquet.arrow.columnname", column_name},

{"parquet.arrow.physicaltype", phys_type},

{"parquet.arrow.records_to_read", records_to_read}});

#endif

return reader->NextBatch(records_to_read, out);

END_PARQUET_CATCH_EXCEPTIONS

}

Status ReadColumn(int i, const std::vector<int>& row_groups,

std::shared_ptr<ChunkedArray>* out) {

std::unique_ptr<ColumnReader> flat_column_reader;

RETURN_NOT_OK(GetColumn(i, SomeRowGroupsFactory(row_groups), &flat_column_reader));

return ReadColumn(i, row_groups, flat_column_reader.get(), out);

}

Status ReadColumn(int i, std::shared_ptr<ChunkedArray>* out) override {

return ReadColumn(i, Iota(reader_->metadata()->num_row_groups()), out);

}

Status ReadTable(std::shared_ptr<Table>* table) override {

return ReadTable(Iota(reader_->metadata()->num_columns()), table);

}

Status ReadRowGroups(const std::vector<int>& row_groups,

const std::vector<int>& indices,

std::shared_ptr<Table>* table) override;

// Helper method used by ReadRowGroups - read the given row groups/columns, skipping

// bounds checks and pre-buffering. Takes a shared_ptr to self to keep the reader

// alive in async contexts.

Future<std::shared_ptr<Table>> DecodeRowGroups(

std::shared_ptr<FileReaderImpl> self, const std::vector<int>& row_groups,

const std::vector<int>& column_indices, ::arrow::internal::Executor* cpu_executor);

Status ReadRowGroups(const std::vector<int>& row_groups,

std::shared_ptr<Table>* table) override {

return ReadRowGroups(row_groups, Iota(reader_->metadata()->num_columns()), table);

}

Status ReadRowGroup(int row_group_index, const std::vector<int>& column_indices,

std::shared_ptr<Table>* out) override {

return ReadRowGroups({row_group_index}, column_indices, out);

}

Status ReadRowGroup(int i, std::shared_ptr<Table>* table) override {

return ReadRowGroup(i, Iota(reader_->metadata()->num_columns()), table);

}

Status GetRecordBatchReader(const std::vector<int>& row_group_indices,

const std::vector<int>& column_indices,

std::unique_ptr<RecordBatchReader>* out) override;

Status GetRecordBatchReader(const std::vector<int>& row_group_indices,

std::unique_ptr<RecordBatchReader>* out) override {

return GetRecordBatchReader(row_group_indices,

Iota(reader_->metadata()->num_columns()), out);

}

Status GetRecordBatchReader(std::unique_ptr<RecordBatchReader>* out) override {

return GetRecordBatchReader(Iota(num_row_groups()),

Iota(reader_->metadata()->num_columns()), out);

}

::arrow::Result<::arrow::AsyncGenerator<std::shared_ptr<::arrow::RecordBatch>>>

GetRecordBatchGenerator(std::shared_ptr<FileReader> reader,

const std::vector<int> row_group_indices,

const std::vector<int> column_indices,

::arrow::internal::Executor* cpu_executor,

int64_t rows_to_readahead) override;

int num_columns() const { return reader_->metadata()->num_columns(); }

ParquetFileReader* parquet_reader() const override { return reader_.get(); }

int num_row_groups() const override { return reader_->metadata()->num_row_groups(); }

void set_use_threads(bool use_threads) override {

reader_properties_.set_use_threads(use_threads);

}

void set_batch_size(int64_t batch_size) override {

reader_properties_.set_batch_size(batch_size);

}

const ArrowReaderProperties& properties() const override { return reader_properties_; }

const SchemaManifest& manifest() const override { return manifest_; }

Status ScanContents(std::vector<int> columns, const int32_t column_batch_size,

int64_t* num_rows) override {

BEGIN_PARQUET_CATCH_EXCEPTIONS

*num_rows = ScanFileContents(columns, column_batch_size, reader_.get());

return Status::OK();

END_PARQUET_CATCH_EXCEPTIONS

}

MemoryPool* pool_;

std::unique_ptr<ParquetFileReader> reader_;

ArrowReaderProperties reader_properties_;

SchemaManifest manifest_;

public:

RowGroupRecordBatchReader(::arrow::RecordBatchIterator batches,

std::shared_ptr<::arrow::Schema> schema)

: batches_(std::move(batches)), schema_(std::move(schema)) {}

~RowGroupRecordBatchReader() override {}

Status ReadNext(std::shared_ptr<::arrow::RecordBatch>* out) override {

return batches_.Next().Value(out);

}

std::shared_ptr<::arrow::Schema> schema() const override { return schema_; }

private:

::arrow::Iterator<std::shared_ptr<::arrow::RecordBatch>> batches_;

std::shared_ptr<::arrow::Schema> schema_;

public:

ColumnChunkReaderImpl(FileReaderImpl* impl, int row_group_index, int column_index)

: impl_(impl), column_index_(column_index), row_group_index_(row_group_index) {}

Status Read(std::shared_ptr<::arrow::ChunkedArray>* out) override {

return impl_->ReadColumn(column_index_, {row_group_index_}, out);

}

private:

FileReaderImpl* impl_;

int column_index_;

int row_group_index_;

public:

RowGroupReaderImpl(FileReaderImpl* impl, int row_group_index)

: impl_(impl), row_group_index_(row_group_index) {}

std::shared_ptr<ColumnChunkReader> Column(int column_index) override {

return std::make_shared<ColumnChunkReaderImpl>(impl_, row_group_index_, column_index);

}

Status ReadTable(const std::vector<int>& column_indices,

std::shared_ptr<::arrow::Table>* out) override {

return impl_->ReadRowGroup(row_group_index_, column_indices, out);

}

Status ReadTable(std::shared_ptr<::arrow::Table>* out) override {

return impl_->ReadRowGroup(row_group_index_, out);

}

private:

FileReaderImpl* impl_;

int row_group_index_;

public:

LeafReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,

std::unique_ptr<FileColumnIterator> input,

::parquet::internal::LevelInfo leaf_info)

: ctx_(std::move(ctx)),

field_(std::move(field)),

input_(std::move(input)),

descr_(input_->descr()) {

record_reader_ = RecordReader::Make(

descr_, leaf_info, ctx_->pool, field_->type()->id() == ::arrow::Type::DICTIONARY);

NextRowGroup();

}

Status GetDefLevels(const int16_t** data, int64_t* length) final {

*data = record_reader_->def_levels();

*length = record_reader_->levels_position();

return Status::OK();

}

Status GetRepLevels(const int16_t** data, int64_t* length) final {

*data = record_reader_->rep_levels();

*length = record_reader_->levels_position();

return Status::OK();

}

bool IsOrHasRepeatedChild() const final { return false; }

Status LoadBatch(int64_t records_to_read) final {

BEGIN_PARQUET_CATCH_EXCEPTIONS

out_ = nullptr;

record_reader_->Reset();

// Pre-allocation gives much better performance for flat columns

record_reader_->Reserve(records_to_read);

while (records_to_read > 0) {

if (!record_reader_->HasMoreData()) {

break;

}

int64_t records_read = record_reader_->ReadRecords(records_to_read);

records_to_read -= records_read;

if (records_read == 0) {

NextRowGroup();

}

}

RETURN_NOT_OK(

TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_));

return Status::OK();

END_PARQUET_CATCH_EXCEPTIONS

}

::arrow::Status BuildArray(int64_t length_upper_bound,

std::shared_ptr<::arrow::ChunkedArray>* out) final {

*out = out_;

return Status::OK();

}

const std::shared_ptr<Field> field() override { return field_; }

private:

std::shared_ptr<ChunkedArray> out_;

void NextRowGroup() {

std::unique_ptr<PageReader> page_reader = input_->NextChunk();

record_reader_->SetPageReader(std::move(page_reader));

}

std::shared_ptr<ReaderContext> ctx_;

std::shared_ptr<Field> field_;

std::unique_ptr<FileColumnIterator> input_;

const ColumnDescriptor* descr_;

std::shared_ptr<RecordReader> record_reader_;

};

class ExtensionReader : public ColumnReaderImpl {

public:

ExtensionReader(std::shared_ptr<Field> field,

std::unique_ptr<ColumnReaderImpl> storage_reader)

: field_(std::move(field)), storage_reader_(std::move(storage_reader)) {}

Status GetDefLevels(const int16_t** data, int64_t* length) override {

return storage_reader_->GetDefLevels(data, length);

}

Status GetRepLevels(const int16_t** data, int64_t* length) override {

return storage_reader_->GetRepLevels(data, length);

}

Status LoadBatch(int64_t number_of_records) final {

return storage_reader_->LoadBatch(number_of_records);

}

Status BuildArray(int64_t length_upper_bound,

std::shared_ptr<ChunkedArray>* out) override {

std::shared_ptr<ChunkedArray> storage;

RETURN_NOT_OK(storage_reader_->BuildArray(length_upper_bound, &storage));

*out = ExtensionType::WrapArray(field_->type(), storage);

return Status::OK();

}

bool IsOrHasRepeatedChild() const final {

return storage_reader_->IsOrHasRepeatedChild();

}

const std::shared_ptr<Field> field() override { return field_; }

private:

std::shared_ptr<Field> field_;

std::unique_ptr<ColumnReaderImpl> storage_reader_;

};

template <typename IndexType>

class ListReader : public ColumnReaderImpl {

public:

ListReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,

::parquet::internal::LevelInfo level_info,

std::unique_ptr<ColumnReaderImpl> child_reader)

: ctx_(std::move(ctx)),

field_(std::move(field)),

level_info_(level_info),

item_reader_(std::move(child_reader)) {}

Status GetDefLevels(const int16_t** data, int64_t* length) override {

return item_reader_->GetDefLevels(data, length);

}

Status GetRepLevels(const int16_t** data, int64_t* length) override {

return item_reader_->GetRepLevels(data, length);

}

bool IsOrHasRepeatedChild() const final { return true; }

Status LoadBatch(int64_t number_of_records) final {

return item_reader_->LoadBatch(number_of_records);

}

virtual ::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(

std::shared_ptr<ArrayData> data) {

if (field_->type()->id() == ::arrow::Type::MAP) {

// Error out if data is not map-compliant instead of aborting in MakeArray below

RETURN_NOT_OK(::arrow::MapArray::ValidateChildData(data->child_data));

}

std::shared_ptr<Array> result = ::arrow::MakeArray(data);

return std::make_shared<ChunkedArray>(result);

}

Status BuildArray(int64_t length_upper_bound,

std::shared_ptr<ChunkedArray>* out) override {

const int16_t* def_levels;

const int16_t* rep_levels;

int64_t num_levels;

RETURN_NOT_OK(item_reader_->GetDefLevels(&def_levels, &num_levels));

RETURN_NOT_OK(item_reader_->GetRepLevels(&rep_levels, &num_levels));

std::shared_ptr<ResizableBuffer> validity_buffer;

::parquet::internal::ValidityBitmapInputOutput validity_io;

validity_io.values_read_upper_bound = length_upper_bound;

if (field_->nullable()) {

ARROW_ASSIGN_OR_RAISE(validity_buffer,

AllocateResizableBuffer(

bit_util::BytesForBits(length_upper_bound), ctx_->pool));

validity_io.valid_bits = validity_buffer->mutable_data();

}

ARROW_ASSIGN_OR_RAISE(

std::shared_ptr<ResizableBuffer> offsets_buffer,

AllocateResizableBuffer(

sizeof(IndexType) * std::max(int64_t{1}, length_upper_bound + 1),

ctx_->pool));

// Ensure zero initialization in case we have reached a zero length list (and

// because first entry is always zero).

IndexType* offset_data = reinterpret_cast<IndexType*>(offsets_buffer->mutable_data());

offset_data[0] = 0;

BEGIN_PARQUET_CATCH_EXCEPTIONS

::parquet::internal::DefRepLevelsToList(def_levels, rep_levels, num_levels,

level_info_, &validity_io, offset_data);

END_PARQUET_CATCH_EXCEPTIONS

RETURN_NOT_OK(item_reader_->BuildArray(offset_data[validity_io.values_read], out));

// Resize to actual number of elements returned.

RETURN_NOT_OK(

offsets_buffer->Resize((validity_io.values_read + 1) * sizeof(IndexType)));

if (validity_buffer != nullptr) {

RETURN_NOT_OK(

validity_buffer->Resize(bit_util::BytesForBits(validity_io.values_read)));

validity_buffer->ZeroPadding();

}

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayData> item_chunk, ChunksToSingle(**out));

std::vector<std::shared_ptr<Buffer>> buffers{

validity_io.null_count > 0 ? validity_buffer : nullptr, offsets_buffer};

auto data = std::make_shared<ArrayData>(

field_->type(),

/*length=*/validity_io.values_read, std::move(buffers),

std::vector<std::shared_ptr<ArrayData>>{item_chunk}, validity_io.null_count);

ARROW_ASSIGN_OR_RAISE(*out, AssembleArray(std::move(data)));

return Status::OK();

}

const std::shared_ptr<Field> field() override { return field_; }

private:

std::shared_ptr<ReaderContext> ctx_;

std::shared_ptr<Field> field_;

::parquet::internal::LevelInfo level_info_;

std::unique_ptr<ColumnReaderImpl> item_reader_;

};

class PARQUET_NO_EXPORT FixedSizeListReader : public ListReader<int32_t> {

public:

FixedSizeListReader(std::shared_ptr<ReaderContext> ctx, std::shared_ptr<Field> field,

::parquet::internal::LevelInfo level_info,

std::unique_ptr<ColumnReaderImpl> child_reader)

: ListReader(std::move(ctx), std::move(field), level_info,

std::move(child_reader)) {}

::arrow::Result<std::shared_ptr<ChunkedArray>> AssembleArray(

std::shared_ptr<ArrayData> data) final {

DCHECK_EQ(data->buffers.size(), 2);

DCHECK_EQ(field()->type()->id(), ::arrow::Type::FIXED_SIZE_LIST);

const auto& type = checked_cast<::arrow::FixedSizeListType&>(*field()->type());

const int32_t* offsets = reinterpret_cast<const int32_t*>(data->buffers[1]->data());

for (int x = 1; x <= data->length; x++) {

int32_t size = offsets[x] - offsets[x - 1];

if (size != type.list_size()) {

return Status::Invalid("Expected all lists to be of size=", type.list_size(),

" but index ", x, " had size=", size);

}

}

data->buffers.resize(1);

std::shared_ptr<Array> result = ::arrow::MakeArray(data);

return std::make_shared<ChunkedArray>(result);

for (const std::unique_ptr<ColumnReaderImpl>& reader : children_) {

RETURN_NOT_OK(reader->LoadBatch(records_to_read));

}

return Status::OK();

*data = nullptr;

if (children_.size() == 0) {

*length = 0;

return Status::Invalid("StructReader had no children");

}

// This method should only be called when this struct or one of its parents

// are optional/repeated or it has a repeated child.

// Meaning all children must have rep/def levels associated

// with them.

RETURN_NOT_OK(def_rep_level_child_->GetDefLevels(data, length));

return Status::OK();

*data = nullptr;

if (children_.size() == 0) {

*length = 0;

return Status::Invalid("StructReader had no childre");

}

// This method should only be called when this struct or one of its parents

// are optional/repeated or it has repeated child.

// Meaning all children must have rep/def levels associated

// with them.

RETURN_NOT_OK(def_rep_level_child_->GetRepLevels(data, length));

return Status::OK();

std::shared_ptr<ChunkedArray>* out) {

std::vector<std::shared_ptr<ArrayData>> children_array_data;

std::shared_ptr<ResizableBuffer> null_bitmap;

::parquet::internal::ValidityBitmapInputOutput validity_io;

validity_io.values_read_upper_bound = length_upper_bound;

// This simplifies accounting below.

validity_io.values_read = length_upper_bound;

BEGIN_PARQUET_CATCH_EXCEPTIONS

const int16_t* def_levels;

const int16_t* rep_levels;

int64_t num_levels;

if (has_repeated_child_) {

ARROW_ASSIGN_OR_RAISE(

null_bitmap,

AllocateResizableBuffer(bit_util::BytesForBits(length_upper_bound), ctx_->pool));

validity_io.valid_bits = null_bitmap->mutable_data();

RETURN_NOT_OK(GetDefLevels(&def_levels, &num_levels));

RETURN_NOT_OK(GetRepLevels(&rep_levels, &num_levels));

DefRepLevelsToBitmap(def_levels, rep_levels, num_levels, level_info_, &validity_io);

} else if (filtered_field_->nullable()) {

ARROW_ASSIGN_OR_RAISE(

null_bitmap,

AllocateResizableBuffer(bit_util::BytesForBits(length_upper_bound), ctx_->pool));

validity_io.valid_bits = null_bitmap->mutable_data();

RETURN_NOT_OK(GetDefLevels(&def_levels, &num_levels));

DefLevelsToBitmap(def_levels, num_levels, level_info_, &validity_io);

}

// Ensure all values are initialized.

if (null_bitmap) {

RETURN_NOT_OK(null_bitmap->Resize(bit_util::BytesForBits(validity_io.values_read)));

null_bitmap->ZeroPadding();

}

END_PARQUET_CATCH_EXCEPTIONS

// Gather children arrays and def levels

for (auto& child : children_) {

std::shared_ptr<ChunkedArray> field;

RETURN_NOT_OK(child->BuildArray(validity_io.values_read, &field));

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrayData> array_data, ChunksToSingle(*field));

children_array_data.push_back(std::move(array_data));

}

if (!filtered_field_->nullable() && !has_repeated_child_) {

validity_io.values_read = children_array_data.front()->length;

}

std::vector<std::shared_ptr<Buffer>> buffers{validity_io.null_count > 0 ? null_bitmap

: nullptr};

auto data =

std::make_shared<ArrayData>(filtered_field_->type(),

/*length=*/validity_io.values_read, std::move(buffers),

std::move(children_array_data));

std::shared_ptr<Array> result = ::arrow::MakeArray(data);

*out = std::make_shared<ChunkedArray>(result);

return Status::OK();

const std::shared_ptr<ReaderContext>& ctx,

std::unique_ptr<ColumnReaderImpl>* out) {

BEGIN_PARQUET_CATCH_EXCEPTIONS

auto type_id = arrow_field->type()->id();

if (type_id == ::arrow::Type::EXTENSION) {

auto storage_field = arrow_field->WithType(

checked_cast<const ExtensionType&>(*arrow_field->type()).storage_type());

RETURN_NOT_OK(GetReader(field, storage_field, ctx, out));

*out = std::make_unique<ExtensionReader>(arrow_field, std::move(*out));

return Status::OK();

}

if (field.children.size() == 0) {

if (!field.is_leaf()) {

return Status::Invalid("Parquet non-leaf node has no children");

}

if (!ctx->IncludesLeaf(field.column_index)) {

*out = nullptr;

return Status::OK();

}

std::unique_ptr<FileColumnIterator> input(

ctx->iterator_factory(field.column_index, ctx->reader));

*out = std::make_unique<LeafReader>(ctx, arrow_field, std::move(input),

field.level_info);

} else if (type_id == ::arrow::Type::LIST || type_id == ::arrow::Type::MAP ||

type_id == ::arrow::Type::FIXED_SIZE_LIST ||

type_id == ::arrow::Type::LARGE_LIST) {

auto list_field = arrow_field;

auto child = &field.children[0];

std::unique_ptr<ColumnReaderImpl> child_reader;

RETURN_NOT_OK(GetReader(*child, ctx, &child_reader));

if (child_reader == nullptr) {

*out = nullptr;

return Status::OK();

}

// These two types might not be equal if there column pruning occurred.

// further down the stack.

const std::shared_ptr<DataType> reader_child_type = child_reader->field()->type();

// This should really never happen but was raised as a question on the code

// review, this should be pretty cheap check so leave it in.

if (ARROW_PREDICT_FALSE(list_field->type()->num_fields() != 1)) {

return Status::Invalid("expected exactly one child field for: ",

list_field->ToString());

}

const DataType& schema_child_type = *(list_field->type()->field(0)->type());

if (type_id == ::arrow::Type::MAP) {

if (reader_child_type->num_fields() != 2 ||

!reader_child_type->field(0)->type()->Equals(

*schema_child_type.field(0)->type())) {

// This case applies if either key or value are completed filtered

// out so we can take the type as is or the key was partially

// so keeping it as a map no longer makes sence.

list_field = list_field->WithType(::arrow::list(child_reader->field()));

} else if (!reader_child_type->field(1)->type()->Equals(

*schema_child_type.field(1)->type())) {

list_field = list_field->WithType(std::make_shared<::arrow::MapType>(

reader_child_type->field(

0), // field 0 is unchanged baed on previous if statement

reader_child_type->field(1)));

}

// Map types are list<struct<key, value>> so use ListReader

// for reconstruction.

*out = std::make_unique<ListReader<int32_t>>(ctx, list_field, field.level_info,

std::move(child_reader));

} else if (type_id == ::arrow::Type::LIST) {

if (!reader_child_type->Equals(schema_child_type)) {

list_field = list_field->WithType(::arrow::list(reader_child_type));

}

*out = std::make_unique<ListReader<int32_t>>(ctx, list_field, field.level_info,

std::move(child_reader));

} else if (type_id == ::arrow::Type::LARGE_LIST) {

if (!reader_child_type->Equals(schema_child_type)) {

list_field = list_field->WithType(::arrow::large_list(reader_child_type));

}

*out = std::make_unique<ListReader<int64_t>>(ctx, list_field, field.level_info,

std::move(child_reader));

} else if (type_id == ::arrow::Type::FIXED_SIZE_LIST) {

if (!reader_child_type->Equals(schema_child_type)) {

auto& fixed_list_type =

checked_cast<const ::arrow::FixedSizeListType&>(*list_field->type());

int32_t list_size = fixed_list_type.list_size();

list_field =

list_field->WithType(::arrow::fixed_size_list(reader_child_type, list_size));

}

*out = std::make_unique<FixedSizeListReader>(ctx, list_field, field.level_info,

std::move(child_reader));

} else {

return Status::UnknownError("Unknown list type: ", field.field->ToString());

}

} else if (type_id == ::arrow::Type::STRUCT) {

std::vector<std::shared_ptr<Field>> child_fields;

int arrow_field_idx = 0;

std::vector<std::unique_ptr<ColumnReaderImpl>> child_readers;

for (const auto& child : field.children) {

std::unique_ptr<ColumnReaderImpl> child_reader;

RETURN_NOT_OK(GetReader(child, ctx, &child_reader));

if (!child_reader) {

arrow_field_idx++;

// If all children were pruned, then we do not try to read this field

continue;

}

std::shared_ptr<::arrow::Field> child_field = child.field;

const DataType& reader_child_type = *child_reader->field()->type();

const DataType& schema_child_type =

*arrow_field->type()->field(arrow_field_idx++)->type();

// These might not be equal if column pruning occurred.

if (!schema_child_type.Equals(reader_child_type)) {

child_field = child_field->WithType(child_reader->field()->type());

}

child_fields.push_back(child_field);

child_readers.emplace_back(std::move(child_reader));

}

if (child_fields.empty()) {

*out = nullptr;

return Status::OK();

}

auto filtered_field =

::arrow::field(arrow_field->name(), ::arrow::struct_(child_fields),

arrow_field->nullable(), arrow_field->metadata());

*out = std::make_unique<StructReader>(ctx, filtered_field, field.level_info,

std::move(child_readers));

} else {

return Status::Invalid("Unsupported nested type: ", arrow_field->ToString());

}

return Status::OK();

END_PARQUET_CATCH_EXCEPTIONS

const std::vector<int>& column_indices,

std::unique_ptr<RecordBatchReader>* out) {

RETURN_NOT_OK(BoundsCheck(row_groups, column_indices));

if (reader_properties_.pre_buffer()) {

// PARQUET-1698/PARQUET-1820: pre-buffer row groups/column chunks if enabled

BEGIN_PARQUET_CATCH_EXCEPTIONS

reader_->PreBuffer(row_groups, column_indices, reader_properties_.io_context(),

reader_properties_.cache_options());

END_PARQUET_CATCH_EXCEPTIONS

}

std::vector<std::shared_ptr<ColumnReaderImpl>> readers;

std::shared_ptr<::arrow::Schema> batch_schema;

RETURN_NOT_OK(GetFieldReaders(column_indices, row_groups, &readers, &batch_schema));

if (readers.empty()) {

// Just generate all batches right now; they're cheap since they have no columns.

int64_t batch_size = properties().batch_size();

auto max_sized_batch =

::arrow::RecordBatch::Make(batch_schema, batch_size, ::arrow::ArrayVector{});

::arrow::RecordBatchVector batches;