if (descr->max_repetition_level() > 0) {

// repeated+flat case

return !descr->schema_node()->is_required();

} else {

// non-repeated+nested case

// Find if a node forces nulls in the lowest level along the hierarchy

const schema::Node* node = descr->schema_node().get();

while (node) {

if (node->is_optional()) {

return true;

}

node = node->parent();

}

return false;

}

if (ARROW_PREDICT_FALSE(number_decoded != expected)) {

ParquetException::EofException("Decoded values " + std::to_string(number_decoded) +

" does not match expected " +

std::to_string(expected));

}

int num_buffered_values, const uint8_t* data,

int32_t data_size) {

max_level_ = max_level;

int32_t num_bytes = 0;

encoding_ = encoding;

num_values_remaining_ = num_buffered_values;

bit_width_ = bit_util::Log2(max_level + 1);

switch (encoding) {

case Encoding::RLE: {

if (data_size < 4) {

throw ParquetException("Received invalid levels (corrupt data page?)");

}

num_bytes = ::arrow::util::SafeLoadAs<int32_t>(data);

if (num_bytes < 0 || num_bytes > data_size - 4) {

throw ParquetException("Received invalid number of bytes (corrupt data page?)");

}

const uint8_t* decoder_data = data + 4;

if (!rle_decoder_) {

rle_decoder_ = std::make_unique<::arrow::util::RleDecoder>(decoder_data,

num_bytes, bit_width_);

} else {

rle_decoder_->Reset(decoder_data, num_bytes, bit_width_);

}

return 4 + num_bytes;

}

case Encoding::BIT_PACKED: {

int num_bits = 0;

if (MultiplyWithOverflow(num_buffered_values, bit_width_, &num_bits)) {

throw ParquetException(

"Number of buffered values too large (corrupt data page?)");

}

num_bytes = static_cast<int32_t>(bit_util::BytesForBits(num_bits));

if (num_bytes < 0 || num_bytes > data_size - 4) {

throw ParquetException("Received invalid number of bytes (corrupt data page?)");

}

if (!bit_packed_decoder_) {

bit_packed_decoder_ =

std::make_unique<::arrow::bit_util::BitReader>(data, num_bytes);

} else {

bit_packed_decoder_->Reset(data, num_bytes);

}

return num_bytes;

}

default:

throw ParquetException("Unknown encoding type for levels.");

}

return -1;

int num_buffered_values, const uint8_t* data) {

max_level_ = max_level;

// Repetition and definition levels always uses RLE encoding

// in the DataPageV2 format.

if (num_bytes < 0) {

throw ParquetException("Invalid page header (corrupt data page?)");

}

encoding_ = Encoding::RLE;

num_values_remaining_ = num_buffered_values;

bit_width_ = bit_util::Log2(max_level + 1);

if (!rle_decoder_) {

rle_decoder_ =

std::make_unique<::arrow::util::RleDecoder>(data, num_bytes, bit_width_);

} else {

rle_decoder_->Reset(data, num_bytes, bit_width_);

}

int num_decoded = 0;

int num_values = std::min(num_values_remaining_, batch_size);

if (encoding_ == Encoding::RLE) {

num_decoded = rle_decoder_->GetBatch(levels, num_values);

} else {

num_decoded = bit_packed_decoder_->GetBatch(bit_width_, levels, num_values);

}

if (num_decoded > 0) {

internal::MinMax min_max = internal::FindMinMax(levels, num_decoded);

if (ARROW_PREDICT_FALSE(min_max.min < 0 || min_max.max > max_level_)) {

std::stringstream ss;

ss << "Malformed levels. min: " << min_max.min << " max: " << min_max.max

<< " out of range. Max Level: " << max_level_;

throw ParquetException(ss.str());

}

}

num_values_remaining_ -= num_decoded;

return num_decoded;

EncodedStatistics page_statistics;

if (!header.__isset.statistics) {

return page_statistics;

}

const format::Statistics& stats = header.statistics;

if (stats.__isset.max) {

page_statistics.set_max(stats.max);

}

if (stats.__isset.min) {

page_statistics.set_min(stats.min);

}

if (stats.__isset.null_count) {

page_statistics.set_null_count(stats.null_count);

}

if (stats.__isset.distinct_count) {

page_statistics.set_distinct_count(stats.distinct_count);

}

return page_statistics;

if (num_values < 0) {

throw ParquetException("Invalid page header (negative number of values)");

}

public:

SerializedPageReader(std::shared_ptr<ArrowInputStream> stream, int64_t total_num_values,

Compression::type codec, const ReaderProperties& properties,

const CryptoContext* crypto_ctx, bool always_compressed)

: properties_(properties),

stream_(std::move(stream)),

decompression_buffer_(AllocateBuffer(properties_.memory_pool(), 0)),

page_ordinal_(0),

seen_num_values_(0),

total_num_values_(total_num_values),

decryption_buffer_(AllocateBuffer(properties_.memory_pool(), 0)) {

if (crypto_ctx != nullptr) {

crypto_ctx_ = *crypto_ctx;

InitDecryption();

}

max_page_header_size_ = kDefaultMaxPageHeaderSize;

decompressor_ = GetCodec(codec);

always_compressed_ = always_compressed;

}

// Implement the PageReader interface

std::shared_ptr<Page> NextPage() override;

void set_max_page_header_size(uint32_t size) override { max_page_header_size_ = size; }

private:

void UpdateDecryption(const std::shared_ptr<Decryptor>& decryptor, int8_t module_type,

std::string* page_aad);

void InitDecryption();

std::shared_ptr<Buffer> DecompressIfNeeded(std::shared_ptr<Buffer> page_buffer,

int compressed_len, int uncompressed_len,

int levels_byte_len = 0);

// Returns true for non-data pages, and if we should skip based on

// data_page_filter_. Performs basic checks on values in the page header.

// Fills in data_page_statistics.

bool ShouldSkipPage(EncodedStatistics* data_page_statistics);

const ReaderProperties properties_;

std::shared_ptr<ArrowInputStream> stream_;

format::PageHeader current_page_header_;

std::shared_ptr<Page> current_page_;

// Compression codec to use.

std::unique_ptr<::arrow::util::Codec> decompressor_;

std::shared_ptr<ResizableBuffer> decompression_buffer_;

bool always_compressed_;

// The fields below are used for calculation of AAD (additional authenticated data)

// suffix which is part of the Parquet Modular Encryption.

// The AAD suffix for a parquet module is built internally by

// concatenating different parts some of which include

// the row group ordinal, column ordinal and page ordinal.

// Please refer to the encryption specification for more details:

// https://github.com/apache/parquet-format/blob/encryption/Encryption.md#44-additional-authenticated-data

// The ordinal fields in the context below are used for AAD suffix calculation.

CryptoContext crypto_ctx_;

int32_t page_ordinal_; // page ordinal does not count the dictionary page

// Maximum allowed page size

uint32_t max_page_header_size_;

// Number of values read in data pages so far

int64_t seen_num_values_;

// Number of values in all the data pages

int64_t total_num_values_;

// data_page_aad_ and data_page_header_aad_ contain the AAD for data page and data page

// header in a single column respectively.

// While calculating AAD for different pages in a single column the pages AAD is

// updated by only the page ordinal.

std::string data_page_aad_;

std::string data_page_header_aad_;

// Encryption

std::shared_ptr<ResizableBuffer> decryption_buffer_;

// Prepare the AAD for quick update later.

if (crypto_ctx_.data_decryptor != nullptr) {

DCHECK(!crypto_ctx_.data_decryptor->file_aad().empty());

data_page_aad_ = encryption::CreateModuleAad(

crypto_ctx_.data_decryptor->file_aad(), encryption::kDataPage,

crypto_ctx_.row_group_ordinal, crypto_ctx_.column_ordinal, kNonPageOrdinal);

}

if (crypto_ctx_.meta_decryptor != nullptr) {

DCHECK(!crypto_ctx_.meta_decryptor->file_aad().empty());

data_page_header_aad_ = encryption::CreateModuleAad(

crypto_ctx_.meta_decryptor->file_aad(), encryption::kDataPageHeader,

crypto_ctx_.row_group_ordinal, crypto_ctx_.column_ordinal, kNonPageOrdinal);

}

int8_t module_type, std::string* page_aad) {

DCHECK(decryptor != nullptr);

if (crypto_ctx_.start_decrypt_with_dictionary_page) {

std::string aad = encryption::CreateModuleAad(

decryptor->file_aad(), module_type, crypto_ctx_.row_group_ordinal,

crypto_ctx_.column_ordinal, kNonPageOrdinal);

decryptor->UpdateAad(aad);

} else {

encryption::QuickUpdatePageAad(page_ordinal_, page_aad);

decryptor->UpdateAad(*page_aad);

}

const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);

if (page_type == PageType::DATA_PAGE) {

const format::DataPageHeader& header = current_page_header_.data_page_header;

CheckNumValuesInHeader(header.num_values);

*data_page_statistics = ExtractStatsFromHeader(header);

seen_num_values_ += header.num_values;

if (data_page_filter_) {

const EncodedStatistics* filter_statistics =

data_page_statistics->is_set() ? data_page_statistics : nullptr;

DataPageStats data_page_stats(filter_statistics, header.num_values,

/*num_rows=*/std::nullopt);

if (data_page_filter_(data_page_stats)) {

return true;

}

}

} else if (page_type == PageType::DATA_PAGE_V2) {

const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;

CheckNumValuesInHeader(header.num_values);

if (header.num_rows < 0) {

throw ParquetException("Invalid page header (negative number of rows)");

}

if (header.definition_levels_byte_length < 0 ||

header.repetition_levels_byte_length < 0) {

throw ParquetException("Invalid page header (negative levels byte length)");

}

*data_page_statistics = ExtractStatsFromHeader(header);

seen_num_values_ += header.num_values;

if (data_page_filter_) {

const EncodedStatistics* filter_statistics =

data_page_statistics->is_set() ? data_page_statistics : nullptr;

DataPageStats data_page_stats(filter_statistics, header.num_values,

header.num_rows);

if (data_page_filter_(data_page_stats)) {

return true;

}

}

} else if (page_type == PageType::DICTIONARY_PAGE) {

const format::DictionaryPageHeader& dict_header =

current_page_header_.dictionary_page_header;

CheckNumValuesInHeader(dict_header.num_values);

} else {

// We don't know what this page type is. We're allowed to skip non-data

// pages.

return true;

}

return false;

ThriftDeserializer deserializer(properties_);

// Loop here because there may be unhandled page types that we skip until

// finding a page that we do know what to do with

while (seen_num_values_ < total_num_values_) {

uint32_t header_size = 0;

uint32_t allowed_page_size = kDefaultPageHeaderSize;

// Page headers can be very large because of page statistics

// We try to deserialize a larger buffer progressively

// until a maximum allowed header limit

while (true) {

PARQUET_ASSIGN_OR_THROW(auto view, stream_->Peek(allowed_page_size));

if (view.size() == 0) {

return std::shared_ptr<Page>(nullptr);

}

// This gets used, then set by DeserializeThriftMsg

header_size = static_cast<uint32_t>(view.size());

try {

if (crypto_ctx_.meta_decryptor != nullptr) {

UpdateDecryption(crypto_ctx_.meta_decryptor, encryption::kDictionaryPageHeader,

&data_page_header_aad_);

}

deserializer.DeserializeMessage(reinterpret_cast<const uint8_t*>(view.data()),

&header_size, &current_page_header_,

crypto_ctx_.meta_decryptor);

break;

} catch (std::exception& e) {

// Failed to deserialize. Double the allowed page header size and try again

std::stringstream ss;

ss << e.what();

allowed_page_size *= 2;

if (allowed_page_size > max_page_header_size_) {

ss << "Deserializing page header failed.\n";

throw ParquetException(ss.str());

}

}

}

// Advance the stream offset

PARQUET_THROW_NOT_OK(stream_->Advance(header_size));

int compressed_len = current_page_header_.compressed_page_size;

int uncompressed_len = current_page_header_.uncompressed_page_size;

if (compressed_len < 0 || uncompressed_len < 0) {

throw ParquetException("Invalid page header");

}

EncodedStatistics data_page_statistics;

if (ShouldSkipPage(&data_page_statistics)) {

PARQUET_THROW_NOT_OK(stream_->Advance(compressed_len));

continue;

}

if (crypto_ctx_.data_decryptor != nullptr) {

UpdateDecryption(crypto_ctx_.data_decryptor, encryption::kDictionaryPage,

&data_page_aad_);

}

// Read the compressed data page.

PARQUET_ASSIGN_OR_THROW(auto page_buffer, stream_->Read(compressed_len));

if (page_buffer->size() != compressed_len) {

std::stringstream ss;

ss << "Page was smaller (" << page_buffer->size() << ") than expected ("

<< compressed_len << ")";

ParquetException::EofException(ss.str());

}

// Decrypt it if we need to

if (crypto_ctx_.data_decryptor != nullptr) {

PARQUET_THROW_NOT_OK(decryption_buffer_->Resize(

compressed_len - crypto_ctx_.data_decryptor->CiphertextSizeDelta(),

/*shrink_to_fit=*/false));

compressed_len = crypto_ctx_.data_decryptor->Decrypt(

page_buffer->data(), compressed_len, decryption_buffer_->mutable_data());

page_buffer = decryption_buffer_;

}

// Uncompress and construct the pages to return.

const PageType::type page_type = LoadEnumSafe(&current_page_header_.type);

if (page_type == PageType::DICTIONARY_PAGE) {

crypto_ctx_.start_decrypt_with_dictionary_page = false;

const format::DictionaryPageHeader& dict_header =

current_page_header_.dictionary_page_header;

bool is_sorted = dict_header.__isset.is_sorted ? dict_header.is_sorted : false;

page_buffer =

DecompressIfNeeded(std::move(page_buffer), compressed_len, uncompressed_len);

return std::make_shared<DictionaryPage>(page_buffer, dict_header.num_values,

LoadEnumSafe(&dict_header.encoding),

is_sorted);

} else if (page_type == PageType::DATA_PAGE) {

++page_ordinal_;

const format::DataPageHeader& header = current_page_header_.data_page_header;

page_buffer =

DecompressIfNeeded(std::move(page_buffer), compressed_len, uncompressed_len);

return std::make_shared<DataPageV1>(page_buffer, header.num_values,

LoadEnumSafe(&header.encoding),

LoadEnumSafe(&header.definition_level_encoding),

LoadEnumSafe(&header.repetition_level_encoding),

uncompressed_len, data_page_statistics);

} else if (page_type == PageType::DATA_PAGE_V2) {

++page_ordinal_;

const format::DataPageHeaderV2& header = current_page_header_.data_page_header_v2;

// Arrow prior to 3.0.0 set is_compressed to false but still compressed.

bool is_compressed =

(header.__isset.is_compressed ? header.is_compressed : false) ||

always_compressed_;

// Uncompress if needed

int levels_byte_len;

if (AddWithOverflow(header.definition_levels_byte_length,

header.repetition_levels_byte_length, &levels_byte_len)) {

throw ParquetException("Levels size too large (corrupt file?)");

}

// DecompressIfNeeded doesn't take `is_compressed` into account as

// it's page type-agnostic.

if (is_compressed) {

page_buffer = DecompressIfNeeded(std::move(page_buffer), compressed_len,

uncompressed_len, levels_byte_len);

}

return std::make_shared<DataPageV2>(

page_buffer, header.num_values, header.num_nulls, header.num_rows,

LoadEnumSafe(&header.encoding), header.definition_levels_byte_length,

header.repetition_levels_byte_length, uncompressed_len, is_compressed,

data_page_statistics);

} else {

throw ParquetException(

"Internal error, we have already skipped non-data pages in ShouldSkipPage()");

}

}

return std::shared_ptr<Page>(nullptr);

std::shared_ptr<Buffer> page_buffer, int compressed_len, int uncompressed_len,

int levels_byte_len) {

if (decompressor_ == nullptr) {

return page_buffer;

}

if (compressed_len < levels_byte_len || uncompressed_len < levels_byte_len) {

throw ParquetException("Invalid page header");

}

// Grow the uncompressed buffer if we need to.

if (uncompressed_len > static_cast<int>(decompression_buffer_->size())) {

PARQUET_THROW_NOT_OK(

decompression_buffer_->Resize(uncompressed_len, /*shrink_to_fit=*/false));

}

if (levels_byte_len > 0) {

// First copy the levels as-is

uint8_t* decompressed = decompression_buffer_->mutable_data();

memcpy(decompressed, page_buffer->data(), levels_byte_len);

}

// Decompress the values

PARQUET_THROW_NOT_OK(decompressor_->Decompress(

compressed_len - levels_byte_len, page_buffer->data() + levels_byte_len,

uncompressed_len - levels_byte_len,

decompression_buffer_->mutable_data() + levels_byte_len));

return decompression_buffer_;

int64_t total_num_values,

Compression::type codec,

const ReaderProperties& properties,

bool always_compressed,

const CryptoContext* ctx) {

return std::unique_ptr<PageReader>(new SerializedPageReader(

std::move(stream), total_num_values, codec, properties, ctx, always_compressed));

int64_t total_num_values,

Compression::type codec,

bool always_compressed,

::arrow::MemoryPool* pool,

const CryptoContext* ctx) {

return std::unique_ptr<PageReader>(

new SerializedPageReader(std::move(stream), total_num_values, codec,

ReaderProperties(pool), ctx, always_compressed));

public:

using T = typename DType::c_type;

ColumnReaderImplBase(const ColumnDescriptor* descr, ::arrow::MemoryPool* pool)

: descr_(descr),

max_def_level_(descr->max_definition_level()),

max_rep_level_(descr->max_repetition_level()),

num_buffered_values_(0),

num_decoded_values_(0),

pool_(pool),

current_decoder_(nullptr),

current_encoding_(Encoding::UNKNOWN) {}

virtual ~ColumnReaderImplBase() = default;

protected:

// Read up to batch_size values from the current data page into the

// pre-allocated memory T*

//

// @returns: the number of values read into the out buffer

int64_t ReadValues(int64_t batch_size, T* out) {

int64_t num_decoded = current_decoder_->Decode(out, static_cast<int>(batch_size));

return num_decoded;

}

// Read up to batch_size values from the current data page into the

// pre-allocated memory T*, leaving spaces for null entries according

// to the def_levels.

//

// @returns: the number of values read into the out buffer

int64_t ReadValuesSpaced(int64_t batch_size, T* out, int64_t null_count,

uint8_t* valid_bits, int64_t valid_bits_offset) {

return current_decoder_->DecodeSpaced(out, static_cast<int>(batch_size),

static_cast<int>(null_count), valid_bits,

valid_bits_offset);

}

// Read multiple definition levels into preallocated memory

//

// Returns the number of decoded definition levels

int64_t ReadDefinitionLevels(int64_t batch_size, int16_t* levels) {

if (max_def_level_ == 0) {

return 0;

}

return definition_level_decoder_.Decode(static_cast<int>(batch_size), levels);

}

bool HasNextInternal() {

// Either there is no data page available yet, or the data page has been

// exhausted

if (num_buffered_values_ == 0 || num_decoded_values_ == num_buffered_values_) {

if (!ReadNewPage() || num_buffered_values_ == 0) {

return false;

}

}

return true;

}

// Read multiple repetition levels into preallocated memory

// Returns the number of decoded repetition levels

int64_t ReadRepetitionLevels(int64_t batch_size, int16_t* levels) {

if (max_rep_level_ == 0) {

return 0;

}

return repetition_level_decoder_.Decode(static_cast<int>(batch_size), levels);

}

// Advance to the next data page

bool ReadNewPage() {

// Loop until we find the next data page.

while (true) {

current_page_ = pager_->NextPage();

if (!current_page_) {

// EOS

return false;

}

if (current_page_->type() == PageType::DICTIONARY_PAGE) {

ConfigureDictionary(static_cast<const DictionaryPage*>(current_page_.get()));

continue;

} else if (current_page_->type() == PageType::DATA_PAGE) {

const auto page = std::static_pointer_cast<DataPageV1>(current_page_);

const int64_t levels_byte_size = InitializeLevelDecoders(

*page, page->repetition_level_encoding(), page->definition_level_encoding());

InitializeDataDecoder(*page, levels_byte_size);

return true;

} else if (current_page_->type() == PageType::DATA_PAGE_V2) {

const auto page = std::static_pointer_cast<DataPageV2>(current_page_);

int64_t levels_byte_size = InitializeLevelDecodersV2(*page);

InitializeDataDecoder(*page, levels_byte_size);

return true;

} else {

// We don't know what this page type is. We're allowed to skip non-data

// pages.

continue;

}

}

return true;

}

void ConfigureDictionary(const DictionaryPage* page) {

int encoding = static_cast<int>(page->encoding());

if (page->encoding() == Encoding::PLAIN_DICTIONARY ||

page->encoding() == Encoding::PLAIN) {

encoding = static_cast<int>(Encoding::RLE_DICTIONARY);

}

auto it = decoders_.find(encoding);

if (it != decoders_.end()) {

throw ParquetException("Column cannot have more than one dictionary.");

}

if (page->encoding() == Encoding::PLAIN_DICTIONARY ||

page->encoding() == Encoding::PLAIN) {

auto dictionary = MakeTypedDecoder<DType>(Encoding::PLAIN, descr_);

dictionary->SetData(page->num_values(), page->data(), page->size());

// The dictionary is fully decoded during DictionaryDecoder::Init, so the

// DictionaryPage buffer is no longer required after this step

//

// TODO(wesm): investigate whether this all-or-nothing decoding of the

// dictionary makes sense and whether performance can be improved

std::unique_ptr<DictDecoder<DType>> decoder = MakeDictDecoder<DType>(descr_, pool_);

decoder->SetDict(dictionary.get());

decoders_[encoding] =

std::unique_ptr<DecoderType>(dynamic_cast<DecoderType*>(decoder.release()));

} else {

ParquetException::NYI("only plain dictionary encoding has been implemented");

}

new_dictionary_ = true;

current_decoder_ = decoders_[encoding].get();

DCHECK(current_decoder_);

}

// Initialize repetition and definition level decoders on the next data page.

// If the data page includes repetition and definition levels, we

// initialize the level decoders and return the number of encoded level bytes.

// The return value helps determine the number of bytes in the encoded data.

int64_t InitializeLevelDecoders(const DataPage& page,

Encoding::type repetition_level_encoding,

Encoding::type definition_level_encoding) {

// Read a data page.

num_buffered_values_ = page.num_values();

// Have not decoded any values from the data page yet

num_decoded_values_ = 0;

const uint8_t* buffer = page.data();

int32_t levels_byte_size = 0;

int32_t max_size = page.size();

// Data page Layout: Repetition Levels - Definition Levels - encoded values.

// Levels are encoded as rle or bit-packed.

// Init repetition levels

if (max_rep_level_ > 0) {

int32_t rep_levels_bytes = repetition_level_decoder_.SetData(

repetition_level_encoding, max_rep_level_,

static_cast<int>(num_buffered_values_), buffer, max_size);

buffer += rep_levels_bytes;

levels_byte_size += rep_levels_bytes;

max_size -= rep_levels_bytes;

}

// TODO figure a way to set max_def_level_ to 0

// if the initial value is invalid

// Init definition levels

if (max_def_level_ > 0) {

int32_t def_levels_bytes = definition_level_decoder_.SetData(

definition_level_encoding, max_def_level_,

static_cast<int>(num_buffered_values_), buffer, max_size);

levels_byte_size += def_levels_bytes;

max_size -= def_levels_bytes;

}

return levels_byte_size;

}

int64_t InitializeLevelDecodersV2(const DataPageV2& page) {

// Read a data page.

num_buffered_values_ = page.num_values();

// Have not decoded any values from the data page yet

num_decoded_values_ = 0;

const uint8_t* buffer = page.data();

const int64_t total_levels_length =

static_cast<int64_t>(page.repetition_levels_byte_length()) +

page.definition_levels_byte_length();

if (total_levels_length > page.size()) {

throw ParquetException("Data page too small for levels (corrupt header?)");

}

if (max_rep_level_ > 0) {

repetition_level_decoder_.SetDataV2(page.repetition_levels_byte_length(),

max_rep_level_,

static_cast<int>(num_buffered_values_), buffer);

}

// ARROW-17453: Even if max_rep_level_ is 0, there may still be

// repetition level bytes written and/or reported in the header by

// some writers (e.g. Athena)

buffer += page.repetition_levels_byte_length();

if (max_def_level_ > 0) {

definition_level_decoder_.SetDataV2(page.definition_levels_byte_length(),

max_def_level_,

static_cast<int>(num_buffered_values_), buffer);

}

return total_levels_length;

}

// Get a decoder object for this page or create a new decoder if this is the

// first page with this encoding.

void InitializeDataDecoder(const DataPage& page, int64_t levels_byte_size) {

const uint8_t* buffer = page.data() + levels_byte_size;

const int64_t data_size = page.size() - levels_byte_size;

if (data_size < 0) {

throw ParquetException("Page smaller than size of encoded levels");

}

Encoding::type encoding = page.encoding();

if (IsDictionaryIndexEncoding(encoding)) {

encoding = Encoding::RLE_DICTIONARY;

}

auto it = decoders_.find(static_cast<int>(encoding));

if (it != decoders_.end()) {

DCHECK(it->second.get() != nullptr);

current_decoder_ = it->second.get();

} else {

switch (encoding) {

case Encoding::PLAIN: {

auto decoder = MakeTypedDecoder<DType>(Encoding::PLAIN, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

case Encoding::BYTE_STREAM_SPLIT: {

auto decoder = MakeTypedDecoder<DType>(Encoding::BYTE_STREAM_SPLIT, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

case Encoding::RLE: {

auto decoder = MakeTypedDecoder<DType>(Encoding::RLE, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

case Encoding::RLE_DICTIONARY:

throw ParquetException("Dictionary page must be before data page.");

case Encoding::DELTA_BINARY_PACKED: {

auto decoder = MakeTypedDecoder<DType>(Encoding::DELTA_BINARY_PACKED, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

case Encoding::DELTA_BYTE_ARRAY: {

auto decoder = MakeTypedDecoder<DType>(Encoding::DELTA_BYTE_ARRAY, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

case Encoding::DELTA_LENGTH_BYTE_ARRAY: {

auto decoder =

MakeTypedDecoder<DType>(Encoding::DELTA_LENGTH_BYTE_ARRAY, descr_);

current_decoder_ = decoder.get();

decoders_[static_cast<int>(encoding)] = std::move(decoder);

break;

}

default:

throw ParquetException("Unknown encoding type.");

}

}

current_encoding_ = encoding;

current_decoder_->SetData(static_cast<int>(num_buffered_values_), buffer,

static_cast<int>(data_size));

}

int64_t available_values_current_page() const {

return num_buffered_values_ - num_decoded_values_;

}

const ColumnDescriptor* descr_;

const int16_t max_def_level_;

const int16_t max_rep_level_;

std::unique_ptr<PageReader> pager_;

std::shared_ptr<Page> current_page_;

// Not set if full schema for this field has no optional or repeated elements

LevelDecoder definition_level_decoder_;

// Not set for flat schemas.

LevelDecoder repetition_level_decoder_;

// The total number of values stored in the data page. This is the maximum of

// the number of encoded definition levels or encoded values. For

// non-repeated, required columns, this is equal to the number of encoded

// values. For repeated or optional values, there may be fewer data values

// than levels, and this tells you how many encoded levels there are in that

// case.

int64_t num_buffered_values_;

// The number of values from the current data page that have been decoded

// into memory

int64_t num_decoded_values_;

::arrow::MemoryPool* pool_;

using DecoderType = TypedDecoder<DType>;

DecoderType* current_decoder_;

Encoding::type current_encoding_;

/// Flag to signal when a new dictionary has been set, for the benefit of

/// DictionaryRecordReader

bool new_dictionary_;

// The exposed encoding

ExposedEncoding exposed_encoding_ = ExposedEncoding::NO_ENCODING;

// Map of encoding type to the respective decoder object. For example, a

// column chunk's data pages may include both dictionary-encoded and

// plain-encoded data.

std::unordered_map<int, std::unique_ptr<DecoderType>> decoders_;

void ConsumeBufferedValues(int64_t num_values) { num_decoded_values_ += num_values; }

public ColumnReaderImplBase<DType> {

public:

using T = typename DType::c_type;

TypedColumnReaderImpl(const ColumnDescriptor* descr, std::unique_ptr<PageReader> pager,

::arrow::MemoryPool* pool)

: ColumnReaderImplBase<DType>(descr, pool) {

this->pager_ = std::move(pager);

}

bool HasNext() override { return this->HasNextInternal(); }

int64_t ReadBatch(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,

T* values, int64_t* values_read) override;

int64_t ReadBatchSpaced(int64_t batch_size, int16_t* def_levels, int16_t* rep_levels,

T* values, uint8_t* valid_bits, int64_t valid_bits_offset,

int64_t* levels_read, int64_t* values_read,

int64_t* null_count) override;

int64_t Skip(int64_t num_values_to_skip) override;

Type::type type() const override { return this->descr_->physical_type(); }

const ColumnDescriptor* descr() const override { return this->descr_; }

ExposedEncoding GetExposedEncoding() override { return this->exposed_encoding_; };

int64_t ReadBatchWithDictionary(int64_t batch_size, int16_t* def_levels,

int16_t* rep_levels, int32_t* indices,

int64_t* indices_read, const T** dict,

int32_t* dict_len) override;

protected:

void SetExposedEncoding(ExposedEncoding encoding) override {

this->exposed_encoding_ = encoding;

}

// Allocate enough scratch space to accommodate skipping 16-bit levels or any

// value type.

void InitScratchForSkip();

// Scratch space for reading and throwing away rep/def levels and values when

// skipping.

std::shared_ptr<ResizableBuffer> scratch_for_skip_;

private: