const FragmentScanRequest& request, const InspectedFragment& inspected_fragment,

const FragmentScanOptions* format_options, compute::ExecContext* exec_context) {

return Status::NotImplemented("New scan method");

{

auto lock = physical_schema_mutex_.Lock();

if (physical_schema_ != nullptr) return physical_schema_;

}

// allow ReadPhysicalSchemaImpl to lock mutex_, if necessary

ARROW_ASSIGN_OR_RAISE(auto physical_schema, ReadPhysicalSchemaImpl());

auto lock = physical_schema_mutex_.Lock();

if (physical_schema_ == nullptr) {

physical_schema_ = std::move(physical_schema);

}

return physical_schema_;

RecordBatchVector record_batches,

compute::Expression partition_expression)

: Fragment(std::move(partition_expression), std::move(schema)),

record_batches_(std::move(record_batches)) {

DCHECK_NE(physical_schema_, nullptr);

compute::Expression partition_expression)

: Fragment(std::move(partition_expression), /*schema=*/nullptr),

record_batches_(std::move(record_batches)) {

// Order of argument evaluation is undefined, so compute physical_schema here

physical_schema_ = record_batches_.empty() ? schema({}) : record_batches_[0]->schema();

const std::shared_ptr<ScanOptions>& options) {

struct State {

State(std::shared_ptr<InMemoryFragment> fragment, int64_t batch_size)

: fragment(std::move(fragment)),

batch_index(0),

offset(0),

batch_size(batch_size) {}

std::shared_ptr<RecordBatch> Next() {

const auto& next_parent = fragment->record_batches_[batch_index];

if (offset < next_parent->num_rows()) {

auto next = next_parent->Slice(offset, batch_size);

offset += batch_size;

return next;

}

batch_index++;

offset = 0;

return nullptr;

}

bool Finished() { return batch_index >= fragment->record_batches_.size(); }

std::shared_ptr<InMemoryFragment> fragment;

std::size_t batch_index;

int64_t offset;

int64_t batch_size;

};

struct Generator {

Generator(std::shared_ptr<InMemoryFragment> fragment, int64_t batch_size)

: state(std::make_shared<State>(std::move(fragment), batch_size)) {}

Future<std::shared_ptr<RecordBatch>> operator()() {

while (!state->Finished()) {

auto next = state->Next();

if (next) {

return Future<std::shared_ptr<RecordBatch>>::MakeFinished(std::move(next));

}

}

return AsyncGeneratorEnd<std::shared_ptr<RecordBatch>>();

}

std::shared_ptr<State> state;

};

return Generator(checked_pointer_cast<InMemoryFragment>(shared_from_this()),

options->batch_size);

compute::Expression predicate, const std::shared_ptr<ScanOptions>& options) {

if (ExpressionHasFieldRefs(predicate)) {

return Future<std::optional<int64_t>>::MakeFinished(std::nullopt);

}

int64_t total = 0;

for (const auto& batch : record_batches_) {

total += batch->num_rows();

}

return Future<std::optional<int64_t>>::MakeFinished(total);

const FragmentScanRequest& request, const InspectedFragment& inspected_fragment,

const FragmentScanOptions* format_options, compute::ExecContext* exec_context) {

return Future<std::shared_ptr<FragmentScanner>>::MakeFinished(

std::make_shared<InMemoryFragment::Scanner>(this));

ARROW_ASSIGN_OR_RAISE(

predicate, SimplifyWithGuarantee(std::move(predicate), partition_expression_));

return predicate.IsSatisfiable() ? GetFragmentsImpl(std::move(predicate))

: MakeEmptyIterator<std::shared_ptr<Fragment>>();

ARROW_ASSIGN_OR_RAISE(

predicate, SimplifyWithGuarantee(std::move(predicate), partition_expression_));

return predicate.IsSatisfiable()

? GetFragmentsAsyncImpl(std::move(predicate),

arrow::internal::GetCpuThreadPool())

: MakeEmptyGenerator<std::shared_ptr<Fragment>>();

compute::Expression predicate, arrow::internal::Executor* executor) {

ARROW_ASSIGN_OR_RAISE(auto iter, GetFragmentsImpl(std::move(predicate)));

ARROW_ASSIGN_OR_RAISE(

auto background_gen,

MakeBackgroundGenerator(std::move(iter), io::default_io_context().executor()));

auto transferred_gen = MakeTransferredGenerator(std::move(background_gen), executor);

return transferred_gen;

explicit VectorRecordBatchGenerator(RecordBatchVector batches)

: batches_(std::move(batches)) {}

RecordBatchIterator Get() const final { return MakeVectorIterator(batches_); }

RecordBatchVector batches_;

explicit TableRecordBatchGenerator(std::shared_ptr<Table> table)

: table_(std::move(table)) {}

RecordBatchIterator Get() const final {

auto reader = std::make_shared<TableBatchReader>(*table_);

auto table = table_;

return MakeFunctionIterator([reader, table] { return reader->Next(); });

}

std::shared_ptr<Table> table_;

std::shared_ptr<Schema> schema) const {

RETURN_NOT_OK(CheckProjectable(*schema_, *schema));

return std::make_shared<InMemoryDataset>(std::move(schema), get_batches_);

auto schema = this->schema();

auto create_fragment =

[schema](std::shared_ptr<RecordBatch> batch) -> Result<std::shared_ptr<Fragment>> {

RETURN_NOT_OK(CheckProjectable(*schema, *batch->schema()));

return std::make_shared<InMemoryFragment>(RecordBatchVector{std::move(batch)});

};

auto batches_it = get_batches_->Get();

return MakeMaybeMapIterator(std::move(create_fragment), std::move(batches_it));

DatasetVector children) {

for (const auto& child : children) {

if (!child->schema()->Equals(*schema)) {

return Status::TypeError("child Dataset had schema ", *child->schema(),

" but the union schema was ", *schema);

}

}

return std::shared_ptr<UnionDataset>(

new UnionDataset(std::move(schema), std::move(children)));

std::shared_ptr<Schema> schema) const {

auto children = children_;

for (auto& child : children) {

ARROW_ASSIGN_OR_RAISE(child, child->ReplaceSchema(schema));

}

return std::shared_ptr<Dataset>(

new UnionDataset(std::move(schema), std::move(children)));

public:

BasicFragmentEvolution(std::vector<int> ds_to_frag_map, Schema* dataset_schema)

: ds_to_frag_map(std::move(ds_to_frag_map)), dataset_schema(dataset_schema) {}

Result<compute::Expression> GetGuarantee(

const std::vector<FieldPath>& dataset_schema_selection) const override {

std::vector<compute::Expression> missing_fields;

for (const FieldPath& path : dataset_schema_selection) {

int top_level_field_idx = path[0];

if (ds_to_frag_map[top_level_field_idx] < 0) {

missing_fields.push_back(

compute::is_null(compute::field_ref(top_level_field_idx)));

}

}

if (missing_fields.empty()) {

return compute::literal(true);

}

if (missing_fields.size() == 1) {

return missing_fields[0];

}

return compute::and_(std::move(missing_fields));

}

Result<std::vector<FragmentSelectionColumn>> DevolveSelection(

const std::vector<FieldPath>& dataset_schema_selection) const override {

std::vector<FragmentSelectionColumn> desired_columns;

for (std::size_t selection_idx = 0; selection_idx < dataset_schema_selection.size();

selection_idx++) {

const FieldPath& path = dataset_schema_selection[selection_idx];

int top_level_field_idx = path[0];

int dest_top_level_idx = ds_to_frag_map[top_level_field_idx];

if (dest_top_level_idx >= 0) {

ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Field> field, path.Get(*dataset_schema));

std::vector<int> dest_path_indices(path.indices());

dest_path_indices[0] = dest_top_level_idx;

desired_columns.push_back(

FragmentSelectionColumn{FieldPath(dest_path_indices), field->type().get(),

static_cast<int>(selection_idx)});

}

}

return std::move(desired_columns);

};

Result<compute::Expression> DevolveFilter(

const compute::Expression& filter) const override {

return compute::ModifyExpression(

filter,

[&](compute::Expression expr) -> Result<compute::Expression> {

const FieldRef* ref = expr.field_ref();

if (ref) {

ARROW_ASSIGN_OR_RAISE(FieldPath path, ref->FindOne(*dataset_schema));

int top_level_idx = path[0];

std::vector<int> modified_indices(path.indices());

modified_indices[0] = ds_to_frag_map[top_level_idx];

if (modified_indices[0] < 0) {

return Status::Invalid(

"Filter cannot be applied. It refers to a missing field ",

ref->ToString(),

" in a way that cannot be satisfied even though we know that field is "

"null");

}

return compute::field_ref(FieldRef(std::move(modified_indices)));

}

return std::move(expr);

},

[](compute::Expression expr, compute::Expression* old_expr) { return expr; });

};

Result<compute::ExecBatch> EvolveBatch(

const std::shared_ptr<RecordBatch>& batch,

const std::vector<FieldPath>& dataset_selection,

const std::vector<FragmentSelectionColumn>& selection) const override {

std::vector<Datum> columns(dataset_selection.size());

DCHECK_EQ(batch->num_columns(), static_cast<int>(selection.size()));

// First go through and populate the columns we retrieved

for (int idx = 0; idx < batch->num_columns(); idx++) {

columns[selection[idx].selection_index] = batch->column(idx);

}

// Next go through and fill in the null columns

for (std::size_t idx = 0; idx < dataset_selection.size(); idx++) {

int top_level_idx = dataset_selection[idx][0];

if (ds_to_frag_map[top_level_idx] < 0) {

columns[idx] = MakeNullScalar(

dataset_schema->field(static_cast<int>(top_level_idx))->type());

}

}

return compute::ExecBatch(columns, batch->num_rows());

};

std::string ToString() const override { return "basic-fragment-evolution"; }

std::vector<int> ds_to_frag_map;

Schema* dataset_schema;

static std::unique_ptr<BasicFragmentEvolution> Make(

const std::shared_ptr<Schema>& dataset_schema,

const std::vector<std::string>& fragment_column_names) {

std::vector<int> ds_to_frag_map;

std::unordered_map<std::string, int> column_names_map;

for (size_t i = 0; i < fragment_column_names.size(); i++) {

column_names_map[fragment_column_names[i]] = static_cast<int>(i);

}

for (int idx = 0; idx < dataset_schema->num_fields(); idx++) {

const std::string& field_name = dataset_schema->field(idx)->name();

auto column_idx_itr = column_names_map.find(field_name);

if (column_idx_itr == column_names_map.end()) {

ds_to_frag_map.push_back(-1);

} else {

ds_to_frag_map.push_back(column_idx_itr->second);

}

}

return std::make_unique<BasicFragmentEvolution>(std::move(ds_to_frag_map),

dataset_schema.get());

}

std::unique_ptr<FragmentEvolutionStrategy> GetStrategy(

const Dataset& dataset, const Fragment& fragment,

const InspectedFragment& inspected_fragment) override {

return BasicFragmentEvolution::Make(dataset.schema(),

inspected_fragment.column_names);

}

std::string ToString() const override { return "basic-dataset-evolution"; }