VectorizedFnCall* fn_call, VExprContext* expr_ctx, Field& constant_val,

const std::set<std::string> fn_name) const {

if (!fn_name.contains(fn_call->fn().name.function_name)) {

return PushDownType::UNACCEPTABLE;

}

const auto& children = fn_call->children();

DCHECK(children.size() == 2);

DCHECK_EQ(children[0]->node_type(), TExprNodeType::SLOT_REF);

if (children[1]->is_constant()) {

std::shared_ptr<ColumnPtrWrapper> const_col_wrapper;

THROW_IF_ERROR(children[1]->get_const_col(expr_ctx, &const_col_wrapper));

const auto* const_column =

assert_cast<const ColumnConst*>(const_col_wrapper->column_ptr.get());

constant_val = const_column->operator[](0);

return PushDownType::PARTIAL_ACCEPTABLE;

} else {

// only handle constant value

return PushDownType::UNACCEPTABLE;

}

if (!predicate.target_is_slot(_parent->node_id())) {

return false;

}

auto& p = _parent->cast<FileScanOperatorX>();

const auto slot_id = predicate.get_texpr(_parent->node_id()).nodes[0].slot_ref.slot_id;

auto* slot = p._slot_id_to_slot_desc[slot_id];

DCHECK(slot != nullptr);

return p.can_push_down_column_predicate(slot);

// For select * from table limit 10; should just use one thread.

if (should_run_serial()) {

return 1;

}

/*

return (state->max_file_scanners_concurrency() > 0 ? state->max_file_scanners_concurrency()

: 16) *

(state->query_parallel_instance_num() / _parent->parallelism(state));

if (should_run_serial()) {

return 1;

}

/*

return (state->min_file_scanners_concurrency() > 0 ? state->min_file_scanners_concurrency()

: 1) *

(state->query_parallel_instance_num() / _parent->parallelism(state));

bool is_load,

const TFileScanRangeParams& scan_params) {

return _should_use_file_scanner_v2(query_options, is_load, scan_params);

bool is_load,

const TFileScanRangeParams& scan_params) {

const bool is_transactional_hive =

scan_params.__isset.table_format_params &&

scan_params.table_format_params.table_format_type == "transactional_hive";

// JNI reader selection is stored per split, but this scan-level selector cannot inspect the

// split yet. Older FEs may omit both the scan-level Paimon marker and split-level reader_type,

// so keep JNI scans on V1 until scanner selection can distinguish every compatibility shape.

return query_options.__isset.enable_file_scanner_v2 && query_options.enable_file_scanner_v2 &&

!is_load && scan_params.format_type != TFileFormatType::FORMAT_WAL &&

scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP &&

scan_params.format_type != TFileFormatType::FORMAT_LANCE &&

scan_params.format_type != TFileFormatType::FORMAT_JNI && !is_transactional_hive;

if (_split_source->num_scan_ranges() == 0) {

_eos = true;

return Status::OK();

}

auto& id_file_map = state()->get_id_file_map();

if (id_file_map != nullptr) {

id_file_map->set_external_scan_params(state()->get_query_ctx(), _max_scanners);

}

auto& p = _parent->cast<FileScanOperatorX>();

// There's only one scan range for each backend in batch split mode. Each backend only starts up one ScanNode instance.

uint32_t shard_num =

std::min(ScannerScheduler::default_remote_scan_thread_num() / p.parallelism(state()),

_max_scanners);

shard_num = std::max(shard_num, 1U);

_kv_cache = std::make_unique<ShardedKVCache>(shard_num);

const TFileScanRangeParams* scan_params = nullptr;

if (state()->get_query_ctx() != nullptr &&

state()->get_query_ctx()->file_scan_range_params_map.count(parent_id()) > 0) {

scan_params = &state()->get_query_ctx()->file_scan_range_params_map[parent_id()];

} else {

scan_params = _split_source->get_params();

}

const bool is_load =

state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr;

// TODO: Use scanner v2 for all queries.

const bool use_file_scanner_v2 =

_should_use_file_scanner_v2(state()->query_options(), is_load, *scan_params);

_operator_profile->add_info_string("UseScannerV2", use_file_scanner_v2 ? "true" : "false");

for (int i = 0; i < _max_scanners; ++i) {

ScannerSPtr scanner;

if (use_file_scanner_v2) {

scanner = FileScannerV2::create_shared(state(), this, p._limit, _split_source,

_scanner_profile.get(), _kv_cache.get(),

&p._colname_to_slot_id);

} else {

scanner = FileScanner::create_shared(state(), this, p._limit, _split_source,

_scanner_profile.get(), _kv_cache.get(),

&p._colname_to_slot_id);

}

RETURN_IF_ERROR(scanner->init(state(), _conjuncts));

scanners->push_back(std::move(scanner));

}

return Status::OK();

if (_parent->nereids_id() == -1) {

return fmt::format("(id={}, table_name={})", _parent->node_id(),

_parent->cast<FileScanOperatorX>()._table_name);

}

return fmt::format("(nereids_id={}, id={}, table_name={})", _parent->nereids_id(),

_parent->node_id(), _parent->cast<FileScanOperatorX>()._table_name);

const std::vector<TScanRangeParams>& scan_ranges) {

auto& p = _parent->cast<FileScanOperatorX>();

auto calc_max_scanners = [&](int parallel_instance_num) -> int {

int max_scanners =

ScannerScheduler::default_remote_scan_thread_num() / parallel_instance_num;

// For external tables, each scanner is not bound to specific splits.

// Instead, when a scanner is scheduled, it dynamically fetches the next scan range

// from a unified split source for scanning.

// Therefore, the number of scanners only needs to match "max_scanners_concurrency"

// to ensure full-speed execution.

// For 32 core node, the default "max_scanners_concurrency" should be 16

max_scanners = std::min(max_scanners, max_scanners_concurrency(state));

return max_scanners;

};

if (scan_ranges.size() == 1) {

auto scan_range = scan_ranges[0].scan_range.ext_scan_range.file_scan_range;

if (scan_range.__isset.split_source) {

p._batch_split_mode = true;

custom_profile()->add_info_string("BatchSplitMode", "true");

auto split_source = scan_range.split_source;

RuntimeProfile::Counter* get_split_timer = ADD_TIMER(custom_profile(), "GetSplitTime");

_max_scanners = calc_max_scanners(p.parallelism(state));

_split_source = std::make_shared<RemoteSplitSourceConnector>(

state, get_split_timer, split_source.split_source_id, split_source.num_splits,

_max_scanners);

}

}

if (!p._batch_split_mode) {

_max_scanners = calc_max_scanners(p.parallelism(state));

if (_split_source == nullptr) {

_split_source = std::make_shared<LocalSplitSourceConnector>(scan_ranges, _max_scanners);

}

// currently the total number of splits in the bach split mode cannot be accurately obtained,

// so we don't do it in the batch split mode.

_max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges());

}

if (!scan_ranges.empty() &&

scan_ranges[0].scan_range.ext_scan_range.file_scan_range.__isset.params) {

// for compatibility.

// in new implement, the tuple id is set in prepare phase

_output_tuple_id =

scan_ranges[0].scan_range.ext_scan_range.file_scan_range.params.dest_tuple_id;

}

RETURN_IF_ERROR(ScanLocalState<FileScanLocalState>::init(state, info));

SCOPED_TIMER(_init_timer);

auto& p = _parent->cast<FileScanOperatorX>();

_output_tuple_id = p._output_tuple_id;

return Status::OK();

RETURN_IF_ERROR(ScanLocalState<FileScanLocalState>::_process_conjuncts(state));

if (Base::_eos) {

return Status::OK();

}

// TODO: Push conjuncts down to reader.

return Status::OK();

RETURN_IF_ERROR(ScanOperatorX<FileScanLocalState>::prepare(state));

if (state->get_query_ctx() != nullptr &&

state->get_query_ctx()->file_scan_range_params_map.contains(node_id())) {

TFileScanRangeParams& params =

state->get_query_ctx()->file_scan_range_params_map[node_id()];

_output_tuple_id = params.dest_tuple_id;

}

return Status::OK();