switch (type) {

case TYPE_BOOLEAN:

return PColumnType::COLUMN_TYPE_BOOL;

case TYPE_TINYINT:

return PColumnType::COLUMN_TYPE_TINY_INT;

case TYPE_SMALLINT:

return PColumnType::COLUMN_TYPE_SMALL_INT;

case TYPE_INT:

return PColumnType::COLUMN_TYPE_INT;

case TYPE_BIGINT:

return PColumnType::COLUMN_TYPE_BIGINT;

case TYPE_LARGEINT:

return PColumnType::COLUMN_TYPE_LARGEINT;

case TYPE_FLOAT:

return PColumnType::COLUMN_TYPE_FLOAT;

case TYPE_DOUBLE:

return PColumnType::COLUMN_TYPE_DOUBLE;

case TYPE_DATE:

return PColumnType::COLUMN_TYPE_DATE;

case TYPE_DATEV2:

return PColumnType::COLUMN_TYPE_DATEV2;

case TYPE_DATETIMEV2:

return PColumnType::COLUMN_TYPE_DATETIMEV2;

case TYPE_DATETIME:

return PColumnType::COLUMN_TYPE_DATETIME;

case TYPE_DECIMALV2:

return PColumnType::COLUMN_TYPE_DECIMALV2;

case TYPE_DECIMAL32:

return PColumnType::COLUMN_TYPE_DECIMAL32;

case TYPE_DECIMAL64:

return PColumnType::COLUMN_TYPE_DECIMAL64;

case TYPE_DECIMAL128I:

return PColumnType::COLUMN_TYPE_DECIMAL128I;

case TYPE_DECIMAL256:

return PColumnType::COLUMN_TYPE_DECIMAL256;

case TYPE_CHAR:

return PColumnType::COLUMN_TYPE_CHAR;

case TYPE_VARCHAR:

return PColumnType::COLUMN_TYPE_VARCHAR;

case TYPE_STRING:

return PColumnType::COLUMN_TYPE_STRING;

case TYPE_IPV4:

return PColumnType::COLUMN_TYPE_IPV4;

case TYPE_IPV6:

return PColumnType::COLUMN_TYPE_IPV6;

default:

throw Exception(ErrorCode::INTERNAL_ERROR,

"runtime filter meet invalid PrimitiveType type {}", int(type));

}

switch (type) {

case PColumnType::COLUMN_TYPE_BOOL:

return TYPE_BOOLEAN;

case PColumnType::COLUMN_TYPE_TINY_INT:

return TYPE_TINYINT;

case PColumnType::COLUMN_TYPE_SMALL_INT:

return TYPE_SMALLINT;

case PColumnType::COLUMN_TYPE_INT:

return TYPE_INT;

case PColumnType::COLUMN_TYPE_BIGINT:

return TYPE_BIGINT;

case PColumnType::COLUMN_TYPE_LARGEINT:

return TYPE_LARGEINT;

case PColumnType::COLUMN_TYPE_FLOAT:

return TYPE_FLOAT;

case PColumnType::COLUMN_TYPE_DOUBLE:

return TYPE_DOUBLE;

case PColumnType::COLUMN_TYPE_DATE:

return TYPE_DATE;

case PColumnType::COLUMN_TYPE_DATEV2:

return TYPE_DATEV2;

case PColumnType::COLUMN_TYPE_DATETIMEV2:

return TYPE_DATETIMEV2;

case PColumnType::COLUMN_TYPE_DATETIME:

return TYPE_DATETIME;

case PColumnType::COLUMN_TYPE_DECIMALV2:

return TYPE_DECIMALV2;

case PColumnType::COLUMN_TYPE_DECIMAL32:

return TYPE_DECIMAL32;

case PColumnType::COLUMN_TYPE_DECIMAL64:

return TYPE_DECIMAL64;

case PColumnType::COLUMN_TYPE_DECIMAL128I:

return TYPE_DECIMAL128I;

case PColumnType::COLUMN_TYPE_DECIMAL256:

return TYPE_DECIMAL256;

case PColumnType::COLUMN_TYPE_VARCHAR:

return TYPE_VARCHAR;

case PColumnType::COLUMN_TYPE_CHAR:

return TYPE_CHAR;

case PColumnType::COLUMN_TYPE_STRING:

return TYPE_STRING;

case PColumnType::COLUMN_TYPE_IPV4:

return TYPE_IPV4;

case PColumnType::COLUMN_TYPE_IPV6:

return TYPE_IPV6;

default:

throw Exception(ErrorCode::INTERNAL_ERROR,

"runtime filter meet invalid PColumnType type {}", int(type));

}

switch (filter_type) {

case PFilterType::IN_FILTER: {

return RuntimeFilterType::IN_FILTER;

}

case PFilterType::BLOOM_FILTER: {

return RuntimeFilterType::BLOOM_FILTER;

}

case PFilterType::MINMAX_FILTER:

return RuntimeFilterType::MINMAX_FILTER;

case PFilterType::MIN_FILTER:

return RuntimeFilterType::MIN_FILTER;

case PFilterType::MAX_FILTER:

return RuntimeFilterType::MAX_FILTER;

default:

return RuntimeFilterType::UNKNOWN_FILTER;

}

switch (type) {

case RuntimeFilterType::IN_FILTER:

return PFilterType::IN_FILTER;

case RuntimeFilterType::BLOOM_FILTER:

return PFilterType::BLOOM_FILTER;

case RuntimeFilterType::MIN_FILTER:

return PFilterType::MIN_FILTER;

case RuntimeFilterType::MAX_FILTER:

return PFilterType::MAX_FILTER;

case RuntimeFilterType::MINMAX_FILTER:

return PFilterType::MINMAX_FILTER;

case RuntimeFilterType::IN_OR_BLOOM_FILTER:

return PFilterType::IN_OR_BLOOM_FILTER;

default:

return PFilterType::UNKNOW_FILTER;

}

try {

TExprNode node = create_texpr_node_from(data, type.type, type.precision, type.scale);

expr = vectorized::VLiteral::create_shared(node);

} catch (const Exception& e) {

return e.to_status();

}

return Status::OK();

vectorized::VExprSPtr& expr, TExprNode* tnode,

bool contain_null = false) {

TExprNode node;

TScalarType tscalar_type;

tscalar_type.__set_type(TPrimitiveType::BOOLEAN);

TTypeNode ttype_node;

ttype_node.__set_type(TTypeNodeType::SCALAR);

ttype_node.__set_scalar_type(tscalar_type);

TTypeDesc t_type_desc;

t_type_desc.types.push_back(ttype_node);

node.__set_type(t_type_desc);

node.__set_opcode(opcode);

node.__set_child_type(to_thrift(type.type));

node.__set_num_children(2);

node.__set_output_scale(type.scale);

node.__set_node_type(contain_null ? TExprNodeType::NULL_AWARE_BINARY_PRED

: TExprNodeType::BINARY_PRED);

TFunction fn;

TFunctionName fn_name;

fn_name.__set_db_name("");

switch (opcode) {

case TExprOpcode::LE:

fn_name.__set_function_name("le");

break;

case TExprOpcode::GE:

fn_name.__set_function_name("ge");

break;

default:

return Status::InternalError(

strings::Substitute("Invalid opcode for max_min_runtimefilter: '$0'", opcode));

}

fn.__set_name(fn_name);

fn.__set_binary_type(TFunctionBinaryType::BUILTIN);

TTypeNode type_node;

type_node.__set_type(TTypeNodeType::SCALAR);

TScalarType scalar_type;

scalar_type.__set_type(to_thrift(type.type));

scalar_type.__set_precision(type.precision);

scalar_type.__set_scale(type.scale);

type_node.__set_scalar_type(scalar_type);

std::vector<TTypeNode> type_nodes;

type_nodes.push_back(type_node);

TTypeDesc type_desc;

type_desc.__set_types(type_nodes);

std::vector<TTypeDesc> arg_types;

arg_types.push_back(type_desc);

arg_types.push_back(type_desc);

fn.__set_arg_types(arg_types);

fn.__set_ret_type(t_type_desc);

fn.__set_has_var_args(false);

node.__set_fn(fn);

*tnode = node;

return vectorized::VExpr::create_expr(node, expr);

RuntimePredicateWrapper(const RuntimeFilterParams* params)

: RuntimePredicateWrapper(params->column_return_type, params->filter_type,

params->filter_id) {};

// for a 'tmp' runtime predicate wrapper

// only could called assign method or as a param for merge

RuntimePredicateWrapper(PrimitiveType column_type, RuntimeFilterType type, uint32_t filter_id)

: _column_return_type(column_type),

_filter_type(type),

_context(new RuntimeFilterContext()),

_filter_id(filter_id) {}

// init runtime filter wrapper

// alloc memory to init runtime filter function

Status init(const RuntimeFilterParams* params) {

_max_in_num = params->max_in_num;

switch (_filter_type) {

case RuntimeFilterType::IN_FILTER: {

_context->hybrid_set.reset(create_set(_column_return_type));

_context->hybrid_set->set_null_aware(params->null_aware);

break;

}

// Only use in nested loop join not need set null aware

case RuntimeFilterType::MIN_FILTER:

case RuntimeFilterType::MAX_FILTER: {

_context->minmax_func.reset(create_minmax_filter(_column_return_type));

break;

}

case RuntimeFilterType::MINMAX_FILTER: {

_context->minmax_func.reset(create_minmax_filter(_column_return_type));

_context->minmax_func->set_null_aware(params->null_aware);

break;

}

case RuntimeFilterType::BLOOM_FILTER: {

_context->bloom_filter_func.reset(create_bloom_filter(_column_return_type));

_context->bloom_filter_func->init_params(params);

return Status::OK();

}

case RuntimeFilterType::IN_OR_BLOOM_FILTER: {

_context->hybrid_set.reset(create_set(_column_return_type));

_context->hybrid_set->set_null_aware(params->null_aware);

_context->bloom_filter_func.reset(create_bloom_filter(_column_return_type));

_context->bloom_filter_func->init_params(params);

return Status::OK();

}

case RuntimeFilterType::BITMAP_FILTER: {

_context->bitmap_filter_func.reset(create_bitmap_filter(_column_return_type));

_context->bitmap_filter_func->set_not_in(params->bitmap_filter_not_in);

return Status::OK();

}

default:

return Status::InternalError("Unknown Filter type");

}

return Status::OK();

}

Status change_to_bloom_filter() {

if (_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) {

return Status::InternalError(

"Can not change to bloom filter because of runtime filter type is {}",

IRuntimeFilter::to_string(_filter_type));

}

BloomFilterFuncBase* bf = _context->bloom_filter_func.get();

if (bf != nullptr) {

insert_to_bloom_filter(bf);

} else if (_context->hybrid_set != nullptr && _context->hybrid_set->size() != 0) {

return Status::InternalError("change to bloom filter need empty set ",

IRuntimeFilter::to_string(_filter_type));

}

// release in filter

_context->hybrid_set.reset();

return Status::OK();

}

Status init_bloom_filter(const size_t build_bf_cardinality) {

if (_filter_type != RuntimeFilterType::BLOOM_FILTER &&

_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER) {

throw Exception(ErrorCode::INTERNAL_ERROR,

"init_bloom_filter meet invalid input type {}", int(_filter_type));

}

return _context->bloom_filter_func->init_with_cardinality(build_bf_cardinality);

}

bool get_build_bf_cardinality() const {

if (_filter_type == RuntimeFilterType::BLOOM_FILTER ||

_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {

return _context->bloom_filter_func->get_build_bf_cardinality();

}

return false;

}

void insert_to_bloom_filter(BloomFilterFuncBase* bloom_filter) const {

if (_context->hybrid_set->size() > 0) {

auto* it = _context->hybrid_set->begin();

while (it->has_next()) {

bloom_filter->insert(it->get_value());

it->next();

}

}

if (_context->hybrid_set->contain_null()) {

bloom_filter->set_contain_null_and_null_aware();

}

}

BloomFilterFuncBase* get_bloomfilter() const { return _context->bloom_filter_func.get(); }

void insert_fixed_len(const vectorized::ColumnPtr& column, size_t start) {

if (is_ignored()) {

throw Exception(ErrorCode::INTERNAL_ERROR, "insert_fixed_len meet ignored rf");

}

switch (_filter_type) {

case RuntimeFilterType::IN_FILTER: {

_context->hybrid_set->insert_fixed_len(column, start);

break;

}

case RuntimeFilterType::MIN_FILTER:

case RuntimeFilterType::MAX_FILTER:

case RuntimeFilterType::MINMAX_FILTER: {

_context->minmax_func->insert_fixed_len(column, start);

break;

}

case RuntimeFilterType::BLOOM_FILTER: {

_context->bloom_filter_func->insert_fixed_len(column, start);

break;

}

case RuntimeFilterType::IN_OR_BLOOM_FILTER: {

if (is_bloomfilter()) {

_context->bloom_filter_func->insert_fixed_len(column, start);

} else {

_context->hybrid_set->insert_fixed_len(column, start);

}

break;

}

default:

DCHECK(false);

break;

}

}

void insert_batch(const vectorized::ColumnPtr& column, size_t start) {

if (get_real_type() == RuntimeFilterType::BITMAP_FILTER) {

bitmap_filter_insert_batch(column, start);

} else {

insert_fixed_len(column, start);

}

}

void bitmap_filter_insert_batch(const vectorized::ColumnPtr column, size_t start) {

std::vector<const BitmapValue*> bitmaps;

if (column->is_nullable()) {

const auto* nullable = assert_cast<const vectorized::ColumnNullable*>(column.get());

const auto& col =

assert_cast<const vectorized::ColumnBitmap&>(nullable->get_nested_column());

const auto& nullmap =

assert_cast<const vectorized::ColumnUInt8&>(nullable->get_null_map_column())

.get_data();

for (size_t i = start; i < column->size(); i++) {

if (!nullmap[i]) {

bitmaps.push_back(&(col.get_data()[i]));

}

}

} else {

const auto* col = assert_cast<const vectorized::ColumnBitmap*>(column.get());

for (size_t i = start; i < column->size(); i++) {

bitmaps.push_back(&(col->get_data()[i]));

}

}

_context->bitmap_filter_func->insert_many(bitmaps);

}

RuntimeFilterType get_real_type() const {

if (_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {

if (_context->hybrid_set) {

return RuntimeFilterType::IN_FILTER;

}

return RuntimeFilterType::BLOOM_FILTER;

}

return _filter_type;

}

size_t get_bloom_filter_size() const {

return _context->bloom_filter_func ? _context->bloom_filter_func->get_size() : 0;

}

Status get_push_exprs(std::list<vectorized::VExprContextSPtr>& probe_ctxs,

std::vector<vectorized::VRuntimeFilterPtr>& push_exprs,

const TExpr& probe_expr);

Status merge(const RuntimePredicateWrapper* wrapper) {

if (wrapper->is_ignored()) {

return Status::OK();

}

_context->ignored = false;

bool can_not_merge_in_or_bloom =

_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER &&

(wrapper->_filter_type != RuntimeFilterType::IN_FILTER &&

wrapper->_filter_type != RuntimeFilterType::BLOOM_FILTER &&

wrapper->_filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER);

bool can_not_merge_other = _filter_type != RuntimeFilterType::IN_OR_BLOOM_FILTER &&

_filter_type != wrapper->_filter_type;

CHECK(!can_not_merge_in_or_bloom && !can_not_merge_other)

<< " can not merge runtime filter(id=" << _filter_id

<< "), current is filter type is " << IRuntimeFilter::to_string(_filter_type)

<< ", other filter type is " << IRuntimeFilter::to_string(wrapper->_filter_type);

switch (_filter_type) {

case RuntimeFilterType::IN_FILTER: {

if (!_context->hybrid_set) {

set_ignored();

return Status::OK();

}

_context->hybrid_set->insert(wrapper->_context->hybrid_set.get());

if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {

set_ignored();

// release in filter

_context->hybrid_set.reset();

}

break;

}

case RuntimeFilterType::MIN_FILTER:

case RuntimeFilterType::MAX_FILTER:

case RuntimeFilterType::MINMAX_FILTER: {

RETURN_IF_ERROR(_context->minmax_func->merge(wrapper->_context->minmax_func.get()));

break;

}

case RuntimeFilterType::BLOOM_FILTER: {

RETURN_IF_ERROR(

_context->bloom_filter_func->merge(wrapper->_context->bloom_filter_func.get()));

break;

}

case RuntimeFilterType::IN_OR_BLOOM_FILTER: {

auto real_filter_type = get_real_type();

auto other_filter_type = wrapper->_filter_type;

if (other_filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER) {

other_filter_type = wrapper->get_real_type();

}

if (real_filter_type == RuntimeFilterType::IN_FILTER) {

// when we meet base rf is in-filter, threre only have two case:

// case1: all input-filter's build_bf_exactly is true, inited by synced global size

// case2: all input-filter's build_bf_exactly is false, inited by default size

if (other_filter_type == RuntimeFilterType::IN_FILTER) {

_context->hybrid_set->insert(wrapper->_context->hybrid_set.get());

if (_max_in_num >= 0 && _context->hybrid_set->size() >= _max_in_num) {

// case2: use default size to init bf

RETURN_IF_ERROR(_context->bloom_filter_func->init_with_fixed_length());

RETURN_IF_ERROR(change_to_bloom_filter());

}

} else {

// case1&case2: use input bf directly and insert hybrid set data into bf

_context->bloom_filter_func = wrapper->_context->bloom_filter_func;

RETURN_IF_ERROR(change_to_bloom_filter());

}

} else {

if (other_filter_type == RuntimeFilterType::IN_FILTER) {

// case2: insert data to global filter

wrapper->insert_to_bloom_filter(_context->bloom_filter_func.get());

} else {

// case1&case2: all input bf must has same size

RETURN_IF_ERROR(_context->bloom_filter_func->merge(

wrapper->_context->bloom_filter_func.get()));

}

}

break;

}

case RuntimeFilterType::BITMAP_FILTER: {

// use input bitmap directly because we assume bitmap filter join always have full data

_context->bitmap_filter_func = wrapper->_context->bitmap_filter_func;

break;

}

default:

return Status::InternalError("unknown runtime filter");

}

return Status::OK();

}

Status assign(const PInFilter* in_filter, bool contain_null) {

_context->hybrid_set.reset(create_set(_column_return_type));

if (contain_null) {

_context->hybrid_set->set_null_aware(true);

_context->hybrid_set->insert((const void*)nullptr);

}

switch (_column_return_type) {

case TYPE_BOOLEAN: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

bool bool_val = column.boolval();

set->insert(&bool_val);

});

break;

}

case TYPE_TINYINT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto int_val = static_cast<int8_t>(column.intval());

set->insert(&int_val);

});

break;

}

case TYPE_SMALLINT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto int_val = static_cast<int16_t>(column.intval());

set->insert(&int_val);

});

break;

}

case TYPE_INT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

int32_t int_val = column.intval();

set->insert(&int_val);

});

break;

}

case TYPE_BIGINT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

int64_t long_val = column.longval();

set->insert(&long_val);

});

break;

}

case TYPE_LARGEINT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto string_val = column.stringval();

StringParser::ParseResult result;

auto int128_val = StringParser::string_to_int<int128_t>(

string_val.c_str(), string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

set->insert(&int128_val);

});

break;

}

case TYPE_FLOAT: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto float_val = static_cast<float>(column.doubleval());

set->insert(&float_val);

});

break;

}

case TYPE_DOUBLE: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

double double_val = column.doubleval();

set->insert(&double_val);

});

break;

}

case TYPE_DATEV2: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto date_v2_val = column.intval();

set->insert(&date_v2_val);

});

break;

}

case TYPE_DATETIMEV2: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto date_v2_val = column.longval();

set->insert(&date_v2_val);

});

break;

}

case TYPE_DATETIME:

case TYPE_DATE: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

const auto& string_val_ref = column.stringval();

VecDateTimeValue datetime_val;

datetime_val.from_date_str(string_val_ref.c_str(), string_val_ref.length());

set->insert(&datetime_val);

});

break;

}

case TYPE_DECIMALV2: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

const auto& string_val_ref = column.stringval();

DecimalV2Value decimal_val(string_val_ref);

set->insert(&decimal_val);

});

break;

}

case TYPE_DECIMAL32: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

int32_t decimal_32_val = column.intval();

set->insert(&decimal_32_val);

});

break;

}

case TYPE_DECIMAL64: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

int64_t decimal_64_val = column.longval();

set->insert(&decimal_64_val);

});

break;

}

case TYPE_DECIMAL128I: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto string_val = column.stringval();

StringParser::ParseResult result;

auto int128_val = StringParser::string_to_int<int128_t>(

string_val.c_str(), string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

set->insert(&int128_val);

});

break;

}

case TYPE_DECIMAL256: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto string_val = column.stringval();

StringParser::ParseResult result;

auto int_val = StringParser::string_to_int<wide::Int256>(

string_val.c_str(), string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

set->insert(&int_val);

});

break;

}

case TYPE_VARCHAR:

case TYPE_CHAR:

case TYPE_STRING: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

const std::string& string_value = column.stringval();

// string_value is std::string, call insert(data, size) function in StringSet will not cast as StringRef

// so could avoid some cast error at different class object.

set->insert((void*)string_value.data(), string_value.size());

});

break;

}

case TYPE_IPV4: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

int32_t tmp = column.intval();

set->insert(&tmp);

});

break;

}

case TYPE_IPV6: {

batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {

auto string_val = column.stringval();

StringParser::ParseResult result;

auto int128_val = StringParser::string_to_int<uint128_t>(

string_val.c_str(), string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

set->insert(&int128_val);

});

break;

}

default: {

return Status::InternalError("not support assign to in filter, type: " +

type_to_string(_column_return_type));

}

}

return Status::OK();

}

void set_enable_fixed_len_to_uint32_v2() {

if (is_bloomfilter()) {

_context->bloom_filter_func->set_enable_fixed_len_to_uint32_v2();

}

}

// used by shuffle runtime filter

// assign this filter by protobuf

Status assign(const PBloomFilter* bloom_filter, butil::IOBufAsZeroCopyInputStream* data,

bool contain_null) {

// we won't use this class to insert or find any data

// so any type is ok

_context->bloom_filter_func.reset(create_bloom_filter(_column_return_type == INVALID_TYPE

? PrimitiveType::TYPE_INT

: _column_return_type));

RETURN_IF_ERROR(_context->bloom_filter_func->assign(data, bloom_filter->filter_length(),

contain_null));

return Status::OK();

}

// used by shuffle runtime filter

// assign this filter by protobuf

Status assign(const PMinMaxFilter* minmax_filter, bool contain_null) {

_context->minmax_func.reset(create_minmax_filter(_column_return_type));

if (contain_null) {

_context->minmax_func->set_null_aware(true);

_context->minmax_func->set_contain_null();

}

switch (_column_return_type) {

case TYPE_BOOLEAN: {

bool min_val = minmax_filter->min_val().boolval();

bool max_val = minmax_filter->max_val().boolval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_TINYINT: {

auto min_val = static_cast<int8_t>(minmax_filter->min_val().intval());

auto max_val = static_cast<int8_t>(minmax_filter->max_val().intval());

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_SMALLINT: {

auto min_val = static_cast<int16_t>(minmax_filter->min_val().intval());

auto max_val = static_cast<int16_t>(minmax_filter->max_val().intval());

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_INT: {

int32_t min_val = minmax_filter->min_val().intval();

int32_t max_val = minmax_filter->max_val().intval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_BIGINT: {

int64_t min_val = minmax_filter->min_val().longval();

int64_t max_val = minmax_filter->max_val().longval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_LARGEINT: {

auto min_string_val = minmax_filter->min_val().stringval();

auto max_string_val = minmax_filter->max_val().stringval();

StringParser::ParseResult result;

auto min_val = StringParser::string_to_int<int128_t>(min_string_val.c_str(),

min_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

auto max_val = StringParser::string_to_int<int128_t>(max_string_val.c_str(),

max_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_FLOAT: {

auto min_val = static_cast<float>(minmax_filter->min_val().doubleval());

auto max_val = static_cast<float>(minmax_filter->max_val().doubleval());

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DOUBLE: {

auto min_val = static_cast<double>(minmax_filter->min_val().doubleval());

auto max_val = static_cast<double>(minmax_filter->max_val().doubleval());

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DATEV2: {

int32_t min_val = minmax_filter->min_val().intval();

int32_t max_val = minmax_filter->max_val().intval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DATETIMEV2: {

int64_t min_val = minmax_filter->min_val().longval();

int64_t max_val = minmax_filter->max_val().longval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DATETIME:

case TYPE_DATE: {

const auto& min_val_ref = minmax_filter->min_val().stringval();

const auto& max_val_ref = minmax_filter->max_val().stringval();

VecDateTimeValue min_val;

VecDateTimeValue max_val;

min_val.from_date_str(min_val_ref.c_str(), min_val_ref.length());

max_val.from_date_str(max_val_ref.c_str(), max_val_ref.length());

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DECIMALV2: {

const auto& min_val_ref = minmax_filter->min_val().stringval();

const auto& max_val_ref = minmax_filter->max_val().stringval();

DecimalV2Value min_val(min_val_ref);

DecimalV2Value max_val(max_val_ref);

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DECIMAL32: {

int32_t min_val = minmax_filter->min_val().intval();

int32_t max_val = minmax_filter->max_val().intval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DECIMAL64: {

int64_t min_val = minmax_filter->min_val().longval();

int64_t max_val = minmax_filter->max_val().longval();

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DECIMAL128I: {

auto min_string_val = minmax_filter->min_val().stringval();

auto max_string_val = minmax_filter->max_val().stringval();

StringParser::ParseResult result;

auto min_val = StringParser::string_to_int<int128_t>(min_string_val.c_str(),

min_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

auto max_val = StringParser::string_to_int<int128_t>(max_string_val.c_str(),

max_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_DECIMAL256: {

auto min_string_val = minmax_filter->min_val().stringval();

auto max_string_val = minmax_filter->max_val().stringval();

StringParser::ParseResult result;

auto min_val = StringParser::string_to_int<wide::Int256>(

min_string_val.c_str(), min_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

auto max_val = StringParser::string_to_int<wide::Int256>(

max_string_val.c_str(), max_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

return _context->minmax_func->assign(&min_val, &max_val);

}

case TYPE_VARCHAR:

case TYPE_CHAR:

case TYPE_STRING: {

auto min_val_ref = minmax_filter->min_val().stringval();

auto max_val_ref = minmax_filter->max_val().stringval();

return _context->minmax_func->assign(&min_val_ref, &max_val_ref);

}

case TYPE_IPV4: {

int tmp_min = minmax_filter->min_val().intval();

int tmp_max = minmax_filter->max_val().intval();

return _context->minmax_func->assign(&tmp_min, &tmp_max);

}

case TYPE_IPV6: {

auto min_string_val = minmax_filter->min_val().stringval();

auto max_string_val = minmax_filter->max_val().stringval();

StringParser::ParseResult result;

auto min_val = StringParser::string_to_int<uint128_t>(min_string_val.c_str(),

min_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

auto max_val = StringParser::string_to_int<uint128_t>(max_string_val.c_str(),

max_string_val.length(), &result);

DCHECK(result == StringParser::PARSE_SUCCESS);

return _context->minmax_func->assign(&min_val, &max_val);

}

default:

break;

}

return Status::InternalError("not support!");

}

void get_bloom_filter_desc(char** data, int* filter_length) {

_context->bloom_filter_func->get_data(data, filter_length);

}

PrimitiveType column_type() { return _column_return_type; }

bool is_bloomfilter() const { return get_real_type() == RuntimeFilterType::BLOOM_FILTER; }

bool contain_null() const {

if (is_bloomfilter()) {

return _context->bloom_filter_func->contain_null();

}

if (_context->hybrid_set) {

if (get_real_type() != RuntimeFilterType::IN_FILTER) {

throw Exception(ErrorCode::INTERNAL_ERROR, "rf has hybrid_set but real type is {}",

int(get_real_type()));

}

return _context->hybrid_set->contain_null();

}

if (_context->minmax_func) {

return _context->minmax_func->contain_null();

}

return false;

}

bool is_ignored() const { return _context->ignored; }

void set_ignored() { _context->ignored = true; }

void batch_assign(const PInFilter* filter,

void (*assign_func)(std::shared_ptr<HybridSetBase>& _hybrid_set,

PColumnValue&)) {

for (int i = 0; i < filter->values_size(); ++i) {

PColumnValue column = filter->values(i);

assign_func(_context->hybrid_set, column);

}

}

size_t get_in_filter_size() const {

return _context->hybrid_set ? _context->hybrid_set->size() : 0;

}

std::shared_ptr<BitmapFilterFuncBase> get_bitmap_filter() const {

return _context->bitmap_filter_func;

}

friend class IRuntimeFilter;

void set_filter_id(int id) {

if (_context->bloom_filter_func) {

_context->bloom_filter_func->set_filter_id(id);

}

if (_context->bitmap_filter_func) {

_context->bitmap_filter_func->set_filter_id(id);

}

if (_context->hybrid_set) {

_context->hybrid_set->set_filter_id(id);

}

}

// When a runtime filter received from remote and it is a bloom filter, _column_return_type will be invalid.

PrimitiveType _column_return_type; // column type

RuntimeFilterType _filter_type;

int32_t _max_in_num = -1;

RuntimeFilterContextSPtr _context;

uint32_t _filter_id;

const TQueryOptions* query_options, const RuntimeFilterRole role,

int node_id, std::shared_ptr<IRuntimeFilter>* res) {

*res = std::make_shared<IRuntimeFilter>(state, desc);

(*res)->set_role(role);

return (*res)->init_with_desc(desc, query_options, node_id);

DCHECK(is_producer());