Skip to content

Commit b38b8b4

Browse files
Gabriel39Gabriel39
authored
[pipelineX](fix) Fix BE crash caused by join and constant expr (apache#24862)
1 parent 6502da8 commit b38b8b4

14 files changed

Lines changed: 96 additions & 18 deletions

‎be/src/pipeline/exec/hashjoin_build_sink.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,6 @@ Status HashJoinBuildSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* st
410410
_build_expr_ctxs.push_back(ctx);
411411

412412
const auto vexpr = _build_expr_ctxs.back()->root();
413-
const auto& data_type = vexpr->data_type();
414413

415414
bool null_aware = eq_join_conjunct.__isset.opcode &&
416415
eq_join_conjunct.opcode == TExprOpcode::EQ_FOR_NULL;
@@ -421,7 +420,10 @@ Status HashJoinBuildSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* st
421420
_store_null_in_hash_table.emplace_back(
422421
null_aware ||
423422
(_build_expr_ctxs.back()->root()->is_nullable() && build_stores_null));
423+
}
424424

425+
for (const auto& expr : _build_expr_ctxs) {
426+
const auto& data_type = expr->root()->data_type();
425427
if (!data_type->have_maximum_size_of_value()) {
426428
break;
427429
}
@@ -589,6 +591,12 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
589591

590592
local_state.init_short_circuit_for_probe();
591593
if (source_state == SourceState::FINISHED) {
594+
// Since the comparison of null values is meaningless, null aware left anti join should not output null
595+
// when the build side is not empty.
596+
if (!local_state._shared_state->build_blocks->empty() &&
597+
_join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
598+
local_state._shared_state->probe_ignore_null = true;
599+
}
592600
local_state._dependency->set_ready_for_read();
593601
}
594602

‎be/src/pipeline/exec/hashjoin_probe_operator.cpp‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ Status HashJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo& info)
3434
SCOPED_TIMER(profile()->total_time_counter());
3535
SCOPED_TIMER(_open_timer);
3636
auto& p = _parent->cast<HashJoinProbeOperatorX>();
37-
_probe_ignore_null = p._probe_ignore_null;
37+
_shared_state->probe_ignore_null = p._probe_ignore_null;
3838
_probe_expr_ctxs.resize(p._probe_expr_ctxs.size());
3939
for (size_t i = 0; i < _probe_expr_ctxs.size(); i++) {
4040
RETURN_IF_ERROR(p._probe_expr_ctxs[i]->clone(state, _probe_expr_ctxs[i]));
@@ -43,11 +43,6 @@ Status HashJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo& info)
4343
for (size_t i = 0; i < _other_join_conjuncts.size(); i++) {
4444
RETURN_IF_ERROR(p._other_join_conjuncts[i]->clone(state, _other_join_conjuncts[i]));
4545
}
46-
// Since the comparison of null values is meaningless, null aware left anti join should not output null
47-
// when the build side is not empty.
48-
if (!_shared_state->build_blocks->empty() && p._join_op == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
49-
_probe_ignore_null = true;
50-
}
5146
_construct_mutable_join_block();
5247
_probe_column_disguise_null.reserve(_probe_expr_ctxs.size());
5348
_probe_arena_memory_usage =
@@ -189,6 +184,42 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
189184
local_state.init_for_probe(state);
190185
SCOPED_TIMER(local_state._probe_timer);
191186
if (local_state._shared_state->short_circuit_for_probe) {
187+
/// If `_short_circuit_for_probe` is true, this indicates no rows
188+
/// match the join condition, and this is 'mark join', so we need to create a column as mark
189+
/// with all rows set to 0.
190+
if (_is_mark_join) {
191+
auto block_rows = local_state._probe_block.rows();
192+
if (block_rows == 0) {
193+
if (local_state._probe_eos) {
194+
source_state = SourceState::FINISHED;
195+
}
196+
return Status::OK();
197+
}
198+
199+
vectorized::Block temp_block;
200+
//get probe side output column
201+
for (int i = 0; i < _left_output_slot_flags.size(); ++i) {
202+
if (_left_output_slot_flags[i]) {
203+
temp_block.insert(local_state._probe_block.get_by_position(i));
204+
}
205+
}
206+
auto mark_column = vectorized::ColumnUInt8::create(block_rows, 0);
207+
temp_block.insert(
208+
{std::move(mark_column), std::make_shared<vectorized::DataTypeUInt8>(), ""});
209+
210+
{
211+
SCOPED_TIMER(local_state._join_filter_timer);
212+
RETURN_IF_ERROR(vectorized::VExprContext::filter_block(
213+
local_state._conjuncts, &temp_block, temp_block.columns()));
214+
}
215+
216+
RETURN_IF_ERROR(local_state._build_output_block(&temp_block, output_block, false));
217+
temp_block.clear();
218+
local_state._probe_block.clear_column_data(
219+
_child_x->row_desc().num_materialized_slots());
220+
local_state.reached_limit(output_block, source_state);
221+
return Status::OK();
222+
}
192223
// If we use a short-circuit strategy, should return empty block directly.
193224
source_state = SourceState::FINISHED;
194225
return Status::OK();
@@ -241,7 +272,7 @@ Status HashJoinProbeOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
241272
*local_state._shared_state->hash_table_variants,
242273
*local_state._process_hashtable_ctx_variants,
243274
vectorized::make_bool_variant(local_state._need_null_map_for_probe),
244-
vectorized::make_bool_variant(local_state._probe_ignore_null));
275+
vectorized::make_bool_variant(local_state._shared_state->probe_ignore_null));
245276
});
246277
} else if (local_state._probe_eos) {
247278
if (_is_right_semi_anti || (_is_outer_join && _join_op != TJoinOp::LEFT_OUTER_JOIN)) {
@@ -299,7 +330,8 @@ bool HashJoinProbeOperatorX::need_more_input_data(RuntimeState* state) const {
299330
auto& local_state = state->get_local_state(id())->cast<HashJoinProbeLocalState>();
300331
return (local_state._probe_block.rows() == 0 ||
301332
local_state._probe_index == local_state._probe_block.rows()) &&
302-
!local_state._probe_eos && !local_state._shared_state->short_circuit_for_probe;
333+
!local_state._probe_eos &&
334+
(!local_state._shared_state->short_circuit_for_probe || _is_mark_join);
303335
}
304336

305337
Status HashJoinProbeOperatorX::_do_evaluate(vectorized::Block& block,

‎be/src/pipeline/exec/hashjoin_probe_operator.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ class HashJoinProbeLocalState final
8585

8686
bool _need_null_map_for_probe = false;
8787
bool _has_set_need_null_map_for_probe = false;
88-
bool _probe_ignore_null = false;
8988
std::unique_ptr<vectorized::HashJoinProbeContext> _probe_context;
9089
vectorized::ColumnUInt8::MutablePtr _null_map_column;
9190
// for cases when a probe row matches more than batch size build rows.

‎be/src/pipeline/exec/nested_loop_join_probe_operator.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ Status NestedLoopJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo&
5959
RETURN_IF_ERROR(p._join_conjuncts[i]->clone(state, _join_conjuncts[i]));
6060
}
6161
_construct_mutable_join_block();
62+
63+
_loop_join_timer = ADD_TIMER(profile(), "LoopGenerateJoin");
6264
return Status::OK();
6365
}
6466

@@ -349,7 +351,7 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::MutableB
349351
DCHECK_LE(_left_block_start_pos + _left_side_process_count, _child_block->rows());
350352
for (int j = _left_block_start_pos;
351353
j < _left_block_start_pos + _left_side_process_count; ++j) {
352-
mark_data.emplace_back(IsSemi != _cur_probe_row_visited_flags[j]);
354+
mark_data.emplace_back(IsSemi == _cur_probe_row_visited_flags[j]);
353355
}
354356
for (size_t i = 0; i < p._num_probe_side_columns; ++i) {
355357
const vectorized::ColumnWithTypeAndName src_column =
@@ -562,6 +564,7 @@ Status NestedLoopJoinProbeOperatorX::pull(RuntimeState* state, vectorized::Block
562564
set_build_side_flag, set_probe_side_flag>(
563565
state, join_op_variants);
564566
};
567+
SCOPED_TIMER(local_state._loop_join_timer);
565568
RETURN_IF_ERROR(std::visit(
566569
func, local_state._shared_state->join_op_variants,
567570
vectorized::make_bool_variant(_match_all_build || _is_right_semi_anti),

‎be/src/pipeline/exec/nested_loop_join_probe_operator.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,8 @@ class NestedLoopJoinProbeLocalState final
198198
std::stack<uint16_t> _probe_offset_stack;
199199
uint64_t _output_null_idx_build_side = 0;
200200
vectorized::VExprContextSPtrs _join_conjuncts;
201+
202+
RuntimeProfile::Counter* _loop_join_timer;
201203
};
202204

203205
class NestedLoopJoinProbeOperatorX final

‎be/src/pipeline/exec/scan_operator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ template <typename Derived>
194194
class ScanLocalState : public ScanLocalStateBase {
195195
ENABLE_FACTORY_CREATOR(ScanLocalState);
196196
ScanLocalState(RuntimeState* state, OperatorXBase* parent);
197-
virtual ~ScanLocalState() = default;
197+
~ScanLocalState() override = default;
198198

199199
Status init(RuntimeState* state, LocalStateInfo& info) override;
200200
Status open(RuntimeState* state) override;

‎be/src/pipeline/pipeline_x/dependency.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,7 @@ struct HashJoinSharedState : public JoinSharedState {
530530
size_t build_exprs_size = 0;
531531
std::shared_ptr<std::vector<vectorized::Block>> build_blocks =
532532
std::make_shared<std::vector<vectorized::Block>>();
533+
bool probe_ignore_null = false;
533534
};
534535

535536
class HashJoinDependency final : public WriteDependency {

‎be/src/vec/exprs/vcase_expr.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,13 @@ Status VCaseExpr::prepare(RuntimeState* state, const RowDescriptor& desc, VExprC
7979

8080
Status VCaseExpr::open(RuntimeState* state, VExprContext* context,
8181
FunctionContext::FunctionStateScope scope) {
82-
RETURN_IF_ERROR(VExpr::open(state, context, scope));
82+
for (int i = 0; i < _children.size(); ++i) {
83+
RETURN_IF_ERROR(_children[i]->open(state, context, scope));
84+
}
8385
RETURN_IF_ERROR(VExpr::init_function_context(context, scope, _function));
86+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
87+
RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr));
88+
}
8489
return Status::OK();
8590
}
8691

‎be/src/vec/exprs/vcast_expr.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,13 @@ doris::Status VCastExpr::prepare(doris::RuntimeState* state, const doris::RowDes
8181

8282
doris::Status VCastExpr::open(doris::RuntimeState* state, VExprContext* context,
8383
FunctionContext::FunctionStateScope scope) {
84-
RETURN_IF_ERROR(VExpr::open(state, context, scope));
84+
for (int i = 0; i < _children.size(); ++i) {
85+
RETURN_IF_ERROR(_children[i]->open(state, context, scope));
86+
}
8587
RETURN_IF_ERROR(VExpr::init_function_context(context, scope, _function));
88+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
89+
RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr));
90+
}
8691
return Status::OK();
8792
}
8893

‎be/src/vec/exprs/vectorized_fn_call.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,13 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc,
121121

122122
Status VectorizedFnCall::open(RuntimeState* state, VExprContext* context,
123123
FunctionContext::FunctionStateScope scope) {
124-
RETURN_IF_ERROR(VExpr::open(state, context, scope));
124+
for (int i = 0; i < _children.size(); ++i) {
125+
RETURN_IF_ERROR(_children[i]->open(state, context, scope));
126+
}
125127
RETURN_IF_ERROR(VExpr::init_function_context(context, scope, _function));
128+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
129+
RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr));
130+
}
126131
return Status::OK();
127132
}
128133

0 commit comments

Comments
 (0)