Skip to content

Commit fe3b56c

Browse files
HappenLeeHappenLee
authored
[Refactor](exec) Remove unless code and add comment (apache#46503)
Remove unless code and add comment be/src/pipeline/pipeline_task.h/ be/src/vec/runtime/vdatetime_value.h
1 parent 8e2cf8a commit fe3b56c

11 files changed

Lines changed: 33 additions & 112 deletions

File tree

‎be/src/pipeline/pipeline.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
148148
std::vector<std::shared_ptr<Pipeline>> _children;
149149

150150
PipelineId _pipeline_id;
151-
int _previous_schedule_id = -1;
152151

153152
// pipline id + operator names. init when:
154153
// build_operators(), if pipeline;

‎be/src/pipeline/pipeline_task.h‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,14 @@ class PipelineTask {
6767

6868
QueryContext* query_context();
6969

70-
int get_previous_core_id() const {
71-
return _previous_schedule_id != -1 ? _previous_schedule_id
72-
: _pipeline->_previous_schedule_id;
73-
}
70+
int get_core_id() const { return _core_id; }
7471

75-
void set_previous_core_id(int id) {
76-
if (id != _previous_schedule_id) {
77-
if (_previous_schedule_id != -1) {
72+
void set_core_id(int id) {
73+
if (id != _core_id) {
74+
if (_core_id != -1) {
7875
COUNTER_UPDATE(_core_change_times, 1);
7976
}
80-
_previous_schedule_id = id;
77+
_core_id = id;
8178
}
8279
}
8380

@@ -175,10 +172,6 @@ class PipelineTask {
175172
void update_queue_level(int queue_level) { this->_queue_level = queue_level; }
176173
int get_queue_level() const { return this->_queue_level; }
177174

178-
// 1.3 priority queue's core id
179-
void set_core_id(int core_id) { this->_core_id = core_id; }
180-
int get_core_id() const { return this->_core_id; }
181-
182175
/**
183176
* Return true if:
184177
* 1. `enable_force_spill` is true which forces this task to spill data.
@@ -254,7 +247,7 @@ class PipelineTask {
254247
bool _has_exceed_timeout = false;
255248
bool _opened;
256249
RuntimeState* _state = nullptr;
257-
int _previous_schedule_id = -1;
250+
int _core_id = -1;
258251
uint32_t _schedule_time = 0;
259252
std::unique_ptr<doris::vectorized::Block> _block;
260253
PipelineFragmentContext* _fragment_context = nullptr;
@@ -269,7 +262,6 @@ class PipelineTask {
269262
// 2 exe task
270263
// 3 update task statistics(update _queue_level/_core_id)
271264
int _queue_level = 0;
272-
int _core_id = 0;
273265

274266
RuntimeProfile* _parent_profile = nullptr;
275267
std::unique_ptr<RuntimeProfile> _task_profile;

‎be/src/pipeline/task_queue.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ PipelineTask* MultiCoreTaskQueue::take(int core_id) {
153153
<< " _core_size: " << _core_size << " _next_core: " << _next_core.load();
154154
task = _prio_task_queues[core_id].try_take(false);
155155
if (task) {
156-
task->set_core_id(core_id);
157156
break;
158157
}
159158
task = _steal_take(core_id);
@@ -162,7 +161,6 @@ PipelineTask* MultiCoreTaskQueue::take(int core_id) {
162161
}
163162
task = _prio_task_queues[core_id].take(WAIT_CORE_TASK_TIMEOUT_MS /* timeout_ms */);
164163
if (task) {
165-
task->set_core_id(core_id);
166164
break;
167165
}
168166
}
@@ -183,15 +181,14 @@ PipelineTask* MultiCoreTaskQueue::_steal_take(int core_id) {
183181
DCHECK(next_id < _core_size);
184182
auto task = _prio_task_queues[next_id].try_take(true);
185183
if (task) {
186-
task->set_core_id(next_id);
187184
return task;
188185
}
189186
}
190187
return nullptr;
191188
}
192189

193190
Status MultiCoreTaskQueue::push_back(PipelineTask* task) {
194-
int core_id = task->get_previous_core_id();
191+
int core_id = task->get_core_id();
195192
if (core_id < 0) {
196193
core_id = _next_core.fetch_add(1) % _core_size;
197194
}
@@ -205,9 +202,12 @@ Status MultiCoreTaskQueue::push_back(PipelineTask* task, int core_id) {
205202
}
206203

207204
void MultiCoreTaskQueue::update_statistics(PipelineTask* task, int64_t time_spent) {
208-
task->inc_runtime_ns(time_spent);
209-
_prio_task_queues[task->get_core_id()].inc_sub_queue_runtime(task->get_queue_level(),
210-
time_spent);
205+
// if the task not execute but exception early close, core_id == -1
206+
// should not do update_statistics
207+
if (auto core_id = task->get_core_id(); core_id >= 0) {
208+
task->inc_runtime_ns(time_spent);
209+
_prio_task_queues[core_id].inc_sub_queue_runtime(task->get_queue_level(), time_spent);
210+
}
211211
}
212212

213213
} // namespace doris::pipeline

‎be/src/pipeline/task_scheduler.cpp‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
#include "common/logging.h"
3535
#include "pipeline/pipeline_task.h"
36-
#include "pipeline/task_queue.h"
3736
#include "pipeline_fragment_context.h"
3837
#include "runtime/exec_env.h"
3938
#include "runtime/query_context.h"
@@ -103,6 +102,9 @@ void TaskScheduler::_do_work(int index) {
103102
if (!task) {
104103
continue;
105104
}
105+
// The task is already running, maybe block in now dependency wake up by other thread
106+
// but the block thread still hold the task, so put it back to the queue, until the hold
107+
// thread set task->set_running(false)
106108
if (task->is_running()) {
107109
static_cast<void>(_task_queue.push_back(task, index));
108110
continue;
@@ -129,12 +131,8 @@ void TaskScheduler::_do_work(int index) {
129131
// task exec
130132
bool eos = false;
131133
auto status = Status::OK();
134+
task->set_core_id(index);
132135

133-
#ifdef __APPLE__
134-
uint32_t core_id = 0;
135-
#else
136-
uint32_t core_id = sched_getcpu();
137-
#endif
138136
ASSIGN_STATUS_IF_CATCH_EXCEPTION(
139137
//TODO: use a better enclose to abstracting these
140138
if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) {
@@ -149,12 +147,11 @@ void TaskScheduler::_do_work(int index) {
149147

150148
uint64_t end_time = MonotonicMicros();
151149
ExecEnv::GetInstance()->pipeline_tracer_context()->record(
152-
{query_id, task_name, core_id, thread_id, start_time, end_time});
150+
{query_id, task_name, static_cast<uint32_t>(index), thread_id,
151+
start_time, end_time});
153152
} else { status = task->execute(&eos); },
154153
status);
155154

156-
task->set_previous_core_id(index);
157-
158155
if (!status.ok()) {
159156
// Print detail informations below when you debugging here.
160157
//
@@ -173,14 +170,11 @@ void TaskScheduler::_do_work(int index) {
173170
if (eos) {
174171
// is pending finish will add the task to dependency's blocking queue, and then the task will be
175172
// added to running queue when dependency is ready.
176-
if (task->is_pending_finish()) {
177-
// Only meet eos, should set task to PENDING_FINISH state
178-
task->set_running(false);
179-
} else {
173+
if (!task->is_pending_finish()) {
180174
Status exec_status = fragment_ctx->get_query_ctx()->exec_status();
181175
_close_task(task, exec_status);
176+
continue;
182177
}
183-
continue;
184178
}
185179

186180
task->set_running(false);

‎be/src/vec/runtime/vdatetime_value.cpp‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2823,40 +2823,6 @@ int date_day_offset_dict::daynr(int year, int month, int day) const {
28232823
return DATE_DAY_OFFSET_DICT[year - START_YEAR][month - 1][day - 1];
28242824
}
28252825

2826-
template <typename T>
2827-
uint32_t DateV2Value<T>::set_date_uint32(uint32_t int_val) {
2828-
union DateV2UInt32Union {
2829-
DateV2Value<T> dt;
2830-
uint32_t ui32;
2831-
~DateV2UInt32Union() {}
2832-
};
2833-
DateV2UInt32Union conv = {.ui32 = int_val};
2834-
if (is_invalid(conv.dt.year(), conv.dt.month(), conv.dt.day(), 0, 0, 0, 0)) {
2835-
return 0;
2836-
}
2837-
this->unchecked_set_time(conv.dt.year(), conv.dt.month(), conv.dt.day(), 0, 0, 0, 0);
2838-
2839-
return int_val;
2840-
}
2841-
2842-
template <typename T>
2843-
uint64_t DateV2Value<T>::set_datetime_uint64(uint64_t int_val) {
2844-
union DateTimeV2UInt64Union {
2845-
DateV2Value<T> dt;
2846-
uint64_t ui64;
2847-
~DateTimeV2UInt64Union() {}
2848-
};
2849-
DateTimeV2UInt64Union conv = {.ui64 = int_val};
2850-
if (is_invalid(conv.dt.year(), conv.dt.month(), conv.dt.day(), conv.dt.hour(), conv.dt.minute(),
2851-
conv.dt.second(), conv.dt.microsecond())) {
2852-
return 0;
2853-
}
2854-
this->unchecked_set_time(conv.dt.year(), conv.dt.month(), conv.dt.day(), conv.dt.hour(),
2855-
conv.dt.minute(), conv.dt.second(), conv.dt.microsecond());
2856-
2857-
return int_val;
2858-
}
2859-
28602826
template <typename T>
28612827
uint8_t DateV2Value<T>::week(uint8_t mode) const {
28622828
uint16_t year = 0;
@@ -3685,26 +3651,6 @@ bool DateV2Value<T>::to_format_string_conservative(const char* format, size_t le
36853651
return true;
36863652
}
36873653

3688-
template <typename T>
3689-
bool DateV2Value<T>::from_date(uint32_t value) {
3690-
DCHECK(!is_datetime);
3691-
if (value < MIN_DATE_V2 || value > MAX_DATE_V2) {
3692-
return false;
3693-
}
3694-
3695-
return set_date_uint32(value);
3696-
}
3697-
3698-
template <typename T>
3699-
bool DateV2Value<T>::from_datetime(uint64_t value) {
3700-
DCHECK(is_datetime);
3701-
if (value < MIN_DATETIME_V2 || value > MAX_DATETIME_V2) {
3702-
return false;
3703-
}
3704-
3705-
return set_datetime_uint64(value);
3706-
}
3707-
37083654
template <typename T>
37093655
int64_t DateV2Value<T>::standardize_timevalue(int64_t value) {
37103656
if (value <= 0) {

‎be/src/vec/runtime/vdatetime_value.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,12 +1181,7 @@ class DateV2Value {
11811181

11821182
underlying_value to_date_int_val() const { return int_val_; }
11831183

1184-
bool from_date(uint32_t value);
1185-
bool from_datetime(uint64_t value);
1186-
11871184
bool from_date_int64(int64_t value);
1188-
uint32_t set_date_uint32(uint32_t int_val);
1189-
uint64_t set_datetime_uint64(uint64_t int_val);
11901185

11911186
bool get_date_from_daynr(uint64_t);
11921187

‎be/test/vec/core/block_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -771,7 +771,7 @@ TEST(BlockTest, dump_data) {
771771
auto& date_v2_data = column_vector_date_v2->get_data();
772772
for (int i = 0; i < 1024; ++i) {
773773
DateV2Value<DateV2ValueType> value;
774-
value.from_date((uint32_t)((2022 << 9) | (6 << 5) | 6));
774+
value.unchecked_set_time(2022, 6, 6, 0, 0, 0, 0);
775775
date_v2_data.push_back(*reinterpret_cast<vectorized::UInt32*>(&value));
776776
}
777777
vectorized::DataTypePtr date_v2_type(std::make_shared<vectorized::DataTypeDateV2>());

‎be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ void serialize_and_deserialize_mysql_test() {
204204
auto& date_v2_data = column_vector_date_v2->get_data();
205205
for (int i = 0; i < row_num; ++i) {
206206
DateV2Value<DateV2ValueType> value;
207-
value.from_date((uint32_t)((2022 << 9) | (6 << 5) | 6));
207+
value.unchecked_set_time(2022, 6, 6, 0, 0, 0, 0);
208208
date_v2_data.push_back(*reinterpret_cast<vectorized::UInt32*>(&value));
209209
}
210210
vectorized::DataTypePtr date_v2_type(

‎be/test/vec/data_types/serde/data_type_serde_pb_test.cpp‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -668,12 +668,9 @@ TEST(DataTypeSerDePbTest, DataTypeScalaSerDeTestDateTime) {
668668
uint8_t minute = i;
669669
uint8_t second = 0;
670670
uint32_t microsecond = 123000;
671-
auto value = ((uint64_t)(((uint64_t)year << 46) | ((uint64_t)month << 42) |
672-
((uint64_t)day << 37) | ((uint64_t)hour << 32) |
673-
((uint64_t)minute << 26) | ((uint64_t)second << 20) |
674-
(uint64_t)microsecond));
671+
675672
DateV2Value<DateTimeV2ValueType> datetime_v2;
676-
datetime_v2.from_datetime(value);
673+
datetime_v2.unchecked_set_time(year, month, day, hour, minute, second, microsecond);
677674
auto datetime_val = binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(datetime_v2);
678675
data.push_back(datetime_val);
679676
}

‎be/test/vec/jsonb/serialize_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@ TEST(BlockSerializeTest, JsonbBlock) {
483483
auto& date_v2_data = column_vector_date_v2->get_data();
484484
for (int i = 0; i < 1024; ++i) {
485485
DateV2Value<DateV2ValueType> value;
486-
value.from_date((uint32_t)((2022 << 9) | (6 << 5) | 6));
486+
value.unchecked_set_time(2022, 6, 6, 0, 0, 0, 0);
487487
date_v2_data.push_back(*reinterpret_cast<vectorized::UInt32*>(&value));
488488
}
489489
vectorized::DataTypePtr date_v2_type(std::make_shared<vectorized::DataTypeDateV2>());

0 commit comments

Comments
 (0)