Skip to content

Commit 3894de4

Browse files
eldenmooneldenmoon
authored
[Enhancement](topn) support two phase read for topn query (apache#15642)
This PR optimize topn query like `SELECT * FROM tableX ORDER BY columnA ASC/DESC LIMIT N`. TopN is is compose of SortNode and ScanNode, when user table is wide like 100+ columns the order by clause is just a few columns.But ScanNode need to scan all data from storage engine even if the limit is very small.This may lead to lots of read amplification.So In this PR I devide TopN query into two phase: 1. The first phase we just need to read `columnA`'s data from storage engine along with an extra RowId column called `__DORIS_ROWID_COL__`.The other columns are pruned from ScanNode. 2. The second phase I put it in the ExchangeNode beacuase it's the central node for topn nodes in the cluster.The ExchangeNode will spawn a RPC to other nodes using the RowIds(sorted and limited from SortNode) read from the first phase and read row by row from storage engine. After the second phase read, Block will contain all the data needed for the query
1 parent c7a7243 commit 3894de4

53 files changed

Lines changed: 829 additions & 33 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/config.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ CONF_Int32(doris_scanner_thread_pool_thread_num, "48");
178178
CONF_Int32(doris_scanner_thread_pool_queue_size, "102400");
179179
// default thrift client connect timeout(in seconds)
180180
CONF_mInt32(thrift_connect_timeout_seconds, "3");
181+
CONF_mInt32(fetch_rpc_timeout_seconds, "20");
181182
// default thrift client retry interval (in milliseconds)
182183
CONF_mInt64(thrift_client_retry_interval_ms, "1000");
183184
// max row count number for single scan range, used in segmentv1

‎be/src/common/consts.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ const std::string CSV = "csv";
2525
const std::string CSV_WITH_NAMES = "csv_with_names";
2626
const std::string CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types";
2727
const std::string BLOCK_TEMP_COLUMN_PREFIX = "__TEMP__";
28+
const std::string ROWID_COL = "__DORIS_ROWID_COL__";
2829

2930
constexpr int MAX_DECIMAL32_PRECISION = 9;
3031
constexpr int MAX_DECIMAL64_PRECISION = 18;

‎be/src/exec/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ set(EXEC_FILES
6363
odbc_connector.cpp
6464
table_connector.cpp
6565
schema_scanner.cpp
66+
rowid_fetcher.cpp
6667
)
6768

6869
if (WITH_LZO)

‎be/src/exec/rowid_fetcher.cpp‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/rowid_fetcher.h"
19+
20+
#include "bthread/countdown_event.h"
21+
#include "exec/tablet_info.h" // DorisNodesInfo
22+
#include "gen_cpp/Types_types.h"
23+
#include "gen_cpp/internal_service.pb.h"
24+
#include "runtime/exec_env.h" // ExecEnv
25+
#include "runtime/runtime_state.h" // RuntimeState
26+
#include "util/brpc_client_cache.h" // BrpcClientCache
27+
#include "util/defer_op.h"
28+
#include "vec/core/block.h" // Block
29+
30+
namespace doris {
31+
32+
Status RowIDFetcher::init(DorisNodesInfo* nodes_info) {
33+
for (auto [node_id, node_info] : nodes_info->nodes_info()) {
34+
auto client = ExecEnv::GetInstance()->brpc_internal_client_cache()->get_client(
35+
node_info.host, node_info.brpc_port);
36+
if (!client) {
37+
LOG(WARNING) << "Get rpc stub failed, host=" << node_info.host
38+
<< ", port=" << node_info.brpc_port;
39+
return Status::InternalError("RowIDFetcher failed to init rpc client");
40+
}
41+
_stubs.push_back(client);
42+
}
43+
return Status::OK();
44+
}
45+
46+
static std::string format_rowid(const GlobalRowLoacation& location) {
47+
return fmt::format("{} {} {} {}", location.tablet_id,
48+
location.row_location.rowset_id.to_string(),
49+
location.row_location.segment_id, location.row_location.row_id);
50+
}
51+
52+
PMultiGetRequest RowIDFetcher::_init_fetch_request(const vectorized::ColumnString& row_ids) {
53+
PMultiGetRequest mget_req;
54+
_tuple_desc->to_protobuf(mget_req.mutable_desc());
55+
for (auto slot : _tuple_desc->slots()) {
56+
slot->to_protobuf(mget_req.add_slots());
57+
}
58+
for (size_t i = 0; i < row_ids.size(); ++i) {
59+
PMultiGetRequest::RowId row_id;
60+
StringRef row_id_rep = row_ids.get_data_at(i);
61+
auto location = reinterpret_cast<const GlobalRowLoacation*>(row_id_rep.data);
62+
row_id.set_tablet_id(location->tablet_id);
63+
row_id.set_rowset_id(location->row_location.rowset_id.to_string());
64+
row_id.set_segment_id(location->row_location.segment_id);
65+
row_id.set_ordinal_id(location->row_location.row_id);
66+
*mget_req.add_rowids() = std::move(row_id);
67+
}
68+
mget_req.set_be_exec_version(_st->be_exec_version());
69+
return mget_req;
70+
}
71+
72+
static void fetch_callback(bthread::CountdownEvent* counter) {
73+
Defer __defer([&] { counter->signal(); });
74+
}
75+
76+
static Status MergeRPCResults(const std::vector<PMultiGetResponse>& rsps,
77+
const std::vector<brpc::Controller>& cntls,
78+
vectorized::MutableBlock* output_block) {
79+
for (const auto& cntl : cntls) {
80+
if (cntl.Failed()) {
81+
LOG(WARNING) << "Failed to fetch meet rpc error:" << cntl.ErrorText()
82+
<< ", host:" << cntl.remote_side();
83+
return Status::InternalError(cntl.ErrorText());
84+
}
85+
}
86+
for (const auto& resp : rsps) {
87+
Status st(resp.status());
88+
if (!st.ok()) {
89+
LOG(WARNING) << "Failed to fetch " << st.to_string();
90+
return st;
91+
}
92+
vectorized::Block partial_block(resp.block());
93+
output_block->merge(partial_block);
94+
}
95+
return Status::OK();
96+
}
97+
98+
Status RowIDFetcher::fetch(const vectorized::ColumnPtr& row_ids,
99+
vectorized::MutableBlock* res_block) {
100+
CHECK(!_stubs.empty());
101+
res_block->clear_column_data();
102+
vectorized::MutableBlock mblock({_tuple_desc}, row_ids->size());
103+
PMultiGetRequest mget_req = _init_fetch_request(assert_cast<const vectorized::ColumnString&>(
104+
*vectorized::remove_nullable(row_ids).get()));
105+
std::vector<PMultiGetResponse> resps(_stubs.size());
106+
std::vector<brpc::Controller> cntls(_stubs.size());
107+
bthread::CountdownEvent counter(_stubs.size());
108+
for (size_t i = 0; i < _stubs.size(); ++i) {
109+
cntls[i].set_timeout_ms(config::fetch_rpc_timeout_seconds * 1000);
110+
auto callback = brpc::NewCallback(fetch_callback, &counter);
111+
_stubs[i]->multiget_data(&cntls[i], &mget_req, &resps[i], callback);
112+
}
113+
counter.wait();
114+
RETURN_IF_ERROR(MergeRPCResults(resps, cntls, &mblock));
115+
// final sort by row_ids sequence, since row_ids is already sorted
116+
vectorized::Block tmp = mblock.to_block();
117+
std::unordered_map<std::string, uint32_t> row_order;
118+
vectorized::ColumnPtr row_id_column = tmp.get_columns().back();
119+
for (size_t x = 0; x < row_id_column->size(); ++x) {
120+
auto location =
121+
reinterpret_cast<const GlobalRowLoacation*>(row_id_column->get_data_at(x).data);
122+
row_order[format_rowid(*location)] = x;
123+
}
124+
for (size_t x = 0; x < row_ids->size(); ++x) {
125+
auto location = reinterpret_cast<const GlobalRowLoacation*>(row_ids->get_data_at(x).data);
126+
res_block->add_row(&tmp, row_order[format_rowid(*location)]);
127+
}
128+
return Status::OK();
129+
}
130+
131+
} // namespace doris

‎be/src/exec/rowid_fetcher.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "gen_cpp/internal_service.pb.h"
21+
#include "vec/core/block.h"
22+
23+
namespace doris {
24+
25+
class DorisNodesInfo;
26+
class RuntimeState;
27+
28+
// fetch rows by global rowid
29+
// tablet_id/rowset_name/segment_id/ordinal_id
30+
class RowIDFetcher {
31+
public:
32+
RowIDFetcher(TupleDescriptor* desc, RuntimeState* st) : _tuple_desc(desc), _st(st) {}
33+
Status init(DorisNodesInfo* nodes_info);
34+
Status fetch(const vectorized::ColumnPtr& row_ids, vectorized::MutableBlock* block);
35+
36+
private:
37+
PMultiGetRequest _init_fetch_request(const vectorized::ColumnString& row_ids);
38+
39+
std::vector<std::shared_ptr<PBackendService_Stub>> _stubs;
40+
TupleDescriptor* _tuple_desc;
41+
RuntimeState* _st;
42+
};
43+
44+
} // namespace doris

‎be/src/exec/tablet_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,8 @@ class DorisNodesInfo {
249249
return nullptr;
250250
}
251251

252+
const std::unordered_map<int64_t, NodeInfo>& nodes_info() { return _nodes; }
253+
252254
private:
253255
std::unordered_map<int64_t, NodeInfo> _nodes;
254256
};

‎be/src/exprs/runtime_filter.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class IOBufAsZeroCopyInputStream;
2929
namespace doris {
3030
class Predicate;
3131
class ObjectPool;
32-
class RuntimeState;
32+
class ExprContext;
3333
class RuntimePredicateWrapper;
3434
class MemTracker;
3535
class TupleRow;

‎be/src/olap/iterators.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ class StorageReadOptions {
119119
vectorized::VExpr* remaining_vconjunct_root = nullptr;
120120
// runtime state
121121
RuntimeState* runtime_state = nullptr;
122+
RowsetId rowset_id;
123+
int32_t tablet_id = 0;
122124
};
123125

124126
class RowwiseIterator {

‎be/src/olap/rowset/beta_rowset_reader.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
6464
_read_options.stats = _stats;
6565
_read_options.push_down_agg_type_opt = _context->push_down_agg_type_opt;
6666
_read_options.remaining_vconjunct_root = _context->remaining_vconjunct_root;
67+
_read_options.rowset_id = _rowset->rowset_id();
68+
_read_options.tablet_id = _rowset->rowset_meta()->tablet_id();
6769
if (read_context->lower_bound_keys != nullptr) {
6870
for (int i = 0; i < read_context->lower_bound_keys->size(); ++i) {
6971
_read_options.key_ranges.emplace_back(&read_context->lower_bound_keys->at(i),

‎be/src/olap/rowset/segment_v2/column_reader.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,56 @@ class ArrayFileColumnIterator final : public ColumnIterator {
417417
vectorized::ColumnArray::ColumnOffsets& column_offsets);
418418
};
419419

420+
class RowIdColumnIterator : public ColumnIterator {
421+
public:
422+
RowIdColumnIterator() = delete;
423+
RowIdColumnIterator(int32_t tid, RowsetId rid, int32_t segid)
424+
: _tablet_id(tid), _rowset_id(rid), _segment_id(segid) {}
425+
426+
Status seek_to_first() override {
427+
_current_rowid = 0;
428+
return Status::OK();
429+
}
430+
431+
Status seek_to_ordinal(ordinal_t ord_idx) override {
432+
_current_rowid = ord_idx;
433+
return Status::OK();
434+
}
435+
436+
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) {
437+
bool has_null;
438+
return next_batch(n, dst, &has_null);
439+
}
440+
441+
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override {
442+
for (size_t i = 0; i < *n; ++i) {
443+
rowid_t row_id = _current_rowid + i;
444+
GlobalRowLoacation location(_tablet_id, _rowset_id, _segment_id, row_id);
445+
dst->insert_data(reinterpret_cast<const char*>(&location), sizeof(GlobalRowLoacation));
446+
}
447+
_current_rowid += *n;
448+
return Status::OK();
449+
}
450+
451+
Status read_by_rowids(const rowid_t* rowids, const size_t count,
452+
vectorized::MutableColumnPtr& dst) override {
453+
for (size_t i = 0; i < count; ++i) {
454+
rowid_t row_id = rowids[i];
455+
GlobalRowLoacation location(_tablet_id, _rowset_id, _segment_id, row_id);
456+
dst->insert_data(reinterpret_cast<const char*>(&location), sizeof(GlobalRowLoacation));
457+
}
458+
return Status::OK();
459+
}
460+
461+
ordinal_t get_current_ordinal() const override { return _current_rowid; }
462+
463+
private:
464+
rowid_t _current_rowid = 0;
465+
int32_t _tablet_id = 0;
466+
RowsetId _rowset_id;
467+
int32_t _segment_id = 0;
468+
};
469+
420470
// This iterator is used to read default value column
421471
class DefaultValueColumnIterator : public ColumnIterator {
422472
public:

0 commit comments

Comments
 (0)