if (tasks.empty()) {

return Status::OK();

}

bthread::Mutex lock;

bthread::ConditionVariable cond;

Status status; // Guard by lock

int count = 0; // Guard by lock

auto* run_bthread_work = +[](void* arg) -> void* {

auto* f = reinterpret_cast<std::function<void()>*>(arg);

(*f)();

delete f;

return nullptr;

};

for (const auto& task : tasks) {

{

std::unique_lock lk(lock);

// Wait until there are available slots

while (status.ok() && count >= concurrency) {

cond.wait(lk);

}

if (!status.ok()) {

break;

}

// Increase running task count

++count;

}

// dispatch task into bthreads

auto* fn = new std::function<void()>([&, &task = task] {

auto st = task();

{

std::lock_guard lk(lock);

--count;

if (!st.ok()) {

std::swap(st, status);

}

cond.notify_one();

}

});

bthread_t bthread_id;

if (bthread_start_background(&bthread_id, nullptr, run_bthread_work, fn) != 0) {

run_bthread_work(fn);

}

}

// Wait until all running tasks have done

{

std::unique_lock lk(lock);

while (count > 0) {

cond.wait(lk);

}

}

return status;

static Status get_client(std::shared_ptr<MetaService_Stub>* stub) {

TEST_SYNC_POINT_RETURN_WITH_VALUE("MetaServiceProxy::get_client", Status::OK(), stub);

return get_pooled_client(stub, nullptr);

}

static Status get_proxy(MetaServiceProxy** proxy) {

// The 'stub' is a useless parameter, added only to reuse the `get_pooled_client` function.

std::shared_ptr<MetaService_Stub> stub;

return get_pooled_client(&stub, proxy);

}

void set_unhealthy() {

std::unique_lock lock(_mutex);

maybe_unhealthy = true;

}

bool need_reconn(long now) {

return maybe_unhealthy && ((now - last_reconn_time_ms.front()) >

config::meta_service_rpc_reconnect_interval_ms);

}

Status get(std::shared_ptr<MetaService_Stub>* stub) {

using namespace std::chrono;

auto now = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();

{

std::shared_lock lock(_mutex);

if (_deadline_ms >= now && !is_idle_timeout(now) && !need_reconn(now)) {

_last_access_at_ms.store(now, std::memory_order_relaxed);

*stub = _stub;

return Status::OK();

}

}

auto channel = std::make_unique<brpc::Channel>();

Status s = init_channel(channel.get());

if (!s.ok()) [[unlikely]] {

return s;

}

*stub = std::make_shared<MetaService_Stub>(channel.release(),

google::protobuf::Service::STUB_OWNS_CHANNEL);

long deadline = now;

// connection age only works without list endpoint.

if (!is_meta_service_endpoint_list() &&

config::meta_service_connection_age_base_seconds > 0) {

std::default_random_engine rng(static_cast<uint32_t>(now));

std::uniform_int_distribution<> uni(

config::meta_service_connection_age_base_seconds,

config::meta_service_connection_age_base_seconds * 2);

deadline = now + duration_cast<milliseconds>(seconds(uni(rng))).count();

} else {

deadline = LONG_MAX;

}

// Last one WIN

std::unique_lock lock(_mutex);

_last_access_at_ms.store(now, std::memory_order_relaxed);

_deadline_ms = deadline;

_stub = *stub;

last_reconn_time_ms.push(now);

last_reconn_time_ms.pop();

maybe_unhealthy = false;

return Status::OK();

}

static bool is_meta_service_endpoint_list() {

return config::meta_service_endpoint.find(',') != std::string::npos;

}

/**

static Status get_pooled_client(std::shared_ptr<MetaService_Stub>* stub,

MetaServiceProxy** proxy) {

static std::once_flag proxies_flag;

static size_t num_proxies = 1;

static std::atomic<size_t> index(0);

static std::unique_ptr<MetaServiceProxy[]> proxies;

if (config::meta_service_endpoint.empty()) {

return Status::InvalidArgument(

"Meta service endpoint is empty. Please configure manually or wait for "

"heartbeat to obtain.");

}

std::call_once(

proxies_flag, +[]() {

if (config::meta_service_connection_pooled) {

num_proxies = config::meta_service_connection_pool_size;

}

proxies = std::make_unique<MetaServiceProxy[]>(num_proxies);

});

for (size_t i = 0; i + 1 < num_proxies; ++i) {

size_t next_index = index.fetch_add(1, std::memory_order_relaxed) % num_proxies;

Status s = proxies[next_index].get(stub);

if (proxy != nullptr) {

*proxy = &(proxies[next_index]);

}

if (s.ok()) return Status::OK();

}

size_t next_index = index.fetch_add(1, std::memory_order_relaxed) % num_proxies;

if (proxy != nullptr) {

*proxy = &(proxies[next_index]);

}

return proxies[next_index].get(stub);

}

static Status init_channel(brpc::Channel* channel) {

static std::atomic<size_t> index = 1;

const char* load_balancer_name = nullptr;

std::string endpoint;

if (is_meta_service_endpoint_list()) {

endpoint = fmt::format("list://{}", config::meta_service_endpoint);

load_balancer_name = "random";

} else {

std::string ip;

uint16_t port;

Status s = get_meta_service_ip_and_port(&ip, &port);

if (!s.ok()) {

LOG(WARNING) << "fail to get meta service ip and port: " << s;

return s;

}

endpoint = get_host_port(ip, port);

}

brpc::ChannelOptions options;

options.connection_group =

fmt::format("ms_{}", index.fetch_add(1, std::memory_order_relaxed));

if (channel->Init(endpoint.c_str(), load_balancer_name, &options) != 0) {

return Status::InvalidArgument("failed to init brpc channel, endpoint: {}", endpoint);

}

return Status::OK();

}

static Status get_meta_service_ip_and_port(std::string* ip, uint16_t* port) {

std::string parsed_host;

if (!parse_endpoint(config::meta_service_endpoint, &parsed_host, port)) {

return Status::InvalidArgument("invalid meta service endpoint: {}",

config::meta_service_endpoint);

}

if (is_valid_ip(parsed_host)) {

*ip = std::move(parsed_host);

return Status::OK();

}

return hostname_to_ip(parsed_host, *ip);

}

bool is_idle_timeout(long now) {

auto idle_timeout_ms = config::meta_service_idle_connection_timeout_ms;

// idle timeout only works without list endpoint.

return !is_meta_service_endpoint_list() && idle_timeout_ms > 0 &&

_last_access_at_ms.load(std::memory_order_relaxed) + idle_timeout_ms < now;

}

std::shared_mutex _mutex;

std::atomic<long> _last_access_at_ms {0};

long _deadline_ms {0};

std::shared_ptr<MetaService_Stub> _stub;

std::queue<long> last_reconn_time_ms {std::deque<long> {0, 0, 0}};

bool maybe_unhealthy = false;

if constexpr (is_any_v<Request, CommitTxnRequest, AbortTxnRequest, PrecommitTxnRequest>) {

return fmt::format(" txn_id={}", req.txn_id());

} else if constexpr (is_any_v<Request, StartTabletJobRequest, FinishTabletJobRequest>) {

return fmt::format(" tablet_id={}", req.job().idx().tablet_id());

} else if constexpr (is_any_v<Request, UpdateDeleteBitmapRequest>) {

return fmt::format(" tablet_id={}, lock_id={}", req.tablet_id(), req.lock_id());

} else if constexpr (is_any_v<Request, GetDeleteBitmapUpdateLockRequest>) {

return fmt::format(" table_id={}, lock_id={}", req.table_id(), req.lock_id());

} else if constexpr (is_any_v<Request, GetTabletRequest>) {

return fmt::format(" tablet_id={}", req.tablet_id());

} else if constexpr (is_any_v<Request, GetObjStoreInfoRequest>) {

return "";

} else if constexpr (is_any_v<Request, CreateRowsetRequest>) {

return fmt::format(" tablet_id={}", req.rowset_meta().tablet_id());

} else if constexpr (is_any_v<Request, RemoveDeleteBitmapRequest>) {

return fmt::format(" tablet_id={}", req.tablet_id());

} else if constexpr (is_any_v<Request, RemoveDeleteBitmapUpdateLockRequest>) {

return fmt::format(" table_id={}, tablet_id={}, lock_id={}", req.table_id(),

req.tablet_id(), req.lock_id());

} else if constexpr (is_any_v<Request, GetDeleteBitmapRequest>) {

return fmt::format(" tablet_id={}", req.tablet_id());

} else if constexpr (is_any_v<Request, GetSchemaDictRequest>) {

return fmt::format(" index_id={}", req.index_id());

} else {

static_assert(!sizeof(Request));

}

MetaServiceMethod<Request, Response> method) {

static_assert(std::is_base_of_v<::google::protobuf::Message, Request>);

static_assert(std::is_base_of_v<::google::protobuf::Message, Response>);

int retry_times = 0;

uint32_t duration_ms = 0;

std::string error_msg;

std::default_random_engine rng = make_random_engine();

std::uniform_int_distribution<uint32_t> u(20, 200);

std::uniform_int_distribution<uint32_t> u2(500, 1000);

MetaServiceProxy* proxy;

RETURN_IF_ERROR(MetaServiceProxy::get_proxy(&proxy));

while (true) {

std::shared_ptr<MetaService_Stub> stub;

RETURN_IF_ERROR(proxy->get(&stub));

brpc::Controller cntl;

if (op_name == "get delete bitmap") {

cntl.set_timeout_ms(3 * config::meta_service_brpc_timeout_ms);

} else {

cntl.set_timeout_ms(config::meta_service_brpc_timeout_ms);

}

cntl.set_max_retry(kBrpcRetryTimes);

res->Clear();

int error_code = 0;

(stub.get()->*method)(&cntl, &req, res, nullptr);

if (cntl.Failed()) [[unlikely]] {

error_msg = cntl.ErrorText();

error_code = cntl.ErrorCode();

proxy->set_unhealthy();

} else if (res->status().code() == MetaServiceCode::OK) {

return Status::OK();

} else if (res->status().code() == MetaServiceCode::INVALID_ARGUMENT) {

return Status::Error<ErrorCode::INVALID_ARGUMENT, false>("failed to {}: {}", op_name,

res->status().msg());

} else if (res->status().code() != MetaServiceCode::KV_TXN_CONFLICT) {

return Status::Error<ErrorCode::INTERNAL_ERROR, false>("failed to {}: {}", op_name,

res->status().msg());

} else {

error_msg = res->status().msg();

}

++retry_times;

if (retry_times > config::meta_service_rpc_retry_times ||

(retry_times > config::meta_service_rpc_timeout_retry_times &&

error_code == brpc::ERPCTIMEDOUT) ||

(retry_times > config::meta_service_conflict_error_retry_times &&

res->status().code() == MetaServiceCode::KV_TXN_CONFLICT)) {

break;

}

duration_ms = retry_times <= 100 ? u(rng) : u2(rng);

LOG(WARNING) << "failed to " << op_name << debug_info(req) << " retry_times=" << retry_times

<< " sleep=" << duration_ms << "ms : " << cntl.ErrorText();

bthread_usleep(duration_ms * 1000);

}

return Status::RpcError("failed to {}: rpc timeout, last msg={}", op_name, error_msg);

const SchemaCloudDictionary& dict) {

std::unordered_map<int32_t, ColumnPB*> unique_id_map;

//init map

for (ColumnPB& column : *out->mutable_tablet_schema()->mutable_column()) {

unique_id_map[column.unique_id()] = &column;

}

// column info

for (int i = 0; i < in.schema_dict_key_list().column_dict_key_list_size(); ++i) {

int dict_key = in.schema_dict_key_list().column_dict_key_list(i);

if (dict.column_dict().find(dict_key) == dict.column_dict().end()) {

return Status::NotFound("Not found entry {}", dict_key);

}

const ColumnPB& dict_val = dict.column_dict().at(dict_key);

ColumnPB& to_add = *out->mutable_tablet_schema()->add_column();

to_add = dict_val;

VLOG_DEBUG << "fill dict column " << dict_val.ShortDebugString();

}

// index info

for (int i = 0; i < in.schema_dict_key_list().index_info_dict_key_list_size(); ++i) {

int dict_key = in.schema_dict_key_list().index_info_dict_key_list(i);

if (dict.index_dict().find(dict_key) == dict.index_dict().end()) {

return Status::NotFound("Not found entry {}", dict_key);

}

const doris::TabletIndexPB& dict_val = dict.index_dict().at(dict_key);

*out->mutable_tablet_schema()->add_index() = dict_val;

VLOG_DEBUG << "fill dict index " << dict_val.ShortDebugString();

}

// sparse column info

for (int i = 0; i < in.schema_dict_key_list().sparse_column_dict_key_list_size(); ++i) {

int dict_key = in.schema_dict_key_list().sparse_column_dict_key_list(i);

if (dict.column_dict().find(dict_key) == dict.column_dict().end()) {

return Status::NotFound("Not found entry {}", dict_key);

}

const ColumnPB& dict_val = dict.column_dict().at(dict_key);

*unique_id_map.at(dict_val.parent_unique_id())->add_sparse_columns() = dict_val;

VLOG_DEBUG << "fill dict sparse column" << dict_val.ShortDebugString();

}

return Status::OK();

VLOG_DEBUG << "send GetTabletRequest, tablet_id: " << tablet_id;

TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::get_tablet_meta", Status::OK(), tablet_id,

tablet_meta);

GetTabletRequest req;

GetTabletResponse resp;

req.set_cloud_unique_id(config::cloud_unique_id);

req.set_tablet_id(tablet_id);

Status st = retry_rpc("get tablet meta", req, &resp, &MetaService_Stub::get_tablet);

if (!st.ok()) {

if (resp.status().code() == MetaServiceCode::TABLET_NOT_FOUND) {

return Status::NotFound("failed to get tablet meta: {}", resp.status().msg());

}

return st;

}

*tablet_meta = std::make_shared<TabletMeta>();

(*tablet_meta)

->init_from_pb(cloud_tablet_meta_to_doris(std::move(*resp.mutable_tablet_meta())));

VLOG_DEBUG << "get tablet meta, tablet_id: " << (*tablet_meta)->tablet_id();

return Status::OK();

SyncRowsetStats* sync_stats) {

std::unique_lock lock {tablet->get_sync_meta_lock()};

return sync_tablet_rowsets_unlocked(tablet, lock, options, sync_stats);

std::unique_lock<bthread::Mutex>& lock,

const SyncOptions& options,

SyncRowsetStats* sync_stats) {

using namespace std::chrono;

TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets", Status::OK(), tablet);

MetaServiceProxy* proxy;

RETURN_IF_ERROR(MetaServiceProxy::get_proxy(&proxy));

int tried = 0;

while (true) {

std::shared_ptr<MetaService_Stub> stub;

RETURN_IF_ERROR(proxy->get(&stub));

brpc::Controller cntl;

cntl.set_timeout_ms(config::meta_service_brpc_timeout_ms);

GetRowsetRequest req;

GetRowsetResponse resp;

int64_t tablet_id = tablet->tablet_id();

int64_t table_id = tablet->table_id();

int64_t index_id = tablet->index_id();

req.set_cloud_unique_id(config::cloud_unique_id);

auto* idx = req.mutable_idx();

idx->set_tablet_id(tablet_id);

idx->set_table_id(table_id);

idx->set_index_id(index_id);

idx->set_partition_id(tablet->partition_id());

{

std::shared_lock rlock(tablet->get_header_lock());

if (options.full_sync) {

req.set_start_version(0);

} else {

req.set_start_version(tablet->max_version_unlocked() + 1);

}

req.set_base_compaction_cnt(tablet->base_compaction_cnt());

req.set_cumulative_compaction_cnt(tablet->cumulative_compaction_cnt());

req.set_cumulative_point(tablet->cumulative_layer_point());

}

req.set_end_version(-1);

// backend side use schema dict in cache if enable cloud schema dict cache

req.set_schema_op(config::variant_use_cloud_schema_dict_cache

? GetRowsetRequest::NO_DICT

: GetRowsetRequest::RETURN_DICT);

VLOG_DEBUG << "send GetRowsetRequest: " << req.ShortDebugString();

auto start = std::chrono::steady_clock::now();

stub->get_rowset(&cntl, &req, &resp, nullptr);

auto end = std::chrono::steady_clock::now();

int64_t latency = cntl.latency_us();

_get_rowset_latency << latency;

int retry_times = config::meta_service_rpc_retry_times;

if (cntl.Failed()) {

proxy->set_unhealthy();

if (tried++ < retry_times) {

auto rng = make_random_engine();

std::uniform_int_distribution<uint32_t> u(20, 200);

std::uniform_int_distribution<uint32_t> u1(500, 1000);

uint32_t duration_ms = tried >= 100 ? u(rng) : u1(rng);

std::this_thread::sleep_for(milliseconds(duration_ms));

LOG_INFO("failed to get rowset meta")

.tag("reason", cntl.ErrorText())

.tag("tablet_id", tablet_id)

.tag("table_id", table_id)

.tag("index_id", index_id)

.tag("partition_id", tablet->partition_id())

.tag("tried", tried)

.tag("sleep", duration_ms);

continue;

}

return Status::RpcError("failed to get rowset meta: {}", cntl.ErrorText());

}

if (resp.status().code() == MetaServiceCode::TABLET_NOT_FOUND) {

return Status::NotFound("failed to get rowset meta: {}", resp.status().msg());

}

if (resp.status().code() != MetaServiceCode::OK) {

return Status::InternalError("failed to get rowset meta: {}", resp.status().msg());

}

if (latency > 100 * 1000) { // 100ms

LOG(INFO) << "finish get_rowset rpc. rowset_meta.size()=" << resp.rowset_meta().size()

<< ", latency=" << latency << "us";

} else {

LOG_EVERY_N(INFO, 100)

<< "finish get_rowset rpc. rowset_meta.size()=" << resp.rowset_meta().size()

<< ", latency=" << latency << "us";

}

int64_t now = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();

tablet->last_sync_time_s = now;

if (sync_stats) {

sync_stats->get_remote_rowsets_rpc_ns +=

std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();

sync_stats->get_remote_rowsets_num += resp.rowset_meta().size();

}

// If is mow, the tablet has no delete bitmap in base rowsets.

// So dont need to sync it.

if (options.sync_delete_bitmap && tablet->enable_unique_key_merge_on_write() &&

tablet->tablet_state() == TABLET_RUNNING) {

DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.sync_tablet_delete_bitmap.block",

DBUG_BLOCK);

DeleteBitmap delete_bitmap(tablet_id);

int64_t old_max_version = req.start_version() - 1;

auto st = sync_tablet_delete_bitmap(tablet, old_max_version, resp.rowset_meta(),

resp.stats(), req.idx(), &delete_bitmap,

options.full_sync, sync_stats);

if (st.is<ErrorCode::ROWSETS_EXPIRED>() && tried++ < retry_times) {

LOG_INFO("rowset meta is expired, need to retry")

.tag("tablet", tablet->tablet_id())

.tag("tried", tried)

.error(st);

continue;

}

if (!st.ok()) {

LOG_WARNING("failed to get delete bitmap")

.tag("tablet", tablet->tablet_id())

.error(st);

return st;

}

tablet->tablet_meta()->delete_bitmap().merge(delete_bitmap);

}

DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.before.modify_tablet_meta", {

auto target_tablet_id = dp->param<int64_t>("tablet_id", -1);

if (target_tablet_id == tablet->tablet_id()) {

DBUG_BLOCK

}

});

{

const auto& stats = resp.stats();

std::unique_lock wlock(tablet->get_header_lock());

// ATTN: we are facing following data race

//

// resp_base_compaction_cnt=0|base_compaction_cnt=0|resp_cumulative_compaction_cnt=0|cumulative_compaction_cnt=1|resp_max_version=11|max_version=8

//

// BE-compaction-thread meta-service BE-query-thread

// | | |

// local | commit cumu-compaction | |

// cc_cnt=0 | ---------------------------> | sync rowset (long rpc, local cc_cnt=0 ) | local

// | | <----------------------------------------- | cc_cnt=0

// | | -. |

// local | done cc_cnt=1 | \ |

// cc_cnt=1 | <--------------------------- | \ |

// | | \ returned with resp cc_cnt=0 (snapshot) |

// | | '------------------------------------> | local

// | | | cc_cnt=1

// | | |

// | | | CHECK FAIL

// | | | need retry

// To get rid of just retry syncing tablet

if (stats.base_compaction_cnt() < tablet->base_compaction_cnt() ||

stats.cumulative_compaction_cnt() < tablet->cumulative_compaction_cnt())

[[unlikely]] {

// stale request, ignore

LOG_WARNING("stale get rowset meta request")

.tag("resp_base_compaction_cnt", stats.base_compaction_cnt())

.tag("base_compaction_cnt", tablet->base_compaction_cnt())

.tag("resp_cumulative_compaction_cnt", stats.cumulative_compaction_cnt())

.tag("cumulative_compaction_cnt", tablet->cumulative_compaction_cnt())

.tag("tried", tried);

if (tried++ < 10) continue;

return Status::OK();

}

std::vector<RowsetSharedPtr> rowsets;

rowsets.reserve(resp.rowset_meta().size());

for (const auto& cloud_rs_meta_pb : resp.rowset_meta()) {

VLOG_DEBUG << "get rowset meta, tablet_id=" << cloud_rs_meta_pb.tablet_id()

<< ", version=[" << cloud_rs_meta_pb.start_version() << '-'

<< cloud_rs_meta_pb.end_version() << ']';

auto existed_rowset = tablet->get_rowset_by_version(

{cloud_rs_meta_pb.start_version(), cloud_rs_meta_pb.end_version()});

if (existed_rowset &&

existed_rowset->rowset_id().to_string() == cloud_rs_meta_pb.rowset_id_v2()) {

continue; // Same rowset, skip it

}

RowsetMetaPB meta_pb;

// Check if the rowset meta contains a schema dictionary key list.

if (cloud_rs_meta_pb.has_schema_dict_key_list() && !resp.has_schema_dict()) {

// Use the locally cached dictionary.

RowsetMetaCloudPB copied_cloud_rs_meta_pb = cloud_rs_meta_pb;

CloudStorageEngine& engine =

ExecEnv::GetInstance()->storage_engine().to_cloud();

{

wlock.unlock();

RETURN_IF_ERROR(

engine.get_schema_cloud_dictionary_cache()

.replace_dict_keys_to_schema(cloud_rs_meta_pb.index_id(),

&copied_cloud_rs_meta_pb));

wlock.lock();

}

meta_pb = cloud_rowset_meta_to_doris(copied_cloud_rs_meta_pb);

} else {

// Otherwise, use the schema dictionary from the response (if available).

meta_pb = cloud_rowset_meta_to_doris(cloud_rs_meta_pb);

if (resp.has_schema_dict()) {

RETURN_IF_ERROR(fill_schema_with_dict(cloud_rs_meta_pb, &meta_pb,

resp.schema_dict()));

}

}

auto rs_meta = std::make_shared<RowsetMeta>();

rs_meta->init_from_pb(meta_pb);

RowsetSharedPtr rowset;

// schema is nullptr implies using RowsetMeta.tablet_schema

Status s = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset);

if (!s.ok()) {

LOG_WARNING("create rowset").tag("status", s);

return s;

}

rowsets.push_back(std::move(rowset));

}

if (!rowsets.empty()) {

// `rowsets.empty()` could happen after doing EMPTY_CUMULATIVE compaction. e.g.:

// BE has [0-1][2-11][12-12], [12-12] is delete predicate, cp is 2;

// after doing EMPTY_CUMULATIVE compaction, MS cp is 13, get_rowset will return [2-11][12-12].

bool version_overlap =

tablet->max_version_unlocked() >= rowsets.front()->start_version();

tablet->add_rowsets(std::move(rowsets), version_overlap, wlock,

options.warmup_delta_data);

if (options.merge_schema) {

RETURN_IF_ERROR(tablet->merge_rowsets_schema());

}

}

tablet->last_base_compaction_success_time_ms = stats.last_base_compaction_time_ms();

tablet->last_cumu_compaction_success_time_ms = stats.last_cumu_compaction_time_ms();

tablet->set_base_compaction_cnt(stats.base_compaction_cnt());

tablet->set_cumulative_compaction_cnt(stats.cumulative_compaction_cnt());

tablet->set_cumulative_layer_point(stats.cumulative_point());

tablet->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),

stats.num_rows(), stats.data_size());

}

return Status::OK();

}

std::ranges::range auto&& rs_metas,

DeleteBitmap* delete_bitmap) {

std::set<int64_t> txn_processed;

for (auto& rs_meta : rs_metas) {

auto txn_id = rs_meta.txn_id();

if (txn_processed.find(txn_id) != txn_processed.end()) {

continue;

}

txn_processed.insert(txn_id);

DeleteBitmapPtr tmp_delete_bitmap;

std::shared_ptr<PublishStatus> publish_status =

std::make_shared<PublishStatus>(PublishStatus::INIT);

CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud();

Status status = engine.txn_delete_bitmap_cache().get_delete_bitmap(

txn_id, tablet->tablet_id(), &tmp_delete_bitmap, nullptr, &publish_status);

// CloudMetaMgr::sync_tablet_delete_bitmap_by_cache() is called after we sync rowsets from meta services.

// If the control flows reaches here, it's gauranteed that the rowsets is commited in meta services, so we can

// use the delete bitmap from cache directly if *publish_status == PublishStatus::SUCCEED without checking other

// stats(version or compaction stats)

if (status.ok() && *publish_status == PublishStatus::SUCCEED) {

// tmp_delete_bitmap contains sentinel marks, we should remove it before merge it to delete bitmap.

// Also, the version of delete bitmap key in tmp_delete_bitmap is DeleteBitmap::TEMP_VERSION_COMMON,

// we should replace it with the rowset's real version

DCHECK(rs_meta.start_version() == rs_meta.end_version());

int64_t rowset_version = rs_meta.start_version();

for (const auto& [delete_bitmap_key, bitmap_value] : tmp_delete_bitmap->delete_bitmap) {

// skip sentinel mark, which is used for delete bitmap correctness check

if (std::get<1>(delete_bitmap_key) != DeleteBitmap::INVALID_SEGMENT_ID) {

delete_bitmap->merge({std::get<0>(delete_bitmap_key),

std::get<1>(delete_bitmap_key), rowset_version},

bitmap_value);

}

}

engine.txn_delete_bitmap_cache().remove_unused_tablet_txn_info(txn_id,

tablet->tablet_id());

} else {

LOG_EVERY_N(INFO, 20)

<< "delete bitmap not found in cache, will sync rowset to get. tablet_id= "

<< tablet->tablet_id() << ", txn_id=" << txn_id << ", status=" << status;

return false;

}

}

return true;

std::ranges::range auto&& rs_metas,

const TabletStatsPB& stats, const TabletIndexPB& idx,

DeleteBitmap* delete_bitmap, bool full_sync,

SyncRowsetStats* sync_stats) {

if (rs_metas.empty()) {

return Status::OK();

}

if (!full_sync &&

sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas, delete_bitmap)) {

if (sync_stats) {

sync_stats->get_local_delete_bitmap_rowsets_num += rs_metas.size();

}

return Status::OK();

} else {

DeleteBitmapPtr new_delete_bitmap = std::make_shared<DeleteBitmap>(tablet->tablet_id());

*delete_bitmap = *new_delete_bitmap;

}

int64_t new_max_version = std::max(old_max_version, rs_metas.rbegin()->end_version());

// When there are many delete bitmaps that need to be synchronized, it

// may take a longer time, especially when loading the tablet for the

// first time, so set a relatively long timeout time.

GetDeleteBitmapRequest req;

GetDeleteBitmapResponse res;

req.set_cloud_unique_id(config::cloud_unique_id);

req.set_tablet_id(tablet->tablet_id());

req.set_base_compaction_cnt(stats.base_compaction_cnt());

req.set_cumulative_compaction_cnt(stats.cumulative_compaction_cnt());

req.set_cumulative_point(stats.cumulative_point());

*(req.mutable_idx()) = idx;

// New rowset sync all versions of delete bitmap

for (const auto& rs_meta : rs_metas) {

req.add_rowset_ids(rs_meta.rowset_id_v2());

req.add_begin_versions(0);

req.add_end_versions(new_max_version);

}

// old rowset sync incremental versions of delete bitmap

if (old_max_version > 0 && old_max_version < new_max_version) {

RowsetIdUnorderedSet all_rs_ids;

RETURN_IF_ERROR(tablet->get_all_rs_id(old_max_version, &all_rs_ids));

for (const auto& rs_id : all_rs_ids) {

req.add_rowset_ids(rs_id.to_string());

req.add_begin_versions(old_max_version + 1);

req.add_end_versions(new_max_version);

}

}

if (sync_stats) {

sync_stats->get_remote_delete_bitmap_rowsets_num += req.rowset_ids_size();

}

VLOG_DEBUG << "send GetDeleteBitmapRequest: " << req.ShortDebugString();

auto start = std::chrono::steady_clock::now();

auto st = retry_rpc("get delete bitmap", req, &res, &MetaService_Stub::get_delete_bitmap);

auto end = std::chrono::steady_clock::now();

if (st.code() == ErrorCode::THRIFT_RPC_ERROR) {

return st;

}

if (res.status().code() == MetaServiceCode::TABLET_NOT_FOUND) {

return Status::NotFound("failed to get delete bitmap: {}", res.status().msg());

}

// The delete bitmap of stale rowsets will be removed when commit compaction job,

// then delete bitmap of stale rowsets cannot be obtained. But the rowsets obtained

// by sync_tablet_rowsets may include these stale rowsets. When this case happend, the

// error code of ROWSETS_EXPIRED will be returned, we need to retry sync rowsets again.

//

// Be query thread meta-service Be compaction thread

// | | |

// | get rowset | |

// |--------------------------->| |

// | return get rowset | |

// |<---------------------------| |

// | | commit job |

// | |<------------------------|

// | | return commit job |

// | |------------------------>|

// | get delete bitmap | |

// |--------------------------->| |

// | return get delete bitmap | |

// |<---------------------------| |

// | | |

if (res.status().code() == MetaServiceCode::ROWSETS_EXPIRED) {

return Status::Error<ErrorCode::ROWSETS_EXPIRED, false>("failed to get delete bitmap: {}",

res.status().msg());

}

if (res.status().code() != MetaServiceCode::OK) {

return Status::Error<ErrorCode::INTERNAL_ERROR, false>("failed to get delete bitmap: {}",

res.status().msg());

}

const auto& rowset_ids = res.rowset_ids();

const auto& segment_ids = res.segment_ids();

const auto& vers = res.versions();

const auto& delete_bitmaps = res.segment_delete_bitmaps();

if (rowset_ids.size() != segment_ids.size() || rowset_ids.size() != vers.size() ||

rowset_ids.size() != delete_bitmaps.size()) {

return Status::Error<ErrorCode::INTERNAL_ERROR, false>(

"get delete bitmap data wrong,"

"rowset_ids.size={},segment_ids.size={},vers.size={},delete_bitmaps.size={}",

rowset_ids.size(), segment_ids.size(), vers.size(), delete_bitmaps.size());

}

if (sync_stats) {

sync_stats->get_remote_delete_bitmap_rpc_ns +=

std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();

sync_stats->get_remote_delete_bitmap_key_count += delete_bitmaps.size();

for (const auto& dbm : delete_bitmaps) {

sync_stats->get_remote_delete_bitmap_bytes += dbm.length();

}

}

for (int i = 0; i < rowset_ids.size(); i++) {

RowsetId rst_id;

rst_id.init(rowset_ids[i]);

delete_bitmap->merge(

{rst_id, segment_ids[i], vers[i]},

roaring::Roaring::readSafe(delete_bitmaps[i].data(), delete_bitmaps[i].length()));

}

int64_t latency = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();

if (latency > 100 * 1000) { // 100ms

LOG(INFO) << "finish get_delete_bitmap rpc. rowset_ids.size()=" << rowset_ids.size()

<< ", delete_bitmaps.size()=" << delete_bitmaps.size() << ", latency=" << latency

<< "us";

} else {

LOG_EVERY_N(INFO, 100) << "finish get_delete_bitmap rpc. rowset_ids.size()="

<< rowset_ids.size()

<< ", delete_bitmaps.size()=" << delete_bitmaps.size()

<< ", latency=" << latency << "us";

}

return Status::OK();

RowsetMetaSharedPtr* existed_rs_meta) {

VLOG_DEBUG << "prepare rowset, tablet_id: " << rs_meta.tablet_id()

<< ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id();

{

Status ret_st;

TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::prepare_rowset", ret_st);

}

CreateRowsetRequest req;

CreateRowsetResponse resp;

req.set_cloud_unique_id(config::cloud_unique_id);

req.set_txn_id(rs_meta.txn_id());

RowsetMetaPB doris_rs_meta = rs_meta.get_rowset_pb(/*skip_schema=*/true);

doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(doris_rs_meta));

Status st = retry_rpc("prepare rowset", req, &resp, &MetaService_Stub::prepare_rowset);

if (!st.ok() && resp.status().code() == MetaServiceCode::ALREADY_EXISTED) {

if (existed_rs_meta != nullptr && resp.has_existed_rowset_meta()) {

RowsetMetaPB doris_rs_meta_tmp =

cloud_rowset_meta_to_doris(std::move(*resp.mutable_existed_rowset_meta()));

*existed_rs_meta = std::make_shared<RowsetMeta>();

(*existed_rs_meta)->init_from_pb(doris_rs_meta_tmp);

}

return Status::AlreadyExist("failed to prepare rowset: {}", resp.status().msg());

}

return st;

RowsetMetaSharedPtr* existed_rs_meta) {

VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id()

<< ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id();

{

Status ret_st;

TEST_INJECTION_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_rowset", ret_st);

}

check_table_size_correctness(rs_meta);

CreateRowsetRequest req;

CreateRowsetResponse resp;

req.set_cloud_unique_id(config::cloud_unique_id);

req.set_txn_id(rs_meta.txn_id());

RowsetMetaPB rs_meta_pb = rs_meta.get_rowset_pb();

doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(rs_meta_pb));

// Replace schema dictionary keys based on the rowset's index ID to maintain schema consistency.

CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud();

// if not enable dict cache, then directly return true to avoid refresh

bool replaced =

config::variant_use_cloud_schema_dict_cache

? engine.get_schema_cloud_dictionary_cache().replace_schema_to_dict_keys(

rs_meta_pb.index_id(), req.mutable_rowset_meta())

: true;

Status st = retry_rpc("commit rowset", req, &resp, &MetaService_Stub::commit_rowset);

if (!st.ok() && resp.status().code() == MetaServiceCode::ALREADY_EXISTED) {

if (existed_rs_meta != nullptr && resp.has_existed_rowset_meta()) {

RowsetMetaPB doris_rs_meta =

cloud_rowset_meta_to_doris(std::move(*resp.mutable_existed_rowset_meta()));

*existed_rs_meta = std::make_shared<RowsetMeta>();

(*existed_rs_meta)->init_from_pb(doris_rs_meta);

}

return Status::AlreadyExist("failed to commit rowset: {}", resp.status().msg());

}

// If dictionary replacement fails, it may indicate that the local schema dictionary is outdated.

// Refreshing the dictionary here ensures that the rowset metadata is updated with the latest schema definitions,

// which is critical for maintaining consistency between the rowset and its corresponding schema.

if (!replaced) {

RETURN_IF_ERROR(

engine.get_schema_cloud_dictionary_cache().refresh_dict(rs_meta_pb.index_id()));

}

return st;

VLOG_DEBUG << "update committed rowset, tablet_id: " << rs_meta.tablet_id()

<< ", rowset_id: " << rs_meta.rowset_id();

CreateRowsetRequest req;

CreateRowsetResponse resp;

req.set_cloud_unique_id(config::cloud_unique_id);

// Variant schema maybe updated, so we need to update the schema as well.

// The updated rowset meta after `rowset->merge_rowset_meta` in `BaseTablet::update_delete_bitmap`

// will be lost in `update_tmp_rowset` if skip_schema.So in order to keep the latest schema we should keep schema in update_tmp_rowset

// for variant type

bool skip_schema = rs_meta.tablet_schema()->num_variant_columns() == 0;

RowsetMetaPB rs_meta_pb = rs_meta.get_rowset_pb(skip_schema);

doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(rs_meta_pb));

Status st =

retry_rpc("update committed rowset", req, &resp, &MetaService_Stub::update_tmp_rowset);

if (!st.ok() && resp.status().code() == MetaServiceCode::ROWSET_META_NOT_FOUND) {