5. pthread_kill@@GLIBC_2.34 @ 0x00000000000969fd
6. gsignal @ 0x0000000000042476
7. __lgamma_r_finite@GLIBC_2.15 @ 0x00000000000287f3
8. __interceptor_abort @ 0x000000000cdac045
9. src/Common/Exception.cpp:60:5: DB::abortOnFailedAssertion(String const&, std::basic_string_view<char, std::char_traits<char>>, void* const*, unsigned long, unsigned long) @ 0x0000000017fad31b
10. src/Common/Exception.cpp:109:13: DB::Exception::handleErrorCode(String const&, std::basic_string_view<char, std::char_traits<char>>, int, bool, std::vector<void*, std::allocator<void*>> const&) @ 0x0000000017fae31a
11. src/Common/Exception.cpp:165:19: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x0000000017faef69
12. src/Common/Exception.h:201:100: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000ce5eaae
13. src/Common/Exception.h:57:54: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000ce5e1ca
14. src/Common/Exception.h:219:77: DB::Exception::Exception<String, String>(int, FormatStringHelperImpl<std::type_identity<String>::type, std::type_identity<String>::type>, String&&, String&&) @ 0x000000000ce5d986
15. src/Common/typeid_cast.cpp:15:11: throwBadTypeidCast(std::type_info const&, std::type_info const&) @ 0x00000000181fe7d4
16.0. inlined from src/Common/typeid_cast.h:22: T typeid_cast<DB::ColumnString const&, DB::IColumn const>(DB::IColumn const&)
16. src/DataTypes/Serializations/SerializationString.cpp:135:5: DB::SerializationString::serializeBinaryBulk(DB::IColumn const&, DB::WriteBuffer&, unsigned long, unsigned long) const @ 0x0000000021c58e69
17. src/DataTypes/Serializations/ISerialization.cpp:258:9: DB::ISerialization::serializeBinaryBulkWithMultipleStreams(DB::IColumn const&, unsigned long, unsigned long, DB::ISerialization::SerializeBinaryBulkSettings&, std::shared_ptr<DB::ISerialization::SerializeBinaryBulkState>&) const @ 0x0000000021a09405
18. src/DataTypes/Serializations/SerializationString.cpp:302:29: DB::SerializationString::serializeBinaryBulkWithMultipleStreams(DB::IColumn const&, unsigned long, unsigned long, DB::ISerialization::SerializeBinaryBulkSettings&, std::shared_ptr<DB::ISerialization::SerializeBinaryBulkState>&) const @ 0x0000000021c5eb9c
19. src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp:733:24: DB::MergeTreeDataPartWriterOnDisk::initColumnsSubstreamsIfNeeded() @ 0x000000002ad4357d
20. src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp:328:5: DB::MergeTreeDataPartWriterWide::write(DB::Block const&, DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const*, DB::Block*) @ 0x000000002ad47a66
21.0. inlined from src/Storages/MergeTree/MergedBlockOutputStream.cpp:479: DB::MergedBlockOutputStream::writeImpl(DB::Block const&, DB::PODArray<unsigned long, 4096ul, Allocator<false, false>, 63ul, 64ul> const*, DB::Block*)
21. src/Storages/MergeTree/MergedBlockOutputStream.cpp:94:13: DB::MergedBlockOutputStream::write(DB::Block const&) @ 0x000000002b06c81a
22. src/Storages/MergeTree/MutateTask.cpp:1844:19: DB::PartMergerWriter::mutateOriginalPartAndPrepareProjections() @ 0x000000002b088104
23.0. inlined from src/Storages/MergeTree/MutateTask.cpp:1703: DB::PartMergerWriter::execute()
23. src/Storages/MergeTree/MutateTask.cpp:2163:21: DB::MutateAllPartColumnsTask::executeStep() @ 0x000000002b0bd29e
24. src/Storages/MergeTree/MutateTask.cpp:3053:23: DB::MutateTask::execute() @ 0x000000002b08f82d
25. src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp:114:34: DB::MutatePlainMergeTreeTask::executeStep() @ 0x000000002b084ad3
26.0. inlined from src/Storages/MergeTree/MergeTreeBackgroundExecutor.h:74: DB::TaskRuntimeData::executeStep() const
26. src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp:383:22: DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::routine(std::shared_ptr<DB::TaskRuntimeData>) @ 0x000000002ab4d46e
27. src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp:446:13: DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::threadFunction() @ 0x000000002ab4fe55
28.0. inlined from src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp:91: operator()
28.1. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:90: std::__invoke_result_impl<void, DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&>::type std::__invoke[abi:fqe220101]<DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&>(DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&)
28.2. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:350: void std::__invoke_void_return_wrapper<void, true>::__call[abi:fqe220101]<DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&>(DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&)
28.3. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:356: void std::__invoke_r[abi:fqe220101]<void, DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&>(DB::MergeTreeBackgroundExecutor<DB::DynamicRuntimeQueue>::MergeTreeBackgroundExecutor(DB::ThreadName, unsigned long, unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, StrongTypedef<unsigned long, ProfileEvents::EventTag>, std::basic_string_view<char, std::char_traits<char>>)::'lambda'()&)
28. contrib/llvm-project/libcxx/include/__functional/function.h:443:47: ? @ 0x000000002ab528e2
29.0. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:502: ?
29.1. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:754: ?
29. src/Common/ThreadPool.cpp:1103:12: ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::worker() @ 0x00000000182ef5ff
30.0. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:0: std::__invoke_result_impl<void, void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&>::type std::__invoke[abi:fqe220101]<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&)
30.1. inlined from contrib/llvm-project/libcxx/include/tuple:1400: decltype(auto) std::__apply_tuple_impl[abi:fqe220101]<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), std::tuple<ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>&, 0ul>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), std::tuple<ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>&, std::integer_sequence<unsigned long, 0ul>)
30.2. inlined from contrib/llvm-project/libcxx/include/tuple:1404: decltype(auto) std::apply[abi:fqe220101]<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), std::tuple<ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>&>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&)(), std::tuple<ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>&)
30.3. inlined from src/Common/ThreadPool.h:363: operator()
30.4. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:90: std::__invoke_result_impl<void, ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&>::type std::__invoke[abi:fqe220101]<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&)
30.5. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:350: void std::__invoke_void_return_wrapper<void, true>::__call[abi:fqe220101]<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&)
30.6. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:356: void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*std::__invoke_r[abi:fqe220101]<void, ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*>(void (ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&)::'lambda'()&>(ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::ThreadFromThreadPool*&&))()
30. contrib/llvm-project/libcxx/include/__functional/function.h:443: ? @ 0x00000000182f9d22
31.0. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:502: ?
31.1. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:754: ?
31.2. inlined from src/Common/ThreadPool.cpp:1293: operator()
31.3. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:90: std::__invoke_result_impl<void, startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&>::type std::__invoke[abi:fqe220101]<startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&>(startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&)
31.4. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:350: void std::__invoke_void_return_wrapper<void, true>::__call[abi:fqe220101]<startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&>(startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&)
31.5. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:356: void std::__invoke_r[abi:fqe220101]<void, startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&>(startThreadFromGlobalPool(std::shared_ptr<ThreadFromGlobalPoolState>, std::function<void ()>, unsigned long, unsigned long, bool, bool)::$_0&)
31. contrib/llvm-project/libcxx/include/__functional/function.h:443:12: ? @ 0x00000000182e95ef
32.0. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:502: ?
32.1. inlined from contrib/llvm-project/libcxx/include/__functional/function.h:754: ?
32. src/Common/ThreadPool.cpp:1113:12: ThreadPoolImpl<std::thread>::ThreadFromThreadPool::worker() @ 0x00000000182eac26
33.0. inlined from contrib/llvm-project/libcxx/include/__type_traits/invoke.h:0: std::__invoke_result_impl<void, void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*>::type std::__invoke[abi:fqe220101]<void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*>(void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*&&)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*&&)
33.1. inlined from contrib/llvm-project/libcxx/include/__thread/thread.h:161: void std::__thread_execute[abi:fqe220101]<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*, 0ul, 1ul>(std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*>&, std::__integer_sequence<unsigned long, 0ul, 1ul>)
33. contrib/llvm-project/libcxx/include/__thread/thread.h:169: void* std::__thread_proxy[abi:fqe220101]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void (ThreadPoolImpl<std::thread>::ThreadFromThreadPool::*)(), ThreadPoolImpl<std::thread>::ThreadFromThreadPool*>>(void*) @ 0x00000000182f659c
34. __tsan_thread_start_func @ 0x000000000cda5493
35. start_thread @ 0x0000000000094ac3
36. clone3 @ 0x00000000001268d0
Stack trace details
The sipHash64(st.trace_full) is 15699613485465201805
The trace is from the master or release branch: True
The query for CIDB to compare the trace with the known one:
WITH ( SELECT groupArrayDistinct(cleanStackTrace(trace_full) AS trace) FROM default.stack_traces WHERE sipHash64(trace) IN (15699613485465201805, {ANOTHER_TRACE_HASH}) -- FIXME: replace with the known hash ) AS traces, 1.97 AS alpha, stack_frame_weights AS ( WITH ( SELECT count() FROM default.stack_traces FINAL ) AS total, 2.0 AS beta, 3.7 AS gamma SELECT arrayJoin(cleanStackTrace(trace_full)) AS frame, countDistinct(trace_full) AS count, log(total / count) AS IDF, sigmoid(beta * (IDF - gamma)) AS weight FROM default.stack_traces FINAL GROUP BY frame ), (SELECT groupArray(weight) AS w, groupArray(frame) AS f FROM stack_frame_weights) AS weights, (trace -> arrayMap((_frame, pos) -> (pow(pos, -alpha) * arrayFirst(w, f -> (f = _frame), weights.w, weights.f)), trace, arrayEnumerate(trace))) AS get_trace_weights, (arr -> arrayStringConcat(arr, '\n')) AS joinArr SELECT arraySimilarity(traces[1], traces[2], get_trace_weights(traces[1]) AS weights1, get_trace_weights(traces[2]) AS weights2) AS similarity, arrayLevenshteinDistanceWeighted(traces[1], traces[2], weights1, weights2), joinArr(traces[1]), joinArr(traces[2]), joinArr(weights1), joinArr(weights2)The following new stack trace from CI Logs
system.crash_logfound:Raw stack trace (with frame addresses)
Possible causes:
The stack trace appeared in the following checks: