【详述】从3.3.16版本,1台BE节点连续崩溃,重新拉起后服务正常,CPU、IO负载仍然异常高。
【背景】1台BE节点,CPU、IO负载高,异常崩溃
【业务影响】
【是否存算分离】
【StarRocks版本】3.3.16
【集群规模】3fe+3be
【机器信息】24C/256G/万兆
【JAVA版本】 java version “11.0.23” 2024-04-16 LTS
【附件】
be.out : 下面是一次崩溃的日志
*** Aborted at 1765983827 (unix time) try “date -d @1765983827” if you are using GNU date ***
PC: @ 0x39b34ba std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
*** SIGSEGV (@0x3b) received by PID 390047 (TID 0x7f4d067ff700) from PID 59; stack trace: ***
@ 0x7f80e415d20b __pthread_once_slow
@ 0x7d054a0 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
@ 0x7f80e4f8faaa os::Linux::chained_handler(int, siginfo*, void*)
@ 0x7f80e4f950ae JVM_handle_linux_signal
@ 0x7f80e4f88fb8 signalHandler(int, siginfo*, void*)
@ 0x7f80e4166630 (/usr/lib64/libpthread-2.17.so+0xf62f)
@ 0x39b34ba std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b33c7 std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b33c7 std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b330f std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b330f std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b330f std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b33c7 std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b33c7 std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b330f std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39b330f std::_Rb_tree_node<std::pair<long const, starrocks::TTabletStat> >* std::_Rb_tree<long, std::pair<long const,
starrocks::TTabletStat>, std::_Select1st<std::pair<long const, starrocks::TTabletStat> >, std::less, std::allocator<std::pair<l
ong const, st
@ 0x39aca42 starrocks::TTabletStatResult::__set_tablets_stats(std::map<long, starrocks::TTabletStat, std::less, std
::allocator<std::pair<long const, starrocks::TTabletStat> > > const&)
@ 0x3999fee starrocks::BackendServiceProcessor::process_get_tablet_stat(int, apache::thrift::protocol::TProtocol*, apache
::thrift::protocol::TProtocol*, void*)
@ 0x399d770 starrocks::BackendServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protoc
ol::TProtocol*, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&, int, void*)
@ 0x399e302 apache::thrift::TDispatchProcessor::process(std::shared_ptrapache::thrift::protocol::TProtocol, std::shared
_ptrapache::thrift::protocol::TProtocol, void*)
@ 0x7cf0168 apache::thrift::server::TConnectedClient::run()
@ 0x7ce8354 apache::thrift::server::TThreadedServer::TConnectedClientRunner::run()
@ 0x7ceaf0d apache::thrift::concurrency:
:threadMain(std::shared_ptrapache::thrift::concurrency::Thread)
@ 0x7ccdc96 std:
:_State_impl<std:
:_Invoker<std::tuple<void (*)(std::shared_ptr<apache::thrift::concurrenc
y::Thread>), std::shared_ptrapache::thrift::concurrency::Thread > > >::_M_run()
@ 0xc45e700 execute_native_thread_routine
@ 0x7f80e415eea5 start_thread
@ 0x7f80e355f96d __clone

