常见 Crash / BUG / 优化 查询

💬 StarRocks 用户问答
#315

  1. ForkJoinPool 死锁

"pull-hive-remote-files-1" #761 prio=5 os_prio=0 cpu=41322.65ms elapsed=3199967.09s tid=0x00007fb6e4ac1000 nid=0x2649 in Object.wait()  [0x00007fb6504cb000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.20.1/Native Method)
        - waiting on <no object reference available>
        at java.util.concurrent.ForkJoinTask.externalAwaitDone(java.base@11.0.20.1/ForkJoinTask.java:330)
        - waiting to re-lock in wait() <0x00007fba6fded9c0> (a java.util.stream.ReduceOps$ReduceTask)
        at java.util.concurrent.ForkJoinTask.doInvoke(java.base@11.0.20.1/ForkJoinTask.java:412)
        at java.util.concurrent.ForkJoinTask.invoke(java.base@11.0.20.1/ForkJoinTask.java:736)
        at java.util.stream.ReduceOps$ReduceOp.evaluateParallel(java.base@11.0.20.1/ReduceOps.java:919)
        at java.util.stream.AbstractPipeline.evaluate(java.base@11.0.20.1/AbstractPipeline.java:233)
        at java.util.stream.ReferencePipeline.collect(java.base@11.0.20.1/ReferencePipeline.java:578)
        at org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap.entrySet(EvaluatingStatisticsMap.java:166)
        - locked <0x00007fba6fdeda58> (a org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap)
        at java.util.Collections$UnmodifiableMap.entrySet(java.base@11.0.20.1/Collections.java:1481)
        at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.aggregateMaps(IOStatisticsBinding.java:240)
        at org.apache.hadoop.fs.statistics.IOStatisticsSnapshot.aggregate(IOStatisticsSnapshot.java:184)
        - locked <0x00007fba6fdeda90> (a org.apache.hadoop.fs.statistics.IOStatisticsSnapshot)
        at org.apache.hadoop.fs.s3a.Listing$ObjectListingIterator.close(Listing.java:718)
        at org.apache.hadoop.fs.s3a.Listing$FileStatusListingIterator.close(Listing.java:414)
        at org.apache.hadoop.util.functional.RemoteIterators$MaybeClose.close(RemoteIterators.java:744)
        at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.close(RemoteIterators.java:456)
        at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.sourceHasNext(RemoteIterators.java:476)
        at org.apache.hadoop.util.functional.RemoteIterators$MappingRemoteIterator.hasNext(RemoteIterators.java:530)
        at com.starrocks.connector.hive.HiveRemoteFileIO$1.hasNext(HiveRemoteFileIO.java:138)
        at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:98)
        at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:67)
        at com.starrocks.connector.CachingRemoteFileIO.loadRemoteFiles(CachingRemoteFileIO.java:87)
        at com.starrocks.connector.CachingRemoteFileIO$Lambda$297/0x00007fb76d6fe4c8.apply(Unknown Source)
        at com.google.common.cache.CacheLoader$FunctionToCacheLoader.load(CacheLoader.java:169)
        at com.google.common.cache.CacheLoader$1.load(CacheLoader.java:192)
        at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3570)
        at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2312)
        at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2189)
        - locked <0x00007fba6fdedcc8> (a com.google.common.cache.LocalCache$StrongAccessEntry)
        at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2079)
        at com.google.common.cache.LocalCache.get(LocalCache.java:4011)
        at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4034)
        at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:5010)
        at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:5017)
        at com.starrocks.connector.CachingRemoteFileIO.getRemoteFiles(CachingRemoteFileIO.java:78)
        at com.starrocks.connector.RemoteFileOperations.lambda$getRemoteFiles$0(RemoteFileOperations.java:86)
        at com.starrocks.connector.RemoteFileOperations$Lambda$1716/0x00007fb6d234b500.call(Unknown Source)
        at java.util.concurrent.FutureTask.run(java.base@11.0.20.1/FutureTask.java:264)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.20.1/ThreadPoolExecutor.java:1128)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.20.1/ThreadPoolExecutor.java:628)
        at java.lang.Thread.run(java.base@11.0.20.1/Thread.java:829)

"pull-hive-remote-files-1" #761 prio=5 os_prio=0 cpu=41322.65ms elapsed=3199967.09s tid=0x00007fb6e4ac1000 nid=0x2649 in Object.wait()  [0x00007fb6504cb000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.20.1/Native Method)
        - waiting on <no object reference available>
        at java.util.concurrent.ForkJoinTask.externalAwaitDone(java.base@11.0.20.1/ForkJoinTask.java:330)
        - waiting to re-lock in wait() <0x00007fba6fded9c0> (a java.util.stream.ReduceOps$ReduceTask)
        at java.util.concurrent.ForkJoinTask.doInvoke(java.base@11.0.20.1/ForkJoinTask.java:412)
        at java.util.concurrent.ForkJoinTask.invoke(java.base@11.0.20.1/ForkJoinTask.java:736)
        at java.util.stream.ReduceOps$ReduceOp.evaluateParallel(java.base@11.0.20.1/ReduceOps.java:919)
        at java.util.stream.AbstractPipeline.evaluate(java.base@11.0.20.1/AbstractPipeline.java:233)
        at java.util.stream.ReferencePipeline.collect(java.base@11.0.20.1/ReferencePipeline.java:578)
        at org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap.entrySet(EvaluatingStatisticsMap.java:166)
        - locked <0x00007fba6fdeda58> (a org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap)
        at java.util.Collections$UnmodifiableMap.entrySet(java.base@11.0.20.1/Collections.java:1481)
        at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.aggregateMaps(IOStatisticsBinding.java:240)
        at org.apache.hadoop.fs.statistics.IOStatisticsSnapshot.aggregate(IOStatisticsSnapshot.java:184)
        - locked <0x00007fba6fdeda90> (a org.apache.hadoop.fs.statistics.IOStatisticsSnapshot)
        at org.apache.hadoop.fs.s3a.Listing$ObjectListingIterator.close(Listing.java:718)
        at org.apache.hadoop.fs.s3a.Listing$FileStatusListingIterator.close(Listing.java:414)
        at org.apache.hadoop.util.functional.RemoteIterators$MaybeClose.close(RemoteIterators.java:744)
        at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.close(RemoteIterators.java:456)
        at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.sourceHasNext(RemoteIterators.java:476)
        at org.apache.hadoop.util.functional.RemoteIterators$MappingRemoteIterator.hasNext(RemoteIterators.java:530)
        at com.starrocks.connector.hive.HiveRemoteFileIO$1.hasNext(HiveRemoteFileIO.java:138)
        at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:98)
        at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:67)
        at com.starrocks.connector.CachingRemoteFileIO.loadRemoteFiles(CachingRemoteFileIO.java:87)
        at com.starrocks.connector.CachingRemoteFileIO$Lambda$297/0x00007fb76d6fe4c8.apply(Unknown Source)
        at com.google.common.cache.CacheLoader$FunctionToCacheLoader.load(CacheLoader.java:169)
        at com.google.common.cache.CacheLoader$1.load(CacheLoader.java:192)
        at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3570)
        at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2312)
        at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2189)
        - locked <0x00007fba6fdedcc8> (a com.google.common.cache.LocalCache$StrongAccessEntry)
        at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2079)
        at com.google.common.cache.LocalCache.get(LocalCache.java:4011)
        at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4034)
        at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:5010)
        at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:5017)
        at com.starrocks.connector.CachingRemoteFileIO.getRemoteFiles(CachingRemoteFileIO.java:78)
        at com.starrocks.connector.RemoteFileOperations.lambda$getRemoteFiles$0(RemoteFileOperations.java:86)
        at com.starrocks.connector.RemoteFileOperations$Lambda$1716/0x00007fb6d234b500.call(Unknown Source)
        at java.util.concurrent.FutureTask.run(java.base@11.0.20.1/FutureTask.java:264)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.20.1/ThreadPoolExecutor.java:1128)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.20.1/ThreadPoolExecutor.java:628)
        at java.lang.Thread.run(java.base@11.0.20.1/Thread.java:829)

  • Github Issue:

  • Github Fix PR:

  • Jira

  • 问题版本:

    • 3.1.0 ~ 3.1.17

    • 3.2.0 ~ 3.2.12

    • 3.3.0 ~ 3.3.8

  • 修复版本:

    • 3.1.18+

    • 3.2.13+

    • 3.3.9+

  • 问题原因:

  • 临时解决办法:

    • Set global cbo_enable_low_cardinality_optimize=false;

    • Set global low_cardinality_optimize_v2=false;

#316

  1. 开启 query cache 带有 having 的查询 crash

*** Aborted at 1712672237 (unix time) try "date -d @1712672237" if you are using GNU date ***
PC: @          0x39c1538 starrocks::UnpackConstColumnBinaryFunction<>::evaluate<>()
*** SIGSEGV (@0x0) received by PID 1650236 (TID 0x7fd6c9ff9640) from PID 0; stack trace: ***
    @          0x5fa5742 google::(anonymous namespace)::FailureSignalHandler()
    @     0x7fd722195520 (unknown)
    @          0x39c1538 starrocks::UnpackConstColumnBinaryFunction<>::evaluate<>()
    @          0x39c180c starrocks::VectorizedBinaryPredicate<>::evaluate_checked()
    @          0x40c0e4e starrocks::VectorizedOrCompoundPredicate::evaluate_checked()
    @          0x40c0e4e starrocks::VectorizedOrCompoundPredicate::evaluate_checked()
    @          0x40c0e4e starrocks::VectorizedOrCompoundPredicate::evaluate_checked()
    @          0x385daa3 starrocks::ExprContext::evaluate()
    @          0x385ddef starrocks::ExprContext::evaluate()
    @          0x28d3432 starrocks::ExecNode::eval_conjuncts()
    @          0x28bf6ff starrocks::pipeline::Operator::eval_conjuncts_and_in_filters()
    @          0x2bfa677 starrocks::pipeline::AggregateBlockingSourceOperator::pull_chunk()
    @          0x2c53915 starrocks::query_cache::ConjugateOperator::pull_chunk()
    @          0x28aa8bc starrocks::query_cache::MultilaneOperator::_pull_chunk_from_lane()
    @          0x28aae25 starrocks::query_cache::MultilaneOperator::pull_chunk()
    @          0x2891f64 starrocks::pipeline::PipelineDriver::process()
    @          0x550125e starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x4db08fa starrocks::ThreadPool::dispatch_thread()
    @          0x4dab38a starrocks::Thread::supervise_thread()
    @     0x7fd7221e7ac3 (unknown)
    @     0x7fd722279850 (unknown)
    @                0x0 (unknown)

  • Github Issue:

  • Github Fix PR:

  • Jira

  • 问题版本:

    • 2.5.0 ~ 2.5.20

    • 3.1.0 ~ 3.1.10

    • 3.2.0 ~ 3.2.4

  • 修复版本:

    • 2.5.21+

    • 3.1.11+

    • 3.2.5+

  • 问题原因:

  • 临时解决办法:

    • Set global enable_query_cache=false;
#317

  1. Parquet 查询 Decimal 列 crash

*** Aborted at 1734387117 (unix time) try "date -d @1734387117" if you are using GNU date ***
PC: @          0x7e96840 starrocks::parquet::BinaryToDecimalConverter<(starrocks::LogicalType)49>::convert(std::shared_ptr<starrocks::Column> const&, starrocks::Column*)
*** SIGSEGV (@0x0) received by PID 2866 (TID 0x7fafe599d700) from PID 0; stack trace: ***
    @     0x7fb077ef920b __pthread_once_slow
    @          0xb204694 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7fb07759c54f os::Linux::chained_handler(int, siginfo_t*, void*)
    @     0x7fb0775a23b8 JVM_handle_linux_signal
    @     0x7fb077593db8 signalHandler(int, siginfo_t*, void*)
    @     0x7fb077f02630 (/usr/lib64/libpthread-2.17.so+0xf62f)
    @          0x7e96840 starrocks::parquet::BinaryToDecimalConverter<(starrocks::LogicalType)49>::convert(std::shared_ptr<starrocks::Column> const&, starrocks::Column*)
    @          0x7e858a7 starrocks::parquet::StatisticsHelper::decode_value_into_column(std::shared_ptr<starrocks::Column> const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char@
    @          0x7eb97dd starrocks::parquet::ScalarColumnReader::page_index_zone_map_filter(std::vector<starrocks::ColumnPredicate const*, std::allocator<starrocks::ColumnPredicate const*> > const&, starrocks::SparseRange<unsigned long>*, starrocks::CompoundNodeType, unsigned long@
    @          0x7eb0e8d starrocks::StatusOr<std::optional<starrocks::SparseRange<unsigned long> > > starrocks::parquet::ZoneMapEvaluator<(starrocks::parquet::FilterLevel)1>::operator()<(starrocks::CompoundNodeType)0>(starrocks::PredicateCompoundNode<(starrocks::CompoundNodeType)0@
    @          0x7eaa89a starrocks::parquet::GroupReader::_deal_with_pageindex()
    @          0x7eac030 starrocks::parquet::GroupReader::prepare()
    @          0x7e740be starrocks::parquet::FileReader::_init_group_readers()
    @          0x7e74893 starrocks::parquet::FileReader::init(starrocks::HdfsScannerContext*)
    @          0x7c44992 starrocks::HdfsParquetScanner::do_open(starrocks::RuntimeState*)
    @          0x7c35727 starrocks::HdfsScanner::open(starrocks::RuntimeState*)
    @          0x7ba8ca2 starrocks::connector::HiveDataSource::_init_scanner(starrocks::RuntimeState*)
    @          0x7baad91 starrocks::connector::HiveDataSource::open(starrocks::RuntimeState*)
    @          0x49327dc starrocks::pipeline::ConnectorChunkSource::_open_data_source(starrocks::RuntimeState*, bool*)
    @          0x4932a1d starrocks::pipeline::ConnectorChunkSource::_read_chunk(starrocks::RuntimeState*, std::shared_ptr<starrocks::Chunk>*)
    @          0x4ccc1ea starrocks::pipeline::ChunkSource::buffer_next_batch_chunks_blocking(starrocks::RuntimeState*, unsigned long, starrocks::workgroup::WorkGroup const*)
    @          0x4922a4e auto starrocks::pipeline::ScanOperator::_trigger_next_scan(starrocks::RuntimeState*, int)::{lambda(auto:1&)#1}::operator()<starrocks::workgroup::YieldContext>(starrocks::workgroup::YieldContext&) const [clone .isra.0]
    @          0x4a39e19 starrocks::workgroup::ScanExecutor::worker_thread()
    @          0x3cf933f starrocks::ThreadPool::dispatch_thread()
    @          0x3cf01c0 starrocks::Thread::supervise_thread(void*)
    @     0x7fb077efaea5 start_thread
    @     0x7fb075b2bb0d __clone
查HIVE表空值字段条件导致所有BE Crash
#318

这个有临时修复方案吗？

#319

2.4版本不支持，只能升级吗？mysql> set global enable_rbo_table_prune = false;
ERROR 1064 (HY000): Unknown system variable ‘enable_rbo_table_prune’

#320

是的，只能升级，2.4版本不推荐使用，升级到2.5版本吧

#321

  1. Left anti join crash

*** Aborted at 1733316816 (unix time) try "date -d @1733316816" if you are using GNU date ***
PC: @          0x37bacdb starrocks::JoinHashMap&lt;&gt;::_ZN9starrocks11JoinHashMapILNS_11LogicalTypeE17ENS_23SerializedJoinBuildFuncENS_23SerializedJoinProbeFuncEE33_probe_from_ht_for_left_anti_joinEPNS_12RuntimeStateERKSt6vectorINS_5SliceESaIS8_EESC_.actor()
*** SIGSEGV (@0x0) received by PID 44699 (TID 0x1465a04da640) from PID 0; stack trace: ***
    @          0x6cb5a62 google::(anonymous namespace)::FailureSignalHandler()
    @     0x14691162b9b9 os::Linux::chained_handler()
    @     0x146911631c7a JVM_handle_linux_signal
    @     0x146911623a4c signalHandler()
    @     0x14691063e6f0 (unknown)
    @          0x37bacdb starrocks::JoinHashMap&lt;&gt;::_ZN9starrocks11JoinHashMapILNS_11LogicalTypeE17ENS_23SerializedJoinBuildFuncENS_23SerializedJoinProbeFuncEE33_probe_from_ht_for_left_anti_joinEPNS_12RuntimeStateERKSt6vectorINS_5SliceESaIS8_EESC_.actor()
    @          0x37a9e5c starrocks::JoinHashMap&lt;&gt;::_probe_coroutine&lt;&gt;()
    @          0x3860e84 starrocks::JoinHashMap&lt;&gt;::probe()
    @          0x37a3d46 starrocks::JoinHashTable::probe()
    @          0x3ce25ca starrocks::HashJoinProber::probe_chunk()
    @          0x3cddad4 starrocks::HashJoiner::_pull_probe_output_chunk()
    @          0x3cddd32 starrocks::HashJoiner::pull_chunk()
    @          0x3b16aa9 starrocks::pipeline::HashJoinProbeOperator::pull_chunk()
    @          0x3ac36e6 starrocks::pipeline::PipelineDriver::process()
    @          0x3ab55bf starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x302f5ec starrocks::ThreadPool::dispatch_thread()
    @          0x3028b6a starrocks::Thread::supervise_thread()
#322

  1. 高基数字符串聚合慢

  • Count distinct 单列高基数字符串

  • Group by 单列高基数字符串

  • Multic count distinct 高基数字符串

# c1 是高基数字符串
select count(distinct c1) from t1;
# 或
select xxx from t1 group by c1
# 或
select count(distinct c1), count(distinct c2) from t_10
#323

295.LogMessageFatal

*** Aborted at 1744634903 (unix time) try “date -d @1744634903” if you are using GNU date ***
PC: @ 0x7f74b286d387 __GI_raise
*** SIGABRT (@0x3eb0000859c) received by PID 34204 (TID 0x7f74741fe700) from PID 34204; stack trace: ***
@ 0x6d933a2 google::(anonymous namespace)::FailureSignalHandler()
@ 0x7f74b353c630 (unknown)
@ 0x7f74b286d387 __GI_raise
@ 0x7f74b286ea78 __GI_abort
@ 0x3666087 starrocks::failure_function()
@ 0x6d86d7d google::LogMessage::Fail()
@ 0x6d891ef google::LogMessage::SendToLog()
@ 0x6d868ce google::LogMessage::Flush()
@ 0x6d897f9 google::LogMessageFatal::~LogMessageFatal()
@ 0x54646dd starrocks::DataDir::load()
@ 0x54425eb _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN9starrocks13StorageEngine14load_data_dirsERKSt6vectorIPNS3_7DataDirESaIS7_EEEUlvE_EEEEE6_M_runEv
@ 0x91d36c0 execute_native_thread_routine
@ 0x7f74b3534ea5 start_thread
@ 0x7f74b293596d __clone
@ 0x0 (unknown)

Github Issue:

Github Fix PR:

Jira
问题版本:

3.1.0 ~ latest

3.2.0 ~ latest

3.3.0 ~ latest

3.4.0 ~ latest

修复版本:

问题原因:

临时解决办法：

  1. 检查日志是否存在 load tablets encounter failure 错误：
grep -a 'load tablets encounter failure' be.INFO
  1. 检查日志是否存在 there is failure when scan rockdb tablet metas 错误：
grep -a 'there is failure when scan rockdb tablet metas' be.INFO
  1. 根据检查结果处理：

  • 如果 there is failure when scan rockdb tablet metas 存在，请联系 StarRocks RD 团队处理。

  • 如果 there is failure when scan rockdb tablet metas 不存在，执行以下步骤：

    1. 在 be.conf 文件中添加以下配置：
ignore_load_tablet_failure = true

2. 保存配置后，重启 BE 服务。
1赞
#324

  1. Spill 导致 BE Crash

*** Aborted at 1741768367 (unix time) try "date -d @1741768367" if you are using GNU date ***
PC: @          0x519375e starrocks::ColumnVisitorMutableAdapter<starrocks::ColumnAppendPermutation>::visit(starrocks::FixedLengthColumn<long>*)
*** SIGSEGV (@0x7f5a22e41000) received by PID 25 (TID 0x7f5ac2240640) from PID 585371648; stack trace: ***
    @     0x7f5b741a3ee8 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x99ee7)
    @          0x9af2d89 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7f5b7414c520 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4251f)
    @          0x519375e starrocks::ColumnVisitorMutableAdapter<starrocks::ColumnAppendPermutation>::visit(starrocks::FixedLengthColumn<long>*)
    @          0x4fc742d starrocks::ColumnFactory<starrocks::FixedLengthColumnBase<long>, starrocks::FixedLengthColumn<long>, starrocks::Column>::accept_mutable(starrocks::ColumnVisitorMutable*)
    @          0x519028d starrocks::materialize_column_by_permutation(starrocks::Column*, std::vector<std::shared_ptr<starrocks::Column>, std::allocator<std::shared_ptr<starrocks::Column> > > const&, starrocks::array_view<starrocks::PermutationItem> const&)
    @          0x5190536 starrocks::materialize_by_permutation(starrocks::Chunk*, std::vector<std::shared_ptr<starrocks::Chunk>, std::allocator<std::shared_ptr<starrocks::Chunk> > > const&, starrocks::array_view<starrocks::PermutationItem> const&)
    @          0x528c30e starrocks::MergeTwoCursor::merge_sorted_intersected_cursor(starrocks::SortedRun&, starrocks::SortedRun&)
    @          0x528d9a1 starrocks::MergeTwoCursor::merge_sorted_cursor_two_way()
    @          0x528eb8c starrocks::MergeTwoCursor::next()
    @          0x528ec62 std::_Function_handler<bool (std::unique_ptr<starrocks::Chunk, std::default_delete<starrocks::Chunk> >*, bool*), starrocks::MergeTwoCursor::MergeTwoCursor(starrocks::SortDescs const&, std::unique_ptr<starrocks::SimpleChunkSortCursor, std::default_delete<stO2
    @          0x5293947 starrocks::SimpleChunkSortCursor::try_get_next()
    @          0x528bcd4 starrocks::MergeCursorsCascade::try_get_next()
    @          0x5287f5c starrocks::CascadeChunkMerger::get_next(std::unique_ptr<starrocks::Chunk, std::default_delete<starrocks::Chunk> >*, std::atomic<bool>*, bool*)
    @          0x54bb965 starrocks::spill::OrderedInputStream::get_next(starrocks::workgroup::YieldContext&, starrocks::spill::SerdeContext&)
    @          0x612da77 starrocks::StatusOr<std::shared_ptr<starrocks::Chunk> > starrocks::spill::SpillerReader::restore<starrocks::spill::IOTaskExecutor, starrocks::spill::ResourceMemTrackerGuard<std::weak_ptr<starrocks::pipeline::QueryContext>, std::weak_ptr<starrocks::spill::SO2
    @          0x612e03b starrocks::StatusOr<std::shared_ptr<starrocks::Chunk> > starrocks::spill::Spiller::restore<starrocks::spill::IOTaskExecutor, starrocks::spill::ResourceMemTrackerGuard<std::weak_ptr<starrocks::pipeline::QueryContext>, std::weak_ptr<starrocks::spill::SpillerO2
    @          0x611c9dd starrocks::pipeline::SpillableAggregateBlockingSourceOperator::_pull_spilled_chunk(starrocks::RuntimeState*)
    @          0x611cfb6 starrocks::pipeline::SpillableAggregateBlockingSourceOperator::pull_chunk(starrocks::RuntimeState*)
    @          0x4f8e0c0 starrocks::pipeline::PipelineDriver::process(starrocks::RuntimeState*, int)
    @          0x79fa3c3 starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x872f223 starrocks::ThreadPool::dispatch_thread()
    @          0x87268e9 starrocks::Thread::supervise_thread(void*)
    @     0x7f5b7419eac3 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x94ac2)
    @     0x7f5b7422fa04 clone

Or

*** Aborted at 1741657717 (unix time) try "date -d @1741657717" if you are using GNU date ***
PC: @     0x7f51ec288aca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x1a0ac9)
*** SIGSEGV (@0x7f51e57ff000) received by PID 25 (TID 0x7f5145bf4640) from PID 18446744073264951296; stack trace: ***
    @     0x7f51ec181ee8 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x99ee7)
    @          0xa16e1c9 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7f51ec12a520 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4251f)
    @     0x7f51ec288aca (/usr/lib/x86_64-linux-gnu/libc.so.6+0x1a0ac9)
    @          0x5472aed starrocks::FixedLengthColumnBase<signed char>::append(starrocks::Column const&, unsigned long, unsigned long)
    @          0x53b194a starrocks::Chunk::append(starrocks::Chunk const&, unsigned long, unsigned long)
    @          0x58b5989 starrocks::spill::OrderedMemTable::append(std::shared_ptr<starrocks::Chunk>)
    @          0x5827e8c starrocks::Status starrocks::spill::RawSpillerWriter::spill<starrocks::spill::IOTaskExecutor, starrocks::spill::ResourceMemTrackerGuard<std::weak_ptr<starrocks::pipeline::QueryContext>, std::weak_ptr<starrocks::spill::Spiller> >&>(starrocks::RuntimeState*,¹^Q
    @          0x582a4ad starrocks::Status starrocks::spill::Spiller::spill<starrocks::spill::IOTaskExecutor, starrocks::spill::ResourceMemTrackerGuard<std::weak_ptr<starrocks::pipeline::QueryContext>, std::weak_ptr<starrocks::spill::Spiller> > >(starrocks::RuntimeState*, std::sha¹^Q
    @          0x75cfa3c starrocks::pipeline::SpillProcessOperator::pull_chunk(starrocks::RuntimeState*)
    @          0x5396c5f starrocks::pipeline::PipelineDriver::process(starrocks::RuntimeState*, int)
    @          0x7d94683 starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x8aa57d2 starrocks::ThreadPool::dispatch_thread()
    @          0x8a9db09 starrocks::Thread::supervise_thread(void*)
    @     0x7f51ec17cac3 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x94ac2)
    @     0x7f51ec20da04 clone
#325

  1. Read json crash

*** SIGABRT (@0x2f) received by PID 47 (TID 0x7f424dff8640) from PID 47; stack trace: ***
    @     0x7f439f765ee8 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x99ee7)
    @          0xa28e2a9 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7f439f70e520 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4251f)
    @     0x7f439f7629fc pthread_kill
    @     0x7f439f70e476 raise
    @     0x7f439f6f47f3 abort
    @          0xef8a4f3 __gnu_cxx::__verbose_terminate_handler() [clone .cold]
    @          0xef88b6c __cxxabiv1::__terminate(void (*)())
    @          0xef88bd7 std::terminate()
    @          0xef88d39 __cxa_throw
    @          0x7b6e1b3 arangodb::velocypack::ObjectIterator::ObjectIterator(arangodb::velocypack::Slice, bool)
    @          0x8a8399e void starrocks::JsonMerger::_merge_json_with_remain<true>(starrocks::JsonFlatPath const*, arangodb::velocypack::Slice const*, arangodb::velocypack::Builder*, unsigned long)
    @          0x8a86264 void starrocks::JsonMerger::_merge_impl<true>(unsigned long)
    @          0x8a7187b starrocks::JsonMerger::merge(std::vector<std::shared_ptr<starrocks::Column>, std::allocator<std::shared_ptr<starrocks::Column> > > const&)
    @          0x66eb07d starrocks::JsonMergeIterator::next_batch(starrocks::SparseRange<unsigned int> const&, starrocks::Column*)
    @          0x7548d43 starrocks::SegmentIterator::_do_get_next(starrocks::Chunk*, std::vector<unsigned int, std::allocator<unsigned int> >*) [clone .constprop.0]
    @          0x754b4ee starrocks::SegmentIterator::do_get_next(starrocks::Chunk*)
    @          0x6cde0dd starrocks::MaskMergeIterator::fill(unsigned long)
    @          0x6cde707 starrocks::MergeIterator::init()
    @          0x6ce12a0 starrocks::MaskMergeIterator::do_get_next(starrocks::Chunk*, std::vector<starrocks::RowSourceMask, std::allocator<starrocks::RowSourceMask> >*)
    @          0x750fd4f starrocks::lake::TabletReader::do_get_next(starrocks::Chunk*, std::vector<starrocks::RowSourceMask, std::allocator<starrocks::RowSourceMask> >*)
    @          0x753239f starrocks::lake::VerticalCompactionTask::compact_column_group(bool, int, unsigned long, std::vector<unsigned int, std::allocator<unsigned int> > const&, std::unique_ptr<starrocks::lake::TabletWriter, std::default_delete<starrocks::lake::TabletWriter> >&, s
    @          0x753326a starrocks::lake::VerticalCompactionTask::execute(std::function<starrocks::Status ()>, starrocks::ThreadPool*)
    @          0x6774c5e starrocks::lake::CompactionScheduler::do_compaction(std::unique_ptr<starrocks::lake::CompactionTaskContext, std::default_delete<starrocks::lake::CompactionTaskContext> >)
    @          0x6776da8 starrocks::lake::CompactionScheduler::thread_task(int)
    @          0x8b274d2 starrocks::ThreadPool::dispatch_thread()
    @          0x8b1f809 starrocks::Thread::supervise_thread(void*)
    @     0x7f439f760ac3 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x94ac2)
    @     0x7f439f7f2850 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x12684f)
#326

  1. bdbje int overflow

Caused by: java.lang.ArrayIndexOutOfBoundsException: Index -1716603227 out of bounds for length 1001
        at java.lang.invoke.VarHandle$1.apply(VarHandle.java:2011) ~[?:?]
        at java.lang.invoke.VarHandle$1.apply(VarHandle.java:2008) ~[?:?]
        at jdk.internal.util.Preconditions$1.apply(Preconditions.java:159) ~[?:?]
        at jdk.internal.util.Preconditions$1.apply(Preconditions.java:156) ~[?:?]
        at jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:62) ~[?:?]
        at jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[?:?]
        at jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[?:?]
        at java.lang.invoke.VarHandleLongs$Array.getAndAdd(VarHandleLongs.java:721) ~[?:?]
        at java.lang.invoke.VarHandleGuards.guard_LIJ_J(VarHandleGuards.java:778) ~[?:?]
        at java.util.concurrent.atomic.AtomicLongArray.incrementAndGet(AtomicLongArray.java:234) ~[?:?]
        at com.sleepycat.je.utilint.LatencyPercentile.add(LatencyPercentile.java:288) ~[starrocks-bdb-je-18.3.16.jar:?]
        at com.sleepycat.je.rep.impl.node.Feeder$InputThread.processHeartbeatResponse(Feeder.java:981) ~[starrocks-bdb-je-18.3.16.jar:?]
        at com.sleepycat.je.rep.impl.node.Feeder$InputThread.runResponseLoop(Feeder.java:856) ~[starrocks-bdb-je-18.3.16.jar:?]
        at com.sleepycat.je.rep.impl.node.Feeder$InputThread.run(Feeder.java:755) ~[starrocks-bdb-je-18.3.16.jar:?]
#327

Be异常重启 能帮忙看下这个帖子吗？

#328

  1. SelectOperator crash

starrocks::BinaryColumnBase<unsigned int>::append(starrocks::Column const&, unsigned long, unsigned long)
*** SIGSEGV (@0x0) received by PID 88 (TID 0x7f261abcf640) from PID 0; stack trace: ***
    @     0x7f26dbf4bee8 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x99ee7)
    @          0xa28f3a9 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7f26dbef4520 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x4251f)
    @          0x5495a7b starrocks::BinaryColumnBase<unsigned int>::append(starrocks::Column const&, unsigned long, unsigned long)
    @          0x54bb516 starrocks::NullableColumn::append(starrocks::Column const&, unsigned long, unsigned long)
    @          0x7caa0b8 starrocks::pipeline::SelectOperator::pull_chunk(starrocks::RuntimeState*)
    @          0x53f9fe0 starrocks::pipeline::PipelineDriver::process(starrocks::RuntimeState*, int)
    @          0x7e6e053 starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x8bae9e3 starrocks::ThreadPool::dispatch_thread()
    @          0x8ba60a9 starrocks::Thread::supervise_thread(void*)
    @     0x7f26dbf46ac3 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x94ac2)
    @     0x7f26dbfd8850 (/usr/lib/x86_64-linux-gnu/libc.so.6+0x12684f)
#329

  1. FE 一直给挂掉的 BE 发请求导致导入报错

一台 BE 挂掉，也会导致导入持续报错

#330

  1. Array_unique_agg/array_union_agg crash

*** Aborted at 1740794708 (unix time) try "date -d @1740794708" if you are using GNU date ***
PC: @          0x617302d starrocks::Rowset::comparator(std::shared_ptr<starrocks::Rowset> const&, std::shared_ptr<starrocks::Rowset> const&)
*** SIGSEGV (@0xa8) received by PID 161549 (TID 0x7fec84d57700) from PID 168; stack trace: ***
    @     0x7ff6a0da6647 (/usr/lib64/libpthread-2.28.so+0x10646)
    @          0x7cd6b20 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7ff6a19ddaaa os::Linux::chained_handler(int, siginfo*, void*)
    @     0x7ff6a19e30ae JVM_handle_linux_signal
    @     0x7ff6a19d6fb8 signalHandler(int, siginfo*, void*)
    @     0x7ff6a0da94c0 (/usr/lib64/libpthread-2.28.so+0x134bf)
    @          0x617302d starrocks::Rowset::comparator(std::shared_ptr<starrocks::Rowset> const&, std::shared_ptr<starrocks::Rowset> const&)
    @          0x6329fec void std::__introsort_loop<__gnu_cxx::__normal_iterator<std::shared_ptr<starrocks::Rowset>*, std::vector<std::shared_ptr<starrocks::Rowset>, std::allocator<std::shared_ptr<starrocks::Rowset> > > >, long, __gnu_cxx::__ops::_Iter_comp_iter<bool (*)(std::sharH8?
    @          0x637866d starrocks::SizeTieredCompactionPolicy::_pick_rowsets_to_size_tiered_compact(bool, std::vector<std::shared_ptr<starrocks::Rowset>, std::allocator<std::shared_ptr<starrocks::Rowset> > >*, double*)
    @          0x637b980 starrocks::SizeTieredCompactionPolicy::need_compaction(double*, starrocks::CompactionType*)
    @          0x615fc4a starrocks::Tablet::need_compaction()
    @          0x636dcb3 starrocks::CompactionManager::update_tablet(std::shared_ptr<starrocks::Tablet> const&)
    @          0x394f753 starrocks::ThreadPool::dispatch_thread()
    @          0x39479f6 starrocks::Thread::supervise_thread(void*)
    @     0x7ff6a0d9ef1b (/usr/lib64/libpthread-2.28.so+0x8f1a)
    @     0x7ff6a0b2833f clone  、
#331

  1. Compaction crash

*** Aborted at 1740794708 (unix time) try "date -d @1740794708" if you are using GNU date ***
PC: @          0x617302d starrocks::Rowset::comparator(std::shared_ptr<starrocks::Rowset> const&, std::shared_ptr<starrocks::Rowset> const&)
*** SIGSEGV (@0xa8) received by PID 161549 (TID 0x7fec84d57700) from PID 168; stack trace: ***
    @     0x7ff6a0da6647 (/usr/lib64/libpthread-2.28.so+0x10646)
    @          0x7cd6b20 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x7ff6a19ddaaa os::Linux::chained_handler(int, siginfo*, void*)
    @     0x7ff6a19e30ae JVM_handle_linux_signal
    @     0x7ff6a19d6fb8 signalHandler(int, siginfo*, void*)
    @     0x7ff6a0da94c0 (/usr/lib64/libpthread-2.28.so+0x134bf)
    @          0x617302d starrocks::Rowset::comparator(std::shared_ptr<starrocks::Rowset> const&, std::shared_ptr<starrocks::Rowset> const&)
    @          0x6329fec void std::__introsort_loop<__gnu_cxx::__normal_iterator<std::shared_ptr<starrocks::Rowset>*, std::vector<std::shared_ptr<starrocks::Rowset>, std::allocator<std::shared_ptr<starrocks::Rowset> > > >, long, __gnu_cxx::__ops::_Iter_comp_iter<bool (*)(std::sharH8?
    @          0x637866d starrocks::SizeTieredCompactionPolicy::_pick_rowsets_to_size_tiered_compact(bool, std::vector<std::shared_ptr<starrocks::Rowset>, std::allocator<std::shared_ptr<starrocks::Rowset> > >*, double*)
    @          0x637b980 starrocks::SizeTieredCompactionPolicy::need_compaction(double*, starrocks::CompactionType*)
    @          0x615fc4a starrocks::Tablet::need_compaction()
    @          0x636dcb3 starrocks::CompactionManager::update_tablet(std::shared_ptr<starrocks::Tablet> const&)
    @          0x394f753 starrocks::ThreadPool::dispatch_thread()
    @          0x39479f6 starrocks::Thread::supervise_thread(void*)
    @     0x7ff6a0d9ef1b (/usr/lib64/libpthread-2.28.so+0x8f1a)
    @     0x7ff6a0b2833f clone
#332

  1. last_value crash

*** Aborted at 1746187907 (unix time) try "date -d @1746187907" if you are using GNU date ***
PC: @          0x36e0f84 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
*** SIGSEGV (@0x0) received by PID 192900 (TID 0x14d14d5a1640) from PID 0; stack trace: ***
    @     0x14d5e588ee18 __pthread_once_slow
    @          0x7da16a0 google::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*)
    @     0x14d5e682b9b9 os::Linux::chained_handler(int, siginfo_t*, void*)
    @     0x14d5e6831c7a JVM_handle_linux_signal
    @     0x14d5e6823a4c signalHandler(int, siginfo_t*, void*)
    @     0x14d5e583e6f0 (/usr/lib64/libc.so.6+0x3e6ef)
    @          0x36e0f84 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
    @          0x53bcaf8 starrocks::LastValueWindowFunction<(starrocks::LogicalType)54, false, starrocks::JsonValue*>::get_values(starrocks::FunctionContext*, unsigned char const*, starrocks::Column*, unsigned long, unsigned long) const
    @          0x41204ee starrocks::Analytor::_get_window_function_result(unsigned long, unsigned long)
    @          0x41205db starrocks::Analytor::_materializing_process_for_unbounded_frame(starrocks::RuntimeState*)
    @          0x41223b9 starrocks::Analytor::_materializing_process(starrocks::RuntimeState*)
    @          0x411f712 starrocks::Analytor::finish_process(starrocks::RuntimeState*)
    @          0x451292c starrocks::pipeline::AnalyticSinkOperator::set_finishing(starrocks::RuntimeState*)
    @          0x44d53b4 starrocks::pipeline::PipelineDriver::_mark_operator_finishing(std::shared_ptr<starrocks::pipeline::Operator>&, starrocks::RuntimeState*)
    @          0x44d8c8a starrocks::pipeline::PipelineDriver::process(starrocks::RuntimeState*, int)
    @          0x479b7d3 starrocks::pipeline::GlobalDriverExecutor::_worker_thread()
    @          0x397e973 starrocks::ThreadPool::dispatch_thread()
    @          0x3976c16 starrocks::Thread::supervise_thread(void*)
    @     0x14d5e5889c02 start_thread
    @     0x14d5e590ec40 __clone3
#333

  1. Spill 报磁盘空间不足

no writable spill storage directories
#334

305. capture_consistent_versions error

ERROR 1064 (HY000): capture_consistent_versions error: version not found. tablet_id: 882465, version: 2215 tablet_max_version:2184 backend [id=10002] [host=1.1.1.1]

京ICP备2022026421号-1