-
ForkJoinPool 死锁
"pull-hive-remote-files-1" #761 prio=5 os_prio=0 cpu=41322.65ms elapsed=3199967.09s tid=0x00007fb6e4ac1000 nid=0x2649 in Object.wait() [0x00007fb6504cb000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(java.base@11.0.20.1/Native Method)
- waiting on <no object reference available>
at java.util.concurrent.ForkJoinTask.externalAwaitDone(java.base@11.0.20.1/ForkJoinTask.java:330)
- waiting to re-lock in wait() <0x00007fba6fded9c0> (a java.util.stream.ReduceOps$ReduceTask)
at java.util.concurrent.ForkJoinTask.doInvoke(java.base@11.0.20.1/ForkJoinTask.java:412)
at java.util.concurrent.ForkJoinTask.invoke(java.base@11.0.20.1/ForkJoinTask.java:736)
at java.util.stream.ReduceOps$ReduceOp.evaluateParallel(java.base@11.0.20.1/ReduceOps.java:919)
at java.util.stream.AbstractPipeline.evaluate(java.base@11.0.20.1/AbstractPipeline.java:233)
at java.util.stream.ReferencePipeline.collect(java.base@11.0.20.1/ReferencePipeline.java:578)
at org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap.entrySet(EvaluatingStatisticsMap.java:166)
- locked <0x00007fba6fdeda58> (a org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap)
at java.util.Collections$UnmodifiableMap.entrySet(java.base@11.0.20.1/Collections.java:1481)
at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.aggregateMaps(IOStatisticsBinding.java:240)
at org.apache.hadoop.fs.statistics.IOStatisticsSnapshot.aggregate(IOStatisticsSnapshot.java:184)
- locked <0x00007fba6fdeda90> (a org.apache.hadoop.fs.statistics.IOStatisticsSnapshot)
at org.apache.hadoop.fs.s3a.Listing$ObjectListingIterator.close(Listing.java:718)
at org.apache.hadoop.fs.s3a.Listing$FileStatusListingIterator.close(Listing.java:414)
at org.apache.hadoop.util.functional.RemoteIterators$MaybeClose.close(RemoteIterators.java:744)
at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.close(RemoteIterators.java:456)
at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.sourceHasNext(RemoteIterators.java:476)
at org.apache.hadoop.util.functional.RemoteIterators$MappingRemoteIterator.hasNext(RemoteIterators.java:530)
at com.starrocks.connector.hive.HiveRemoteFileIO$1.hasNext(HiveRemoteFileIO.java:138)
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:98)
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:67)
at com.starrocks.connector.CachingRemoteFileIO.loadRemoteFiles(CachingRemoteFileIO.java:87)
at com.starrocks.connector.CachingRemoteFileIO$Lambda$297/0x00007fb76d6fe4c8.apply(Unknown Source)
at com.google.common.cache.CacheLoader$FunctionToCacheLoader.load(CacheLoader.java:169)
at com.google.common.cache.CacheLoader$1.load(CacheLoader.java:192)
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3570)
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2312)
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2189)
- locked <0x00007fba6fdedcc8> (a com.google.common.cache.LocalCache$StrongAccessEntry)
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2079)
at com.google.common.cache.LocalCache.get(LocalCache.java:4011)
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4034)
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:5010)
at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:5017)
at com.starrocks.connector.CachingRemoteFileIO.getRemoteFiles(CachingRemoteFileIO.java:78)
at com.starrocks.connector.RemoteFileOperations.lambda$getRemoteFiles$0(RemoteFileOperations.java:86)
at com.starrocks.connector.RemoteFileOperations$Lambda$1716/0x00007fb6d234b500.call(Unknown Source)
at java.util.concurrent.FutureTask.run(java.base@11.0.20.1/FutureTask.java:264)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.20.1/ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.20.1/ThreadPoolExecutor.java:628)
at java.lang.Thread.run(java.base@11.0.20.1/Thread.java:829)
"pull-hive-remote-files-1" #761 prio=5 os_prio=0 cpu=41322.65ms elapsed=3199967.09s tid=0x00007fb6e4ac1000 nid=0x2649 in Object.wait() [0x00007fb6504cb000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(java.base@11.0.20.1/Native Method)
- waiting on <no object reference available>
at java.util.concurrent.ForkJoinTask.externalAwaitDone(java.base@11.0.20.1/ForkJoinTask.java:330)
- waiting to re-lock in wait() <0x00007fba6fded9c0> (a java.util.stream.ReduceOps$ReduceTask)
at java.util.concurrent.ForkJoinTask.doInvoke(java.base@11.0.20.1/ForkJoinTask.java:412)
at java.util.concurrent.ForkJoinTask.invoke(java.base@11.0.20.1/ForkJoinTask.java:736)
at java.util.stream.ReduceOps$ReduceOp.evaluateParallel(java.base@11.0.20.1/ReduceOps.java:919)
at java.util.stream.AbstractPipeline.evaluate(java.base@11.0.20.1/AbstractPipeline.java:233)
at java.util.stream.ReferencePipeline.collect(java.base@11.0.20.1/ReferencePipeline.java:578)
at org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap.entrySet(EvaluatingStatisticsMap.java:166)
- locked <0x00007fba6fdeda58> (a org.apache.hadoop.fs.statistics.impl.EvaluatingStatisticsMap)
at java.util.Collections$UnmodifiableMap.entrySet(java.base@11.0.20.1/Collections.java:1481)
at org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.aggregateMaps(IOStatisticsBinding.java:240)
at org.apache.hadoop.fs.statistics.IOStatisticsSnapshot.aggregate(IOStatisticsSnapshot.java:184)
- locked <0x00007fba6fdeda90> (a org.apache.hadoop.fs.statistics.IOStatisticsSnapshot)
at org.apache.hadoop.fs.s3a.Listing$ObjectListingIterator.close(Listing.java:718)
at org.apache.hadoop.fs.s3a.Listing$FileStatusListingIterator.close(Listing.java:414)
at org.apache.hadoop.util.functional.RemoteIterators$MaybeClose.close(RemoteIterators.java:744)
at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.close(RemoteIterators.java:456)
at org.apache.hadoop.util.functional.RemoteIterators$WrappingRemoteIterator.sourceHasNext(RemoteIterators.java:476)
at org.apache.hadoop.util.functional.RemoteIterators$MappingRemoteIterator.hasNext(RemoteIterators.java:530)
at com.starrocks.connector.hive.HiveRemoteFileIO$1.hasNext(HiveRemoteFileIO.java:138)
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:98)
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:67)
at com.starrocks.connector.CachingRemoteFileIO.loadRemoteFiles(CachingRemoteFileIO.java:87)
at com.starrocks.connector.CachingRemoteFileIO$Lambda$297/0x00007fb76d6fe4c8.apply(Unknown Source)
at com.google.common.cache.CacheLoader$FunctionToCacheLoader.load(CacheLoader.java:169)
at com.google.common.cache.CacheLoader$1.load(CacheLoader.java:192)
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3570)
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2312)
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2189)
- locked <0x00007fba6fdedcc8> (a com.google.common.cache.LocalCache$StrongAccessEntry)
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2079)
at com.google.common.cache.LocalCache.get(LocalCache.java:4011)
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4034)
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:5010)
at com.google.common.cache.LocalCache$LocalLoadingCache.getUnchecked(LocalCache.java:5017)
at com.starrocks.connector.CachingRemoteFileIO.getRemoteFiles(CachingRemoteFileIO.java:78)
at com.starrocks.connector.RemoteFileOperations.lambda$getRemoteFiles$0(RemoteFileOperations.java:86)
at com.starrocks.connector.RemoteFileOperations$Lambda$1716/0x00007fb6d234b500.call(Unknown Source)
at java.util.concurrent.FutureTask.run(java.base@11.0.20.1/FutureTask.java:264)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.20.1/ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.20.1/ThreadPoolExecutor.java:628)
at java.lang.Thread.run(java.base@11.0.20.1/Thread.java:829)
-
Github Issue:
-
Github Fix PR:
-
Jira
-
问题版本:
-
3.1.0 ~ 3.1.17
-
3.2.0 ~ 3.2.12
-
3.3.0 ~ 3.3.8
-
-
修复版本:
-
3.1.18+
-
3.2.13+
-
3.3.9+
-
-
问题原因:
-
临时解决办法:
-
Set global cbo_enable_low_cardinality_optimize=false;
-
Set global low_cardinality_optimize_v2=false;
-