为了更快的定位您的问题,请提供以下信息,谢谢
【详述】3.3.13存算分离集群出现慢锁,导致insert作业卡住
【背景】日常ETL作业,Insert提交在20个内
【业务影响】到账StarRocks ETL 卡住
【是否存算分离】是
【StarRocks版本】3.3.13
【集群规模】例如:3fe+15CN
LockManager detects slow lock {“owners”:[{“id”:29717,“name”:“starrocks-taskrun-pool-163”,“type”:“INTENTION_SHARED”,“heldFor”:611623,“queryId”:“86fc9182-4bad-11f0-a41d-d2485e851d9b”,“waitTime”:0,“stack”:[“java.base@11.0.26/jdk.internal.misc.Unsafe.park(Native Method)”,“java.base@11.0.26/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)”,“java.base@11.0.26/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1796)”,“java.base@11.0.26/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3128)”,“java.base@11.0.26/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1830)”,“java.base@11.0.26/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2004)”,“app//com.github.benmanes.caffeine.cache.LocalAsyncCache$AbstractCacheView.resolve(LocalAsyncCache.java:520)”,“app//com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache$LoadingCacheView.getAll(LocalAsyncLoadingCache.java:130)”,“app//com.starrocks.sql.optimizer.statistics.CachedStatisticStorage.getColumnNDVForPartitions(CachedStatisticStorage.java:418)”,“app//com.starrocks.sql.optimizer.statistics.CachedStatisticStorage.getColumnStatisticsOfPartitionLevel(CachedStatisticStorage.java:451)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalcUtils.getPartitionStatistics(StatisticsCalcUtils.java:111)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalculator.adjustPredicateCardinality(StatisticsCalculator.java:404)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalculator.computeOlapScanNode(StatisticsCalculator.java:358)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalOlapScan(StatisticsCalculator.java:300)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalculator.visitLogicalOlapScan(StatisticsCalculator.java:179)”,“app//com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator.accept(LogicalOlapScanOperator.java:163)”,“app//com.starrocks.sql.optimizer.statistics.StatisticsCalculator.estimatorStats(StatisticsCalculator.java:202)”,“app//com.starrocks.sql.optimizer.task.DeriveStatsTask.execute(DeriveStatsTask.java:63)”,“app//com.starrocks.sql.optimizer.task.SeriallyTaskScheduler.executeTasks(SeriallyTaskScheduler.java:65)”,“app//com.starrocks.sql.optimizer.Optimizer.memoOptimize(Optimizer.java:890)”,“app//com.starrocks.sql.optimizer.Optimizer.optimizeByCost(Optimizer.java:272)”,“app//com.starrocks.sql.optimizer.Optimizer.optimize(Optimizer.java:196)”,“app//com.starrocks.sql.optimizer.Optimizer.optimize(Optimizer.java:172)”,“app//com.starrocks.sql.InsertPlanner.buildExecPlan(InsertPlanner.java:517)”,“app//com.starrocks.sql.InsertPlanner.plan(InsertPlanner.java:315)”,“app//com.starrocks.sql.StatementPlanner.planInsertStmt(StatementPlanner.java:228)”,“app//com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:145)”,“app//com.starrocks.sql.StatementPlanner.plan(StatementPlanner.java:95)”,“app//com.starrocks.qe.StmtExecutor.execute(StmtExecutor.java:576)”,“app//com.starrocks.scheduler.SqlTaskRunProcessor.processTaskRun(SqlTaskRunProcessor.java:63)”,“app//com.starrocks.scheduler.TaskRun.executeTaskRun(TaskRun.java:284)”,“app//com.starrocks.scheduler.TaskRunExecutor.lambda$executeTaskRun$0(TaskRunExecutor.java:60)”,“app//com.starrocks.scheduler.TaskRunExecutor$$Lambda$2518/0x00007f9ad614e4b0.get(Unknown Source)”,“java.base@11.0.26/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)”,“java.base@11.0.26/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)”,“java.base@11.0.26/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)”,“java.base@11.0.26/java.lang.Thread.run(Thread.java:829)”]}],“waiter”:[{“id”:29518,“name”:“starrocks-taskrun-pool-150”,“type”:“WRITE”,“waitTime”:611610},{“id”:53,“name”:“autovacuum”,“type”:“INTENTION_SHARED”,“waitTime”:610564},{“id”:27,“name”:“AutoStatistic”,“type”:“INTENTION_SHARED”,“waitTime”:609341,“queryId”:“8872ddf4-4bad-11f0-a41d-d2485e851d9b”},{“id”:30060,“name”:“thrift-server-pool-1973”,“type”:“INTENTION_SHARED”,“waitTime”:608112,“queryId”:“892e3eec-4bad-11f0-924e-16521b0120e6”},{“id”:56,“name”:“MemoryUsageTracker”,“type”:“READ”,“waitTime”:602475},{“id”:30286,“name”:“thrift-server-pool-2007”,“type”:“INTENTION_SHARED”,“waitTime”:601553,“queryId”:“8d158ada-4bad-11f0-853b-466fd4e7e884”},{“id”:29504,“name”:“starrocks-taskrun-pool-143”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:600074},{“id”:566,“name”:“thrift-server-pool-89”,“type”:“READ”,“waitTime”:598365},{“id”:21,“name”:“consistency checker”,“type”:“READ”,“waitTime”:597255},{“id”:198,“name”:“COMPACTION_DISPATCH”,“type”:“INTENTION_SHARED”,“waitTime”:593161},{“id”:29516,“name”:“starrocks-taskrun-pool-148”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:572395},{“id”:29529,“name”:“starrocks-taskrun-pool-154”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:562475},{“id”:29731,“name”:“starrocks-taskrun-pool-164”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:553025},{“id”:29860,“name”:“starrocks-mysql-nio-pool-378”,“type”:“INTENTION_SHARED”,“waitTime”:543125,“queryId”:“afeaa061-4bad-11f0-a41d-d2485e851d9b”},{“id”:29555,“name”:“starrocks-taskrun-pool-159”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:510949},{“id”:29520,“name”:“starrocks-taskrun-pool-152”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:482063},{“id”:29457,“name”:“starrocks-taskrun-pool-137”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:480123},{“id”:29553,“name”:“starrocks-taskrun-pool-157”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:460498},{“id”:29558,“name”:“starrocks-taskrun-pool-162”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:452946},{“id”:30,“name”:“tablet stat mgr”,“type”:“READ”,“waitTime”:425226},{“id”:48,“name”:“StarMgrMetaSyncer”,“type”:“READ”,“waitTime”:397048},{“id”:30483,“name”:“starrocks-mysql-nio-pool-391”,“type”:“INTENTION_SHARED”,“waitTime”:363185,“queryId”:“1b2b7224-4bae-11f0-a41d-d2485e851d9b”},{“id”:46,“name”:“DynamicPartitionScheduler”,“type”:“READ”,“waitTime”:281501},{“id”:29505,“name”:“starrocks-taskrun-pool-144”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:255221},{“id”:29498,“name”:“starrocks-taskrun-pool-142”,“type”:“INTENTION_EXCLUSIVE”,“waitTime”:238118},{“id”:30643,“name”:“starrocks-mysql-nio-pool-406”,“type”:“INTENTION_SHARED”,“waitTime”:3000,“queryId”:“f1db50ad-4bae-11f0-a41d-d2485e851d9b”}]}