【详述】问题详细描述
1.当前streamload 比较耗时2w条数据导入 5-10s不等
2.日志出现slow lock较多
3.同时information_schema.task_runs这张表查询一直卡主
4.以上问题是否和物化视图刷新频繁有关, 同时发现大量物化视图statistic采集任务触发频繁, 其中一个物化建表如下
麻烦帮忙排查下
create …
DISTRIBUTED BY HASH(device_id)
REFRESH ASYNC
PROPERTIES (
“replicated_storage” = “true”,
“replication_num” = “3”,
“storage_medium” = “HDD”
)
【背景】做过哪些操作？无
【业务影响】数据导入耗时, 数据整体产出延迟
【是否存算分离】否
【StarRocks版本】例如：3.3.7
【集群规模】例如：4fe（4 follower）+4be（fe与be混部）
【机器信息】CPU虚拟核/内存/网卡，例如：40C/256G/万兆
【联系方式】k_sudi@163.com
streamload日志: {“Status”:“Success”,“BeginTxnTimeMs”:1134,“Message”:“OK”,“NumberUnselectedRows”:0,“CommitAndPublishTimeMs”:4625,“Label”:“dwd_pv_solar_inverter_power_detail-1741231114333-1”,“LoadBytes”:6057027,“StreamLoadPlanTimeMs”:1159,“NumberTotalRows”:26905,“WriteDataTimeMs”:4938,“TxnId”:13003927,“LoadTimeMs”:11858,“ReadDataTimeMs”:5,“NumberLoadedRows”:26905,“NumberFilteredRows”:0}
slow lock日志
{
“owners”: [
{
“id”: 9095,
“name”: “starrocks-taskrun-pool-15”,
“type”: “INTENTION_EXCLUSIVE”,
“heldFor”: 122491,
“waitTime”: 0,
“stack”: [
“app//com.starrocks.common.util.concurrent.lock.LockManager.release(LockManager.java:284)”,
“app//com.starrocks.common.util.concurrent.lock.Locker.release(Locker.java:106)”,
“app//com.starrocks.common.util.concurrent.lock.Locker.unLockTablesWithIntensiveDbLock(Locker.java:418)”,
“app//com.starrocks.common.util.concurrent.lock.Locker.unLockDatabase(Locker.java:472)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.doCommit(InsertOverwriteJobRunner.java:525)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.doLoad(InsertOverwriteJobRunner.java:171)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.handle(InsertOverwriteJobRunner.java:150)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.transferTo(InsertOverwriteJobRunner.java:211)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.prepare(InsertOverwriteJobRunner.java:255)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.handle(InsertOverwriteJobRunner.java:147)”,
“app//com.starrocks.load.InsertOverwriteJobRunner.run(InsertOverwriteJobRunner.java:135)”,
“app//com.starrocks.load.InsertOverwriteJobMgr.executeJob(InsertOverwriteJobMgr.java:91)”,
“app//com.starrocks.qe.StmtExecutor.handleInsertOverwrite(StmtExecutor.java:2044)”,
“app//com.starrocks.qe.StmtExecutor.handleDMLStmt(StmtExecutor.java:2139)”,
“app//com.starrocks.qe.StmtExecutor.handleDMLStmtWithProfile(StmtExecutor.java:2053)”,
“app//com.starrocks.scheduler.PartitionBasedMvRefreshProcessor.refreshMaterializedView(PartitionBasedMvRefreshProcessor.java:1124)”,
“app//com.starrocks.scheduler.PartitionBasedMvRefreshProcessor.doRefreshMaterializedView(PartitionBasedMvRefreshProcessor.java:462)”,
“app//com.starrocks.scheduler.PartitionBasedMvRefreshProcessor.doRefreshMaterializedViewWithRetry(PartitionBasedMvRefreshProcessor.java:368)”,
“app//com.starrocks.scheduler.PartitionBasedMvRefreshProcessor.doMvRefresh(PartitionBasedMvRefreshProcessor.java:327)”,
“app//com.starrocks.scheduler.PartitionBasedMvRefreshProcessor.processTaskRun(PartitionBasedMvRefreshProcessor.java:199)”,
“app//com.starrocks.scheduler.TaskRun.executeTaskRun(TaskRun.java:270)”,
“app//com.starrocks.scheduler.TaskRunExecutor.lambda$executeTaskRun$0(TaskRunExecutor.java:58)”,
“app//com.starrocks.scheduler.TaskRunExecutor$$Lambda$2926/0x00007f2db7b2c8a8.get(Unknown Source)”,
“java.base@11.0.22/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)”,
“java.base@11.0.22/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)”,
“java.base@11.0.22/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)”,
“java.base@11.0.22/java.lang.Thread.run(Thread.java:834)”
]
}
],
“waiter”: [
{
“id”: 237556,
“name”: “thrift-server-pool-147467”,
“type”: “READ”,
“waitTime”: 122421,
“stack”: [
“java.base/java.lang.Thread.run(Thread.java:834)”
]
},
{
“id”: 9097,
“name”: “starrocks-taskrun-pool-17”,
“type”: “INTENTION_EXCLUSIVE”,
“waitTime”: 122413,
“stack”: [
“java.base@11.0.22/java.lang.Thread.run(Thread.java:834)”
]
},
{
“id”: 42,
“name”: “UpdateDbUsedDataQuota”,
“type”: “READ”,
“waitTime”: 98,
“stack”: [
“app//com.starrocks.common.util.Daemon.run(Daemon.java:107)”
]
},
{
“id”: 53,
“name”: “tablet checker”,
“type”: “READ”,
“waitTime”: 95,
“stack”: [
“app//com.starrocks.common.util.Daemon.run(Daemon.java:107)”
]
}
]
}
