昨天重启fe节点，出现以下附件报错
【是否存算分离】yes
【StarRocks版本】3.2.6
【集群规模】3fe（1 follower+2observer）+ 3cn
【附件】
2024-05-15 09:19:45.143+08:00 ERROR (main|1) [BDBEnvironment.setupEnvironment():302] failed to setup environment after retried 3 times
com.sleepycat.je.EnvironmentFailureException: (JE 18.3.16) starrocks-prod-fe-2.starrocks-prod-fe-search.starrocks.svc.cluster.local_9010_1714448697730(-1):/opt/starrocks/fe/meta/bdb last LSN=0x860/0x85a0eb LOG_INTEGRITY: Log information is incorrect, problem is likely persistent. Environment is invalid and must be closed.
at com.sleepycat.je.recovery.RecoveryManager.traceAndThrowException(RecoveryManager.java:3648) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2236) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.buildTree(RecoveryManager.java:977) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.recover(RecoveryManager.java:380) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.EnvironmentImpl.finishInit(EnvironmentImpl.java:895) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.DbEnvPool.getEnvironment(DbEnvPool.java:222) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.makeEnvironmentImpl(Environment.java:278) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.(Environment.java:258) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:605) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:464) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:538) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setupEnvironment(BDBEnvironment.java:267) ~[starrocks-fe.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setup(BDBEnvironment.java:172) ~[starrocks-fe.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.initBDBEnvironment(BDBEnvironment.java:150) ~[starrocks-fe.jar:?]
at com.starrocks.journal.JournalFactory.create(JournalFactory.java:31) ~[starrocks-fe.jar:?]
at com.starrocks.server.GlobalStateMgr.initJournal(GlobalStateMgr.java:1230) ~[starrocks-fe.jar:?]
at com.starrocks.server.GlobalStateMgr.initialize(GlobalStateMgr.java:1167) ~[starrocks-fe.jar:?]
at com.starrocks.StarRocksFE.start(StarRocksFE.java:136) ~[starrocks-fe.jar:?]
at com.starrocks.StarRocksFE.main(StarRocksFE.java:84) ~[starrocks-fe.jar:?]
Caused by: com.sleepycat.je.EnvironmentFailureException: (JE 18.3.16) starrocks-prod-fe-2.starrocks-prod-fe-search.starrocks.svc.cluster.local_9010_1714448697730(-1):/opt/starrocks/fe/meta/bdb java.io.FileNotFoundException: /opt/starrocks/fe/meta/bdb/00000858.jdb (No such file or directory) LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. fetchRoot of 0x858/0x4e4396 state=0 expires=never
at com.sleepycat.je.log.LogManager.getLogEntryHandleNotFound(LogManager.java:958) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getEntryHandleNotFound(LogManager.java:1192) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.ChildReference.fetchTarget(ChildReference.java:129) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree$RootChildReference.fetchTarget(Tree.java:227) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.searchSplitsAllowed(Tree.java:1679) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.getParentBINForChildLN(Tree.java:1449) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redo(RecoveryManager.java:2628) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoOneLN(RecoveryManager.java:2373) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2226) ~[starrocks-bdb-je-18.3.16.jar:?]
… 17 more
Caused by: java.io.FileNotFoundException: /opt/starrocks/fe/meta/bdb/00000858.jdb (No such file or directory)
at java.io.RandomAccessFile.open0(Native Method) ~[?:?]
at java.io.RandomAccessFile.open(RandomAccessFile.java:345) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:259) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:214) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:127) ~[?:?]
at com.sleepycat.je.log.FileManager$DefaultRandomAccessFile.(FileManager.java:3369) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager$6.createFile(FileManager.java:3397) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager.openFileHandle(FileManager.java:1427) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager.getFileHandle(FileManager.java:1298) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogSource(LogManager.java:1213) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:859) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:802) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntryHandleNotFound(LogManager.java:956) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getEntryHandleNotFound(LogManager.java:1192) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.ChildReference.fetchTarget(ChildReference.java:129) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree$RootChildReference.fetchTarget(Tree.java:227) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.searchSplitsAllowed(Tree.java:1679) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.getParentBINForChildLN(Tree.java:1449) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redo(RecoveryManager.java:2628) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoOneLN(RecoveryManager.java:2373) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2226) ~[starrocks-bdb-je-18.3.16.jar:?]
… 17 more
2024-05-15 09:19:45.144+08:00 ERROR (main|1) [StarRocksFE.start():193] StarRocksFE start failed
com.starrocks.journal.JournalException: failed to setup environment after retried 3 times
at com.starrocks.journal.bdbje.BDBEnvironment.setupEnvironment(BDBEnvironment.java:303) ~[starrocks-fe.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setup(BDBEnvironment.java:172) ~[starrocks-fe.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.initBDBEnvironment(BDBEnvironment.java:150) ~[starrocks-fe.jar:?]
at com.starrocks.journal.JournalFactory.create(JournalFactory.java:31) ~[starrocks-fe.jar:?]
at com.starrocks.server.GlobalStateMgr.initJournal(GlobalStateMgr.java:1230) ~[starrocks-fe.jar:?]
at com.starrocks.server.GlobalStateMgr.initialize(GlobalStateMgr.java:1167) ~[starrocks-fe.jar:?]
at com.starrocks.StarRocksFE.start(StarRocksFE.java:136) ~[starrocks-fe.jar:?]
at com.starrocks.StarRocksFE.main(StarRocksFE.java:84) ~[starrocks-fe.jar:?]
Caused by: com.sleepycat.je.EnvironmentFailureException: (JE 18.3.16) starrocks-prod-fe-2.starrocks-prod-fe-search.starrocks.svc.cluster.local_9010_1714448697730(-1):/opt/starrocks/fe/meta/bdb last LSN=0x860/0x85a0eb LOG_INTEGRITY: Log information is incorrect, problem is likely persistent. Environment is invalid and must be closed. at com.sleepycat.je.recovery.RecoveryManager.traceAndThrowException(RecoveryManager.java:3648) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2236) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.buildTree(RecoveryManager.java:977) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.recover(RecoveryManager.java:380) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.EnvironmentImpl.finishInit(EnvironmentImpl.java:895) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.DbEnvPool.getEnvironment(DbEnvPool.java:222) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.makeEnvironmentImpl(Environment.java:278) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.(Environment.java:258) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:605) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:464) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:538) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setupEnvironment(BDBEnvironment.java:267) ~[starrocks-fe.jar:?]
… 7 more
Caused by: com.sleepycat.je.EnvironmentFailureException: (JE 18.3.16) starrocks-prod-fe-2.starrocks-prod-fe-search.starrocks.svc.cluster.local_9010_1714448697730(-1):/opt/starrocks/fe/meta/bdb java.io.FileNotFoundException: /opt/starrocks/fe/meta/bdb/00000858.jdb (No such file or directory) LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. fetchRoot of 0x858/0x4e4396 state=0 expires=never
at com.sleepycat.je.log.LogManager.getLogEntryHandleNotFound(LogManager.java:958) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getEntryHandleNotFound(LogManager.java:1192) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.ChildReference.fetchTarget(ChildReference.java:129) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree$RootChildReference.fetchTarget(Tree.java:227) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.searchSplitsAllowed(Tree.java:1679) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.getParentBINForChildLN(Tree.java:1449) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redo(RecoveryManager.java:2628) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoOneLN(RecoveryManager.java:2373) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2226) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.buildTree(RecoveryManager.java:977) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.recover(RecoveryManager.java:380) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.EnvironmentImpl.finishInit(EnvironmentImpl.java:895) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.DbEnvPool.getEnvironment(DbEnvPool.java:222) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.makeEnvironmentImpl(Environment.java:278) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.(Environment.java:258) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:605) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:464) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:538) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setupEnvironment(BDBEnvironment.java:267) ~[starrocks-fe.jar:?]
… 7 more
Caused by: java.io.FileNotFoundException: /opt/starrocks/fe/meta/bdb/00000858.jdb (No such file or directory)
at java.io.RandomAccessFile.open0(Native Method) ~[?:?]
at java.io.RandomAccessFile.open(RandomAccessFile.java:345) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:259) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:214) ~[?:?]
at java.io.RandomAccessFile.(RandomAccessFile.java:127) ~[?:?]
at com.sleepycat.je.log.FileManager$DefaultRandomAccessFile.(FileManager.java:3369) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager$6.createFile(FileManager.java:3397) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager.openFileHandle(FileManager.java:1427) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.FileManager.getFileHandle(FileManager.java:1298) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogSource(LogManager.java:1213) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:859) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:802) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getLogEntryHandleNotFound(LogManager.java:956) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.log.LogManager.getEntryHandleNotFound(LogManager.java:1192) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.ChildReference.fetchTarget(ChildReference.java:129) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree$RootChildReference.fetchTarget(Tree.java:227) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.searchSplitsAllowed(Tree.java:1679) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.tree.Tree.getParentBINForChildLN(Tree.java:1449) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redo(RecoveryManager.java:2628) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoOneLN(RecoveryManager.java:2373) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.redoLNs(RecoveryManager.java:2226) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.buildTree(RecoveryManager.java:977) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.recovery.RecoveryManager.recover(RecoveryManager.java:380) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.EnvironmentImpl.finishInit(EnvironmentImpl.java:895) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.dbi.DbEnvPool.getEnvironment(DbEnvPool.java:222) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.makeEnvironmentImpl(Environment.java:278) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.Environment.(Environment.java:258) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:605) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:464) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.sleepycat.je.rep.ReplicatedEnvironment.(ReplicatedEnvironment.java:538) ~[starrocks-bdb-je-18.3.16.jar:?]
at com.starrocks.journal.bdbje.BDBEnvironment.setupEnvironment(BDBEnvironment.java:267) ~[starrocks-fe.jar:?]
… 7 more