为了更快的定位您的问题,请提供以下信息,谢谢
【详述】starrocks3.2.0版本,存算一体部署,4FE,4BE部署;微服务使用Druid连接池连接Starrocks;一开始运行正常,跑着跑着会报错连接超时
【背景】
【业务影响】
【是否存算分离】
【StarRocks版本】3.2.0
【集群规模】例如:4fe(1 Leader+3follower)+4be(fe与be混部)
【附件】
FE warn日志
2024-06-13 12:12:23,582 WARN (starrocks-mysql-nio-pool-299|5120) [ReadListener.lambda$handleEvent$0():81] Exception happened in one session(com.starrocks.mysql.nio.NConnectContext@76900bf1).
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:1.8.0_392]
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:1.8.0_392]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:470) ~[?:1.8.0_392]
at org.xnio.nio.NioSocketConduit.write(NioSocketConduit.java:153) ~[xnio-nio-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.conduits.ConduitStreamSinkChannel.write(ConduitStreamSinkChannel.java:150) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.channels.Channels.writeBlocking(Channels.java:147) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at com.starrocks.mysql.nio.NMysqlChannel.realNetSend(NMysqlChannel.java:69) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.send(MysqlChannel.java:354) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.flush(MysqlChannel.java:377) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.sendAndFlush(MysqlChannel.java:454) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.finalizeCommand(ConnectProcessor.java:660) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:850) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_392]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_392]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_392]
2024-06-13 12:16:26,993 WARN (starrocks-mysql-nio-pool-299|5120) [ConnectProcessor.handleQuery():396] Process one query failed because IOException:
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:1.8.0_392]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:470) ~[?:1.8.0_392]
at org.xnio.nio.NioSocketConduit.write(NioSocketConduit.java:153) ~[xnio-nio-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.conduits.ConduitStreamSinkChannel.write(ConduitStreamSinkChannel.java:150) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.channels.Channels.writeBlocking(Channels.java:147) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at com.starrocks.mysql.nio.NMysqlChannel.realNetSend(NMysqlChannel.java:69) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.send(MysqlChannel.java:354) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.flush(MysqlChannel.java:377) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.sendAndFlush(MysqlChannel.java:454) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.finalizeCommand(ConnectProcessor.java:660) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:850) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_392]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_392]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_392]
2024-06-13 12:16:26,993 WARN (starrocks-mysql-nio-pool-299|5120) [ConnectProcessor.handleQuery():396] Process one query failed because IOException:
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:1.8.0_392]
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:1.8.0_392]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:470) ~[?:1.8.0_392]
at org.xnio.nio.NioSocketConduit.write(NioSocketConduit.java:153) ~[xnio-nio-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.conduits.ConduitStreamSinkChannel.write(ConduitStreamSinkChannel.java:150) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.channels.Channels.writeBlocking(Channels.java:147) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at com.starrocks.mysql.nio.NMysqlChannel.realNetSend(NMysqlChannel.java:69) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.send(MysqlChannel.java:354) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.flush(MysqlChannel.java:377) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.sendAndFlush(MysqlChannel.java:454) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.finalizeCommand(ConnectProcessor.java:660) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:390) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.dispatch(ConnectProcessor.java:570) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:848) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_392]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_392]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_392]
2024-06-13 12:16:26,994 WARN (starrocks-mysql-nio-pool-299|5120) [ReadListener.lambda$handleEvent$0():81] Exception happened in one session(com.starrocks.mysql.nio.NConnectContext@2ad251b0).
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:1.8.0_392]
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) ~[?:1.8.0_392]
at sun.nio.ch.IOUtil.write(IOUtil.java:65) ~[?:1.8.0_392]
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:470) ~[?:1.8.0_392]
at org.xnio.nio.NioSocketConduit.write(NioSocketConduit.java:153) ~[xnio-nio-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.conduits.ConduitStreamSinkChannel.write(ConduitStreamSinkChannel.java:150) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at org.xnio.channels.Channels.writeBlocking(Channels.java:147) ~[xnio-api-3.8.10.Final.jar:3.8.10.Final]
at com.starrocks.mysql.nio.NMysqlChannel.realNetSend(NMysqlChannel.java:69) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.send(MysqlChannel.java:354) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.flush(MysqlChannel.java:377) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlChannel.sendAndFlush(MysqlChannel.java:454) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.finalizeCommand(ConnectProcessor.java:660) ~[starrocks-fe.jar:?]
at com.starrocks.qe.ConnectProcessor.processOnce(ConnectProcessor.java:850) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:69) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_392]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_392]
客户端日志
Communications link failure with primary host ..14.191:9030. Connection timed out; nested exception is java.sql.SQLInvalidAuthorizationSpecException: (conn=9) Communications link failure with primary host ..14.191:9030. Connection timed out
at org.springframework.jdbc.support.SQLExceptionSubclassTranslator.doTranslate(SQLExceptionSubclassTranslator.java:90)
at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:73)
at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:82)
at org.springframework.jdbc.core.JdbcTemplate.translateException(JdbcTemplate.java:1575)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:398)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:434)
at com.huawei.winscloud.ai.api.datawarehouse.service.StarRocksService.execute(StarRocksService.java:54)
at com.huawei.winscloud.ai.api.datawarehouse.service.StarRocksService.setInitData(StarRocksService.java:155)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleElement.invoke(InitDestroyAnnotationBeanPostProcessor.java:389)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor$LifecycleMetadata.invokeInitMethods(InitDestroyAnnotationBeanPostProcessor.java:333)
at org.springframework.beans.factory.annotation.InitDestroyAnnotationBeanPostProcessor.postProcessBeforeInitialization(InitDestroyAnnotationBeanPostProcessor.java:157)
… 55 common frames omitted
Caused by: java.sql.SQLInvalidAuthorizationSpecException: (conn=9) Communications link failure with primary host ..14.191:9030. Connection timed out
at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.createException(ExceptionFactory.java:66)
at org.mariadb.jdbc.internal.util.exceptions.ExceptionFactory.create(ExceptionFactory.java:158)
at org.mariadb.jdbc.MariaDbStatement.executeExceptionEpilogue(MariaDbStatement.java:262)
at org.mariadb.jdbc.MariaDbStatement.executeInternal(MariaDbStatement.java:362)
at org.mariadb.jdbc.MariaDbStatement.execute(MariaDbStatement.java:500)
at com.alibaba.druid.pool.DruidPooledStatement.execute(DruidPooledStatement.java:635)
at org.springframework.jdbc.core.JdbcTemplate$1ExecuteStatementCallback.doInStatement(JdbcTemplate.java:425)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:383)
… 65 common frames omitted
Caused by: java.sql.SQLException: Communications link failure with primary host ..14.191:9030. Connection timed out
on HostAddress{host=‘..14.191’, port=9030, type=‘master’},master=true. Driver has reconnect connection
at org.mariadb.jdbc.internal.failover.AbstractMastersListener.throwFailoverMessage(AbstractMastersListener.java:563)
at org.mariadb.jdbc.internal.failover.FailoverProxy.handleFailOver(FailoverProxy.java:391)
at org.mariadb.jdbc.internal.failover.FailoverProxy.executeInvocation(FailoverProxy.java:324)
at org.mariadb.jdbc.internal.failover.FailoverProxy.invoke(FailoverProxy.java:294)
at com.sun.proxy.$Proxy171.executeQuery(Unknown Source)
at org.mariadb.jdbc.MariaDbStatement.executeInternal(MariaDbStatement.java:356)
… 69 common frames omitted
Caused by: java.sql.SQLNonTransientConnectionException: Read timed out
at org.mariadb.jdbc.internal.protocol.AbstractQueryProtocol.handleIoException(AbstractQueryProtocol.java:2091)
at org.mariadb.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1541)
at org.mariadb.jdbc.internal.protocol.AbstractQueryProtocol.getResult(AbstractQueryProtocol.java:1520)
at org.mariadb.jdbc.internal.protocol.AbstractQueryProtocol.executeQuery(AbstractQueryProtocol.java:256)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.mariadb.jdbc.internal.failover.AbstractMastersListener.invoke(AbstractMastersListener.java:408)
at org.mariadb.jdbc.internal.failover.FailoverProxy.executeInvocation(FailoverProxy.java:301)
… 72 common frames omitted
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at java.io.FilterInputStream.read(FilterInputStream.java:133)
at org.mariadb.jdbc.internal.io.input.ReadAheadBufferedStream.fillBuffer(ReadAheadBufferedStream.java:131)
at org.mariadb.jdbc.internal.io.input.ReadAheadBufferedStream.read(ReadAheadBufferedStream.java:104)
at org.mariadb.jdbc.internal.io.input.StandardPacketInputStream.getPacketArray(StandardPacketInputStream.java:247)
at org.mariadb.jdbc.internal.io.input.StandardPacketInputStream.getPacket(StandardPacketInputStream.java:218)
at org.mariadb.jdbc.internal.protocol.AbstractQueryProtocol.readPacket(AbstractQueryProtocol.java:1539)