详细日志
2024-08-23 12:39:25.950+08:00 WARN (starrocks-mysql-nio-pool-2379|676113) [MysqlChannel.fetchOnePacket():194] receive packet sequence id[32] want to get[1]
2024-08-23 12:39:25.950+08:00 WARN (starrocks-mysql-nio-pool-2379|676113) [AcceptListener.lambda$handleEvent$1():119] connect processor exception because
java.io.IOException: Bad packet sequence.
at com.starrocks.mysql.MysqlChannel.fetchOnePacket(MysqlChannel.java:195) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.readAuthPacket(MysqlProto.java:241) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.negotiate(MysqlProto.java:136) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.AcceptListener.lambda$handleEvent$1(AcceptListener.java:92) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:834) ~[?:?]
2024-08-23 12:39:38.074+08:00 ERROR (hive-remote-files-refresh-38|676104) [HiveRemoteFileIO.getRemoteFiles():120] Failed to get hive remote file’s metadata on path: hdfs://easyops-cluster/user/easydata/hive_db/easydata_dwd.db/dwd_fin_xj_net_realizable_value_i_1d/pardate=2024-07
java.net.SocketException: Call From localhost.localhost/127.0.0.1 to bigdata002.digitalchina.com:8020 failed on socket exception: java.net.SocketException: Network is unreachable; For more details see: http://wiki.apache.org/hadoop/SocketException
at jdk.internal.reflect.GeneratedConstructorAccessor133.newInstance(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:490) ~[?:?]
at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:930) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:888) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1571) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1513) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1410) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.3.6.jar:?]
at com.sun.proxy.$Proxy50.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getListing(ClientNamenodeProtocolTranslatorPB.java:689) ~[hadoop-hdfs-client-3.3.6.jar:?]
at jdk.internal.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-3.3.6.jar:?]
at com.sun.proxy.$Proxy51.getListing(Unknown Source) ~[?:?]
at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:1702) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1286) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1269) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1214) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1210) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1228) ~[hadoop-hdfs-client-3.3.6.jar:?]
at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2352) ~[starrocks-hadoop-ext.jar:?]
at com.starrocks.connector.hive.HiveRemoteFileIO$1.(HiveRemoteFileIO.java:132) ~[starrocks-fe.jar:?]
at com.starrocks.connector.hive.HiveRemoteFileIO.listFilesRecursive(HiveRemoteFileIO.java:130) ~[starrocks-fe.jar:?]
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:96) ~[starrocks-fe.jar:?]
at com.starrocks.connector.hive.HiveRemoteFileIO.getRemoteFiles(HiveRemoteFileIO.java:67) ~[starrocks-fe.jar:?]
at com.starrocks.connector.CachingRemoteFileIO.loadRemoteFiles(CachingRemoteFileIO.java:93) ~[starrocks-fe.jar:?]
at com.starrocks.connector.CachingRemoteFileIO$1.load(CachingRemoteFileIO.java:54) ~[starrocks-fe.jar:?]
at com.starrocks.connector.CachingRemoteFileIO$1.load(CachingRemoteFileIO.java:51) ~[starrocks-fe.jar:?]
at com.google.common.cache.CacheLoader.reload(CacheLoader.java:100) ~[spark-dpp-1.0.0.jar:?]
at com.google.common.cache.CacheLoader$1.lambda$reload$0(CacheLoader.java:198) ~[spark-dpp-1.0.0.jar:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
at com.starrocks.connector.ReentrantExecutor.lambda$execute$1(ReentrantExecutor.java:46) ~[starrocks-fe.jar:?]
at com.starrocks.connector.BoundedExecutor.releaseQueue(BoundedExecutor.java:77) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:834) ~[?:?]
Caused by: java.net.SocketException: Network is unreachable
at sun.nio.ch.Net.connect0(Native Method) ~[?:?]
at sun.nio.ch.Net.connect(Net.java:483) ~[?:?]
at sun.nio.ch.Net.connect(Net.java:472) ~[?:?]
at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:694) ~[?:?]
at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:191) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:600) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:652) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:773) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:347) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client.getConnection(Client.java:1632) ~[hadoop-common-3.3.6.jar:?]
at org.apache.hadoop.ipc.Client.call(Client.java:1457) ~[hadoop-common-3.3.6.jar:?]
… 37 more
2024-08-23 12:39:39.993+08:00 WARN (thrift-server-pool-664935|676146) [DefaultCoordinator.updateFragmentExecStatus():915] exec state report failed status=errorCode INTERNAL_ERROR hdfsOpenFile failed, file=hdfs://easyops-cluster/user/easydata/hive_db/easydata_dwd.db/dwd_fin_xj_net_realizable_value_i_1d/pardate=2024-07/000000_0. err_msg: error=Error(255): Unknown error 255, root_cause=SocketException: Network is unreachable: file = hdfs://easyops-cluster/user/easydata/hive_db/easydata_dwd.db/dwd_fin_xj_net_realizable_value_i_1d/pardate=2024-07/000000_0, query_id=55b48ac0-6109-11ef-959e-525400a2c314, instance_id=55b48ac0-6109-11ef-959e-525400a2c316, backend_id=10140
2024-08-23 12:39:39.993+08:00 WARN (thrift-server-pool-664935|676146) [DefaultCoordinator.updateStatus():731] one instance report fail throw updateStatus(), need cancel. job id: -1, query id: 55b48ac0-6109-11ef-959e-525400a2c314, instance id: 55b48ac0-6109-11ef-959e-525400a2c316
2024-08-23 12:39:39.996+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [DefaultCoordinator.getNext():756] get next fail, need cancel. status errorCode CANCELLED InternalError, query id: 55b48ac0-6109-11ef-959e-525400a2c314
2024-08-23 12:39:39.996+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [DefaultCoordinator.getNext():782] query failed: hdfsOpenFile failed, file=hdfs://easyops-cluster/user/easydata/hive_db/easydata_dwd.db/dwd_fin_xj_net_realizable_value_i_1d/pardate=2024-07/000000_0. err_msg: error=Error(255): Unknown error 255, root_cause=SocketException: Network is unreachable: file = hdfs://easyops-cluster/user/easydata/hive_db/easydata_dwd.db/dwd_fin_xj_net_realizable_value_i_1d/pardate=2024-07/000000_0
2024-08-23 12:39:39.996+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [StmtExecutor.execute():644] Query 55b48ac0-6109-11ef-959e-525400a2c314 failed. Planner profile : Planner:
Reason:
2024-08-23 12:39:39.996+08:00 WARN (thrift-server-pool-664936|676147) [DefaultCoordinator.updateFragmentExecStatus():915] exec state report failed status=errorCode CANCELLED InternalError, query_id=55b48ac0-6109-11ef-959e-525400a2c314, instance_id=55b48ac0-6109-11ef-959e-525400a2c315, backend_id=10170
2024-08-23 12:39:40.950+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [MysqlChannel.fetchOnePacket():194] receive packet sequence id[32] want to get[1]
2024-08-23 12:39:40.950+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [AcceptListener.lambda$handleEvent$1():119] connect processor exception because
java.io.IOException: Bad packet sequence.
at com.starrocks.mysql.MysqlChannel.fetchOnePacket(MysqlChannel.java:195) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.readAuthPacket(MysqlProto.java:241) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.negotiate(MysqlProto.java:136) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.AcceptListener.lambda$handleEvent$1(AcceptListener.java:92) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:834) ~[?:?]
2024-08-23 12:39:55.951+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [MysqlChannel.fetchOnePacket():194] receive packet sequence id[32] want to get[1]
2024-08-23 12:39:55.951+08:00 WARN (starrocks-mysql-nio-pool-2378|676098) [AcceptListener.lambda$handleEvent$1():119] connect processor exception because
java.io.IOException: Bad packet sequence.
at com.starrocks.mysql.MysqlChannel.fetchOnePacket(MysqlChannel.java:195) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.readAuthPacket(MysqlProto.java:241) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.MysqlProto.negotiate(MysqlProto.java:136) ~[starrocks-fe.jar:?]
at com.starrocks.mysql.nio.AcceptListener.lambda$handleEvent$1(AcceptListener.java:92) ~[starrocks-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
at java.lang.Thread.run(Thread.java:834) ~[?:?]
