Hudi作为目的端时，如果作业启动失败，并且MRS集群安全组仅开放了TCP协议端口，遇到“javax.security.auth.login.LoginException: Receive timed out”错误，怎么办？

问题描述

此类问题大多出现在首次启动Hudi作业的场景中，特别是在MRS集群未开放UDF协议端口的情况下。

报错关键字包括：“javax.security.auth.login.LoginException: Receive timed out” 和 “UDPClient.receive”。

org.apache.hadoop.security.KerberosAuthException: failure to login: for principal: tc_admin@2A899CBE_5062_4F0C_A466_65F788289EB8.COM from keytab /tmp/cdm-hadoop-config/user.keytab 
javax.security.auth.login.LoginException: Receive timed out
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1998) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytabAndReturnUGI(UserGroupInformation.java:1361) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:1122) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at com.huawei.clouds.dataarts.migration.mrs.auth.utils.SecurityPrincipalUtils.securityLogin(SecurityPrincipalUtils.java:90) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.mrs.auth.utils.MrsUtils.securityLogin(MrsUtils.java:165) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.mrs.auth.utils.MrsUtils.downloadConfAndLogin(MrsUtils.java:137) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.mrs.auth.utils.MrsUtils.downloadConfAndLogin(MrsUtils.java:67) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.connector.hudi.table.HoodieTableFactory.createDynamicTableSink(HoodieTableFactory.java:185) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at org.apache.flink.table.factories.FactoryUtil.createDynamicTableSink(FactoryUtil.java:304) ~[flink-table-api-java-uber-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.planner.delegation.PlannerBase.getTableSink(PlannerBase.scala:434) ~[flink-table-planner_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.planner.delegation.PlannerBase.translateToRel(PlannerBase.scala:233) ~[flink-table-planner_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.planner.delegation.PlannerBase.$anonfun$translate$1(PlannerBase.scala:182) ~[flink-table-planner_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.Iterator.foreach(Iterator.scala:937) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.Iterator.foreach$(Iterator.scala:937) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.AbstractIterator.foreach(Iterator.scala:1425) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.IterableLike.foreach(IterableLike.scala:70) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.IterableLike.foreach$(IterableLike.scala:69) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.AbstractIterable.foreach(Iterable.scala:54) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.TraversableLike.map(TraversableLike.scala:233) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.TraversableLike.map$(TraversableLike.scala:226) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at scala.collection.AbstractTraversable.map(Traversable.scala:104) ~[flink-scala_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.planner.delegation.PlannerBase.translate(PlannerBase.scala:182) ~[flink-table-planner_2.12-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.api.internal.TableEnvironmentImpl.translate(TableEnvironmentImpl.java:1678) ~[flink-table-api-java-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:785) ~[flink-table-api-java-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.table.api.internal.StatementSetImpl.execute(StatementSetImpl.java:108) ~[flink-table-api-java-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.inlong.sort.parser.result.FlinkSqlParseResult.executeLoadSqls(FlinkSqlParseResult.java:83) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at org.apache.inlong.sort.parser.result.FlinkSqlParseResult.execute(FlinkSqlParseResult.java:62) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.engine.converter.Entrance.run(Entrance.java:152) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at com.huawei.clouds.dataarts.migration.engine.converter.Entrance.main(Entrance.java:95) ~[engine-dist-secret-1.0.0-SNAPSHOT-assembly.jar:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:381) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:232) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:335) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$2(ApplicationDispatcherBootstrap.java:263) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:171) ~[flink-rpc-akka_2ba332f7-1d0f-4b86-90ab-9fc3c40a4c46.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) ~[flink-rpc-akka_2ba332f7-1d0f-4b86-90ab-9fc3c40a4c46.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.lambda$withContextClassLoader$0(ClassLoadingUtils.java:41) ~[flink-rpc-akka_2ba332f7-1d0f-4b86-90ab-9fc3c40a4c46.jar:1.15.0-h0.cbu.dli.321.r2]
at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:49) [flink-rpc-akka_2ba332f7-1d0f-4b86-90ab-9fc3c40a4c46.jar:1.15.0-h0.cbu.dli.321.r2]
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:48) [flink-rpc-akka_2ba332f7-1d0f-4b86-90ab-9fc3c40a4c46.jar:1.15.0-h0.cbu.dli.321.r2]
at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) [?:1.8.0_362]
at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) [?:1.8.0_362]
at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) [?:1.8.0_362]
at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) [?:1.8.0_362]
Caused by: javax.security.auth.login.LoginException: Receive timed out
at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:814) ~[?:1.8.0_362]
at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:618) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) ~[?:1.8.0_362]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) ~[?:1.8.0_362]
at javax.security.auth.login.LoginContext.login(LoginContext.java:587) ~[?:1.8.0_362]
at org.apache.hadoop.security.UserGroupInformation$HadoopLoginContext.login(UserGroupInformation.java:2077) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
at org.apache.hadoop.security.UserGroupInformation.doSubjectLogin(UserGroupInformation.java:1987) ~[flink-dist-1.15.0-h0.cbu.dli.321.r2.jar:1.15.0-h0.cbu.dli.321.r2]
... 49 more
Caused by: java.net.SocketTimeoutException: Receive timed out
at java.net.PlainDatagramSocketImpl.receive0(Native Method) ~[?:1.8.0_362]
at java.net.AbstractPlainDatagramSocketImpl.receive(AbstractPlainDatagramSocketImpl.java:143) ~[?:1.8.0_362]
at java.net.DatagramSocket.receive(DatagramSocket.java:812) ~[?:1.8.0_362]
at sun.security.krb5.internal.
UDPClient.receive
(NetClient.java:206) ~[?:1.8.0_362]
at sun.security.krb5.KdcComm$KdcCommunication.run(KdcComm.java:404) ~[?:1.8.0_362]
at sun.security.krb5.KdcComm$KdcCommunication.run(KdcComm.java:364) ~[?:1.8.0_362]

原因分析

写MRS-Hudi，需要在MRS安全组中开放对Migration资源组的UDP协议端口，否则在作业启动时会提示登录超时。

解决方案

安全组协议端口需要全放开，参考数据源安全组放通哪些端口可满足Migration访问？。