分布式训练docker启动不了

2019-03-28 19:50:51,417 INFO xdl.AppMasterBase: Launching worker container [container_1553773796023_0001_01_000005]
2019-03-28 19:50:51,440 INFO xdl.AppMasterBase: container Id is container_1553773796023_0001_01_000005,node Id is slave3:42716
2019-03-28 19:50:53,848 INFO xdl.AppMasterRunner$ApplicationMasterSignalHandler: Application Master is killed by signal:15
2019-03-28 19:50:53,861 INFO impl.AMRMClientImpl: Waiting for application to be successfully unregistered.
2019-03-28 19:50:53,975 ERROR xdl.AppMasterRunner: run error!
org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException: Application attempt appattempt_1553773796023_0001_000001 doesn't exist in ApplicationMasterService cache.
at org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:404)
at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:872)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:818)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2678)

at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)
at org.apache.hadoop.yarn.ipc.RPCUtil.instantiateYarnException(RPCUtil.java:75)
at org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:116)
at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:79)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy13.allocate(Unknown Source)
at org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:320)
at com.alibaba.xdl.AppMasterBase.waitForWorkerFinish(AppMasterBase.java:303)
at com.alibaba.xdl.AppMasterBase.run(AppMasterBase.java:182)
at com.alibaba.xdl.AppMasterRunner.main(AppMasterRunner.java:81)
Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException): Application attempt appattempt_1553773796023_0001_000001 doesn't exist in ApplicationMasterService cache.
at org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:404)
at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:523)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:991)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:872)
at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:818)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2678)

at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1511)
at org.apache.hadoop.ipc.Client.call(Client.java:1457)
at org.apache.hadoop.ipc.Client.call(Client.java:1367)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
at com.sun.proxy.$Proxy12.allocate(Unknown Source)
at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)
... 13 more
Exception in thread "SIGTERM handler" java.lang.ExceptionInInitializerError
at java.nio.file.FileSystems.getDefault(FileSystems.java:176)
at java.nio.file.Paths.get(Paths.java:84)
at com.alibaba.xdl.AppMasterBase.removeZkRootPath(AppMasterBase.java:1000)
at com.alibaba.xdl.AppMasterBase.close(AppMasterBase.java:1018)
at com.alibaba.xdl.AppMasterBase.dealWithExit(AppMasterBase.java:978)
at com.alibaba.xdl.AppMasterRunner$ApplicationMasterSignalHandler.handle(AppMasterRunner.java:121)
at sun.misc.Signal$1.run(Signal.java:212)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.security.PrivilegedActionException: sun.nio.fs.UnixException: No such file or directory
at java.security.AccessController.doPrivileged(Native Method)
at java.nio.file.FileSystems$DefaultFileSystemHolder.defaultFileSystem(FileSystems.java:96)
at java.nio.file.FileSystems$DefaultFileSystemHolder.(FileSystems.java:90)
... 8 more
Caused by: sun.nio.fs.UnixException: No such file or directory
at sun.nio.fs.UnixNativeDispatcher.getcwd(Native Method)
at sun.nio.fs.UnixFileSystem.(UnixFileSystem.java:67)
at sun.nio.fs.LinuxFileSystem.(LinuxFileSystem.java:39)
at sun.nio.fs.LinuxFileSystemProvider.newFileSystem(LinuxFileSystemProvider.java:46)
at sun.nio.fs.LinuxFileSystemProvider.newFileSystem(LinuxFileSystemProvider.java:39)
at sun.nio.fs.UnixFileSystemProvider.(UnixFileSystemProvider.java:56)
at sun.nio.fs.LinuxFileSystemProvider.(LinuxFileSystemProvider.java:41)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at sun.nio.fs.DefaultFileSystemProvider.createProvider(DefaultFileSystemProvider.java:48)
at sun.nio.fs.DefaultFileSystemProvider.create(DefaultFileSystemProvider.java:63)
at java.nio.file.FileSystems$DefaultFileSystemHolder.getDefaultProvider(FileSystems.java:108)
at java.nio.file.FileSystems$DefaultFileSystemHolder.access$000(FileSystems.java:89)
at java.nio.file.FileSystems$DefaultFileSystemHolder$1.run(FileSystems.java:98)
at java.nio.file.FileSystems$DefaultFileSystemHolder$1.run(FileSystems.java:96)
... 11 more
Exception in thread "main" java.lang.NoClassDefFoundError: Could not initialize class java.nio.file.FileSystems$DefaultFileSystemHolder
at java.nio.file.FileSystems.getDefault(FileSystems.java:176)
at java.nio.file.Paths.get(Paths.java:84)
at com.alibaba.xdl.AppMasterBase.removeZkRootPath(AppMasterBase.java:1000)
at com.alibaba.xdl.AppMasterBase.close(AppMasterBase.java:1018)
at com.alibaba.xdl.AppMasterBase.dealWithExit(AppMasterBase.java:978)
at com.alibaba.xdl.AppMasterRunner.main(AppMasterRunner.java:85)

Start at AppMasterBase.run, waitForWorkerFinish, removeZkRootPath, and close, then inspect AppMasterRunner and its signal handler using the reported YARN and Java stack traces. Reproduce the distributed Docker launch and determine why the worker startup and cleanup fail; done means the application launches its worker container and exits without these errors.

Written by the indexing model from the issue text.