alibaba / alibaba/otter

node响应出问题了

Open
#464 1 comment 0 reactions 0 assignees View on GitHub
Dominant language
Java
Stars
8.1k
Forks
2.5k
PR merge metrics
No merged PRs in 30d

Description

Caused by: com.alibaba.otter.shared.communication.core.exception.CommunicationException: call[10.99.98.231:2088] , Event[NotifyChannelEvent[channel=Channel[id=1,name=232-231,status=START,description=,pipelines=[Pipeline[id=1,channelId=1,name=pl-232-231,description=,selectNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],extractNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],loadNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],pairs=[DataMediaPair[id=7,pipelineId=1,source=DbDataMedia[id=11,namespace=ymdb,name=tb_activity,source=DbMediaSource[url=jdbc:mysql://10.99.98.232:3306,username=ymdb,password=123456,driver=com.mysql.jdbc.Driver,properties=,id=1,name=jyd-232,type=MYSQL,encode=UTF8MB4,gmtCreate=2018-02-02 02:13:32,gmtModified=2018-02-02 15:05:39],encode=,gmtCreate=2018-02-02 06:51:41,gmtModified=2018-02-02 06:51:41,mode=SINGLE],target=DbDataMedia[id=12,namespace=ymdb,name=tb_activity,source=DbMediaSource[url=jdbc:mysql://10.99.98.231:3306,username=ymdb,password=123456,driver=com.mysql.jdbc.Driver,properties=,id=2,name=jyd-231,type=MYSQL,encode=UTF8MB4,gmtCreate=2018-02-02 02:13:49,gmtModified=2018-02-02 08:15:13],encode=,gmtCreate=2018-02-02 06:51:41,gmtModified=2018-02-02 06:51:41,mode=SINGLE],pullWeight=,pushWeight=5,resolverData=,filterData=,columnPairMode=INCLUDE,columnPairs=[],columnGroups=[],gmtCreate=2018-02-02 06:51:41,gmtModified=2018-02-02 06:51:41],
Caused by: com.alibaba.dubbo.rpc.RpcException: Invoke remote method timeout. method: acceptEvent, provider: dubbo://10.99.98.231:2088/endpoint?acceptEvent.timeout=50000&client=netty&codec=dubbo&connections=30&iothreads=4&lazy=true&serialization=java&threads=50, cause: Waiting server-side response timeout by scan timer. start time: 2018-02-03 00:45:54.936, end time: 2018-02-03 00:46:44.947, client elapsed: 4 ms, server elapsed: 50007 ms, timeout: 50000 ms, request: Request [id=168617, version=2.0.0, twoway=true, event=false, broken=false, data=RpcInvocation [methodName=acceptEvent, parameterTypes=[class com.alibaba.otter.shared.communication.core.model.Event], arguments=[NotifyChannelEvent[channel=Channel[id=1,name=232-231,status=START,description=,pipelines=[Pipeline[id=1,channelId=1,name=pl-232-231,description=,selectNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],extractNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],loadNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],pairs=[DataMediaPair[id=7,pipelineId=1,source=DbDataMedia[id=11,namespace=ymdb,name=tb_activity,source=DbMediaSource[url=jdbc:mysql://10.99.98.232:3306,username=ymdb,password=123456,driver=com.mysql.jdbc.Driver,properties=,id=1,name=jyd-232,type=MYSQL,encode=UTF8MB4,gmtCreate=2018-02-02 02:13:32,gmtModified=2018-02-02 15:05:39],
at com.alibaba.dubbo.rpc.protocol.AbstractInvoker.invoke(AbstractInvoker.java:144)
at com.alibaba.dubbo.rpc.proxy.InvokerInvocationHandler.invoke(InvokerInvocationHandler.java:52)
at com.alibaba.dubbo.common.bytecode.proxy0.acceptEvent(proxy0.java)
at com.alibaba.otter.shared.communication.core.impl.dubbo.DubboCommunicationConnection.call(DubboCommunicationConnection.java:45)
at com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl.call(DefaultCommunicationClientImpl.java:96)
at com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl$2.call(DefaultCommunicationClientImpl.java:142)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.alibaba.dubbo.remoting.TimeoutException: Waiting server-side response timeout by scan timer. start time: 2018-02-03 00:45:54.936, end time: 2018-02-03 00:46:44.947, client elapsed: 4 ms, server elapsed: 50007 ms, timeout: 50000 ms, request: Request [id=168617, version=2.0.0, twoway=true, event=false, broken=false, data=RpcInvocation [methodName=acceptEvent, parameterTypes=[class com.alibaba.otter.shared.communication.core.model.Event], arguments=[NotifyChannelEvent[channel=Channel[id=1,name=232-231,status=START,description=,pipelines=[Pipeline[id=1,channelId=1,name=pl-232-231,description=,selectNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],extractNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],loadNodes=[Node[id=1,name=node-231,ip=10.99.98.231,port=2088,status=START,description=,parameters=NodeParameter[mbeanPort=2090,downloadPort=9090,zkCluster=AutoKeeperCluster[id=1,clusterName=jyd-232,serverList=[10.99.98.232:2181],description=,gmtCreate=2018-02-02 02:12:05,gmtModified=2018-02-02 02:12:05],externalIp=,useExternalIp=false],gmtCreate=2018-02-02 02:12:29,gmtModified=2018-02-02 02:12:29]],pairs=[DataMediaPair[id=7,pipelineId=1,source=DbDataMedia[id=11,namespace=ymdb,name=tb_activity,source=DbMediaSource[url=jdbc:mysql://10.99.98.232:3306,username=ymdb,password=123456,driver=com.mysql.jdbc.Driver,properties=,id=1,name=jyd-232,type=MYSQL,encode=UTF8MB4,gmtCreate=2018-02-02 02:13:32,gmtModified=2018-02-02 15:05:39],encode=,gmtCreate=2018-02-02 06:51:41,gmtModified=2018-02-02 06:51:41,mode=SINGLE],target=DbDataMedia[id=12,namespace=ymdb,name=tb_activity,source=DbMediaSource[url=jdbc:mysql://10.99.98.231:3306,username=ymdb,password=123456,driver=com.mysql.jdbc.Driver,properties=,id=2,name=jyd-231,type=MYSQL,encode=UTF8MB4,gmtCreate=2018-02-02 02:13:49,
at com.alibaba.dubbo.remoting.exchange.support.DefaultFuture.returnFromResponse(DefaultFuture.java:188)
at com.alibaba.dubbo.remoting.exchange.support.DefaultFuture.get(DefaultFuture.java:110)
at com.alibaba.dubbo.remoting.exchange.support.DefaultFuture.get(DefaultFuture.java:84)
at com.alibaba.dubbo.rpc.protocol.dubbo.DubboInvoker.doInvoke(DubboInvoker.java:96)
... 12 more

Contributor guide

No contributing guide indexed for this repository

Research direction

Start with the stack trace in shared/communication/core/impl/dubbo/DubboCommunicationConnection.java and DefaultCommunicationClientImpl.java, then inspect the Dubbo acceptEvent timeout path. Use the reported 50-second server-side timeout and node 10.99.98.231:2088 as the reproduction context. Done means the cause is identified and the timeout is resolved or documented with a regression test.

Written by the indexing model from the issue text.

Assessment

Tech stack
java
Domain
databases, distributed-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
20/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.