Node failure - thread in timed_waiting - affecting latency of the application
We are running a cluster using ISPN 5.3.1.Final in sync replication mode.we have 2 nodes in the cluster and Infinispan is embedded in our standalone Java application.The Java application receives webservice requests and Infinispan is used to store some data of the request in the context of application thread.when we test failover of nodes ,we could see that the application thread in one node is halted for some time at the moment the other node is killed.we couldnt findout exactly the time period for which the stoppage continues,but this affects the latency of the application.
i have seen that application thread is in timed_waiting for some time when the second node is killed and see that the below thread continues in this state for 1-2 seconds.we get this issue very consistently.
"http-192.168.10.103-8084-Processor49_DIS1" daemon prio=10 tid=0x00007eff40012800 nid=0x28da waiting on condition [0x00007efff5c58000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000076e3bd728> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2116)
at org.jgroups.blocks.Request.responsesComplete(Request.java:197)
at org.jgroups.blocks.Request.execute(Request.java:89)
at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:370)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:301)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:179)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:515)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:175)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:197)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:254)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:241)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:236)
at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:215)
at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:131)
at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:123)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:106)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:107)
at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitPrepareCommand(PessimisticLockingInterceptor.java:103)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:128)
at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:115)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126)
at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at com.suntecgroup.tbms.tpe.io.InputProcessor.publishEvent(InputProcessor.java:1848)
at com.suntecgroup.tbms.tpe.io.InputProcessor.updateRepositories(InputProcessor.java:1759)
at com.suntecgroup.tbms.tpe.io.InputProcessor.processServiceEvents(InputProcessor.java:773)
at com.suntecgroup.tbms.tpe.io.InputProcessor.processEvent(InputProcessor.java:222)
at com.suntecgroup.tbms.tpe.service.monitor.WebServiceEventGenerator.pushEvent(WebServiceEventGenerator.java:447)
at com.suntecgroup.tbms.tpe.service.monitor.WebServiceEventGenerator.executeCommand(WebServiceEventGenerator.java:261)
at com.suntecgroup.tbms.tpe.core.server.ServerManager.execute(ServerManager.java:2838)
at com.suntecgroup.tbms.container.core.ServiceContainerCommandDespatcher.run(ServiceContainerCommandDespatcher.java:41)