0 Replies Latest reply on Jun 11, 2017 4:31 PM by bseam

    Timed out waiting for topology

    bseam

      There is also Replication timeout. Both these exceptions for synchronous PUT for a cluster of just 6 nodes with 40 seconds replication delay configured. Does the Topology timeout also use this same config param? There is also flurry of TopologyChanged events seen in the logs if that indicates anything.

       

      Of course, reason can be network latency, locks, etc,. All we have is just logs to analyze.

       

      Any possible solutions other than upgrading Infinispan version not that it will solve this issue? We could increase replication delay.

       

      Exception traces below:

       

      Caused by: org.infinispan.util.concurrent.TimeoutException: Timed out waiting for topology 111

        at org.infinispan.statetransfer.StateTransferLockImpl.waitForTransactionData(StateTransferLockImpl.java:92) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.BaseStateTransferInterceptor.waitForTransactionData(BaseStateTransferInterceptor.java:96) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:334) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:249) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:108) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:191) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:177) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1619) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.cache.impl.CacheImpl.putInternal(CacheImpl.java:1099) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.cache.impl.CacheImpl.put(CacheImpl.java:1090) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.cache.impl.CacheImpl.put(CacheImpl.java:1689) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.cache.impl.CacheImpl.put(CacheImpl.java:256) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

       

       

       

      Caused by: org.infinispan.util.concurrent.TimeoutException: Replication timeout for node23

        at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:87) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:586) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:287) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.handleNonTxWriteCommand(BaseDistributionInterceptor.java:339) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.distribution.NonTxDistributionInterceptor.visitPutKeyValueCommand(NonTxDistributionInterceptor.java:105) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.DistCacheWriterInterceptor.visitPutKeyValueCommand(DistCacheWriterInterceptor.java:72) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.CacheLoaderInterceptor.visitDataCommand(CacheLoaderInterceptor.java:150) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.CacheLoaderInterceptor.visitPutKeyValueCommand(CacheLoaderInterceptor.java:96) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.EntryWrappingInterceptor.invokeNextAndApplyChanges(EntryWrappingInterceptor.java:388) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.interceptors.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:453) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:195) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:88) ~[infinispan-core-7.2.5.Final.jar:7.2.5.     Final]  at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40) ~[infinispan-core-7.2.5.Final.jar:

      7.2.5.Final]  at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:55) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

        at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]  at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) ~[infinispan-core-7.2.5.Final.jar:7.2.5.Final]

      ....