exceptions during parallel updates
yelin66 Mar 29, 2010 11:08 AMI was testing Infinispan with parallel updates from multiple cache instances, and getting couple exceptions intermittently.
1. The following replication timeout exception occurred in about 10 seconds after the update was kicked off while my replTimeout was set much longer than that. Also, when the update succeeded on average that update took around 60 milliseconds only. Could you please suggest what may potentically cause the timeout?
13:20:53,888 | INFO | Thread-14 | ReadUpdateTestDriver | t.drivers.ReadUpdateTestDriver$5 341 | updating 2500 entries
13:21:03,230 | ERROR | Thread-14 | InvocationContextInterceptor | ors.InvocationContextInterceptor 55 | Execution error:
org.infinispan.util.concurrent.TimeoutException: Replication timeout for SSELabCl07-53048
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:421)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:100)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:124)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:229)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:216)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:212)
at org.infinispan.interceptors.DistributionInterceptor.handleWriteCommand(DistributionInterceptor.java:304)
at org.infinispan.interceptors.DistributionInterceptor.visitPutMapCommand(DistributionInterceptor.java:168)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.LockingInterceptor.visitPutMapCommand(LockingInterceptor.java:210)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:183)
at org.infinispan.interceptors.TxInterceptor.visitPutMapCommand(TxInterceptor.java:152)
at org.infinispan.interceptors.DistTxInterceptor.visitPutMapCommand(DistTxInterceptor.java:104)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:48)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:34)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:269)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:447)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:216)
at ge.energy.smartgrid.applications.omtest.controllers.InfinispanCacheController.putAll(InfinispanCacheController.java:180)
at ge.energy.smartgrid.applications.omtest.drivers.ReadUpdateTestDriver$5.run(ReadUpdateTestDriver.java:370)
13:21:03,232 | ERROR | Thread-14 | jgroups | org.jgroups.util.Util$1 76 | uncaught exception in Thread[Thread-14,5,Thread Pools] (thread group=org.jgroups.util.Util$1[name=JGroups,maxpri=10] )
org.infinispan.util.concurrent.TimeoutException: Replication timeout for SSELabCl07-53048
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:421)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:100)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:124)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:229)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:216)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:212)
at org.infinispan.interceptors.DistributionInterceptor.handleWriteCommand(DistributionInterceptor.java:304)
at org.infinispan.interceptors.DistributionInterceptor.visitPutMapCommand(DistributionInterceptor.java:168)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.LockingInterceptor.visitPutMapCommand(LockingInterceptor.java:210)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:183)
at org.infinispan.interceptors.TxInterceptor.visitPutMapCommand(TxInterceptor.java:152)
at org.infinispan.interceptors.DistTxInterceptor.visitPutMapCommand(DistTxInterceptor.java:104)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:48)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:34)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:269)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:447)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:216)
at ge.energy.smartgrid.applications.omtest.controllers.InfinispanCacheController.putAll(InfinispanCacheController.java:180)
at ge.energy.smartgrid.applications.omtest.drivers.ReadUpdateTestDriver$5.run(ReadUpdateTestDriver.java:370)
13:21:03,492 | WARN | bCluster01-36455 | NAKACK | .jgroups.protocols.pbcast.NAKACK 965 | (requester=SSELabClusterMaster-58229, local_addr=LabCluster01-36455) message SSELabCl04-17481::7670 not found in retransmission table of SSELabCl04-17481:
[7654 : 7671 (7671)]
13:21:03,492 | WARN | bCluster01-36455 | NAKACK | .jgroups.protocols.pbcast.NAKACK 965 | (requester=SSELabCl07-53048, local_addr=LabCluster01-36455) message SSELabCl04-17481::7670 not found in retransmission table of SSELabCl04-17481:
[7654 : 7671 (7671)]
13:21:03,493 | WARN | bCluster01-36455 | NAKACK | .jgroups.protocols.pbcast.NAKACK 965 | (requester=SSELabCl05-11846, local_addr=LabCluster01-36455) message SSELabCl04-17481::7670 not found in retransmission table of SSELabCl04-17481:
[7654 : 7671 (7671)]
13:21:03,492 | WARN | bCluster01-36455 | NAKACK | .jgroups.protocols.pbcast.NAKACK 965 | (requester=SSELabCl08-8505, local_addr=LabCluster01-36455) message SSELabCl04-17481::7670 not found in retransmission table of SSELabCl04-17481:
2. Got the following SuspectException, but not sure what it means. Could you please suggest?
11:35:20,063 | WARN | SSELabCl02-54447 | GMS | org.jgroups.protocols.pbcast.GMS 562 | SSELabCl02-54447: not member of view [SSELabClusterMaster-6376|10] [SSELabClusterMaster-6376, LabCluster01-42248, SSELabCl03-25876, SSELabCl04-52466, SSELabCl05-1712, SSELabCl06-47092, SSELabCl08-59019, SSELabCl09-22842]; discarding it
11:35:22,181 | WARN | SSELabCl02-54447 | GMS | org.jgroups.protocols.pbcast.GMS 562 | SSELabCl02-54447: not member of view [SSELabClusterMaster-6376|11] [SSELabClusterMaster-6376]; discarding it
11:35:24,090 | ERROR | Thread-8 | InvocationContextInterceptor | ors.InvocationContextInterceptor 55 | Execution error:
org.infinispan.remoting.transport.jgroups.SuspectException: Suspected member: SSELabCl08-59019
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:418)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:100)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:124)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:229)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:216)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:212)
at org.infinispan.interceptors.DistributionInterceptor.handleWriteCommand(DistributionInterceptor.java:304)
at org.infinispan.interceptors.DistributionInterceptor.visitPutMapCommand(DistributionInterceptor.java:168)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.LockingInterceptor.visitPutMapCommand(LockingInterceptor.java:210)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:183)
at org.infinispan.interceptors.TxInterceptor.visitPutMapCommand(TxInterceptor.java:152)
at org.infinispan.interceptors.DistTxInterceptor.visitPutMapCommand(DistTxInterceptor.java:104)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:48)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:34)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:269)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:447)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:216)
at ge.energy.smartgrid.applications.omtest.controllers.InfinispanCacheController.putAll(InfinispanCacheController.java:180)
at ge.energy.smartgrid.applications.omtest.drivers.ReadUpdateTestDriver$2.run(ReadUpdateTestDriver.java:182)
11:35:24,092 | ERROR | Thread-8 | jgroups | org.jgroups.util.Util$1 76 | uncaught exception in Thread[Thread-8,5,Thread Pools] (thread group=org.jgroups.util.Util$1[name=JGroups,maxpri=10] )
org.infinispan.remoting.transport.jgroups.SuspectException: Suspected member: SSELabCl08-59019
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:418)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:100)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:124)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:229)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:216)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:212)
at org.infinispan.interceptors.DistributionInterceptor.handleWriteCommand(DistributionInterceptor.java:304)
at org.infinispan.interceptors.DistributionInterceptor.visitPutMapCommand(DistributionInterceptor.java:168)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.LockingInterceptor.visitPutMapCommand(LockingInterceptor.java:210)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:132)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:183)
at org.infinispan.interceptors.TxInterceptor.visitPutMapCommand(TxInterceptor.java:152)
at org.infinispan.interceptors.DistTxInterceptor.visitPutMapCommand(DistTxInterceptor.java:104)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:118)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:48)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:34)
at org.infinispan.commands.AbstractVisitor.visitPutMapCommand(AbstractVisitor.java:73)
at org.infinispan.commands.write.PutMapCommand.acceptVisitor(PutMapCommand.java:64)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:269)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:447)
at org.infinispan.CacheDelegate.putAll(CacheDelegate.java:216)
at ge.energy.smartgrid.applications.omtest.controllers.InfinispanCacheController.putAll(InfinispanCacheController.java:180)
at ge.energy.smartgrid.applications.omtest.drivers.ReadUpdateTestDriver$2.run(ReadUpdateTestDriver.java:182)