I've managed to work out that the thrashing is happening as per this stack trace:
"Incoming-2865,JSLEE_CLUSTER-TZRE-OCS2-43868" #11872 prio=5 os_prio=0 tid=0x00007fcf000e4800 nid=0x7894 runnable [0x00007fce7e9fd000]
java.lang.Thread.State: RUNNABLE
at org.infinispan.commons.util.AbstractMap.hash(AbstractMap.java:32)
at org.infinispan.commons.util.FastCopyHashMap.put(FastCopyHashMap.java:224)
at org.infinispan.atomic.impl.PutOperation.replay(PutOperation.java:46)
at org.infinispan.atomic.impl.AtomicHashMapDelta.merge(AtomicHashMapDelta.java:45)
at org.infinispan.commands.write.PutKeyValueCommand.performPut(PutKeyValueCommand.java:253)
at org.infinispan.commands.write.PutKeyValueCommand.perform(PutKeyValueCommand.java:109)
at org.infinispan.interceptors.CallInterceptor.handleDefault(CallInterceptor.java:125)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:43)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:78)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.handleTxWriteCommand(TxDistributionInterceptor.java:284)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitPutKeyValueCommand(TxDistributionInterceptor.java:112)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:78)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.EntryWrappingInterceptor$EntryWrappingVisitor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:619)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:78)
at org.infinispan.interceptors.EntryWrappingInterceptor.wrapEntriesForPrepare(EntryWrappingInterceptor.java:678)
at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:105)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:37)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:93)
at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:83)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:158)
at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:145)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112)
at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:39)
- locked <0x00000000fc97d320> (a org.infinispan.transaction.impl.RemoteTransaction)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:114)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:83)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:59)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112)
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:176)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:335)
at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:100)
at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92)
at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34)
at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.handleRunnable(BasePerCacheInboundInvocationHandler.java:132)
at org.infinispan.remoting.inboundhandler.NonTotalOrderTxPerCacheInboundInvocationHandler.handle(NonTotalOrderTxPerCacheInboundInvocationHandler.java:99)
at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleCacheRpcCommand(GlobalInboundInvocationHandler.java:126)
at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:75)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommandFromLocalCluster(CommandAwareRpcDispatcher.java:205)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:175)
at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:455)
at org.jgroups.blocks.RequestCorrelator.dispatch(RequestCorrelator.java:406)
at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:357)
at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:245)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:664)
at org.jgroups.JChannel.up(JChannel.java:760)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1034)
at org.jgroups.protocols.FRAG2.up(FRAG2.java:182)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:438)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:438)
at org.jgroups.stack.Protocol.up(Protocol.java:426)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:294)
at org.jgroups.protocols.UNICAST3.deliverBatch(UNICAST3.java:1100)
at org.jgroups.protocols.UNICAST3.removeAndDeliver(UNICAST3.java:899)
at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:803)
at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:433)
at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:649)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155)
at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:325)
at org.jgroups.protocols.MERGE3.up(MERGE3.java:292)
at org.jgroups.protocols.Discovery.up(Discovery.java:296)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1657)
at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1872)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
My hunch is that there are lots of entries being looped through in the changelog and hence line 60 is a large loop.
@Override
53 public DeltaAware More ...merge(DeltaAware d) {
54 AtomicHashMap<Object, Object> other;
55 if (d != null && (d instanceof AtomicHashMap))
56 other = (AtomicHashMap<Object, Object>) d;
57 else
58 other = new AtomicHashMap();
59 if (changeLog != null) {
60 for (Operation<Object, Object> o : changeLog) o.replay(other.delegate);
61 }
62 return other;
63 }
What would be the reason for this?
234,4-11 3%