Infinispan hangs abnormally
We are using ISPN 5.3.0-Final and jgroups-3.3.1 on a system having 3 servers. The first two forms a cluster as primary site, the remaining one located at the remote site as the backup site. We do an async backup from primary site to backup site as backup policy.
It has been running without problems for a year but recently we met an incident that a message with empty header content made the gird hanged. Stack trace of such incident was attached below. It seems like the packet transfer within the cluster is corrupted. We can simulate the incident outputting the same stack trace by setting the header content (with header ID=60) of the message to null in the below method.
protected void send(Message msg, Address dest, boolean multicast) [TP.java]
i.e., by calling msg.putHeader((short)60, null); inside send().
I would like to know if anyone had encountered this problem before and can someone tell me in what occasion would this header (with id 60) become null. Any information is appreciated.
10:54:35.408 ERROR [pool-84-thread-1] o.i.i.InvocationContextInterceptor - ISPN000136: Execution error
java.lang.NullPointerException: null
at org.jgroups.util.Headers.marshalledSize(Headers.java:133) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.Message.size(Message.java:831) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.TP.down(TP.java:1294) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.Discovery.down(Discovery.java:593) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.MERGE2.down(MERGE2.java:182) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FD_SOCK.down(FD_SOCK.java:348) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FD.down(FD.java:307) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.VERIFY_SUSPECT.down(VERIFY_SUSPECT.java:84) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.NAKACK2.down(NAKACK2.java:538) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.UNICAST2.down(UNICAST2.java:547) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.STABLE.down(STABLE.java:361) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.GMS.down(GMS.java:958) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.STATE_TRANSFER.down(STATE_TRANSFER.java:237) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.FLUSH.down(FLUSH.java:277) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.tom.TOA.handleDownMessage(TOA.java:190) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.tom.TOA.down(TOA.java:65) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.UFC.handleDownMessage(UFC.java:130) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FlowControl.down(FlowControl.java:342) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FlowControl.down(FlowControl.java:352) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FRAG2.down(FRAG2.java:143) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.RSVP.down(RSVP.java:142) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.relay.RELAY2.forwardTo(RELAY2.java:574) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.relay.RELAY2.down(RELAY2.java:388) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:1022) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.JChannel.down(JChannel.java:767) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.down(MessageDispatcher.java:683) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.RequestCorrelator.sendUnicastRequest(RequestCorrelator.java:202) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.UnicastRequest.sendRequest(UnicastRequest.java:43) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.Request.execute(Request.java:83) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:399) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.backupRemotely(JGroupsTransport.java:580) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.xsite.BackupSenderImpl.backupCommand(BackupSenderImpl.java:202) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.xsite.BackupSenderImpl.backupPrepare(BackupSenderImpl.java:123) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.xsite.PessimisticBackupInterceptor.visitPrepareCommand(PessimisticBackupInterceptor.java:46) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:116) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitPrepareCommand(PessimisticLockingInterceptor.java:103) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:130) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:117) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:216) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:189) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:93) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:128) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:92) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:343) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:177) [infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81) [infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.afterCompletion(SynchronizationImple.java:96) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.afterCompletion(TwoPhaseCoordinator.java:402) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:103) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:164) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1165) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:117) [jbossjta-4.16.6.Final.jar:na]
...
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) [na:1.7.0_51]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_51]
at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51]
10:54:35.410 ERROR [pool-84-thread-1] o.i.t.TransactionCoordinator - ISPN000097: Error while processing a prepare in a single-phase transaction
java.lang.NullPointerException: null
at org.jgroups.util.Headers.marshalledSize(Headers.java:133) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.Message.size(Message.java:831) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.TP.down(TP.java:1294) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.Discovery.down(Discovery.java:593) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.MERGE2.down(MERGE2.java:182) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FD_SOCK.down(FD_SOCK.java:348) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FD.down(FD.java:307) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.VERIFY_SUSPECT.down(VERIFY_SUSPECT.java:84) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.NAKACK2.down(NAKACK2.java:538) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.UNICAST2.down(UNICAST2.java:547) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.STABLE.down(STABLE.java:361) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.GMS.down(GMS.java:958) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.STATE_TRANSFER.down(STATE_TRANSFER.java:237) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.pbcast.FLUSH.down(FLUSH.java:277) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.tom.TOA.handleDownMessage(TOA.java:190) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.tom.TOA.down(TOA.java:65) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.UFC.handleDownMessage(UFC.java:130) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FlowControl.down(FlowControl.java:342) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FlowControl.down(FlowControl.java:352) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.FRAG2.down(FRAG2.java:143) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.RSVP.down(RSVP.java:142) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.relay.RELAY2.forwardTo(RELAY2.java:574) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.protocols.relay.RELAY2.down(RELAY2.java:388) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:1022) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.JChannel.down(JChannel.java:767) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.down(MessageDispatcher.java:683) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.RequestCorrelator.sendUnicastRequest(RequestCorrelator.java:202) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.UnicastRequest.sendRequest(UnicastRequest.java:43) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.Request.execute(Request.java:83) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:399) ~[jgroups-3.3.1.Final.jar:3.3.1.Final]
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.backupRemotely(JGroupsTransport.java:580) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.xsite.BackupSenderImpl.backupCommand(BackupSenderImpl.java:202) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.xsite.BackupSenderImpl.backupPrepare(BackupSenderImpl.java:123) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.xsite.PessimisticBackupInterceptor.visitPrepareCommand(PessimisticBackupInterceptor.java:46) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:116) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitPrepareCommand(PessimisticLockingInterceptor.java:103) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:130) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:117) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:61) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:216) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:189) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:93) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:134) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:128) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:92) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:118) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:147) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:343) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:177) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.afterCompletion(SynchronizationImple.java:96) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.afterCompletion(TwoPhaseCoordinator.java:402) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:103) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:164) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1165) [jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:117) [jbossjta-4.16.6.Final.jar:na]
...
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) [na:1.7.0_51]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_51]
at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51]
10:54:35.412 WARN [pool-84-thread-1] com.arjuna.ats.jta - ARJUNA016029: SynchronizationImple.afterCompletion - failed for SynchronizationAdapter{localTransaction=LocalTransaction{remoteLockedNodes=[XXX00-16889, XXX01-52844], isMarkedForRollback=false, lockedKeys=null, backupKeyLocks=[Key@b6b7bd7[sequence=20443]], topologyId=2, isFromStateTransfer=false} org.infinispan.transaction.synchronization.SyncLocalTransaction@4de1} org.infinispan.transaction.synchronization.SynchronizationAdapter@4e00 with exception
org.infinispan.CacheException: Could not commit.
at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:83) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.afterCompletion(SynchronizationImple.java:96) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.afterCompletion(TwoPhaseCoordinator.java:402) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:103) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:164) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1165) ~[jbossjta-4.16.6.Final.jar:na]
at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:117) [jbossjta-4.16.6.Final.jar:na]
...
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) [na:1.7.0_51]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) [na:1.7.0_51]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_51]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_51]
at java.lang.Thread.run(Thread.java:744) [na:1.7.0_51]
Caused by: javax.transaction.xa.XAException: null
at org.infinispan.transaction.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:231) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:179) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81) ~[infinispan-core-5.3.0.Final.jar:5.3.0.Final]
... 18 common frames omitted
