Urgent: Replication error with infinispan
khooweihiong Oct 9, 2013 6:16 AMHi all,
I have tested my application in master server in a master-slave server environment and sometime i get the error below. It seems like when I am testing in the master server, at the same it was sending message to the slave server. The error occurred at intervals. Is it something related to infinispan cache? Any advice or suggestion to solving the error below?
Thank You.
[Server:server-three-master] 10:45:08,505 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ajp--10.164.85.170-8259-3) ISPN000136: Execution error: org.infinispan.CacheException: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:184) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:136) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:128) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:80) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:120) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:105) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:207) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:84) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.jboss.as.clustering.infinispan.DefaultEmbeddedCacheManager$ClassLoaderAwareCommandInterceptor.handleDefault(DefaultEmbeddedCacheManager.java:410) [jboss-as-clustering-infinispan-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:174) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75) [jboss-as-clustering-web-spi-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865)
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47)
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133)
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91)
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88)
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56)
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153)
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155)
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368)
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505)
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445)
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930)
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:360) [jgroups-3.0.6.Final.jar:3.0.6.Final]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:263) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] ... 86 more
[Server:server-three-master]
[Server:server-three-master] 10:45:08,537 ERROR [org.infinispan.transaction.TransactionCoordinator] (ajp--10.164.85.170-8259-3) ISPN000097: Error while processing 1PC PrepareCommand: org.infinispan.CacheException: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:184)
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:136)
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:128)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:80)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:120)
[Server:server-three-master] at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:105)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:207)
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:84)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130)
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.jboss.as.clustering.infinispan.DefaultEmbeddedCacheManager$ClassLoaderAwareCommandInterceptor.handleDefault(DefaultEmbeddedCacheManager.java:410)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345)
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:174)
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100)
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50) [jboss-as-jpa-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:360)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:263)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)
[Server:server-three-master] ... 86 more
[Server:server-three-master]
[Server:server-three-master] 10:45:08,562 ERROR [org.infinispan.transaction.tm.DummyTransaction] (ajp--10.164.85.170-8259-3) ISPN000111: afterCompletion() failed for SynchronizationAdapter{localTransaction=LocalTransaction{remoteLockedNodes=null, isMarkedForRollback=false, transaction=DummyTransaction{xid=DummyXid{id=31}, status=3}, lockedKeys=null, backupKeyLocks=null, viewId=1} org.infinispan.transaction.synchronization.SyncLocalTransaction@1c} org.infinispan.transaction.synchronization.SynchronizationAdapter@3b: org.infinispan.CacheException: Could not commit.
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:83)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100)
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50) [jboss-as-jpa-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: javax.transaction.xa.XAException
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:219)
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:176)
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81)
[Server:server-three-master] ... 22 more
[Server:server-three-master]
[Server:server-three-master] 10:45:24,350 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ajp--10.164.85.170-8259-5) ISPN000136: Execution error: org.infinispan.CacheException: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:184) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:136) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:128) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:80) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:120) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:105) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:207) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:84) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.jboss.as.clustering.infinispan.DefaultEmbeddedCacheManager$ClassLoaderAwareCommandInterceptor.handleDefault(DefaultEmbeddedCacheManager.java:410) [jboss-as-clustering-infinispan-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:174) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75) [jboss-as-clustering-web-spi-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865)
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47)
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133)
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91)
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88)
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56)
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153)
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155)
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368)
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505)
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445)
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930)
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:360) [jgroups-3.0.6.Final.jar:3.0.6.Final]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:263) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163) [infinispan-core-5.1.2.FINAL.jar:5.1.2.FINAL]
[Server:server-three-master] ... 86 more
[Server:server-three-master]
[Server:server-three-master] 10:45:24,379 ERROR [org.infinispan.transaction.TransactionCoordinator] (ajp--10.164.85.170-8259-5) ISPN000097: Error while processing 1PC PrepareCommand: org.infinispan.CacheException: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)
[Server:server-three-master] at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:184)
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.broadcastPrepare(ReplicationInterceptor.java:136)
[Server:server-three-master] at org.infinispan.interceptors.ReplicationInterceptor.visitPrepareCommand(ReplicationInterceptor.java:128)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:80)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:120)
[Server:server-three-master] at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:105)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:207)
[Server:server-three-master] at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:84)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130)
[Server:server-three-master] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
[Server:server-three-master] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130)
[Server:server-three-master] at org.jboss.as.clustering.infinispan.DefaultEmbeddedCacheManager$ClassLoaderAwareCommandInterceptor.handleDefault(DefaultEmbeddedCacheManager.java:410)
[Server:server-three-master] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113)
[Server:server-three-master] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)
[Server:server-three-master] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345)
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:174)
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100)
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50) [jboss-as-jpa-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: org.jgroups.TimeoutException: timeout sending message to slave:server-three-slave/web
[Server:server-three-master] at org.jgroups.blocks.MessageDispatcher.sendMessage(MessageDispatcher.java:360)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:263)
[Server:server-three-master] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)
[Server:server-three-master] ... 86 more
[Server:server-three-master]
[Server:server-three-master] 10:45:24,404 ERROR [org.infinispan.transaction.tm.DummyTransaction] (ajp--10.164.85.170-8259-5) ISPN000111: afterCompletion() failed for SynchronizationAdapter{localTransaction=LocalTransaction{remoteLockedNodes=null, isMarkedForRollback=false, transaction=DummyTransaction{xid=DummyXid{id=32}, status=3}, lockedKeys=null, backupKeyLocks=null, viewId=1} org.infinispan.transaction.synchronization.SyncLocalTransaction@1d} org.infinispan.transaction.synchronization.SynchronizationAdapter@3c: org.infinispan.CacheException: Could not commit.
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:83)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90)
[Server:server-three-master] at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100)
[Server:server-three-master] at org.jboss.as.clustering.web.impl.TransactionBatchingManager.endBatch(TransactionBatchingManager.java:75)
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.processSessionRepl(DistributableSessionManager.java:1530) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.DistributableSessionManager.storeSession(DistributableSessionManager.java:865) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:47) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:133) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:91) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.JvmRouteValve.invoke(JvmRouteValve.java:88) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.session.LockingValve.invoke(LockingValve.java:56) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50) [jboss-as-jpa-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153) [jboss-as-web-7.1.1.Final.jar:7.1.1.Final]
[Server:server-three-master] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:505) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:445) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:930) [jbossweb-7.0.13.Final.jar:]
[Server:server-three-master] at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_21]
[Server:server-three-master] Caused by: javax.transaction.xa.XAException
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:219)
[Server:server-three-master] at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:176)
[Server:server-three-master] at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:81)
[Server:server-three-master] ... 22 more
[Server:server-three-master]