HeuristicMixed Exception (javax.jms.IllegalStateException: T
Hello Everyone,
This issue is haunting us. We have recently moved to JBoss 4.2.2 GA from Orion application server.
We have four instances of JBoss Servers on one Linux machine. We are consistently getting the HeuristicMixed Exception. We switched the JBoss JTA to the deprecated in-memory transaction manager to get a more accurate Exception stack trace. In the architecture of our application, the client is a Webstart based Java application. It invokes a Stateless Session Bean which executes a long running transaction. In between, we also send and receive messages to/from ActiveMQ.
During the long running transaction (more than 2 hours), it also may send a JMS message to JBOSS MQ server. "java:/JmsXA" is used to look up the JMS connection. After the message is sent, the jms connection is closed by the java code.
Jboss jta configuration has been modified to allow sub-transactions. And, the datasource used is local-tx-datasource.
We are having this issue in Production for the last few weeks. Any help is greatly appreciated.
Here is the exception:
2008-10-02 06:07:00,215 DEBUG [org.jboss.mq.referenceable.SpyDestinationObjectFactory] SpyDestinationObjectFactory->getObjectInstance()
2008-10-02 06:07:00,216 INFO [com.coslp.riskcenter.server.m2m.M2MDataBO] CompleteM2M for cmdTypeId: 3, systemCode: RC
2008-10-02 06:07:00,220 INFO [com.coslp.riskcenter.server.server.ServerDataBO] RC M2M completed for Power
2008-10-02 06:07:00,220 INFO [com.coslp.riskcenter.server.server.ServerDataBO] M2M completed for Power
2008-10-02 06:07:00,274 WARN [org.jboss.tm.TransactionImpl] XAException: tx=TransactionImpl:XidImpl[FormatId=257, GlobalId=sercprodapp3/166642, BranchQual=, localId=166642] errorCode=XAER_RMFAIL
org.jboss.mq.SpyXAException: Resource manager error during prepare; - nested throwable: (javax.jms.IllegalStateException: The connection is closed)
at org.jboss.mq.SpyXAException.getAsXAException(SpyXAException.java:72)
at org.jboss.mq.SpyXAResource.prepare(SpyXAResource.java:151)
at org.jboss.resource.connectionmanager.xa.JcaXAResourceWrapper.prepare(JcaXAResourceWrapper.java:93)
at org.jboss.tm.TransactionImpl$Resource.prepare(TransactionImpl.java:2212)
at org.jboss.tm.TransactionImpl.prepareResources(TransactionImpl.java:1660)
at org.jboss.tm.TransactionImpl.commit(TransactionImpl.java:347)
at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501)
at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361)
at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181)
at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168)
at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)
at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)
at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648)
at org.jboss.ejb.Container.invoke(Container.java:960)
at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at org.jboss.invocation.unified.server.UnifiedInvoker.invoke(UnifiedInvoker.java:231)
at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:288)
at $Proxy15.invoke(Unknown Source)
at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:769)
at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:573)
at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
Caused by: javax.jms.IllegalStateException: The connection is closed
at org.jboss.mq.Connection.checkClosed(Connection.java:1246)
at org.jboss.mq.Connection.send(Connection.java:1106)
at org.jboss.mq.SpyXAResourceManager.prepare(SpyXAResourceManager.java:255)
at org.jboss.mq.SpyXAResource.prepare(SpyXAResource.java:147)
... 35 more
2008-10-02 06:07:11,156 DEBUG [org.jboss.remoting.transport.socket.ServerThread] WorkerThread#1[10.10.3.51:52753] closing socketWrapper: ServerSocketWrapper[Socket[addr=/10.10.3.51,port=52753,localport=1446].47a4a804]
2008-10-02 06:07:11,156 DEBUG [org.jboss.remoting.transport.socket.ServerSocketWrapper] wrote CLOSING
2008-10-02 06:07:11,156 DEBUG [org.jboss.remoting.transport.socket.SocketWrapper] ServerSocketWrapper[Socket[addr=/10.10.3.51,port=52753,localport=1446].47a4a804] closing
2008-10-02 06:09:03,362 WARN [org.jboss.tm.TransactionImpl] XAException: tx=TransactionImpl:XidImpl[FormatId=257, GlobalId=sercprodapp3/166642, BranchQual=, localId=166642] errorCode=XAER_RMFAIL
org.jboss.mq.SpyXAException: Resource manager error during rollback; - nested throwable: (javax.jms.IllegalStateException: The connection is closed)
at org.jboss.mq.SpyXAException.getAsXAException(SpyXAException.java:72)
at org.jboss.mq.SpyXAResource.rollback(SpyXAResource.java:182)
at org.jboss.resource.connectionmanager.xa.JcaXAResourceWrapper.rollback(JcaXAResourceWrapper.java:103)
at org.jboss.tm.TransactionImpl$Resource.rollback(TransactionImpl.java:2277)
at org.jboss.tm.TransactionImpl.rollbackResources(TransactionImpl.java:1837)
at org.jboss.tm.TransactionImpl.commit(TransactionImpl.java:368)
at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501)
at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361)
at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181)
at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168)
at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)
at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)
at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648)
at org.jboss.ejb.Container.invoke(Container.java:960)
at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at org.jboss.invocation.unified.server.UnifiedInvoker.invoke(UnifiedInvoker.java:231)
at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:288)
at $Proxy15.invoke(Unknown Source)
at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:769)
at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:573)
at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
Caused by: javax.jms.IllegalStateException: The connection is closed
at org.jboss.mq.Connection.checkClosed(Connection.java:1246)
at org.jboss.mq.Connection.send(Connection.java:1106)
at org.jboss.mq.SpyXAResourceManager.rollback(SpyXAResourceManager.java:308)
at org.jboss.mq.SpyXAResource.rollback(SpyXAResource.java:178)
... 35 more
2008-10-02 06:09:03,368 ERROR [org.jboss.ejb.plugins.LogInterceptor] TransactionRolledbackException in method: public abstract java.lang.Object com.coslp.riskcenter.framework.PCIAutomaticEJBObject.executeTransactional(com.coslp.riskcenter.framework.PCExecutionMetadata,java.lang.String,java.lang.String,java.lang.Object[]) throws java.rmi.RemoteException,com.coslp.riskcenter.framework.PCAppException, causedBy:
org.jboss.tm.JBossRollbackException: Unable to commit, tx=TransactionImpl:XidImpl[FormatId=257, GlobalId=sercprodapp3/166642, BranchQual=, localId=166642] status=STATUS_NO_TRANSACTION; - nested throwable: (org.jboss.mq.SpyXAException: Resource manager error during prepare; - nested throwable: (javax.jms.IllegalStateException: The connection is closed))
at org.jboss.tm.TransactionImpl.commit(TransactionImpl.java:372)
at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501)
at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361)
at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181)
at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168)
at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)
at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)
at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648)
at org.jboss.ejb.Container.invoke(Container.java:960)
at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at org.jboss.invocation.unified.server.UnifiedInvoker.invoke(UnifiedInvoker.java:231)
at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
at javax.management.MBeanServerInvocationHandler.invoke(MBeanServerInvocationHandler.java:288)
at $Proxy15.invoke(Unknown Source)
at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:769)
at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:573)
at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:387)
at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:166)
Caused by: org.jboss.mq.SpyXAException: Resource manager error during prepare; - nested throwable: (javax.jms.IllegalStateException: The connection is closed)
at org.jboss.mq.SpyXAException.getAsXAException(SpyXAException.java:72)
at org.jboss.mq.SpyXAResource.prepare(SpyXAResource.java:151)
at org.jboss.resource.connectionmanager.xa.JcaXAResourceWrapper.prepare(JcaXAResourceWrapper.java:93)
at org.jboss.tm.TransactionImpl$Resource.prepare(TransactionImpl.java:2212)
at org.jboss.tm.TransactionImpl.prepareResources(TransactionImpl.java:1660)
at org.jboss.tm.TransactionImpl.commit(TransactionImpl.java:347)
... 31 more
Caused by: javax.jms.IllegalStateException: The connection is closed
at org.jboss.mq.Connection.checkClosed(Connection.java:1246)
at org.jboss.mq.Connection.send(Connection.java:1106)
at org.jboss.mq.SpyXAResourceManager.prepare(SpyXAResourceManager.java:255)
at org.jboss.mq.SpyXAResource.prepare(SpyXAResource.java:147)
... 35 more
