Getting ReplicationException randomly
shunmugaraj Jan 11, 2009 12:01 PMHi:
I am getting this error when we run our application in clustering mode. Initially after restart everything seems to be fine for 15 mins but after sometime we are seeing below exception and then consistently.
09:17:25,899 WARN [arjLoggerI18N] [com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator_2] TwoPhaseCoordinator.beforeCompletion - failed for com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple@1290b68 java.lang.RuntimeException: Unable to broadcast invalidation messages at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:118) at org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) at org.jboss.cache.interceptors.TxInterceptor.runPreparePhase(TxInterceptor.java:873) at org.jboss.cache.interceptors.TxInterceptor$LocalSynchronizationHandler.beforeCompletion(TxInterceptor.java:1138) at org.jboss.cache.interceptors.OrderedSynchronizationHandler.beforeCompletion(OrderedSynchronizationHandler.java:75) at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:114) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:247) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:86) at com.arjuna.ats.arjuna.AtomicAction.end(AtomicAction.java:216) at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commit(TransactionImple.java:238) at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501) at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361) at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181) at org.jboss.ejb.plugins.AbstractInterceptor.invoke(AbstractInterceptor.java:111) at com.appln.foundation.appserver.jboss4.IsolationBoundaryIntercepter.invoke(IsolationBoundaryIntercepter.java:57) at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168) at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205) at org.jboss.ejb.plugins.CleanShutdownInterceptor.invoke(CleanShutdownInterceptor.java:278) at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138) at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648) at org.jboss.ejb.Container.invoke(Container.java:960) at sun.reflect.GeneratedMethodAccessor468.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.invocation.local.LocalInvoker$MBeanServerAction.invoke(LocalInvoker.java:169) at org.jboss.invocation.local.LocalInvoker.invoke(LocalInvoker.java:118) at org.jboss.invocation.InvokerInterceptor.invokeLocal(InvokerInterceptor.java:209) at org.jboss.invocation.InvokerInterceptor.invoke(InvokerInterceptor.java:195) at org.jboss.proxy.ejb.RetryInterceptor.invoke(RetryInterceptor.java:176) at org.jboss.proxy.TransactionInterceptor.invoke(TransactionInterceptor.java:61) at org.jboss.proxy.SecurityInterceptor.invoke(SecurityInterceptor.java:70) at org.jboss.proxy.ejb.StatelessSessionInterceptor.invoke(StatelessSessionInterceptor.java:112) at org.jboss.proxy.ClientContainer.invoke(ClientContainer.java:100) at $Proxy262.getAvailableSearchKeys(Unknown Source) at com.appln.foundation.webapp.common.tags.QueryBuilderTag.doStartTag(QueryBuilderTag.java:891) at org.apache.jsp.jsp.hierarchymanagement.advancedSearch_jsp._jspService(advancedSearch_jsp.java:317) at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590) at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505) at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968) at org.apache.jsp.jsp.cm.hierarchymanagement.abstractUnionListOutput_jsp._jspService(abstractUnionListOutput_jsp.java:672) at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590) at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505) at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968) at org.apache.jasper.runtime.PageContextImpl.include(PageContextImpl.java:614) at org.apache.struts.tiles.TilesUtilImpl.doInclude(TilesUtilImpl.java:99) at org.apache.struts.tiles.TilesUtil.doInclude(TilesUtil.java:135) at org.apache.struts.taglib.tiles.InsertTag.doInclude(InsertTag.java:760) at org.apache.struts.taglib.tiles.InsertTag$InsertHandler.doEndTag(InsertTag.java:892) at org.apache.struts.taglib.tiles.InsertTag.doEndTag(InsertTag.java:462) at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspx_meth_tiles_005finsert_005f0(emptyAjaxLayout_jsp.java:382) at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspService(emptyAjaxLayout_jsp.java:167) at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387) at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320) at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469) at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403) at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301) at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063) at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263) at org.apache.struts.tiles.TilesRequestProcessor.processTilesDefinition(TilesRequestProcessor.java:239) at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:302) at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229) at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133) at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176) at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194) at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432) at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687) at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469) at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403) at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301) at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063) at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263) at org.apache.struts.action.RequestProcessor.processForwardConfig(RequestProcessor.java:386) at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:318) at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229) at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133) at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176) at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194) at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432) at javax.servlet.http.HttpServlet.service(HttpServlet.java:710) at javax.servlet.http.HttpServlet.service(HttpServlet.java:803) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.appln.cm.webapp.common.menu.CmPermissionsFilter.doFilter(CmPermissionsFilter.java:33) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.appln.foundation.webapp.authentication.AuthenticationFilterBase.doFilter(AuthenticationFilterBase.java:108) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at com.appln.foundation.webapp.common.compression.GZIPFilter.doFilter(GZIPFilter.java:149) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175) at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:179) at org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:87) at org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:84) at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104) at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:241) at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:437) at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:381) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) at java.lang.Thread.run(Thread.java:595) Caused by: org.jboss.cache.ReplicationException: rsp=sender=10.196.15.176:55801, retval=null, received=false, suspected=false at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4422) at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4344) at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4455) at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:110) at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:88) at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateAcrossCluster(InvalidationInterceptor.java:160) at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateModifications(InvalidationInterceptor.java:170) at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:102) ... 142 more Caused by: org.jboss.cache.lock.TimeoutException: Response timed out: sender=10.196.15.176:55801, retval=null, received=false, suspected=false at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4420) ... 149 more Any suggestions will be helpful.