2 Replies Latest reply on Jan 12, 2009 9:19 AM by manik

    Getting ReplicationException randomly

    shunmugaraj

      Hi:

      I am getting this error when we run our application in clustering mode. Initially after restart everything seems to be fine for 15 mins but after sometime we are seeing below exception and then consistently.


      09:17:25,899 WARN [arjLoggerI18N] [com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator_2] TwoPhaseCoordinator.beforeCompletion - failed for com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple@1290b68
      java.lang.RuntimeException: Unable to broadcast invalidation messages
       at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:118)
       at org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68)
       at org.jboss.cache.interceptors.TxInterceptor.runPreparePhase(TxInterceptor.java:873)
       at org.jboss.cache.interceptors.TxInterceptor$LocalSynchronizationHandler.beforeCompletion(TxInterceptor.java:1138)
       at org.jboss.cache.interceptors.OrderedSynchronizationHandler.beforeCompletion(OrderedSynchronizationHandler.java:75)
       at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:114)
       at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:247)
       at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:86)
       at com.arjuna.ats.arjuna.AtomicAction.end(AtomicAction.java:216)
       at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commit(TransactionImple.java:238)
       at org.jboss.ejb.plugins.TxInterceptorCMT.endTransaction(TxInterceptorCMT.java:501)
       at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:361)
       at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181)
       at org.jboss.ejb.plugins.AbstractInterceptor.invoke(AbstractInterceptor.java:111)
       at com.appln.foundation.appserver.jboss4.IsolationBoundaryIntercepter.invoke(IsolationBoundaryIntercepter.java:57)
       at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:168)
       at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)
       at org.jboss.ejb.plugins.CleanShutdownInterceptor.invoke(CleanShutdownInterceptor.java:278)
       at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)
       at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:648)
       at org.jboss.ejb.Container.invoke(Container.java:960)
       at sun.reflect.GeneratedMethodAccessor468.invoke(Unknown Source)
       at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
       at java.lang.reflect.Method.invoke(Method.java:585)
       at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
       at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
       at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
       at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
       at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
       at org.jboss.invocation.local.LocalInvoker$MBeanServerAction.invoke(LocalInvoker.java:169)
       at org.jboss.invocation.local.LocalInvoker.invoke(LocalInvoker.java:118)
       at org.jboss.invocation.InvokerInterceptor.invokeLocal(InvokerInterceptor.java:209)
       at org.jboss.invocation.InvokerInterceptor.invoke(InvokerInterceptor.java:195)
       at org.jboss.proxy.ejb.RetryInterceptor.invoke(RetryInterceptor.java:176)
       at org.jboss.proxy.TransactionInterceptor.invoke(TransactionInterceptor.java:61)
       at org.jboss.proxy.SecurityInterceptor.invoke(SecurityInterceptor.java:70)
       at org.jboss.proxy.ejb.StatelessSessionInterceptor.invoke(StatelessSessionInterceptor.java:112)
       at org.jboss.proxy.ClientContainer.invoke(ClientContainer.java:100)
       at $Proxy262.getAvailableSearchKeys(Unknown Source)
       at com.appln.foundation.webapp.common.tags.QueryBuilderTag.doStartTag(QueryBuilderTag.java:891)
       at org.apache.jsp.jsp.hierarchymanagement.advancedSearch_jsp._jspService(advancedSearch_jsp.java:317)
       at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387)
       at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
       at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687)
       at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590)
       at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505)
       at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968)
       at org.apache.jsp.jsp.cm.hierarchymanagement.abstractUnionListOutput_jsp._jspService(abstractUnionListOutput_jsp.java:672)
       at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387)
       at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
       at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687)
       at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:590)
       at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:505)
       at org.apache.jasper.runtime.JspRuntimeLibrary.include(JspRuntimeLibrary.java:968)
       at org.apache.jasper.runtime.PageContextImpl.include(PageContextImpl.java:614)
       at org.apache.struts.tiles.TilesUtilImpl.doInclude(TilesUtilImpl.java:99)
       at org.apache.struts.tiles.TilesUtil.doInclude(TilesUtil.java:135)
       at org.apache.struts.taglib.tiles.InsertTag.doInclude(InsertTag.java:760)
       at org.apache.struts.taglib.tiles.InsertTag$InsertHandler.doEndTag(InsertTag.java:892)
       at org.apache.struts.taglib.tiles.InsertTag.doEndTag(InsertTag.java:462)
       at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspx_meth_tiles_005finsert_005f0(emptyAjaxLayout_jsp.java:382)
       at org.apache.jsp.jsp.hierarchymanagement.emptyAjaxLayout_jsp._jspService(emptyAjaxLayout_jsp.java:167)
       at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:387)
       at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:320)
       at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:266)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687)
       at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469)
       at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403)
       at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301)
       at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063)
       at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263)
       at org.apache.struts.tiles.TilesRequestProcessor.processTilesDefinition(TilesRequestProcessor.java:239)
       at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:302)
       at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229)
       at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133)
       at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176)
       at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194)
       at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:710)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:687)
       at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:469)
       at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:403)
       at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:301)
       at org.apache.struts.action.RequestProcessor.doForward(RequestProcessor.java:1063)
       at org.apache.struts.tiles.TilesRequestProcessor.doForward(TilesRequestProcessor.java:263)
       at org.apache.struts.action.RequestProcessor.processForwardConfig(RequestProcessor.java:386)
       at org.apache.struts.tiles.TilesRequestProcessor.processForwardConfig(TilesRequestProcessor.java:318)
       at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:229)
       at com.appln.foundation.webapp.common.EcmTilesRequestProcessor.process(EcmTilesRequestProcessor.java:133)
       at com.appln.cm.webapp.util.CmRequestProcessor.process(CmRequestProcessor.java:176)
       at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1194)
       at org.apache.struts.action.ActionServlet.doPost(ActionServlet.java:432)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:710)
       at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at com.appln.cm.webapp.common.menu.CmPermissionsFilter.doFilter(CmPermissionsFilter.java:33)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at com.appln.foundation.webapp.authentication.AuthenticationFilterBase.doFilter(AuthenticationFilterBase.java:108)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at com.appln.foundation.webapp.common.compression.GZIPFilter.doFilter(GZIPFilter.java:149)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)
       at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
       at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
       at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230)
       at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
       at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:179)
       at org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:87)
       at org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:84)
       at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84)
       at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
       at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
       at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)
       at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
       at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:241)
       at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:437)
       at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:381)
       at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)
       at java.lang.Thread.run(Thread.java:595)
      Caused by: org.jboss.cache.ReplicationException: rsp=sender=10.196.15.176:55801, retval=null, received=false, suspected=false
       at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4422)
       at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4344)
       at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4455)
       at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:110)
       at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:88)
       at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateAcrossCluster(InvalidationInterceptor.java:160)
       at org.jboss.cache.interceptors.InvalidationInterceptor.invalidateModifications(InvalidationInterceptor.java:170)
       at org.jboss.cache.interceptors.InvalidationInterceptor.invoke(InvalidationInterceptor.java:102)
       ... 142 more
      Caused by: org.jboss.cache.lock.TimeoutException: Response timed out: sender=10.196.15.176:55801, retval=null, received=false, suspected=false
       at org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4420)
       ... 149 more
      
      
      Any suggestions will be helpful.