0 Replies Latest reply on Jan 29, 2014 11:21 AM by kawashingtonjr

    Issues with JBoss Cache/Hibernate, getting waiting for unblock error

    kawashingtonjr

      We are seeing an issue in JBoss 5.1 where the web pages stop responding but jboss is still logging.  On the SQL server side, the user connects keep climbing til they reach around 500 and just stay wating.  Looks like from the cause by with org.jboss.cache it could be related to JGroups.  We do have a setup with two JBoss instance communicating in a cluster.

       

      JGroups - 2.6.10.GA

      JBoss Cache - 3.1.0.GA

       

      Error stack:

       

      2014-01-28 15:29:35,296 ERROR [org.hibernate.event.def.AbstractFlushingEventListener] Could not synchronize database state with session

      org.hibernate.cache.CacheException: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException: State retrieval timed out waiting for flush to unblock. (timeout = 60 seconds)

      at org.hibernate.cache.jbc2.timestamp.TimestampsRegionImpl.put(TimestampsRegionImpl.java:130)

      at org.hibernate.cache.UpdateTimestampsCache.preinvalidate(UpdateTimestampsCache.java:70)

      at org.hibernate.engine.ActionQueue.execute(ActionQueue.java:275)

      at org.hibernate.engine.ActionQueue.executeActions(ActionQueue.java:263)

      at org.hibernate.engine.ActionQueue.executeActions(ActionQueue.java:168)

      at org.hibernate.event.def.AbstractFlushingEventListener.performExecutions(AbstractFlushingEventListener.java:321)

      at org.hibernate.event.def.DefaultFlushEventListener.onFlush(DefaultFlushEventListener.java:50)

      at org.hibernate.impl.SessionImpl.flush(SessionImpl.java:1027)

      at org.hibernate.ejb.AbstractEntityManagerImpl.flush(AbstractEntityManagerImpl.java:304)

      at org.jboss.jpa.tx.TransactionScopedEntityManager.flush(TransactionScopedEntityManager.java:216)

      at com.agilquest.onboard.business.control.ouid.OuidControlBean.getNextBlockAndUpdate(OuidControlBean.java:194)

      at com.agilquest.onboard.business.control.ouid.OuidControlBean.getNextBlock(OuidControlBean.java:150)

      at com.agilquest.onboard.business.control.ouid.OuidControlBean.lookupEntityNextKey(OuidControlBean.java:125)

      at com.agilquest.onboard.business.control.ouid.OuidControlBean.getNextOUID(OuidControlBean.java:69)

      at sun.reflect.GeneratedMethodAccessor608.invoke(Unknown Source)

      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

      at java.lang.reflect.Method.invoke(Method.java:597)

      at org.jboss.invocation.Invocation.performCall(Invocation.java:386)

      at org.jboss.ejb.StatelessSessionContainer$ContainerInterceptor.invoke(StatelessSessionContainer.java:228)

      at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:156)

      at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63)

      at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121)

      at org.jboss.ejb.plugins.AbstractTxInterceptorBMT.invokeNext(AbstractTxInterceptorBMT.java:173)

      at org.jboss.ejb.plugins.TxInterceptorBMT.invoke(TxInterceptorBMT.java:77)

      at org.jboss.ejb.plugins.StatelessSessionInstanceInterceptor.invoke(StatelessSessionInstanceInterceptor.java:173)

      at org.jboss.ejb.plugins.SecurityInterceptor.process(SecurityInterceptor.java:228)

      at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:211)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.process(PreSecurityInterceptor.java:97)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.invoke(PreSecurityInterceptor.java:81)

      at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)

      at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)

      at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:650)

      at org.jboss.ejb.Container.invoke(Container.java:1029)

      at org.jboss.ejb.plugins.local.BaseLocalProxyFactory.invoke(BaseLocalProxyFactory.java:436)

      at org.jboss.ejb.plugins.local.StatelessSessionProxy.invoke(StatelessSessionProxy.java:103)

      at $Proxy412.getNextOUID(Unknown Source)

      at com.agilquest.onboard.business.control.transaction.TransactionManagementControlBean.getNextOuid(TransactionManagementControlBean.java:13510)

      at com.agilquest.onboard.business.control.transaction.TransactionManagementControlBean.doCreateTransaction(TransactionManagementControlBean.java:4967)

      at com.agilquest.onboard.business.control.transaction.TransactionManagementControlBean.createTransaction(TransactionManagementControlBean.java:372)

      at sun.reflect.GeneratedMethodAccessor817.invoke(Unknown Source)

      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

      at java.lang.reflect.Method.invoke(Method.java:597)

      at org.jboss.invocation.Invocation.performCall(Invocation.java:386)

      at org.jboss.ejb.StatelessSessionContainer$ContainerInterceptor.invoke(StatelessSessionContainer.java:228)

      at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:156)

      at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63)

      at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121)

      at org.jboss.ejb.plugins.AbstractTxInterceptorBMT.invokeNext(AbstractTxInterceptorBMT.java:173)

      at org.jboss.ejb.plugins.TxInterceptorBMT.invoke(TxInterceptorBMT.java:77)

      at org.jboss.ejb.plugins.StatelessSessionInstanceInterceptor.invoke(StatelessSessionInstanceInterceptor.java:173)

      at org.jboss.ejb.plugins.SecurityInterceptor.process(SecurityInterceptor.java:228)

      at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:211)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.process(PreSecurityInterceptor.java:97)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.invoke(PreSecurityInterceptor.java:81)

      at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)

      at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)

      at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:650)

      at org.jboss.ejb.Container.invoke(Container.java:1029)

      at org.jboss.ejb.plugins.local.BaseLocalProxyFactory.invoke(BaseLocalProxyFactory.java:436)

      at org.jboss.ejb.plugins.local.StatelessSessionProxy.invoke(StatelessSessionProxy.java:103)

      at $Proxy510.createTransaction(Unknown Source)

      at com.agilquest.onboard.business.control.xpress.XPressControlBean.justInTimeReservation(XPressControlBean.java:319)

      at sun.reflect.GeneratedMethodAccessor847.invoke(Unknown Source)

      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

      at java.lang.reflect.Method.invoke(Method.java:597)

      at org.jboss.invocation.Invocation.performCall(Invocation.java:386)

      at org.jboss.ejb.StatelessSessionContainer$ContainerInterceptor.invoke(StatelessSessionContainer.java:228)

      at org.jboss.resource.connectionmanager.CachedConnectionInterceptor.invoke(CachedConnectionInterceptor.java:156)

      at org.jboss.ejb.plugins.StatelessSessionInstanceInterceptor.invoke(StatelessSessionInstanceInterceptor.java:173)

      at org.jboss.ejb.plugins.CallValidationInterceptor.invoke(CallValidationInterceptor.java:63)

      at org.jboss.ejb.plugins.AbstractTxInterceptor.invokeNext(AbstractTxInterceptor.java:121)

      at org.jboss.ejb.plugins.TxInterceptorCMT.runWithTransactions(TxInterceptorCMT.java:315)

      at org.jboss.ejb.plugins.TxInterceptorCMT.invoke(TxInterceptorCMT.java:181)

      at org.jboss.ejb.plugins.SecurityInterceptor.process(SecurityInterceptor.java:228)

      at org.jboss.ejb.plugins.SecurityInterceptor.invoke(SecurityInterceptor.java:211)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.process(PreSecurityInterceptor.java:97)

      at org.jboss.ejb.plugins.security.PreSecurityInterceptor.invoke(PreSecurityInterceptor.java:81)

      at org.jboss.ejb.plugins.LogInterceptor.invoke(LogInterceptor.java:205)

      at org.jboss.ejb.plugins.ProxyFactoryFinderInterceptor.invoke(ProxyFactoryFinderInterceptor.java:138)

      at org.jboss.ejb.SessionContainer.internalInvoke(SessionContainer.java:650)

      at org.jboss.ejb.Container.invoke(Container.java:1029)

      at sun.reflect.GeneratedMethodAccessor459.invoke(Unknown Source)

      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

      at java.lang.reflect.Method.invoke(Method.java:597)

      at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:157)

      at org.jboss.mx.server.Invocation.dispatch(Invocation.java:96)

      at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)

      at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

      at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)

      at org.jboss.invocation.local.LocalInvoker$MBeanServerAction.invoke(LocalInvoker.java:169)

      at org.jboss.invocation.local.LocalInvoker.invoke(LocalInvoker.java:118)

      at org.jboss.invocation.InvokerInterceptor.invokeLocal(InvokerInterceptor.java:209)

      at org.jboss.invocation.InvokerInterceptor.invoke(InvokerInterceptor.java:195)

      at org.jboss.proxy.TransactionInterceptor.invoke(TransactionInterceptor.java:61)

      at org.jboss.proxy.ejb.SecurityContextInterceptor.invoke(SecurityContextInterceptor.java:64)

      at org.jboss.proxy.SecurityInterceptor.invoke(SecurityInterceptor.java:68)

      at org.jboss.proxy.ejb.StatelessSessionInterceptor.invoke(StatelessSessionInterceptor.java:112)

      at org.jboss.proxy.ClientContainer.invoke(ClientContainer.java:101)

      at $Proxy545.justInTimeReservation(Unknown Source)

      at com.agilquest.onboard.cipphone.CiscoIPPhoneAction.justInTime(CiscoIPPhoneAction.java:160)

      at com.agilquest.onboard.cipphone.CiscoIPPhonePerformAction.doJitAction(CiscoIPPhonePerformAction.java:265)

      at com.agilquest.onboard.cipphone.CiscoIPPhonePerformAction.doAction(CiscoIPPhonePerformAction.java:168)

      at com.agilquest.onboard.cipphone.CiscoIPPhonePerformAction.<init>(CiscoIPPhonePerformAction.java:130)

      at com.agilquest.onboard.cipphone.jsp.justInTimeResults_jsp._jspService(justInTimeResults_jsp.java:50)

      at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)

      at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)

      at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

      at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

      at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)

      at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

      at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

      at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235)

      at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)

      at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:190)

      at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92)

      at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126)

      at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70)

      at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)

      at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)

      at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:158)

      at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)

      at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330)

      at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:829)

      at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:598)

      at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)

      at java.lang.Thread.run(Thread.java:619)

      Caused by: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException: State retrieval timed out waiting for flush to unblock. (timeout = 60 seconds)

      at org.hibernate.cache.jbc2.util.CacheHelper.put(CacheHelper.java:214)

      at org.hibernate.cache.jbc2.timestamp.TimestampsRegionImpl.put(TimestampsRegionImpl.java:128)

      ... 125 more

      Caused by: org.jboss.cache.lock.TimeoutException: State retrieval timed out waiting for flush to unblock. (timeout = 60 seconds)

      at org.jboss.cache.RPCManagerImpl$FlushTracker.waitForFlushCompletion(RPCManagerImpl.java:195)

      at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:745)

      at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:716)

      at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:721)

      at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:161)

      at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:135)

      at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:107)

      at org.jboss.cache.interceptors.ReplicationInterceptor.handleCrudMethod(ReplicationInterceptor.java:160)

      at org.jboss.cache.interceptors.ReplicationInterceptor.visitPutKeyValueCommand(ReplicationInterceptor.java:107)

      at org.jboss.cache.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:100)

      at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)

      at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)

      at org.jboss.cache.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:65)

      at org.jboss.cache.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:100)

      at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)

      at org.jboss.cache.interceptors.TxInterceptor.attachGtxAndPassUpChain(TxInterceptor.java:301)

      at org.jboss.cache.interceptors.TxInterceptor.handleDefault(TxInterceptor.java:283)

      at org.jboss.cache.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:65)

      at org.jboss.cache.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:100)

      at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)

      at org.jboss.cache.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:119)

      at org.jboss.cache.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:100)

      at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)

      at org.jboss.cache.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:178)

      at org.jboss.cache.interceptors.InvocationContextInterceptor.visitPutKeyValueCommand(InvocationContextInterceptor.java:82)

      at org.jboss.cache.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:100)

      at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:287)

      at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:555)

      at org.jboss.ha.cachemanager.CacheManagerManagedCache.put(CacheManagerManagedCache.java:287)

      at org.hibernate.cache.jbc2.util.CacheHelper.put(CacheHelper.java:212)

      ... 126 more