3 Replies Latest reply on Nov 10, 2008 9:10 AM by michalkozakiewicz

Cache locking issue after one node of the cluster is restart

michalkozakiewicz Nov 7, 2008 6:53 AM

Hi,
we are running 2 nodes on Jboss 5.0.0.Beta4. We had to restart one of the nodes and the deployment failed:

Caused by: org.hibernate.HibernateException: Could not instantiate cache implementation

Caused by: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account could not be acquired by Thread[main,5,jboss] after 15000 ms. Locks: Read lock owners: []

Write lock owner: GlobalTransaction:<10.0.0.23:3336>:40235

, lock info: write owner=GlobalTransaction:<10.0.0.23:3336>:40235 (activeReaders=0, activeWriter=Thread[Multiplexer,udp,10.0.0.24:1723,10,MultiplexerThreads], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

Any ideas please?
Michal

The full stack trace:

2:36:24,837 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999822 credits

12:36:34,869 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999822 credits

12:36:44,853 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999822 credits

12:36:54,885 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999822 credits

12:36:55,385 ERROR [RegionManager] failed to activate /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account/cash/AccountJournalEntry

org.jboss.cache.lock.TimeoutException: read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account could not be acquired by Thread[main,5,jboss] after 15000 ms. Locks: Read lock owners: []

Write lock owner: GlobalTransaction:<10.0.0.23:3336>:40235

, lock info: write owner=GlobalTransaction:<10.0.0.23:3336>:40235 (activeReaders=0, activeWriter=Thread[Multiplexer,udp,10.0.0.24:1723,10,MultiplexerThreads], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock0(IdentityLock.java:323)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock(IdentityLock.java:267)

 at org.jboss.cache.lock.IdentityLock.acquire(IdentityLock.java:493)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireNodeLock(PessimisticLockInterceptor.java:526)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.lock(PessimisticLockInterceptor.java:420)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireLocksWithTimeout(PessimisticLockInterceptor.java:348)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.handleGetNodeMethod(PessimisticLockInterceptor.java:301)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:90)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.invoke(PessimisticLockInterceptor.java:83)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:56)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:452)

 at org.jboss.cache.interceptors.TxInterceptor.handleGetNodeMethod(TxInterceptor.java:245)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:90)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.InvocationContextInterceptor.invoke(InvocationContextInterceptor.java:73)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:123)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:64)

 at org.jboss.cache.invocation.CacheInvocationDelegate.getNode(CacheInvocationDelegate.java:450)

 at org.jboss.cache.invocation.NodeInvocationDelegate.getChild(NodeInvocationDelegate.java:307)

 at org.jboss.cache.invocation.NodeInvocationDelegate.addChild(NodeInvocationDelegate.java:278)

 at org.jboss.cache.RegionManager.activateRegion(RegionManager.java:410)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:325)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:288)

 at org.jboss.cache.RegionImpl.activate(RegionImpl.java:82)

 at org.jboss.ejb3.entity.JBCCache.activateCacheRegion(JBCCache.java:321)

 at org.jboss.ejb3.entity.JBCCache.<init>(JBCCache.java:84)

 at org.jboss.ejb3.entity.JBCCacheFactory.buildCache(JBCCacheFactory.java:54)

 at org.jboss.ejb3.entity.TreeCacheProviderHook.buildCache(TreeCacheProviderHook.java:74)

 at org.hibernate.cache.CacheFactory.createCache(CacheFactory.java:61)

 at org.hibernate.impl.SessionFactoryImpl.<init>(SessionFactoryImpl.java:214)

 at org.hibernate.cfg.Configuration.buildSessionFactory(Configuration.java:1294)

 at org.hibernate.cfg.AnnotationConfiguration.buildSessionFactory(AnnotationConfiguration.java:918)

 at org.hibernate.ejb.Ejb3Configuration.buildEntityManagerFactory(Ejb3Configuration.java:668)

 at org.hibernate.ejb.HibernatePersistence.createContainerEntityManagerFactory(HibernatePersistence.java:132)

 at org.jboss.ejb3.entity.PersistenceUnitDeployment.start(PersistenceUnitDeployment.java:259)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:56)

 at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:110)

 at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:214)

 at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:45)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:108)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:69)

 at org.jboss.kernel.plugins.dependency.LifecycleAction.installActionInternal(LifecycleAction.java:221)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:135)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:46)

 at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)

 at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)

 at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:574)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:398)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:96)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:90)

 at org.jboss.ejb3.MCKernelAbstraction.install(MCKernelAbstraction.java:153)

 at org.jboss.ejb3.Ejb3Deployment.startPersistenceUnits(Ejb3Deployment.java:606)

 at org.jboss.ejb3.Ejb3Deployment.start(Ejb3Deployment.java:402)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:53)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:37)

 at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:65)

 at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)

 at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:169)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:853)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:906)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:794)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:498)

 at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:506)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.loadProfile(ProfileServiceBootstrap.java:246)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:131)

 at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:408)

 at org.jboss.Main.boot(Main.java:208)

 at org.jboss.Main$1.run(Main.java:534)

 at java.lang.Thread.run(Thread.java:595)

12:36:55,385 ERROR [RegionManager] failed inactivating /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account/cash/AccountJournalEntry

org.jboss.cache.CacheException: Region /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account/cash/AccountJournalEntry is already being activated/deactivated

 at org.jboss.cache.RegionManager.inactivateRegion(RegionManager.java:523)

 at org.jboss.cache.RegionManager.activateRegion(RegionManager.java:458)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:325)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:288)

 at org.jboss.cache.RegionImpl.activate(RegionImpl.java:82)

 at org.jboss.ejb3.entity.JBCCache.activateCacheRegion(JBCCache.java:321)

 at org.jboss.ejb3.entity.JBCCache.<init>(JBCCache.java:84)

 at org.jboss.ejb3.entity.JBCCacheFactory.buildCache(JBCCacheFactory.java:54)

 at org.jboss.ejb3.entity.TreeCacheProviderHook.buildCache(TreeCacheProviderHook.java:74)

 at org.hibernate.cache.CacheFactory.createCache(CacheFactory.java:61)

 at org.hibernate.impl.SessionFactoryImpl.<init>(SessionFactoryImpl.java:214)

 at org.hibernate.cfg.Configuration.buildSessionFactory(Configuration.java:1294)

 at org.hibernate.cfg.AnnotationConfiguration.buildSessionFactory(AnnotationConfiguration.java:918)

 at org.hibernate.ejb.Ejb3Configuration.buildEntityManagerFactory(Ejb3Configuration.java:668)

 at org.hibernate.ejb.HibernatePersistence.createContainerEntityManagerFactory(HibernatePersistence.java:132)

 at org.jboss.ejb3.entity.PersistenceUnitDeployment.start(PersistenceUnitDeployment.java:259)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:56)

 at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:110)

 at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:214)

 at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:45)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:108)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:69)

 at org.jboss.kernel.plugins.dependency.LifecycleAction.installActionInternal(LifecycleAction.java:221)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:135)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:46)

 at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)

 at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)

 at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:574)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:398)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:96)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:90)

 at org.jboss.ejb3.MCKernelAbstraction.install(MCKernelAbstraction.java:153)

 at org.jboss.ejb3.Ejb3Deployment.startPersistenceUnits(Ejb3Deployment.java:606)

 at org.jboss.ejb3.Ejb3Deployment.start(Ejb3Deployment.java:402)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:53)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:37)

 at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:65)

 at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)

 at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:169)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:853)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:906)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:794)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:498)

 at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:506)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.loadProfile(ProfileServiceBootstrap.java:246)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:131)

 at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:408)

 at org.jboss.Main.boot(Main.java:208)

 at org.jboss.Main$1.run(Main.java:534)

 at java.lang.Thread.run(Thread.java:595)

12:36:55,385 ERROR [AbstractKernelController] Error installing to Start: name=persistence.units:ear=zcapitalspreads.ear,jar=services.jar,unitName=capital.spreads state=Create

javax.persistence.PersistenceException: [PersistenceUnit: capital.spreads] Unable to build EntityManagerFactory

 at org.hibernate.ejb.Ejb3Configuration.buildEntityManagerFactory(Ejb3Configuration.java:676)

 at org.hibernate.ejb.HibernatePersistence.createContainerEntityManagerFactory(HibernatePersistence.java:132)

 at org.jboss.ejb3.entity.PersistenceUnitDeployment.start(PersistenceUnitDeployment.java:259)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:56)

 at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:110)

 at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:214)

 at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:45)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:108)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:69)

 at org.jboss.kernel.plugins.dependency.LifecycleAction.installActionInternal(LifecycleAction.java:221)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:135)

 at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.installAction(KernelControllerContextAction.java:46)

 at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)

 at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)

 at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:574)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:398)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:96)

 at org.jboss.kernel.plugins.dependency.AbstractKernelController.install(AbstractKernelController.java:90)

 at org.jboss.ejb3.MCKernelAbstraction.install(MCKernelAbstraction.java:153)

 at org.jboss.ejb3.Ejb3Deployment.startPersistenceUnits(Ejb3Deployment.java:606)

 at org.jboss.ejb3.Ejb3Deployment.start(Ejb3Deployment.java:402)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:53)

 at org.jboss.ejb3.deployers.EJBStage2Deployer.deploy(EJBStage2Deployer.java:37)

 at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:65)

 at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)

 at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:169)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:853)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:906)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:794)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:498)

 at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:506)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.loadProfile(ProfileServiceBootstrap.java:246)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:131)

 at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:408)

 at org.jboss.Main.boot(Main.java:208)

 at org.jboss.Main$1.run(Main.java:534)

 at java.lang.Thread.run(Thread.java:595)

Caused by: org.hibernate.HibernateException: Could not instantiate cache implementation

 at org.hibernate.cache.CacheFactory.createCache(CacheFactory.java:64)

 at org.hibernate.impl.SessionFactoryImpl.<init>(SessionFactoryImpl.java:214)

 at org.hibernate.cfg.Configuration.buildSessionFactory(Configuration.java:1294)

 at org.hibernate.cfg.AnnotationConfiguration.buildSessionFactory(AnnotationConfiguration.java:918)

 at org.hibernate.ejb.Ejb3Configuration.buildEntityManagerFactory(Ejb3Configuration.java:668)

 ... 54 more

Caused by: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/account could not be acquired by Thread[main,5,jboss] after 15000 ms. Locks: Read lock owners: []

Write lock owner: GlobalTransaction:<10.0.0.23:3336>:40235

, lock info: write owner=GlobalTransaction:<10.0.0.23:3336>:40235 (activeReaders=0, activeWriter=Thread[Multiplexer,udp,10.0.0.24:1723,10,MultiplexerThreads], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock0(IdentityLock.java:323)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock(IdentityLock.java:267)

 at org.jboss.cache.lock.IdentityLock.acquire(IdentityLock.java:493)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireNodeLock(PessimisticLockInterceptor.java:526)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.lock(PessimisticLockInterceptor.java:420)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireLocksWithTimeout(PessimisticLockInterceptor.java:348)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.handleGetNodeMethod(PessimisticLockInterceptor.java:301)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:90)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.invoke(PessimisticLockInterceptor.java:83)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:56)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:452)

 at org.jboss.cache.interceptors.TxInterceptor.handleGetNodeMethod(TxInterceptor.java:245)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:90)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.InvocationContextInterceptor.invoke(InvocationContextInterceptor.java:73)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:123)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:64)

 at org.jboss.cache.invocation.CacheInvocationDelegate.getNode(CacheInvocationDelegate.java:450)

 at org.jboss.cache.invocation.NodeInvocationDelegate.getChild(NodeInvocationDelegate.java:307)

 at org.jboss.cache.invocation.NodeInvocationDelegate.addChild(NodeInvocationDelegate.java:278)

 at org.jboss.cache.RegionManager.activateRegion(RegionManager.java:410)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:325)

 at org.jboss.cache.RegionManager.activate(RegionManager.java:288)

 at org.jboss.cache.RegionImpl.activate(RegionImpl.java:82)

 at org.jboss.ejb3.entity.JBCCache.activateCacheRegion(JBCCache.java:321)

 at org.jboss.ejb3.entity.JBCCache.<init>(JBCCache.java:84)

 at org.jboss.ejb3.entity.JBCCacheFactory.buildCache(JBCCacheFactory.java:54)

 at org.jboss.ejb3.entity.TreeCacheProviderHook.buildCache(TreeCacheProviderHook.java:74)

 at org.hibernate.cache.CacheFactory.createCache(CacheFactory.java:61)

 ... 58 more

1. Re: Cache locking issue after one node of the cluster is res

manik Nov 7, 2008 8:48 AM (in response to michalkozakiewicz)

Is this something you see consistently/reproducibly?
Actions

2. Re: Cache locking issue after one node of the cluster is res

michalkozakiewicz Nov 10, 2008 5:04 AM (in response to michalkozakiewicz)

Hi Manik,
thank you for you reply.

Unfortunately this specific issue is not reproducible. However we are seeing other TimeoutException's (which are displayed as warnings in the log) causing transaction rollbacks.

In the case below, since the transaction is rolled back in
ServletContextListener (LoadTimer) the web app fails to deploy.

I'll be getting more exception examples and posting them here.

This seems to happen only under load, if you need any more info, I ll be happy to provide it.

Thanks for your help.
Michal

13:13:25,190 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999054 credits

13:13:35,206 WARN [FC] Received two credit requests from 10.0.0.23:3336 without any intervening messages; sending 1999054 credits

13:13:40,956 WARN [arjLoggerI18N] [com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator_2] TwoPhaseCoordinator.beforeCompletion - failed for com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple@135a2ca

org.jboss.cache.ReplicationException: rsp=sender=10.0.0.23:3336, retval=null, received=false, suspected=false

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1475)

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1369)

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1362)

 at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:70)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:119)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:80)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:75)

 at org.jboss.cache.interceptors.ReplicationInterceptor.runPreparePhase(ReplicationInterceptor.java:248)

 at org.jboss.cache.interceptors.ReplicationInterceptor.handlePrepareMethod(ReplicationInterceptor.java:59)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:126)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.runPreparePhase(TxInterceptor.java:1013)

 at org.jboss.cache.interceptors.TxInterceptor$LocalSynchronizationHandler.beforeCompletion(TxInterceptor.java:1355)

 at org.jboss.cache.interceptors.OrderedSynchronizationHandler.beforeCompletion(OrderedSynchronizationHandler.java:84)

 at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:114)

 at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:250)

 at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:86)

 at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:177)

 at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1389)

 at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:135)

 at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:87)

 at org.jboss.aspects.tx.TxPolicy.endTransaction(TxPolicy.java:175)

 at org.jboss.aspects.tx.TxPolicy.invokeInOurTx(TxPolicy.java:87)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:193)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RunAsSecurityInterceptorv2.invoke(RunAsSecurityInterceptorv2.java:118)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RoleBasedAuthorizationInterceptorv2.invoke(RoleBasedAuthorizationInterceptorv2.java:190)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:249)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:214)

 at org.jboss.ejb3.stateless.StatelessLocalProxy.invoke(StatelessLocalProxy.java:86)

 at $Proxy268.start(Unknown Source)

 at com.chaucerdigital.capitalspreads.timer.LoadTimer.contextInitialized(LoadTimer.java:78)

 at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:3860)

 at org.apache.catalina.core.StandardContext.start(StandardContext.java:4363)

 at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:791)

 at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:771)

 at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:525)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.apache.tomcat.util.modeler.BaseModelMBean.invoke(BaseModelMBean.java:297)

 at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164)

 at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)

 at org.apache.catalina.core.StandardContext.init(StandardContext.java:5357)

 at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeployInternal(TomcatDeployment.java:346)

 at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeploy(TomcatDeployment.java:142)

 at org.jboss.web.deployers.AbstractWarDeployment.start(AbstractWarDeployment.java:347)

 at org.jboss.web.deployers.WebModule.startModule(WebModule.java:89)

 at org.jboss.web.deployers.WebModule.start(WebModule.java:67)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:157)

 at org.jboss.mx.server.Invocation.dispatch(Invocation.java:96)

 at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)

 at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

 at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)

 at org.jboss.system.microcontainer.ServiceProxy.invoke(ServiceProxy.java:184)

 at $Proxy5.start(Unknown Source)

 at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:42)

 at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:37)

 at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)

 at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)

 at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.system.microcontainer.ServiceControllerContext.install(ServiceControllerContext.java:255)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.system.ServiceController.doChange(ServiceController.java:659)

 at org.jboss.system.ServiceController.start(ServiceController.java:431)

 at org.jboss.system.deployers.ServiceDeployer.start(ServiceDeployer.java:150)

 at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:108)

 at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:46)

 at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:65)

 at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)

 at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:169)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:853)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:874)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:906)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:794)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:498)

 at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:506)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.loadProfile(ProfileServiceBootstrap.java:246)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:131)

 at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:408)

 at org.jboss.Main.boot(Main.java:208)

 at org.jboss.Main$1.run(Main.java:534)

 at java.lang.Thread.run(Thread.java:595)

Caused by: org.jboss.cache.lock.TimeoutException: Replication timeout for 10.0.0.23:3336

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1473)

 ... 116 more

13:13:40,956 ERROR [console]] Exception sending context initialized event to listener instance of class com.chaucerdigital.capitalspreads.timer.LoadTimer

java.lang.RuntimeException: javax.transaction.RollbackException: [com.arjuna.ats.internal.jta.transaction.arjunacore.commitwhenaborted] [com.arjuna.ats.internal.jta.transaction.arjunacore.commitwhenaborted] Can't commit because the transaction is in aborted state

 at org.jboss.aspects.tx.TxPolicy.handleEndTransactionException(TxPolicy.java:198)

 at org.jboss.aspects.tx.TxPolicy.endTransaction(TxPolicy.java:180)

 at org.jboss.aspects.tx.TxPolicy.invokeInOurTx(TxPolicy.java:87)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:193)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RunAsSecurityInterceptorv2.invoke(RunAsSecurityInterceptorv2.java:118)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RoleBasedAuthorizationInterceptorv2.invoke(RoleBasedAuthorizationInterceptorv2.java:190)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:249)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:214)

 at org.jboss.ejb3.stateless.StatelessLocalProxy.invoke(StatelessLocalProxy.java:86)

 at $Proxy268.start(Unknown Source)

 at com.chaucerdigital.capitalspreads.timer.LoadTimer.contextInitialized(LoadTimer.java:78)

 at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:3860)

 at org.apache.catalina.core.StandardContext.start(StandardContext.java:4363)

 at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:791)

 at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:771)

 at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:525)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.apache.tomcat.util.modeler.BaseModelMBean.invoke(BaseModelMBean.java:297)

 at org.jboss.mx.server.RawDynamicInvoker.invoke(RawDynamicInvoker.java:164)

 at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)

 at org.apache.catalina.core.StandardContext.init(StandardContext.java:5357)

 at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeployInternal(TomcatDeployment.java:346)

 at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeploy(TomcatDeployment.java:142)

 at org.jboss.web.deployers.AbstractWarDeployment.start(AbstractWarDeployment.java:347)

 at org.jboss.web.deployers.WebModule.startModule(WebModule.java:89)

 at org.jboss.web.deployers.WebModule.start(WebModule.java:67)

 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:157)

 at org.jboss.mx.server.Invocation.dispatch(Invocation.java:96)

 at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)

 at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

 at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)

 at org.jboss.system.microcontainer.ServiceProxy.invoke(ServiceProxy.java:184)

 at $Proxy5.start(Unknown Source)

 at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:42)

 at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:37)

 at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)

 at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)

 at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.system.microcontainer.ServiceControllerContext.install(ServiceControllerContext.java:255)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.system.ServiceController.doChange(ServiceController.java:659)

 at org.jboss.system.ServiceController.start(ServiceController.java:431)

 at org.jboss.system.deployers.ServiceDeployer.start(ServiceDeployer.java:150)

 at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:108)

 at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:46)

 at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:65)

 at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)

 at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:169)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:853)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:874)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:906)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:794)

 at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:327)

 at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1309)

 at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:734)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:862)

 at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:784)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:622)

 at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:411)

 at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:498)

 at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:506)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.loadProfile(ProfileServiceBootstrap.java:246)

 at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:131)

 at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:408)

 at org.jboss.Main.boot(Main.java:208)

 at org.jboss.Main$1.run(Main.java:534)

 at java.lang.Thread.run(Thread.java:595)

Caused by: javax.transaction.RollbackException: [com.arjuna.ats.internal.jta.transaction.arjunacore.commitwhenaborted] [com.arjuna.ats.internal.jta.transaction.arjunacore.commitwhenaborted] Can't commit because the transaction is in aborted state

 at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1401)

 at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:135)

 at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:87)

 at org.jboss.aspects.tx.TxPolicy.endTransaction(TxPolicy.java:175)

 ... 93 more

Caused by: org.jboss.cache.ReplicationException: rsp=sender=10.0.0.23:3336, retval=null, received=false, suspected=false

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1475)

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1369)

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1362)

 at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:70)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:119)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:80)

 at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:75)

 at org.jboss.cache.interceptors.ReplicationInterceptor.runPreparePhase(ReplicationInterceptor.java:248)

 at org.jboss.cache.interceptors.ReplicationInterceptor.handlePrepareMethod(ReplicationInterceptor.java:59)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:126)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.runPreparePhase(TxInterceptor.java:1013)

 at org.jboss.cache.interceptors.TxInterceptor$LocalSynchronizationHandler.beforeCompletion(TxInterceptor.java:1355)

 at org.jboss.cache.interceptors.OrderedSynchronizationHandler.beforeCompletion(OrderedSynchronizationHandler.java:84)

 at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:114)

 at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:250)

 at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:86)

 at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:177)

 at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1389)

 ... 96 more

Caused by: org.jboss.cache.lock.TimeoutException: Replication timeout for 10.0.0.23:3336

 at org.jboss.cache.CacheImpl.callRemoteMethods(CacheImpl.java:1473)

 ... 116 more

3. Re: Cache locking issue after one node of the cluster is res

michalkozakiewicz Nov 10, 2008 9:10 AM (in response to michalkozakiewicz)

Another locking issue resulting in transaction rollback

2008-11-07 11:03:38,984 INFO [org.hibernate.event.def.DefaultLoadEventListener] Error performing load command

org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/user/User/com.chaucerdigital.capitalspreads.user.User#200003 could not be acquired by GlobalTransaction:<10.0.0.24:4021>:17766 after 15000 ms. Locks: Read lock owners: []
Write lock owner: GlobalTransaction:<10.0.0.24:4021>:17762
, lock info: write owner=GlobalTransaction:<10.0.0.24:4021>:17762 (activeReaders=0, activeWriter=Thread[http-0.0.0.0-8080-2,5,jboss], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock0(IdentityLock.java:323)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock(IdentityLock.java:267)

 at org.jboss.cache.lock.IdentityLock.acquire(IdentityLock.java:493)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireNodeLock(PessimisticLockInterceptor.java:526)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.lock(PessimisticLockInterceptor.java:420)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireLocksWithTimeout(PessimisticLockInterceptor.java:348)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.handleGetKeyValueMethod(PessimisticLockInterceptor.java:295)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.invoke(PessimisticLockInterceptor.java:83)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:452)

 at org.jboss.cache.interceptors.TxInterceptor.handleGetKeyValueMethod(TxInterceptor.java:250)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.CacheMgmtInterceptor.handleGetKeyValueMethod(CacheMgmtInterceptor.java:81)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.InvocationContextInterceptor.invoke(InvocationContextInterceptor.java:73)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:123)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:64)

 at org.jboss.cache.invocation.CacheInvocationDelegate.get(CacheInvocationDelegate.java:403)

 at org.jboss.ejb3.entity.JBCCache.read(JBCCache.java:111)

 at org.hibernate.cache.TransactionalCache.get(TransactionalCache.java:31)

 at org.hibernate.event.def.DefaultLoadEventListener.loadFromSecondLevelCache(DefaultLoadEventListener.java:486)

 at org.hibernate.event.def.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:357)

 at org.hibernate.event.def.DefaultLoadEventListener.load(DefaultLoadEventListener.java:139)

 at org.hibernate.event.def.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:195)

 at org.hibernate.event.def.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:103)

 at org.hibernate.impl.SessionImpl.fireLoad(SessionImpl.java:878)

 at org.hibernate.impl.SessionImpl.internalLoad(SessionImpl.java:846)

 at org.hibernate.type.EntityType.resolveIdentifier(EntityType.java:557)

 at org.hibernate.type.EntityType.resolve(EntityType.java:379)

 at org.hibernate.engine.TwoPhaseLoad.initializeEntity(TwoPhaseLoad.java:116)

 at org.hibernate.loader.Loader.initializeEntitiesAndCollections(Loader.java:854)

 at org.hibernate.loader.Loader.doQuery(Loader.java:729)

 at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:236)

 at org.hibernate.loader.Loader.doList(Loader.java:2220)

 at org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2104)

 at org.hibernate.loader.Loader.list(Loader.java:2099)

 at org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:378)

 at org.hibernate.hql.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:338)

 at org.hibernate.engine.query.HQLQueryPlan.performList(HQLQueryPlan.java:172)

 at org.hibernate.impl.SessionImpl.list(SessionImpl.java:1121)

 at org.hibernate.impl.QueryImpl.list(QueryImpl.java:79)

 at org.hibernate.ejb.QueryImpl.getSingleResult(QueryImpl.java:81)

 at com.chaucerdigital.capitalspreads.security.SecurityManagerBean.loadUserByUsernameUserTypeBrokerCode(SecurityManagerBean.java:70)

 at sun.reflect.GeneratedMethodAccessor414.invoke(Unknown Source)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeTarget(MethodInvocation.java:121)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:110)

 at org.jboss.ejb3.interceptor.InvocationContextImpl.proceed(InvocationContextImpl.java:166)

 at org.jboss.ejb3.interceptor.EJB3InterceptorsInterceptor.invoke(EJB3InterceptorsInterceptor.java:63)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.entity.TransactionScopedEntityManagerInterceptor.invoke(TransactionScopedEntityManagerInterceptor.java:54)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.AllowedOperationsInterceptor.invoke(AllowedOperationsInterceptor.java:47)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPolicy.invokeInCallerTx(TxPolicy.java:126)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:197)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RunAsSecurityInterceptorv2.invoke(RunAsSecurityInterceptorv2.java:118)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RoleBasedAuthorizationInterceptorv2.invoke(RoleBasedAuthorizationInterceptorv2.java:190)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:249)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:214)

 at org.jboss.ejb3.stateless.StatelessLocalProxy.invoke(StatelessLocalProxy.java:86)

 at $Proxy246.loadUserByUsernameUserTypeBrokerCode(Unknown Source)

 at com.chaucerdigital.capitalspreads.webservice.PublicPagesWebService.authenticateUser(PublicPagesWebService.java:227)

 at sun.reflect.GeneratedMethodAccessor413.invoke(Unknown Source)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeTarget(MethodInvocation.java:121)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:110)

 at org.jboss.ejb3.interceptor.InvocationContextImpl.proceed(InvocationContextImpl.java:166)

 at org.jboss.ejb3.interceptor.EJB3InterceptorsInterceptor.invoke(EJB3InterceptorsInterceptor.java:63)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.entity.TransactionScopedEntityManagerInterceptor.invoke(TransactionScopedEntityManagerInterceptor.java:54)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.AllowedOperationsInterceptor.invoke(AllowedOperationsInterceptor.java:47)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPolicy.invokeInOurTx(TxPolicy.java:79)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:193)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.tx.NullInterceptor.invoke(NullInterceptor.java:42)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.wsf.container.jboss50.invocation.InvocationHandlerEJB3.invoke(InvocationHandlerEJB3.java:104)

 at org.jboss.ws.core.server.ServiceEndpointInvoker.invoke(ServiceEndpointInvoker.java:221)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.processRequest(RequestHandlerImpl.java:466)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.handleRequest(RequestHandlerImpl.java:284)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.doPost(RequestHandlerImpl.java:201)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.handleHttpRequest(RequestHandlerImpl.java:134)

 at org.jboss.wsf.stack.jbws.EndpointServlet.service(EndpointServlet.java:84)

 at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)

 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

 at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)

 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

 at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)

 at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)

 at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:189)

 at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:90)

 at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:96)

 at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)

 at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)

 at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)

 at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)

 at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:309)

 at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844)

 at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:601)

 at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)

 at java.lang.Thread.run(Thread.java:595)

2008-11-07 11:03:38,984 ERROR [org.jboss.aspects.tx.TxPolicy] javax.ejb.EJBTransactionRolledbackException: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/user/User/com.chaucerdigital.capitalspreads.user.User#200003 could not be acquired by GlobalTransaction:<10.0.0.24:4021>:17766 after 15000 ms. Locks: Read lock owners: []
Write lock owner: GlobalTransaction:<10.0.0.24:4021>:17762
, lock info: write owner=GlobalTransaction:<10.0.0.24:4021>:17762 (activeReaders=0, activeWriter=Thread[http-0.0.0.0-8080-2,5,jboss], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

2008-11-07 11:03:38,984 ERROR [org.jboss.ws.core.jaxws.SOAPFaultHelperJAXWS] SOAP request exception

javax.ejb.EJBTransactionRolledbackException: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/user/User/com.chaucerdigital.capitalspreads.user.User#200003 could not be acquired by GlobalTransaction:<10.0.0.24:4021>:17766 after 15000 ms. Locks: Read lock owners: []
Write lock owner: GlobalTransaction:<10.0.0.24:4021>:17762
, lock info: write owner=GlobalTransaction:<10.0.0.24:4021>:17762 (activeReaders=0, activeWriter=Thread[http-0.0.0.0-8080-2,5,jboss], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.jboss.ejb3.tx.Ejb3TxPolicy.handleInCallerTx(Ejb3TxPolicy.java:91)

 at org.jboss.aspects.tx.TxPolicy.invokeInCallerTx(TxPolicy.java:130)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:197)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RunAsSecurityInterceptorv2.invoke(RunAsSecurityInterceptorv2.java:118)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.RoleBasedAuthorizationInterceptorv2.invoke(RoleBasedAuthorizationInterceptorv2.java:190)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:249)

 at org.jboss.ejb3.stateless.StatelessContainer.localInvoke(StatelessContainer.java:214)

 at org.jboss.ejb3.stateless.StatelessLocalProxy.invoke(StatelessLocalProxy.java:86)

 at $Proxy246.loadUserByUsernameUserTypeBrokerCode(Unknown Source)

 at com.chaucerdigital.capitalspreads.webservice.PublicPagesWebService.authenticateUser(PublicPagesWebService.java:227)

 at sun.reflect.GeneratedMethodAccessor413.invoke(Unknown Source)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeTarget(MethodInvocation.java:121)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:110)

 at org.jboss.ejb3.interceptor.InvocationContextImpl.proceed(InvocationContextImpl.java:166)

 at org.jboss.ejb3.interceptor.EJB3InterceptorsInterceptor.invoke(EJB3InterceptorsInterceptor.java:63)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.entity.TransactionScopedEntityManagerInterceptor.invoke(TransactionScopedEntityManagerInterceptor.java:54)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.AllowedOperationsInterceptor.invoke(AllowedOperationsInterceptor.java:47)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPolicy.invokeInOurTx(TxPolicy.java:79)

 at org.jboss.aspects.tx.TxInterceptor$Required.invoke(TxInterceptor.java:193)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPropagationInterceptor.invoke(TxPropagationInterceptor.java:76)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.stateless.StatelessInstanceInterceptor.invoke(StatelessInstanceInterceptor.java:64)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.remoting.ReplicantsManagerInterceptor.invoke(ReplicantsManagerInterceptor.java:51)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.tx.NullInterceptor.invoke(NullInterceptor.java:42)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.security.Ejb3AuthenticationInterceptorv2.invoke(Ejb3AuthenticationInterceptorv2.java:166)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.ENCPropagationInterceptor.invoke(ENCPropagationInterceptor.java:46)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.asynchronous.AsynchronousInterceptor.invoke(AsynchronousInterceptor.java:106)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.wsf.container.jboss50.invocation.InvocationHandlerEJB3.invoke(InvocationHandlerEJB3.java:104)

 at org.jboss.ws.core.server.ServiceEndpointInvoker.invoke(ServiceEndpointInvoker.java:221)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.processRequest(RequestHandlerImpl.java:466)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.handleRequest(RequestHandlerImpl.java:284)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.doPost(RequestHandlerImpl.java:201)

 at org.jboss.wsf.stack.jbws.RequestHandlerImpl.handleHttpRequest(RequestHandlerImpl.java:134)

 at org.jboss.wsf.stack.jbws.EndpointServlet.service(EndpointServlet.java:84)

 at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)

 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

 at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)

 at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

 at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

 at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)

 at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)

 at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:189)

 at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:90)

 at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:96)

 at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)

 at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)

 at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)

 at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)

 at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:309)

 at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844)

 at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:601)

 at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)

 at java.lang.Thread.run(Thread.java:595)

Caused by: javax.persistence.PersistenceException: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/user/User/com.chaucerdigital.capitalspreads.user.User#200003 could not be acquired by GlobalTransaction:<10.0.0.24:4021>:17766 after 15000 ms. Locks: Read lock owners: []
Write lock owner: GlobalTransaction:<10.0.0.24:4021>:17762
, lock info: write owner=GlobalTransaction:<10.0.0.24:4021>:17762 (activeReaders=0, activeWriter=Thread[http-0.0.0.0-8080-2,5,jboss], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.hibernate.ejb.AbstractEntityManagerImpl.throwPersistenceException(AbstractEntityManagerImpl.java:630)

 at org.hibernate.ejb.QueryImpl.getSingleResult(QueryImpl.java:107)

 at com.chaucerdigital.capitalspreads.security.SecurityManagerBean.loadUserByUsernameUserTypeBrokerCode(SecurityManagerBean.java:70)

 at sun.reflect.GeneratedMethodAccessor414.invoke(Unknown Source)

 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

 at java.lang.reflect.Method.invoke(Method.java:585)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeTarget(MethodInvocation.java:121)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:110)

 at org.jboss.ejb3.interceptor.InvocationContextImpl.proceed(InvocationContextImpl.java:166)

 at org.jboss.ejb3.interceptor.EJB3InterceptorsInterceptor.invoke(EJB3InterceptorsInterceptor.java:63)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.entity.TransactionScopedEntityManagerInterceptor.invoke(TransactionScopedEntityManagerInterceptor.java:54)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.ejb3.AllowedOperationsInterceptor.invoke(AllowedOperationsInterceptor.java:47)

 at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101)

 at org.jboss.aspects.tx.TxPolicy.invokeInCallerTx(TxPolicy.java:126)

 ... 79 more

Caused by: org.hibernate.cache.CacheException: org.jboss.cache.lock.TimeoutException read lock for /zcapitalspreads_ear,services_jar,capital_spreads/com/chaucerdigital/capitalspreads/user/User/com.chaucerdigital.capitalspreads.user.User#200003 could not be acquired by GlobalTransaction:<10.0.0.24:4021>:17766 after 15000 ms. Locks: Read lock owners: []
Write lock owner: GlobalTransaction:<10.0.0.24:4021>:17762
, lock info: write owner=GlobalTransaction:<10.0.0.24:4021>:17762 (activeReaders=0, activeWriter=Thread[http-0.0.0.0-8080-2,5,jboss], waitingReaders=0, waitingWriters=0, waitingUpgrader=0)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock0(IdentityLock.java:323)

 at org.jboss.cache.lock.IdentityLock.acquireReadLock(IdentityLock.java:267)

 at org.jboss.cache.lock.IdentityLock.acquire(IdentityLock.java:493)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireNodeLock(PessimisticLockInterceptor.java:526)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.lock(PessimisticLockInterceptor.java:420)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.acquireLocksWithTimeout(PessimisticLockInterceptor.java:348)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.handleGetKeyValueMethod(PessimisticLockInterceptor.java:295)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.PessimisticLockInterceptor.invoke(PessimisticLockInterceptor.java:83)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:62)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:452)

 at org.jboss.cache.interceptors.TxInterceptor.handleGetKeyValueMethod(TxInterceptor.java:250)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.CacheMgmtInterceptor.handleGetKeyValueMethod(CacheMgmtInterceptor.java:81)

 at org.jboss.cache.interceptors.MethodDispacherInterceptor.invoke(MethodDispacherInterceptor.java:87)

 at org.jboss.cache.interceptors.Interceptor.nextInterceptor(Interceptor.java:111)

 at org.jboss.cache.interceptors.InvocationContextInterceptor.invoke(InvocationContextInterceptor.java:73)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:123)

 at org.jboss.cache.invocation.AbstractInvocationDelegate.invoke(AbstractInvocationDelegate.java:64)

 at org.jboss.cache.invocation.CacheInvocationDelegate.get(CacheInvocationDelegate.java:403)

 at org.jboss.ejb3.entity.JBCCache.read(JBCCache.java:111)

 at org.hibernate.cache.TransactionalCache.get(TransactionalCache.java:31)

 at org.hibernate.event.def.DefaultLoadEventListener.loadFromSecondLevelCache(DefaultLoadEventListener.java:486)

 at org.hibernate.event.def.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:357)

 at org.hibernate.event.def.DefaultLoadEventListener.load(DefaultLoadEventListener.java:139)

 at org.hibernate.event.def.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:195)

 at org.hibernate.event.def.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:103)

 at org.hibernate.impl.SessionImpl.fireLoad(SessionImpl.java:878)

 at org.hibernate.impl.SessionImpl.internalLoad(SessionImpl.java:846)

 at org.hibernate.type.EntityType.resolveIdentifier(EntityType.java:557)

 at org.hibernate.type.EntityType.resolve(EntityType.java:379)

 at org.hibernate.engine.TwoPhaseLoad.initializeEntity(TwoPhaseLoad.java:116)

 at org.hibernate.loader.Loader.initializeEntitiesAndCollections(Loader.java:854)

 at org.hibernate.loader.Loader.doQuery(Loader.java:729)

 at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:236)

 at org.hibernate.loader.Loader.doList(Loader.java:2220)

 at org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2104)

 at org.hibernate.loader.Loader.list(Loader.java:2099)

 at org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:378)

 at org.hibernate.hql.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:338)

 at org.hibernate.engine.query.HQLQueryPlan.performList(HQLQueryPlan.java:172)

 at org.hibernate.impl.SessionImpl.list(SessionImpl.java:1121)

 at org.hibernate.impl.QueryImpl.list(QueryImpl.java:79)

 at org.hibernate.ejb.QueryImpl.getSingleResult(QueryImpl.java:81)

 ... 93 more

2008-11-07 11:03:39,531 WARN [org.hibernate.engine.loading.LoadContexts] fail-safe cleanup (collections) : org.hibernate.engine.loading.CollectionLoadContext@176df19<rs=net.sourceforge.jtds.jdbc.JtdsResultSet@15f9832>

Go to original post