Java Deadlock during shutdown
lepe Apr 2, 2009 3:34 PMHi,
JBPAPP_4_3_0_GA with jbossesb 4.2.1GA.CP1.
Using ThreadPoolExecutor to spawn a couple of threads/runnable tasks that will start pipelines in the ESB.
What the task to is to call ServiceInvoker.deliverSynch(). A synch call might take some time to execute.
When I got the Shutdown Deadlock I had about 40 concurrent threads calling the ESB. During the threads running I decided to shutdown JBoss. I am trying to find a way to nicely interrupt all running tasks and prohibit new tasks for being submitted.
But I then saw the problem with a deadlock. Both the Shutdown Hook and one of my threads are waiting for the same object. See below, cut from jstack.
Any ideas!?
regards,
/L
Found one Java-level deadlock: ============================= "JBoss Shutdown Hook": waiting to lock monitor 0x000000012db69dd0 (object 0x0000000128895238, a java.lang.Class), which is held by "WorkerThread-79" "WorkerThread-79": waiting for ownable synchronizer 0x000000011290a490, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "JBoss Shutdown Hook" Java stack information for the threads listed above: =================================================== "JBoss Shutdown Hook": at org.jboss.soa.esb.couriers.CourierFactory.deregisterCourier(CourierFactory.java:172) - waiting to lock <0x0000000128895238> (a java.lang.Class for org.jboss.soa.esb.couriers.CourierFactory) at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.cleanup(TwoWayCourierImpl.java:234) at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:228) at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:188) at org.jboss.soa.esb.lifecycle.LifecycleResource.cleanupLifecycleResource(LifecycleResource.java:209) at org.jboss.soa.esb.lifecycle.LifecycleResource.destroyResources(LifecycleResource.java:178) - locked <0x000000011290a360> (a org.jboss.soa.esb.lifecycle.LifecycleResource) at org.jboss.soa.esb.lifecycle.LifecycleResourceManager.cleanupAllResources(LifecycleResourceManager.java:349) at org.jboss.soa.esb.listeners.config.JBoss4ESBDeployer.stopService(JBoss4ESBDeployer.java:164) at org.jboss.system.ServiceMBeanSupport.jbossInternalStop(ServiceMBeanSupport.java:315) at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:247) at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:978) at $Proxy0.stop(Unknown Source) at org.jboss.system.ServiceController.stop(ServiceController.java:508) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at org.jboss.system.ServiceController.stop(ServiceController.java:499) at sun.reflect.GeneratedMethodAccessor325.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210) at $Proxy40.stop(Unknown Source) at org.jboss.deployment.SimpleSubDeployerSupport.stopService(SimpleSubDeployerSupport.java:374) at org.jboss.deployment.SimpleSubDeployerSupport.stop(SimpleSubDeployerSupport.java:141) at org.jboss.deployment.MainDeployer.stop(MainDeployer.java:667) at org.jboss.deployment.MainDeployer.undeploy(MainDeployer.java:638) at org.jboss.deployment.MainDeployer.shutdown(MainDeployer.java:516) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133) at org.jboss.mx.server.Invocation.invoke(Invocation.java:88) at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142) at org.jboss.mx.server.Invocation.invoke(Invocation.java:88) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) at org.jboss.system.server.ServerImpl$ShutdownHook.shutdownDeployments(ServerImpl.java:1058) at org.jboss.system.server.ServerImpl$ShutdownHook.shutdown(ServerImpl.java:1033) at org.jboss.system.server.ServerImpl$ShutdownHook.run(ServerImpl.java:996) "WorkerThread-79": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000011290a490> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:747) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:877) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:594) at org.jboss.soa.esb.lifecycle.LifecycleResource.getLifecycleResource(LifecycleResource.java:93) at org.jboss.soa.esb.couriers.CourierFactory.deregisterCourier(CourierFactory.java:172) - locked <0x0000000128895238> (a java.lang.Class for org.jboss.soa.esb.couriers.CourierFactory) at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.cleanup(TwoWayCourierImpl.java:234) at org.jboss.soa.esb.couriers.CourierUtil.cleanCourier(CourierUtil.java:268) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.attemptDelivery(ServiceInvoker.java:533) at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.access$200(ServiceInvoker.java:430) at org.jboss.soa.esb.client.ServiceInvoker.post(ServiceInvoker.java:307) at org.jboss.soa.esb.client.ServiceInvoker.deliverSync(ServiceInvoker.java:187) at se.sj.ipl.notification.profile.NotificationTask.call(NotificationTask.java:74) at se.sj.ipl.notification.profile.NotificationTask.call(NotificationTask.java:13) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) at java.util.concurrent.FutureTask.run(FutureTask.java:138) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) at java.util.concurrent.FutureTask.run(FutureTask.java:138) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:637)