5 Replies Latest reply on Mar 15, 2012 4:55 AM by adrianzuo

    Met deadlock between the two classes FileDeploymentManager and SecurityDeployer

    adrianzuo

      All

       

      My jboss version is 4.2.2 GA, now I am using HornetQ 2.2.5 final as jms provider, but I met deadlock sometimes when I gracefully stopping JBoss between below two threads:

       

      "pool-2-thread-1" prio=6 tid=0x451bc800 nid=0x131c waiting for monitor entry [0x49dbf000]

         java.lang.Thread.State: BLOCKED (on object monitor)

          at org.hornetq.core.deployers.impl.XmlDeployer.undeploy(XmlDeployer.java:137)

          - waiting to lock <0x0ed454f0> (a org.hornetq.core.deployers.impl.SecurityDeployer)  

          at org.hornetq.core.deployers.impl.FileDeploymentManager.run(FileDeploymentManager.java:240)

          - locked <0x0ed45488> (a org.hornetq.core.deployers.impl.FileDeploymentManager) 

          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

          at java.lang.Thread.run(Thread.java:662)

       

         Locked ownable synchronizers:

          - <0x0f0022c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)

       

      "JBoss Shutdown Hook" daemon prio=6 tid=0x5969a400 nid=0x49c  [0x6bebe000]

         java.lang.Thread.State: BLOCKED (on object monitor)   

          at org.hornetq.core.deployers.impl.FileDeploymentManager.unregisterDeployer(FileDeploymentManager.java:148)   

          - waiting to lock <0x0ed45488> (a org.hornetq.core.deployers.impl.FileDeploymentManager)

          at org.hornetq.core.deployers.impl.XmlDeployer.stop(XmlDeployer.java:238) 

          - locked <0x0ed454f0> (a org.hornetq.core.deployers.impl.SecurityDeployer) 

          at org.hornetq.core.server.impl.HornetQServerImpl.stop(HornetQServerImpl.java:676)

          - locked <0x0ed38230> (a org.hornetq.core.server.impl.HornetQServerImpl)

          at org.hornetq.core.server.impl.HornetQServerImpl.stop(HornetQServerImpl.java:621)

          at org.hornetq.jms.server.impl.JMSServerManagerImpl.stop(JMSServerManagerImpl.java:342)

          - locked <0x0ed382e0> (a org.hornetq.jms.server.impl.JMSServerManagerImpl)

          at org.hornetq.service.HornetQJMSStarterService.stop(HornetQJMSStarterService.java:39)

          at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

          at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

          at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

          at java.lang.reflect.Method.invoke(Method.java:597)

          at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)

          at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)

          at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)

          at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

          at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)

          at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:995)

          at $Proxy0.stop(Unknown Source)

          at org.jboss.system.ServiceController.stop(ServiceController.java:508)

          at sun.reflect.GeneratedMethodAccessor1041.invoke(Unknown Source)

          at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

          at java.lang.reflect.Method.invoke(Method.java:597)

          at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)

          at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)

          at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)

          at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

          at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)

          at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210)

          at $Proxy4.stop(Unknown Source)

          at org.jboss.deployment.SARDeployer.stop(SARDeployer.java:336)

          at org.jboss.deployment.MainDeployer.stop(MainDeployer.java:667)

          at org.jboss.deployment.MainDeployer.undeploy(MainDeployer.java:638)

          at org.jboss.deployment.MainDeployer.shutdown(MainDeployer.java:516)

          at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

          at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

          at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

          at java.lang.reflect.Method.invoke(Method.java:597)

          at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)

          at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)

          at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133)

          at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)

          at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142)

          at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)

          at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)

          at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)

          at org.jboss.system.server.ServerImpl$ShutdownHook.shutdownDeployments(ServerImpl.java:1058)

          at org.jboss.system.server.ServerImpl$ShutdownHook.shutdown(ServerImpl.java:1033)

          at org.jboss.system.server.ServerImpl$ShutdownHook.run(ServerImpl.java:996)

       

         Locked ownable synchronizers:

          - <0x10a784d8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)

          - <0x10ea6bd0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)

       

      Can anyone have a look? Is this a potentinal bug of HornetQ?

      And I also debugged the HornetQ source code and found the method run() in the class org.hornetq.core.deployers.impl.FileDeploymentManager is excueted termly, so I guess it should cause the deadlock happened intermittently and hard to reproduced.

       

      Regards,

      Candice

        • 1. Re: Met deadlock between the two classes FileDeploymentManager and SecurityDeployer
          gaohoward

          Yes I think it's a bug. Can you please create a Jira for that?

           

          Thanks

          Howard

          • 2. Re: Met deadlock between the two classes FileDeploymentManager and SecurityDeployer
            adrianzuo

            Howard

             

            Thanks for your reply.

            Can you give me more detail analysis for the issue?

             

            Regards,

            Candice

            • 3. Re: Met deadlock between the two classes FileDeploymentManager and SecurityDeployer
              adrianzuo

              Howard

               

              I have created a jira for it. See below:

              https://issues.jboss.org/browse/HORNETQ-827

               

              Regards,

              Candice

              • 4. Re: Met deadlock between the two classes FileDeploymentManager and SecurityDeployer
                clebert.suconic

                JBoss 4.2.2?

                 

                Unless you can replicate this on AS5/EAP or AS7.. this will be a won't fix.

                • 5. Re: Met deadlock between the two classes FileDeploymentManager and SecurityDeployer
                  adrianzuo

                  Hi Clebert

                   

                  After investigate, I found the deadlock was caused by the stop apis calling order in the two classes JMSServerManagerImpl and HornetQServerImpl. So I modified them in the 2.2.5 source code, see below diff:

                   

                  Index: src/main/org/hornetq/core/server/impl/HornetQServerImpl.java

                  ===================================================================

                  --- src/main/org/hornetq/core/server/impl/HornetQServerImpl.java    (revision 163078)

                  +++ src/main/org/hornetq/core/server/impl/HornetQServerImpl.java    (working copy)

                  @@ -665,6 +665,8 @@

                               basicUserCredentialsDeployer.stop();

                   

                               addressSettingsDeployer.stop();

                  +           

                  +            deploymentManager.stop();

                   

                               if (queueDeployer != null)

                               {

                  @@ -674,9 +676,7 @@

                               if (securityDeployer != null)

                               {

                                  securityDeployer.stop();

                  -            }

                  -

                  -            deploymentManager.stop();

                  +            }           

                            }

                   

                            managementService.unregisterServer();

                  Index: src/main/org/hornetq/jms/server/impl/JMSServerManagerImpl.java

                  ===================================================================

                  --- src/main/org/hornetq/jms/server/impl/JMSServerManagerImpl.java    (revision 163078)

                  +++ src/main/org/hornetq/jms/server/impl/JMSServerManagerImpl.java    (working copy)

                  @@ -290,15 +290,15 @@

                            return;

                         }

                   

                  -      if (jmsDeployer != null)

                  -      {

                  -         jmsDeployer.stop();

                  -      }

                  -

                         if (deploymentManager != null)

                         {

                            deploymentManager.stop();

                         }

                  +     

                  +      if (jmsDeployer != null)

                  +      {

                  +         jmsDeployer.stop();

                  +      }   

                   

                         // Storage could be null on a shared store backup server before initialization

                         if (storage != null)

                   

                  Is this modification ok?

                  Can you help have a look at it?

                   

                  -Regards

                  -Candice