1 Reply Latest reply on Feb 9, 2011 2:09 PM by jbossinterestcn

    JBOSS ESB 4.7 - ESB hangs during shutdown in Linux

    jbossinterestcn Newbie

      Environment:

       

       

      JBOSS 5.1 (for JDK 1.6) + JBOSS ESB 4.7 - on 64 bit LINUX machine

       

      Process was running in the foreground and I issued Ctrl +C to shutdown the server..

       

      The process hangs after the last message below and I have to issue a kill -9 on the pid.

       

      (Have included relevant parts of the thread dump below)

       

      12:51:26,528 WARN  [SmooksResource] Closing smooks resources for identity ID-8

      12:51:26,535 INFO  [EsbDeployment] Stopping 'jbossesb.esb'

      12:52:26,537 WARN  [StartStopLifecycleAction] Error during stop for jboss.esb.vfsfile:/usr/share/jboss/server/arjis_esb_node/deploy/jbossesb.esb/

      org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleException: Thread still active

              at org.jboss.soa.esb.listeners.lifecycle.AbstractThreadedManagedLifecycle.doDestroy(AbstractThreadedManagedLifecycle.java:168)

              at org.jboss.soa.esb.listeners.lifecycle.AbstractManagedLifecycle.destroy(AbstractManagedLifecycle.java:259)

              at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.destroyInstances(ManagedLifecycleController.java:192)

              at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.stopAndDestroy(ManagedLifecycleController.java:256)

              at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.stop(ManagedLifecycleController.java:92)

              at org.jboss.soa.esb.listeners.deployers.mc.EsbDeployment.stop(EsbDeployment.java:132)

              at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

              at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

              at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

              at java.lang.reflect.Method.invoke(Method.java:597)

              at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)

              at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)

              at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)

              at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:241)

              at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:47)

              at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:109)

              at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:70)

              at org.jboss.kernel.plugins.dependency.LifecycleAction.uninstallActionInternal(LifecycleAction.java:249)

              at org.jboss.kernel.plugins.dependency.InstallsAwareAction.uninstallAction(InstallsAwareAction.java:157)

              at org.jboss.kernel.plugins.dependency.InstallsAwareAction.uninstallAction(InstallsAwareAction.java:42)

              at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleUninstallAction(SimpleControllerContextAction.java:79)

              at org.jboss.dependency.plugins.action.AccessControllerContextAction.uninstall(AccessControllerContextAction.java:131)

              at org.jboss.dependency.plugins.AbstractControllerContextActions.uninstall(AbstractControllerContextActions.java:58)

              at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)

              at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)

              at org.jboss.system.ServiceController.doChange(ServiceController.java:688)

              at org.jboss.system.ServiceController.stop(ServiceController.java:510)

              at org.jboss.system.deployers.ServiceDeployer.stop(ServiceDeployer.java:170)

              at org.jboss.system.deployers.ServiceDeployer.undeploy(ServiceDeployer.java:150)

              at org.jboss.system.deployers.ServiceDeployer.undeploy(ServiceDeployer.java:46)

              at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalUndeploy(AbstractSimpleRealDeployer.java:69)

              at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.undeploy(AbstractRealDeployer.java:112)

              at org.jboss.deployers.plugins.deployers.DeployerWrapper.undeploy(DeployerWrapper.java:196)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.doUndeploy(DeployersImpl.java:1469)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1376)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1369)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1356)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.uninstall(DeployersImpl.java:1331)

              at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)

              at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)

              at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:694)

              at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:679)

              at org.jboss.system.server.profileservice.repository.MainDeployerAdapter.process(MainDeployerAdapter.java:117)

              at org.jboss.system.server.profileservice.repository.ProfileDeployAction.uninstall(ProfileDeployAction.java:95)

              at org.jboss.system.server.profileservice.repository.AbstractProfileAction.uninstall(AbstractProfileAction.java:70)

              at org.jboss.system.server.profileservice.repository.AbstractProfileService.uninstall(AbstractProfileService.java:375)

              at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)

              at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)

              at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)

              at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)

              at org.jboss.system.server.profileservice.repository.AbstractProfileService.deactivateProfile(AbstractProfileService.java:402)

              at org.jboss.system.server.profileservice.ProfileServiceBootstrap.deactivateProfiles(ProfileServiceBootstrap.java:388)

              at org.jboss.system.server.profileservice.ProfileServiceBootstrap.shutdown(ProfileServiceBootstrap.java:345)

              at org.jboss.bootstrap.AbstractServerImpl.shutdownServer(AbstractServerImpl.java:571)

              at org.jboss.bootstrap.AbstractServerImpl$ShutdownHook.run(AbstractServerImpl.java:909)

      12:52:26,539 INFO  [EsbDeployment] Destroying 'jbossesb.esb'

      12:52:26,540 INFO  [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED shutting down.

      12:52:26,540 INFO  [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED paused.

      12:52:26,540 INFO  [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED shutdown complete.

      12:52:26,540 WARN  [ServiceMessageCounterLifecycleResource] Calling cleanup on existing service message counters for identity ID-0

      12:52:26,540 WARN  [CourierFactory] Calling cleanup on existing couriers for identity ID-0

       

       

       

      Thread dump extract:

      (The JMSCourier is locked by the ESBScheduler thread below..)

       

      "JBoss Shutdown Hook" daemon prio=10 tid=0x0000000040d80800 nid=0x534b waiting for monitor entry [0x00007f32757d4000]

         java.lang.Thread.State: BLOCKED (on object monitor)

              at org.jboss.internal.soa.esb.couriers.JmsCourier.cleanup(JmsCourier.java:103)

              - waiting to lock <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)

              at org.jboss.soa.esb.couriers.CourierUtil.cleanCourier(CourierUtil.java:258)

              at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.cleanup(TwoWayCourierImpl.java:235)

              at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:241)

              at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:201)

              at org.jboss.soa.esb.lifecycle.LifecycleResource.cleanupLifecycleResource(LifecycleResource.java:209)

              at org.jboss.soa.esb.lifecycle.LifecycleResource.cleanupResource(LifecycleResource.java:154)

              at org.jboss.soa.esb.lifecycle.LifecycleResourceManager.cleanContextResources(LifecycleResourceManager.java:305)

              at org.jboss.soa.esb.lifecycle.LifecycleResourceManager.disassociateDeployment(LifecycleResourceManager.java:218)

              at org.jboss.soa.esb.listeners.deployers.mc.EsbDeployment.destroy(EsbDeployment.java:139)

              at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

              at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

              at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

              at java.lang.reflect.Method.invoke(Method.java:597)

              at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)

              at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)

              at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)

              at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:241)

       

      ...

       

      "ESBScheduler:jbossesb.esb_Worker-0" prio=10 tid=0x0000000042823000 nid=0x512b runnable [0x00007f327d1c0000]

         java.lang.Thread.State: RUNNABLE

              at java.net.PlainSocketImpl.socketConnect(Native Method)

              at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)

              - locked <0x00000000d3732880> (a java.net.SocksSocketImpl)

              at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)

              at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)

              at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)

              at java.net.Socket.connect(Socket.java:529)

              at java.net.Socket.connect(Socket.java:478)

              at java.net.Socket.<init>(Socket.java:375)

              at java.net.Socket.<init>(Socket.java:189)

              at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:22)

              at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:128)

              at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:595)

              at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:198)

              at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)

              at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:110)

              at org.jnp.server.NamingServer_Stub.lookup(Unknown Source)

              at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:726)

              at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:686)

              at javax.naming.InitialContext.lookup(InitialContext.java:392)

              at org.jboss.soa.esb.helpers.NamingContextPool.createContext(NamingContextPool.java:350)

              at org.jboss.soa.esb.helpers.NamingContextPool.getContext(NamingContextPool.java:304)

              at org.jboss.soa.esb.helpers.NamingContextPool.getContext(NamingContextPool.java:159)

              - locked <0x00000000e763c6d0> (a org.jboss.soa.esb.helpers.NamingContextPool)

              at org.jboss.soa.esb.helpers.NamingContextPool.getNamingContext(NamingContextPool.java:410)

              at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.getFactoryConnection(JmsConnectionPool.java:310)

              at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.internalGetSession(JmsConnectionPool.java:222)

              - locked <0x00000000e7882f00> (a org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool)

              at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.getSession(JmsConnectionPool.java:194)

              - locked <0x00000000e7882f00> (a org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool)

              at org.jboss.internal.soa.esb.couriers.JmsCourier.getJmsSession(JmsCourier.java:159)

              - locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)

              at org.jboss.internal.soa.esb.couriers.JmsCourier.createMessageProducer(JmsCourier.java:385)

              - locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)

              at org.jboss.internal.soa.esb.couriers.JmsCourier.internalDeliver(JmsCourier.java:209)

              - locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)

              at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:181)

              at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.deliver(TwoWayCourierImpl.java:189)

              at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.attemptDelivery(ServiceInvoker.java:667)

              at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.access$200(ServiceInvoker.java:569)

              at org.jboss.soa.esb.client.ServiceInvoker.post(ServiceInvoker.java:359)

              at org.jboss.soa.esb.client.ServiceInvoker.deliverAsync(ServiceInvoker.java:254)

              at org.jboss.internal.soa.esb.persistence.format.db.DBMessageStoreImpl.redeliver(DBMessageStoreImpl.java:415)

              at org.jboss.soa.esb.actions.MessageRedeliverer.process(MessageRedeliverer.java:74)

              at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.processPipeline(ActionProcessingPipeline.java:634)

              at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.processPipeline(ActionProcessingPipeline.java:586)

              at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.process(ActionProcessingPipeline.java:420)

              at org.jboss.soa.esb.listeners.ScheduleListener.onSchedule(ScheduleListener.java:117)

              at org.jboss.soa.esb.listeners.lifecycle.AbstractScheduledManagedLifecycle$1.onSchedule(AbstractScheduledManagedLifecycle.java:68)

              at org.jboss.soa.esb.schedule.SchedulerJob$ESBScheduledJob.execute(SchedulerJob.java:289)

              at org.quartz.core.JobRunShell.run(JobRunShell.java:203)

              at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:520)