JBOSS ESB 4.7 - ESB hangs during shutdown in Linux
Environment:
JBOSS 5.1 (for JDK 1.6) + JBOSS ESB 4.7 - on 64 bit LINUX machine
Process was running in the foreground and I issued Ctrl +C to shutdown the server..
The process hangs after the last message below and I have to issue a kill -9 on the pid.
(Have included relevant parts of the thread dump below)
12:51:26,528 WARN [SmooksResource] Closing smooks resources for identity ID-8
12:51:26,535 INFO [EsbDeployment] Stopping 'jbossesb.esb'
12:52:26,537 WARN [StartStopLifecycleAction] Error during stop for jboss.esb.vfsfile:/usr/share/jboss/server/arjis_esb_node/deploy/jbossesb.esb/
org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleException: Thread still active
at org.jboss.soa.esb.listeners.lifecycle.AbstractThreadedManagedLifecycle.doDestroy(AbstractThreadedManagedLifecycle.java:168)
at org.jboss.soa.esb.listeners.lifecycle.AbstractManagedLifecycle.destroy(AbstractManagedLifecycle.java:259)
at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.destroyInstances(ManagedLifecycleController.java:192)
at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.stopAndDestroy(ManagedLifecycleController.java:256)
at org.jboss.soa.esb.listeners.lifecycle.ManagedLifecycleController.stop(ManagedLifecycleController.java:92)
at org.jboss.soa.esb.listeners.deployers.mc.EsbDeployment.stop(EsbDeployment.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)
at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)
at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:241)
at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:47)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:109)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:70)
at org.jboss.kernel.plugins.dependency.LifecycleAction.uninstallActionInternal(LifecycleAction.java:249)
at org.jboss.kernel.plugins.dependency.InstallsAwareAction.uninstallAction(InstallsAwareAction.java:157)
at org.jboss.kernel.plugins.dependency.InstallsAwareAction.uninstallAction(InstallsAwareAction.java:42)
at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleUninstallAction(SimpleControllerContextAction.java:79)
at org.jboss.dependency.plugins.action.AccessControllerContextAction.uninstall(AccessControllerContextAction.java:131)
at org.jboss.dependency.plugins.AbstractControllerContextActions.uninstall(AbstractControllerContextActions.java:58)
at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)
at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
at org.jboss.system.ServiceController.doChange(ServiceController.java:688)
at org.jboss.system.ServiceController.stop(ServiceController.java:510)
at org.jboss.system.deployers.ServiceDeployer.stop(ServiceDeployer.java:170)
at org.jboss.system.deployers.ServiceDeployer.undeploy(ServiceDeployer.java:150)
at org.jboss.system.deployers.ServiceDeployer.undeploy(ServiceDeployer.java:46)
at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalUndeploy(AbstractSimpleRealDeployer.java:69)
at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.undeploy(AbstractRealDeployer.java:112)
at org.jboss.deployers.plugins.deployers.DeployerWrapper.undeploy(DeployerWrapper.java:196)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doUndeploy(DeployersImpl.java:1469)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1376)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1369)
at org.jboss.deployers.plugins.deployers.DeployersImpl.doUninstallParentLast(DeployersImpl.java:1356)
at org.jboss.deployers.plugins.deployers.DeployersImpl.uninstall(DeployersImpl.java:1331)
at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)
at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:694)
at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:679)
at org.jboss.system.server.profileservice.repository.MainDeployerAdapter.process(MainDeployerAdapter.java:117)
at org.jboss.system.server.profileservice.repository.ProfileDeployAction.uninstall(ProfileDeployAction.java:95)
at org.jboss.system.server.profileservice.repository.AbstractProfileAction.uninstall(AbstractProfileAction.java:70)
at org.jboss.system.server.profileservice.repository.AbstractProfileService.uninstall(AbstractProfileService.java:375)
at org.jboss.dependency.plugins.AbstractControllerContext.uninstall(AbstractControllerContext.java:354)
at org.jboss.dependency.plugins.AbstractController.uninstall(AbstractController.java:1664)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1275)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1179)
at org.jboss.dependency.plugins.AbstractController.uninstallContext(AbstractController.java:1234)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:827)
at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
at org.jboss.system.server.profileservice.repository.AbstractProfileService.deactivateProfile(AbstractProfileService.java:402)
at org.jboss.system.server.profileservice.ProfileServiceBootstrap.deactivateProfiles(ProfileServiceBootstrap.java:388)
at org.jboss.system.server.profileservice.ProfileServiceBootstrap.shutdown(ProfileServiceBootstrap.java:345)
at org.jboss.bootstrap.AbstractServerImpl.shutdownServer(AbstractServerImpl.java:571)
at org.jboss.bootstrap.AbstractServerImpl$ShutdownHook.run(AbstractServerImpl.java:909)
12:52:26,539 INFO [EsbDeployment] Destroying 'jbossesb.esb'
12:52:26,540 INFO [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED shutting down.
12:52:26,540 INFO [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED paused.
12:52:26,540 INFO [QuartzScheduler] Scheduler ESBScheduler:jbossesb.esb_$_NON_CLUSTERED shutdown complete.
12:52:26,540 WARN [ServiceMessageCounterLifecycleResource] Calling cleanup on existing service message counters for identity ID-0
12:52:26,540 WARN [CourierFactory] Calling cleanup on existing couriers for identity ID-0
Thread dump extract:
(The JMSCourier is locked by the ESBScheduler thread below..)
"JBoss Shutdown Hook" daemon prio=10 tid=0x0000000040d80800 nid=0x534b waiting for monitor entry [0x00007f32757d4000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.jboss.internal.soa.esb.couriers.JmsCourier.cleanup(JmsCourier.java:103)
- waiting to lock <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)
at org.jboss.soa.esb.couriers.CourierUtil.cleanCourier(CourierUtil.java:258)
at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.cleanup(TwoWayCourierImpl.java:235)
at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:241)
at org.jboss.soa.esb.couriers.CourierFactory$LifecycleCourierFactory.destroyLifecycleResource(CourierFactory.java:201)
at org.jboss.soa.esb.lifecycle.LifecycleResource.cleanupLifecycleResource(LifecycleResource.java:209)
at org.jboss.soa.esb.lifecycle.LifecycleResource.cleanupResource(LifecycleResource.java:154)
at org.jboss.soa.esb.lifecycle.LifecycleResourceManager.cleanContextResources(LifecycleResourceManager.java:305)
at org.jboss.soa.esb.lifecycle.LifecycleResourceManager.disassociateDeployment(LifecycleResourceManager.java:218)
at org.jboss.soa.esb.listeners.deployers.mc.EsbDeployment.destroy(EsbDeployment.java:139)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)
at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)
at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)
at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:241)
...
"ESBScheduler:jbossesb.esb_Worker-0" prio=10 tid=0x0000000042823000 nid=0x512b runnable [0x00007f327d1c0000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)
- locked <0x00000000d3732880> (a java.net.SocksSocketImpl)
at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)
at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)
at java.net.Socket.connect(Socket.java:529)
at java.net.Socket.connect(Socket.java:478)
at java.net.Socket.<init>(Socket.java:375)
at java.net.Socket.<init>(Socket.java:189)
at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:22)
at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:128)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:595)
at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:198)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:184)
at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:110)
at org.jnp.server.NamingServer_Stub.lookup(Unknown Source)
at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:726)
at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:686)
at javax.naming.InitialContext.lookup(InitialContext.java:392)
at org.jboss.soa.esb.helpers.NamingContextPool.createContext(NamingContextPool.java:350)
at org.jboss.soa.esb.helpers.NamingContextPool.getContext(NamingContextPool.java:304)
at org.jboss.soa.esb.helpers.NamingContextPool.getContext(NamingContextPool.java:159)
- locked <0x00000000e763c6d0> (a org.jboss.soa.esb.helpers.NamingContextPool)
at org.jboss.soa.esb.helpers.NamingContextPool.getNamingContext(NamingContextPool.java:410)
at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.getFactoryConnection(JmsConnectionPool.java:310)
at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.internalGetSession(JmsConnectionPool.java:222)
- locked <0x00000000e7882f00> (a org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool)
at org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool.getSession(JmsConnectionPool.java:194)
- locked <0x00000000e7882f00> (a org.jboss.internal.soa.esb.rosetta.pooling.JmsConnectionPool)
at org.jboss.internal.soa.esb.couriers.JmsCourier.getJmsSession(JmsCourier.java:159)
- locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)
at org.jboss.internal.soa.esb.couriers.JmsCourier.createMessageProducer(JmsCourier.java:385)
- locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)
at org.jboss.internal.soa.esb.couriers.JmsCourier.internalDeliver(JmsCourier.java:209)
- locked <0x00000000e7877de0> (a org.jboss.internal.soa.esb.couriers.JmsCourier)
at org.jboss.internal.soa.esb.couriers.JmsCourier.deliver(JmsCourier.java:181)
at org.jboss.internal.soa.esb.couriers.TwoWayCourierImpl.deliver(TwoWayCourierImpl.java:189)
at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.attemptDelivery(ServiceInvoker.java:667)
at org.jboss.soa.esb.client.ServiceInvoker$EPRInvoker.access$200(ServiceInvoker.java:569)
at org.jboss.soa.esb.client.ServiceInvoker.post(ServiceInvoker.java:359)
at org.jboss.soa.esb.client.ServiceInvoker.deliverAsync(ServiceInvoker.java:254)
at org.jboss.internal.soa.esb.persistence.format.db.DBMessageStoreImpl.redeliver(DBMessageStoreImpl.java:415)
at org.jboss.soa.esb.actions.MessageRedeliverer.process(MessageRedeliverer.java:74)
at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.processPipeline(ActionProcessingPipeline.java:634)
at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.processPipeline(ActionProcessingPipeline.java:586)
at org.jboss.soa.esb.listeners.message.ActionProcessingPipeline.process(ActionProcessingPipeline.java:420)
at org.jboss.soa.esb.listeners.ScheduleListener.onSchedule(ScheduleListener.java:117)
at org.jboss.soa.esb.listeners.lifecycle.AbstractScheduledManagedLifecycle$1.onSchedule(AbstractScheduledManagedLifecycle.java:68)
at org.jboss.soa.esb.schedule.SchedulerJob$ESBScheduledJob.execute(SchedulerJob.java:289)
at org.quartz.core.JobRunShell.run(JobRunShell.java:203)
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:520)