JBoss Messaging Locked Up
rtm333 May 4, 2007 10:59 AMHi,
Today we had a situation where JBoss Messaging (JBM 1.0.1.SP4 on JBoss AS 4.0.5.GA) ceased to send any messages to its clients and required a server restart.
This is an excerpt from a stack dump at the time of the problem:
"WorkerThread#25[10.196.0.39:4848]" prio=10 tid=0x01efa520 nid=0x6bc2 in Object.wait() [0x7d0ff000..0x7d0ffc70] at java.lang.Object.wait(Native Method) - waiting on <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at java.lang.Object.wait(Object.java:474) at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock.acquire(WriterPreferenceReadWriteLock.java:240) - locked <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at org.jboss.messaging.core.local.PointToMultipointRouter.add(PointToMultipointRouter.java:129) at org.jboss.messaging.core.local.Topic.add(Topic.java:106) at org.jboss.messaging.core.local.CoreSubscription.connect(CoreSubscription.java:102) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegate(ServerSessionEndpoint.java:283) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$createConsumerDelegate$aop(SessionAdvised.java:90) at sun.reflect.GeneratedMethodAccessor147.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:112) at org.jboss.jms.server.container.SecurityAspect.handleCreateConsumerDelegate(SecurityAspect.java:118) at sun.reflect.GeneratedMethodAccessor146.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:130) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.aop.Advisor.dynamicInvoke(Advisor.java:723) at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:101) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:127) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:715) at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:552) at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:377) at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:159) "WorkerThread#22[10.196.43.33:2668]" prio=10 tid=0x029100d0 nid=0x385b in Object.wait() [0x8338f000..0x8338fbf0] at java.lang.Object.wait(Native Method) - waiting on <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at java.lang.Object.wait(Object.java:474) at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock.acquire(WriterPreferenceReadWriteLock.java:240) - locked <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at org.jboss.messaging.core.local.PointToMultipointRouter.remove(PointToMultipointRouter.java:161) at org.jboss.messaging.core.local.Topic.remove(Topic.java:126) at org.jboss.messaging.core.local.CoreSubscription.disconnect(CoreSubscription.java:111) at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.close(ServerConsumerEndpoint.java:360) - locked <0x9c4bf920> (a java.lang.Object) at org.jboss.jms.server.endpoint.advised.ConsumerAdvised.org$jboss$jms$server$endpoint$advised$ConsumerAdvised$close$aop(ConsumerAdvised.java:59) at sun.reflect.GeneratedMethodAccessor159.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:112) at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.aop.Advisor.dynamicInvoke(Advisor.java:723) at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:101) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:127) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:715) at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:552) at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:377) at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:159) "WorkerThread#21[10.196.166.169:4603]" prio=10 tid=0x0174a560 nid=0x2716 in Object.wait() [0x8667f000..0x8667fa70] at java.lang.Object.wait(Native Method) - waiting on <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at java.lang.Object.wait(Object.java:474) at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock.acquire(WriterPreferenceReadWriteLock.java:240) - locked <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at org.jboss.messaging.core.local.PointToMultipointRouter.remove(PointToMultipointRouter.java:161) at org.jboss.messaging.core.local.Topic.remove(Topic.java:126) at org.jboss.messaging.core.local.CoreSubscription.disconnect(CoreSubscription.java:111) at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.close(ServerConsumerEndpoint.java:360) - locked <0xa0ede0f8> (a java.lang.Object) at org.jboss.jms.server.endpoint.advised.ConsumerAdvised.org$jboss$jms$server$endpoint$advised$ConsumerAdvised$close$aop(ConsumerAdvised.java:59) at sun.reflect.GeneratedMethodAccessor159.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:112) at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.aop.Advisor.dynamicInvoke(Advisor.java:723) at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:101) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:127) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:715) at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:552) at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:377) at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:159) "WorkerThread#9[10.196.167.238:4254]" prio=10 tid=0x008c6d60 nid=0xa16 in Object.wait() [0x87b1f000..0x87b1fa70] at java.lang.Object.wait(Native Method) - waiting on <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at java.lang.Object.wait(Object.java:474) at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock.acquire(WriterPreferenceReadWriteLock.java:240) - locked <0x984b3e80> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$WriterLock) at org.jboss.messaging.core.local.PointToMultipointRouter.remove(PointToMultipointRouter.java:161) at org.jboss.messaging.core.local.Topic.remove(Topic.java:126) at org.jboss.messaging.core.local.CoreSubscription.disconnect(CoreSubscription.java:111) at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.close(ServerConsumerEndpoint.java:360) - locked <0xa0e40190> (a java.lang.Object) at org.jboss.jms.server.endpoint.advised.ConsumerAdvised.org$jboss$jms$server$endpoint$advised$ConsumerAdvised$close$aop(ConsumerAdvised.java:59) at sun.reflect.GeneratedMethodAccessor159.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:112) at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.aop.Advisor.dynamicInvoke(Advisor.java:723) at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:101) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:127) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:715) at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:552) at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:377) at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:159) "Thread-197" prio=10 tid=0x01eba118 nid=0x46b waiting for monitor entry [0x8504f000..0x8504fbf0] at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:219) - waiting to lock <0xa0e40190> (a java.lang.Object) at org.jboss.messaging.core.local.SingleDestinationRouter.handle(SingleDestinationRouter.java:80) at org.jboss.messaging.core.ChannelSupport.push(ChannelSupport.java:1642) at org.jboss.messaging.core.ChannelSupport.deliverInternal(ChannelSupport.java:682) at org.jboss.messaging.core.ChannelSupport.handleInternal(ChannelSupport.java:843) at org.jboss.messaging.core.ChannelSupport$HandleRunnable.run(ChannelSupport.java:1732) at EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:89) at java.lang.Thread.run(Thread.java:595) "ClientPublishQ-4" prio=10 tid=0x01218828 nid=0xcf in Object.wait() [0x896ee000..0x896ef8f0] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:474) at EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$ReaderLock.acquire(WriterPreferenceReadWriteLock.java:163) - locked <0x984b3e70> (a EDU.oswego.cs.dl.util.concurrent.WriterPreferenceReadWriteLock$ReaderLock) at org.jboss.messaging.core.local.PointToMultipointRouter.handle(PointToMultipointRouter.java:81) at org.jboss.messaging.core.local.Topic.handle(Topic.java:86) at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:707) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:449) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:83) at sun.reflect.GeneratedMethodAccessor111.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:112) at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:152) at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:130) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105) at org.jboss.aop.joinpoint.MethodInvocation.invokeNext(MethodInvocation.java:101) at org.jboss.aop.Advisor.dynamicInvoke(Advisor.java:723) at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:101) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:127) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:715) at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101) at org.jboss.remoting.Client.invoke(Client.java:1414) at org.jboss.remoting.Client.invoke(Client.java:511) at org.jboss.jms.client.delegate.DelegateSupport.invoke(DelegateSupport.java:111) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java) at org.jboss.jms.client.container.TransactionAspect.handleSend(TransactionAspect.java:175) at org.jboss.aop.advice.org.jboss.jms.client.container.TransactionAspect17.invoke(TransactionAspect17.java) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:182) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:117) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java) at org.jboss.jms.client.container.ExceptionInterceptor.invoke(ExceptionInterceptor.java:69) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java) at org.jboss.jms.client.container.ClientLogInterceptor.invoke(ClientLogInterceptor.java:107) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java) at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java) at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:253) at sun.reflect.GeneratedMethodAccessor109.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:130) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:182) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:117) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ExceptionInterceptor.invoke(ExceptionInterceptor.java:69) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClientLogInterceptor.invoke(ClientLogInterceptor.java:107) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:172) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:220) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:147) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:138) at org.jboss.jms.client.JBossMessageProducer.publish(JBossMessageProducer.java:190)
The full stack dump is some 6000 lines long. If required I will happily provide it by mail.
This is an absolute show stopper for us!
Any ideas appreciated.