Messaging blocked by long time-out
rtm333 Mar 10, 2009 8:50 AMHi,
We are using JBAS 4.2.2.GA with JBM 1.4.2.GA-SP1 and jboss-remoting-2.2.2.SP11-brew.jar.
The server (running on Linux) is publishing on several topics to some Windows clients. If the network connection to one of the clients fails, the updates to ALL clients stop until a time-out occurs after about 15 to 18 minutes!
Below is the stack trace of the server-side exception caused by the time-out. It should help to identify the offending operation that takes so long to time out.
Can I do anything to shorten this time-out substantially to, say, 10 secs?
Any help welcome.
13:16:36,249 DEBUG [MicroSocketClientInvoker] SocketClientInvoker[17edafa, bisocket://11.222.33.44:1604734513] got exception java.io.IOException: Connection timed out 13:16:36,249 DEBUG [SocketWrapper] ClientSocketWrapper[Socket[addr=/11.222.33.44,port=4444,localport=53531].112d47] closing 13:16:36,251 ERROR [SocketClientInvoker] Got marshalling exception, exiting java.io.IOException: Connection timed out at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) at java.net.SocketOutputStream.write(SocketOutputStream.java:136) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123) at java.io.DataOutputStream.flush(DataOutputStream.java:106) at org.jboss.jms.wireformat.ClientDelivery.write(ClientDelivery.java:93) at org.jboss.jms.wireformat.JMSWireFormat.write(JMSWireFormat.java:237) at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.versionedWrite(MicroSocketClientInvoker.java:971) at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:606) at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:422) at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:133) at org.jboss.remoting.Client.invoke(Client.java:1640) at org.jboss.remoting.Client.invoke(Client.java:554) at org.jboss.remoting.Client.invokeOneway(Client.java:604) at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallback(ServerInvokerCallbackHandler.java:826) at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallbackOneway(ServerInvokerCallbackHandler.java:697) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.performDelivery(ServerSessionEndpoint.java:1454) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.handleDelivery(ServerSessionEndpoint.java:1366) at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:327) at org.jboss.messaging.core.impl.RoundRobinDistributor.handle(RoundRobinDistributor.java:119) at org.jboss.messaging.core.impl.MessagingQueue$DistributorWrapper.handle(MessagingQueue.java:583) at org.jboss.messaging.core.impl.ClusterRoundRobinDistributor.handle(ClusterRoundRobinDistributor.java:79) at org.jboss.messaging.core.impl.ChannelSupport.deliverInternal(ChannelSupport.java:638) at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:506) at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:239) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2212) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:499) at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:763) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:399) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:866) at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1825) at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1836) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:815) at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101) at org.jboss.remoting.Client.invoke(Client.java:1640) at org.jboss.remoting.Client.invoke(Client.java:554) at org.jboss.remoting.Client.invokeOneway(Client.java:604) at org.jboss.remoting.Client.invokeOneway(Client.java:792) at org.jboss.remoting.Client.invokeOneway(Client.java:782) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:180) at org.jboss.jms.client.delegate.DelegateSupport.doInvokeOneway(DelegateSupport.java:165) at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$send$aop(ClientSessionDelegate.java:495) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.SessionAspect.handleSend(SessionAspect.java:661) at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect28.invoke(SessionAspect28.java) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java) at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:269) at org.jboss.aop.advice.org.jboss.jms.client.container.ProducerAspect39.invoke(ProducerAspect39.java) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:207) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:136) at org.jboss.jms.client.JBossMessageProducer.publish(JBossMessageProducer.java:177) [...] 13:16:36,256 DEBUG [DefaultCallbackErrorHandler] DefaultCallbackErrorHandler[SocketServerInvoker[0.0.0.0:11307]] handling java.rmi.MarshalException: Failed to communicate. Problem during marshalling/unmarshalling; nested exception is: java.io.IOException: Connection timed out. Number of errors so far 1 13:16:36,256 DEBUG [DefaultCallbackErrorHandler] DefaultCallbackErrorHandler[SocketServerInvoker[0.0.0.0:11307]] ignoring the callback error 13:16:36,256 ERROR [ServerInvokerCallbackHandler] Error handling callback java.rmi.MarshalException: Failed to communicate. Problem during marshalling/unmarshalling; nested exception is: java.io.IOException: Connection timed out at org.jboss.remoting.transport.socket.SocketClientInvoker.handleException(SocketClientInvoker.java:127) at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:689) at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:422) at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:133) at org.jboss.remoting.Client.invoke(Client.java:1640) at org.jboss.remoting.Client.invoke(Client.java:554) at org.jboss.remoting.Client.invokeOneway(Client.java:604) at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallback(ServerInvokerCallbackHandler.java:826) at org.jboss.remoting.callback.ServerInvokerCallbackHandler.handleCallbackOneway(ServerInvokerCallbackHandler.java:697) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.performDelivery(ServerSessionEndpoint.java:1454) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.handleDelivery(ServerSessionEndpoint.java:1366) at org.jboss.jms.server.endpoint.ServerConsumerEndpoint.handle(ServerConsumerEndpoint.java:327) at org.jboss.messaging.core.impl.RoundRobinDistributor.handle(RoundRobinDistributor.java:119) at org.jboss.messaging.core.impl.MessagingQueue$DistributorWrapper.handle(MessagingQueue.java:583) at org.jboss.messaging.core.impl.ClusterRoundRobinDistributor.handle(ClusterRoundRobinDistributor.java:79) at org.jboss.messaging.core.impl.ChannelSupport.deliverInternal(ChannelSupport.java:638) at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:506) at org.jboss.messaging.core.impl.ChannelSupport.handle(ChannelSupport.java:239) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.routeInternal(MessagingPostOffice.java:2212) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.route(MessagingPostOffice.java:499) at org.jboss.jms.server.endpoint.ServerConnectionEndpoint.sendMessage(ServerConnectionEndpoint.java:763) at org.jboss.jms.server.endpoint.ServerSessionEndpoint.send(ServerSessionEndpoint.java:399) at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$send$aop(SessionAdvised.java:87) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.container.SecurityAspect.handleSend(SecurityAspect.java:157) at sun.reflect.GeneratedMethodAccessor110.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121) at org.jboss.jms.server.endpoint.advised.SessionAdvised$send_7280680627620114891.invokeNext(SessionAdvised$send_7280680627620114891.java) at org.jboss.jms.server.endpoint.advised.SessionAdvised.send(SessionAdvised.java) at org.jboss.jms.wireformat.SessionSendRequest.serverInvoke(SessionSendRequest.java:95) at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:866) at org.jboss.remoting.ServerInvoker$1.run(ServerInvoker.java:1825) at org.jboss.jms.server.remoting.DirectThreadPool.run(DirectThreadPool.java:63) at org.jboss.remoting.ServerInvoker.handleOnewayInvocation(ServerInvoker.java:1836) at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:815) at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:101) at org.jboss.remoting.Client.invoke(Client.java:1640) at org.jboss.remoting.Client.invoke(Client.java:554) at org.jboss.remoting.Client.invokeOneway(Client.java:604) at org.jboss.remoting.Client.invokeOneway(Client.java:792) at org.jboss.remoting.Client.invokeOneway(Client.java:782) at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:180) at org.jboss.jms.client.delegate.DelegateSupport.doInvokeOneway(DelegateSupport.java:165) at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$send$aop(ClientSessionDelegate.java:495) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.SessionAspect.handleSend(SessionAspect.java:661) at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect28.invoke(SessionAspect28.java) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientSessionDelegate$send_6145266547759487588.invokeNext(ClientSessionDelegate$send_6145266547759487588.java) at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java) at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:269) at org.jboss.aop.advice.org.jboss.jms.client.container.ProducerAspect39.invoke(ProducerAspect39.java) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170) at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105) at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java) at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:207) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145) at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:136) at org.jboss.jms.client.JBossMessageProducer.publish(JBossMessageProducer.java:177) [...] Caused by: java.io.IOException: Connection timed out at java.net.SocketOutputStream.socketWrite0(Native Method) at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92) at java.net.SocketOutputStream.write(SocketOutputStream.java:136) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123) at java.io.DataOutputStream.flush(DataOutputStream.java:106) at org.jboss.jms.wireformat.ClientDelivery.write(ClientDelivery.java:93) at org.jboss.jms.wireformat.JMSWireFormat.write(JMSWireFormat.java:237) at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.versionedWrite(MicroSocketClientInvoker.java:971) at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:606) ... 72 more