0 Replies Latest reply on Sep 10, 2009 11:27 AM by mayankmit2002

    Failover falied with network failure

    mayankmit2002
    mayankmit2002 Sep 10, 2009 11:27 AM

      I have 2 JBoss 4.2.3GA in a cluster with Jboss Messaging 1.4.2 SP1, on network failover following exceptions are comming:


      <log4j:event logger="org.jboss.messaging.util.ExceptionUtil" timestamp="1252592516133" level="ERROR" thread="WorkManager(2)-134">
<log4j:message><![CDATA[SessionEndpoint[ks1-wx38lfzf-1-twjzjfzf-hus5ub-s222e464] createConsumerDelegate [7z1-93r8lfzf-1-twjzjfzf-hus5ub-s222e464]]]></log4j:message>
<log4j:throwable><![CDATA[java.lang.IllegalStateException: org.jboss.messaging.core.impl.postoffice.GroupMember@118d28b response not received from 150.158.74.137:2644 - there may be others
 at org.jboss.messaging.core.impl.postoffice.GroupMember.multicastControl(GroupMember.java:262)
 at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.internalAddBinding(MessagingPostOffice.java:1895)
 at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.addBinding(MessagingPostOffice.java:454)
 at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegateInternal(ServerSessionEndpoint.java:1909)
 at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegate(ServerSessionEndpoint.java:262)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$createConsumerDelegate$aop(SessionAdvised.java:94)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.container.SecurityAspect.handleCreateConsumerDelegate(SecurityAspect.java:123)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
 at java.lang.reflect.Method.invoke(Method.java:597)
 at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised.createConsumerDelegate(SessionAdvised.java)
 at org.jboss.jms.wireformat.SessionCreateConsumerDelegateRequest.serverInvoke(SessionCreateConsumerDelegateRequest.java:100)
 at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143)
 at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:908)
 at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:106)
 at org.jboss.remoting.Client.invoke(Client.java:1683)
 at org.jboss.remoting.Client.invoke(Client.java:607)
 at org.jboss.remoting.Client.invoke(Client.java:593)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
 at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$createConsumerDelegate$aop(ClientSessionDelegate.java:267)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.StateCreationAspect.handleCreateConsumerDelegate(StateCreationAspect.java:142)
 at org.jboss.aop.advice.org.jboss.jms.client.container.StateCreationAspect30.invoke(StateCreationAspect30.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.ConsumerAspect.handleCreateConsumerDelegate(ConsumerAspect.java:76)
 at org.jboss.aop.advice.org.jboss.jms.client.container.ConsumerAspect29.invoke(ConsumerAspect29.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate.createConsumerDelegate(ClientSessionDelegate.java)
 at org.jboss.jms.client.JBossConnectionConsumer.<init>(JBossConnectionConsumer.java:126)
 at org.jboss.jms.client.container.SessionAspect.handleCreateConnectionConsumer(SessionAspect.java:772)
 at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect1.invoke(SessionAspect1.java)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
 at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate.createConnectionConsumer(ClientConnectionDelegate.java)
 at org.jboss.jms.client.JBossConnection.createConnectionConsumer(JBossConnection.java:140)
 at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.setupConsumer(JmsServerSessionPool.java:284)
 at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.start(JmsServerSessionPool.java:89)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation.setupSessionPool(JmsActivation.java:585)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation.setup(JmsActivation.java:323)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation$SetupActivation.run(JmsActivation.java:635)
 at org.jboss.resource.work.WorkWrapper.execute(WorkWrapper.java:204)
 at org.jboss.util.threadpool.BasicTaskWrapper.run(BasicTaskWrapper.java:275)
 at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:756)
 at java.lang.Thread.run(Thread.java:619)
]]></log4j:throwable>
</log4j:event>

<log4j:event logger="org.jboss.resource.adapter.jms.inflow.JmsActivation" timestamp="1252592516133" level="WARN" thread="WorkManager(2)-134">
<log4j:message><![CDATA[Failure in jms activation org.jboss.resource.adapter.jms.inflow.JmsActivationSpec@750455(ra=org.jboss.resource.adapter.jms.JmsResourceAdapter@115d34f destination=topic/deviceDiscovery isTopic=true tx=true durable=false reconnect=10 provider=java:/DefaultJMSProvider user=null maxMessages=1 minSession=1 maxSession=1 keepAlive=60000 useDLQ=true DLQHandler=org.jboss.resource.adapter.jms.inflow.dlq.GenericDLQHandler DLQJndiName=queue/DLQ DLQUser=null DLQMaxResent=5)]]></log4j:message>
<log4j:throwable><![CDATA[org.jboss.jms.exception.MessagingJMSException: A failure has occurred during processing of the request. Please consult the server logs for more details. SessionEndpoint[ks1-wx38lfzf-1-twjzjfzf-hus5ub-s222e464] createConsumerDelegate [7z1-93r8lfzf-1-twjzjfzf-hus5ub-s222e464]
 at org.jboss.messaging.util.ExceptionUtil.handleJMSInvocation(ExceptionUtil.java:72)
 at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegate(ServerSessionEndpoint.java:268)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$createConsumerDelegate$aop(SessionAdvised.java:94)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.container.SecurityAspect.handleCreateConsumerDelegate(SecurityAspect.java:123)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
 at java.lang.reflect.Method.invoke(Method.java:597)
 at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.server.endpoint.advised.SessionAdvised.createConsumerDelegate(SessionAdvised.java)
 at org.jboss.jms.wireformat.SessionCreateConsumerDelegateRequest.serverInvoke(SessionCreateConsumerDelegateRequest.java:100)
 at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143)
 at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:908)
 at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:106)
 at org.jboss.remoting.Client.invoke(Client.java:1683)
 at org.jboss.remoting.Client.invoke(Client.java:607)
 at org.jboss.remoting.Client.invoke(Client.java:593)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
 at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$createConsumerDelegate$aop(ClientSessionDelegate.java:267)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.StateCreationAspect.handleCreateConsumerDelegate(StateCreationAspect.java:142)
 at org.jboss.aop.advice.org.jboss.jms.client.container.StateCreationAspect30.invoke(StateCreationAspect30.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.ConsumerAspect.handleCreateConsumerDelegate(ConsumerAspect.java:76)
 at org.jboss.aop.advice.org.jboss.jms.client.container.ConsumerAspect29.invoke(ConsumerAspect29.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
 at org.jboss.jms.client.delegate.ClientSessionDelegate.createConsumerDelegate(ClientSessionDelegate.java)
 at org.jboss.jms.client.JBossConnectionConsumer.<init>(JBossConnectionConsumer.java:126)
 at org.jboss.jms.client.container.SessionAspect.handleCreateConnectionConsumer(SessionAspect.java:772)
 at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect1.invoke(SessionAspect1.java)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
 at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
 at org.jboss.jms.client.delegate.ClientConnectionDelegate.createConnectionConsumer(ClientConnectionDelegate.java)
 at org.jboss.jms.client.JBossConnection.createConnectionConsumer(JBossConnection.java:140)
 at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.setupConsumer(JmsServerSessionPool.java:284)
 at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.start(JmsServerSessionPool.java:89)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation.setupSessionPool(JmsActivation.java:585)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation.setup(JmsActivation.java:323)
 at org.jboss.resource.adapter.jms.inflow.JmsActivation$SetupActivation.run(JmsActivation.java:635)
 at org.jboss.resource.work.WorkWrapper.execute(WorkWrapper.java:204)
 at org.jboss.util.threadpool.BasicTaskWrapper.run(BasicTaskWrapper.java:275)
 at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:756)
 at java.lang.Thread.run(Thread.java:619)
]]></log4j:throwable>
</log4j:event>

<log4j:event logger="org.jboss.jms.client.container.ClientConsumer" timestamp="1252592518695" level="ERROR" thread="Thread-109">
<log4j:message><![CDATA[Failed to send changeRate message]]></log4j:message>
<log4j:throwable><![CDATA[org.jboss.jms.exception.MessagingNetworkFailureException
 at org.jboss.jms.client.delegate.DelegateSupport.handleThrowable(DelegateSupport.java:240)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:205)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
 at org.jboss.jms.client.delegate.ClientConsumerDelegate.org$jboss$jms$client$delegate$ClientConsumerDelegate$changeRate$aop(ClientConsumerDelegate.java:138)
 at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
 at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
 at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
 at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
 at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
 at org.jboss.jms.client.delegate.ClientConsumerDelegate.changeRate(ClientConsumerDelegate.java)
 at org.jboss.jms.client.container.ClientConsumer.sendChangeRateMessage(ClientConsumer.java:754)
 at org.jboss.jms.client.container.ClientConsumer.resume(ClientConsumer.java:690)
 at org.jboss.messaging.core.impl.clusterconnection.MessageSucker.setConsuming(MessageSucker.java:208)
 at org.jboss.messaging.core.impl.MessagingQueue.informSuckers(MessagingQueue.java:566)
 at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:516)
 at org.jboss.messaging.core.impl.ChannelSupport.deliver(ChannelSupport.java:368)
 at org.jboss.jms.server.endpoint.ServerSessionEndpoint$2.run(ServerSessionEndpoint.java:1561)
 at org.jboss.messaging.util.OrderedExecutorFactory$ChildExecutor.run(OrderedExecutorFactory.java:120)
 at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
 at java.lang.Thread.run(Thread.java:619)
Caused by: org.jboss.remoting.CannotConnectException: Can not get connection to server. Problem establishing socket connection for InvokerLocator [bisocket://150.158.74.137:4457/?JBM_clientMaxPoolSize=200&clientLeasePeriod=5000&clientSocketClass=org.jboss.jms.client.remoting.ClientSocketWrapper&dataType=jms&marshaller=org.jboss.jms.wireformat.JMSWireFormat&numberOfCallRetries=1&numberOfRetries=10&pingFrequency=214748364&pingWindowFactor=10&socket.check_connection=false&stopLeaseOnFailure=true&timeout=0&unmarshaller=org.jboss.jms.wireformat.JMSWireFormat&validatorPingPeriod=2000&validatorPingTimeout=1000]
 at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:771)
 at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:423)
 at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:175)
 at org.jboss.remoting.Client.invoke(Client.java:1683)
 at org.jboss.remoting.Client.invoke(Client.java:607)
 at org.jboss.remoting.Client.invoke(Client.java:593)
 at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
 ... 21 more
Caused by: java.net.ConnectException: Connection timed out: connect
 at java.net.PlainSocketImpl.socketConnect(Native Method)
 at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)
 at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)
 at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)
 at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)
 at java.net.Socket.connect(Socket.java:519)
 at org.jboss.remoting.util.SecurityUtility.connect(SecurityUtility.java:969)
 at org.jboss.remoting.transport.socket.SocketClientInvoker.createSocket(SocketClientInvoker.java:203)
 at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.createSocket(BisocketClientInvoker.java:430)
 at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.getConnection(MicroSocketClientInvoker.java:1084)
 at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:757)
 ... 27 more
]]></log4j:throwable>
</log4j:event>

<log4j:event logger="org.jboss.remoting.transport.socket.MicroSocketClientInvoker" timestamp="1252592519836" level="WARN" thread="WorkerThread#12[150.158.74.235:2671]">
<log4j:message><![CDATA[numberOfRetries is no longer used]]></log4j:message>
</log4j:event>

<log4j:event logger="org.jboss.jms.server.connectionmanager.SimpleConnectionManager" timestamp="1252592520805" level="WARN" thread="Timer-19">
<log4j:message><![CDATA[A problem has been detected with the connection to remote client 464e223t-w3dotv-fzfj1zg6-1-fzfk093t-1ov, jmsClientID=5-gpm2jfzf-1-6gz1jfzf-vtod3w-t322e464. It is possible the client has exited without closing its connection(s) or the network has failed. All associated connection resources will be cleaned up.]]></log4j:message>
</log4j:event>

<log4j:event logger="org.jboss.resource.adapter.jms.inflow.JmsActivation" timestamp="1252592526148" level="INFO" thread="WorkManager(2)-134">
<log4j:message><![CDATA[Attempting to reconnect org.jboss.resource.adapter.jms.inflow.JmsActivationSpec@750455(ra=org.jboss.resource.adapter.jms.JmsResourceAdapter@115d34f destination=topic/deviceDiscovery isTopic=true tx=true durable=false reconnect=10 provider=java:/DefaultJMSProvider user=null maxMessages=1 minSession=1 maxSession=1 keepAlive=60000 useDLQ=true DLQHandler=org.jboss.resource.adapter.jms.inflow.dlq.GenericDLQHandler DLQJndiName=queue/DLQ DLQUser=null DLQMaxResent=5)]]></log4j:message>
</log4j:event>


      • 1585 Views
      • Tags: