0 Replies Latest reply on Sep 10, 2009 11:27 AM by Mayank Mittal

    Failover falied with network failure

    Mayank Mittal Novice

      I have 2 JBoss 4.2.3GA in a cluster with Jboss Messaging 1.4.2 SP1, on network failover following exceptions are comming:


      <log4j:event logger="org.jboss.messaging.util.ExceptionUtil" timestamp="1252592516133" level="ERROR" thread="WorkManager(2)-134">
      <log4j:message><![CDATA[SessionEndpoint[ks1-wx38lfzf-1-twjzjfzf-hus5ub-s222e464] createConsumerDelegate [7z1-93r8lfzf-1-twjzjfzf-hus5ub-s222e464]]]></log4j:message>
      <log4j:throwable><![CDATA[java.lang.IllegalStateException: org.jboss.messaging.core.impl.postoffice.GroupMember@118d28b response not received from 150.158.74.137:2644 - there may be others
       at org.jboss.messaging.core.impl.postoffice.GroupMember.multicastControl(GroupMember.java:262)
       at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.internalAddBinding(MessagingPostOffice.java:1895)
       at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.addBinding(MessagingPostOffice.java:454)
       at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegateInternal(ServerSessionEndpoint.java:1909)
       at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegate(ServerSessionEndpoint.java:262)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$createConsumerDelegate$aop(SessionAdvised.java:94)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.container.SecurityAspect.handleCreateConsumerDelegate(SecurityAspect.java:123)
       at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
       at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
       at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
       at java.lang.reflect.Method.invoke(Method.java:597)
       at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised.createConsumerDelegate(SessionAdvised.java)
       at org.jboss.jms.wireformat.SessionCreateConsumerDelegateRequest.serverInvoke(SessionCreateConsumerDelegateRequest.java:100)
       at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143)
       at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:908)
       at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:106)
       at org.jboss.remoting.Client.invoke(Client.java:1683)
       at org.jboss.remoting.Client.invoke(Client.java:607)
       at org.jboss.remoting.Client.invoke(Client.java:593)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
       at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$createConsumerDelegate$aop(ClientSessionDelegate.java:267)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.StateCreationAspect.handleCreateConsumerDelegate(StateCreationAspect.java:142)
       at org.jboss.aop.advice.org.jboss.jms.client.container.StateCreationAspect30.invoke(StateCreationAspect30.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.ConsumerAspect.handleCreateConsumerDelegate(ConsumerAspect.java:76)
       at org.jboss.aop.advice.org.jboss.jms.client.container.ConsumerAspect29.invoke(ConsumerAspect29.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate.createConsumerDelegate(ClientSessionDelegate.java)
       at org.jboss.jms.client.JBossConnectionConsumer.<init>(JBossConnectionConsumer.java:126)
       at org.jboss.jms.client.container.SessionAspect.handleCreateConnectionConsumer(SessionAspect.java:772)
       at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect1.invoke(SessionAspect1.java)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
       at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate.createConnectionConsumer(ClientConnectionDelegate.java)
       at org.jboss.jms.client.JBossConnection.createConnectionConsumer(JBossConnection.java:140)
       at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.setupConsumer(JmsServerSessionPool.java:284)
       at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.start(JmsServerSessionPool.java:89)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation.setupSessionPool(JmsActivation.java:585)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation.setup(JmsActivation.java:323)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation$SetupActivation.run(JmsActivation.java:635)
       at org.jboss.resource.work.WorkWrapper.execute(WorkWrapper.java:204)
       at org.jboss.util.threadpool.BasicTaskWrapper.run(BasicTaskWrapper.java:275)
       at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:756)
       at java.lang.Thread.run(Thread.java:619)
      ]]></log4j:throwable>
      </log4j:event>
      
      <log4j:event logger="org.jboss.resource.adapter.jms.inflow.JmsActivation" timestamp="1252592516133" level="WARN" thread="WorkManager(2)-134">
      <log4j:message><![CDATA[Failure in jms activation org.jboss.resource.adapter.jms.inflow.JmsActivationSpec@750455(ra=org.jboss.resource.adapter.jms.JmsResourceAdapter@115d34f destination=topic/deviceDiscovery isTopic=true tx=true durable=false reconnect=10 provider=java:/DefaultJMSProvider user=null maxMessages=1 minSession=1 maxSession=1 keepAlive=60000 useDLQ=true DLQHandler=org.jboss.resource.adapter.jms.inflow.dlq.GenericDLQHandler DLQJndiName=queue/DLQ DLQUser=null DLQMaxResent=5)]]></log4j:message>
      <log4j:throwable><![CDATA[org.jboss.jms.exception.MessagingJMSException: A failure has occurred during processing of the request. Please consult the server logs for more details. SessionEndpoint[ks1-wx38lfzf-1-twjzjfzf-hus5ub-s222e464] createConsumerDelegate [7z1-93r8lfzf-1-twjzjfzf-hus5ub-s222e464]
       at org.jboss.messaging.util.ExceptionUtil.handleJMSInvocation(ExceptionUtil.java:72)
       at org.jboss.jms.server.endpoint.ServerSessionEndpoint.createConsumerDelegate(ServerSessionEndpoint.java:268)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised.org$jboss$jms$server$endpoint$advised$SessionAdvised$createConsumerDelegate$aop(SessionAdvised.java:94)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.container.SecurityAspect.handleCreateConsumerDelegate(SecurityAspect.java:123)
       at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
       at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
       at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
       at java.lang.reflect.Method.invoke(Method.java:597)
       at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:121)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.container.ServerLogInterceptor.invoke(ServerLogInterceptor.java:105)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised$createConsumerDelegate_8721389917985689973.invokeNext(SessionAdvised$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.server.endpoint.advised.SessionAdvised.createConsumerDelegate(SessionAdvised.java)
       at org.jboss.jms.wireformat.SessionCreateConsumerDelegateRequest.serverInvoke(SessionCreateConsumerDelegateRequest.java:100)
       at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:143)
       at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:908)
       at org.jboss.remoting.transport.local.LocalClientInvoker.invoke(LocalClientInvoker.java:106)
       at org.jboss.remoting.Client.invoke(Client.java:1683)
       at org.jboss.remoting.Client.invoke(Client.java:607)
       at org.jboss.remoting.Client.invoke(Client.java:593)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
       at org.jboss.jms.client.delegate.ClientSessionDelegate.org$jboss$jms$client$delegate$ClientSessionDelegate$createConsumerDelegate$aop(ClientSessionDelegate.java:267)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.StateCreationAspect.handleCreateConsumerDelegate(StateCreationAspect.java:142)
       at org.jboss.aop.advice.org.jboss.jms.client.container.StateCreationAspect30.invoke(StateCreationAspect30.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.ConsumerAspect.handleCreateConsumerDelegate(ConsumerAspect.java:76)
       at org.jboss.aop.advice.org.jboss.jms.client.container.ConsumerAspect29.invoke(ConsumerAspect29.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientSessionDelegate$createConsumerDelegate_8721389917985689973.invokeNext(ClientSessionDelegate$createConsumerDelegate_8721389917985689973.java)
       at org.jboss.jms.client.delegate.ClientSessionDelegate.createConsumerDelegate(ClientSessionDelegate.java)
       at org.jboss.jms.client.JBossConnectionConsumer.<init>(JBossConnectionConsumer.java:126)
       at org.jboss.jms.client.container.SessionAspect.handleCreateConnectionConsumer(SessionAspect.java:772)
       at org.jboss.aop.advice.org.jboss.jms.client.container.SessionAspect1.invoke(SessionAspect1.java)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
       at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.invokeNext(ClientConnectionDelegate$createConnectionConsumer_7915292773999367429.java)
       at org.jboss.jms.client.delegate.ClientConnectionDelegate.createConnectionConsumer(ClientConnectionDelegate.java)
       at org.jboss.jms.client.JBossConnection.createConnectionConsumer(JBossConnection.java:140)
       at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.setupConsumer(JmsServerSessionPool.java:284)
       at org.jboss.resource.adapter.jms.inflow.JmsServerSessionPool.start(JmsServerSessionPool.java:89)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation.setupSessionPool(JmsActivation.java:585)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation.setup(JmsActivation.java:323)
       at org.jboss.resource.adapter.jms.inflow.JmsActivation$SetupActivation.run(JmsActivation.java:635)
       at org.jboss.resource.work.WorkWrapper.execute(WorkWrapper.java:204)
       at org.jboss.util.threadpool.BasicTaskWrapper.run(BasicTaskWrapper.java:275)
       at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:756)
       at java.lang.Thread.run(Thread.java:619)
      ]]></log4j:throwable>
      </log4j:event>
      
      <log4j:event logger="org.jboss.jms.client.container.ClientConsumer" timestamp="1252592518695" level="ERROR" thread="Thread-109">
      <log4j:message><![CDATA[Failed to send changeRate message]]></log4j:message>
      <log4j:throwable><![CDATA[org.jboss.jms.exception.MessagingNetworkFailureException
       at org.jboss.jms.client.delegate.DelegateSupport.handleThrowable(DelegateSupport.java:240)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:205)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:160)
       at org.jboss.jms.client.delegate.ClientConsumerDelegate.org$jboss$jms$client$delegate$ClientConsumerDelegate$changeRate$aop(ClientConsumerDelegate.java:138)
       at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
       at org.jboss.jms.client.container.FailoverValveInterceptor.invoke(FailoverValveInterceptor.java:92)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
       at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:170)
       at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:105)
       at org.jboss.jms.client.delegate.ClientConsumerDelegate$changeRate_N952316153687074823.invokeNext(ClientConsumerDelegate$changeRate_N952316153687074823.java)
       at org.jboss.jms.client.delegate.ClientConsumerDelegate.changeRate(ClientConsumerDelegate.java)
       at org.jboss.jms.client.container.ClientConsumer.sendChangeRateMessage(ClientConsumer.java:754)
       at org.jboss.jms.client.container.ClientConsumer.resume(ClientConsumer.java:690)
       at org.jboss.messaging.core.impl.clusterconnection.MessageSucker.setConsuming(MessageSucker.java:208)
       at org.jboss.messaging.core.impl.MessagingQueue.informSuckers(MessagingQueue.java:566)
       at org.jboss.messaging.core.impl.MessagingQueue.deliverInternal(MessagingQueue.java:516)
       at org.jboss.messaging.core.impl.ChannelSupport.deliver(ChannelSupport.java:368)
       at org.jboss.jms.server.endpoint.ServerSessionEndpoint$2.run(ServerSessionEndpoint.java:1561)
       at org.jboss.messaging.util.OrderedExecutorFactory$ChildExecutor.run(OrderedExecutorFactory.java:120)
       at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
       at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
       at java.lang.Thread.run(Thread.java:619)
      Caused by: org.jboss.remoting.CannotConnectException: Can not get connection to server. Problem establishing socket connection for InvokerLocator [bisocket://150.158.74.137:4457/?JBM_clientMaxPoolSize=200&clientLeasePeriod=5000&clientSocketClass=org.jboss.jms.client.remoting.ClientSocketWrapper&dataType=jms&marshaller=org.jboss.jms.wireformat.JMSWireFormat&numberOfCallRetries=1&numberOfRetries=10&pingFrequency=214748364&pingWindowFactor=10&socket.check_connection=false&stopLeaseOnFailure=true&timeout=0&unmarshaller=org.jboss.jms.wireformat.JMSWireFormat&validatorPingPeriod=2000&validatorPingTimeout=1000]
       at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:771)
       at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.transport(BisocketClientInvoker.java:423)
       at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:175)
       at org.jboss.remoting.Client.invoke(Client.java:1683)
       at org.jboss.remoting.Client.invoke(Client.java:607)
       at org.jboss.remoting.Client.invoke(Client.java:593)
       at org.jboss.jms.client.delegate.DelegateSupport.doInvoke(DelegateSupport.java:189)
       ... 21 more
      Caused by: java.net.ConnectException: Connection timed out: connect
       at java.net.PlainSocketImpl.socketConnect(Native Method)
       at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)
       at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)
       at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)
       at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)
       at java.net.Socket.connect(Socket.java:519)
       at org.jboss.remoting.util.SecurityUtility.connect(SecurityUtility.java:969)
       at org.jboss.remoting.transport.socket.SocketClientInvoker.createSocket(SocketClientInvoker.java:203)
       at org.jboss.remoting.transport.bisocket.BisocketClientInvoker.createSocket(BisocketClientInvoker.java:430)
       at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.getConnection(MicroSocketClientInvoker.java:1084)
       at org.jboss.remoting.transport.socket.MicroSocketClientInvoker.transport(MicroSocketClientInvoker.java:757)
       ... 27 more
      ]]></log4j:throwable>
      </log4j:event>
      
      <log4j:event logger="org.jboss.remoting.transport.socket.MicroSocketClientInvoker" timestamp="1252592519836" level="WARN" thread="WorkerThread#12[150.158.74.235:2671]">
      <log4j:message><![CDATA[numberOfRetries is no longer used]]></log4j:message>
      </log4j:event>
      
      <log4j:event logger="org.jboss.jms.server.connectionmanager.SimpleConnectionManager" timestamp="1252592520805" level="WARN" thread="Timer-19">
      <log4j:message><![CDATA[A problem has been detected with the connection to remote client 464e223t-w3dotv-fzfj1zg6-1-fzfk093t-1ov, jmsClientID=5-gpm2jfzf-1-6gz1jfzf-vtod3w-t322e464. It is possible the client has exited without closing its connection(s) or the network has failed. All associated connection resources will be cleaned up.]]></log4j:message>
      </log4j:event>
      
      <log4j:event logger="org.jboss.resource.adapter.jms.inflow.JmsActivation" timestamp="1252592526148" level="INFO" thread="WorkManager(2)-134">
      <log4j:message><![CDATA[Attempting to reconnect org.jboss.resource.adapter.jms.inflow.JmsActivationSpec@750455(ra=org.jboss.resource.adapter.jms.JmsResourceAdapter@115d34f destination=topic/deviceDiscovery isTopic=true tx=true durable=false reconnect=10 provider=java:/DefaultJMSProvider user=null maxMessages=1 minSession=1 maxSession=1 keepAlive=60000 useDLQ=true DLQHandler=org.jboss.resource.adapter.jms.inflow.dlq.GenericDLQHandler DLQJndiName=queue/DLQ DLQUser=null DLQMaxResent=5)]]></log4j:message>
      </log4j:event>