11 Replies Latest reply on Mar 7, 2012 10:45 AM by Joseph Fouts

    Exception when removing a node from a cluster

    Joseph Fouts Novice

      I have a 3 node standalone cluster that works fine.

      When I remove a node from the cluster I get the following WARN message then an exception.

      The exception is only displayed on the servers console that was started first.

       

      Any ideas?

       

      Thanks.

       

       

      14:26:26,268 INFO  [class com.lidp.main.BGProcess_ADMB] (Thread-1610 (HornetQ-client-global-threads-372281913)) Processing Message ...... ADMB record work message 3

      14:26:26,277 INFO  [class com.lidp.main.BGController] (EJB default - 3) Ending BGController

      14:29:48,443 WARN  [org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver] (OOB-20,null) Could not write a cluster node removal message to channel Channel ID 24326ab2 (inbound) of Remoting connection 10f43005 to null: org.jboss.remoting3.NotOpenException: Writes closed

          at org.jboss.remoting3.remote.RemoteConnectionChannel.openOutboundMessage(RemoteConnectionChannel.java:107)

          at org.jboss.remoting3.remote.RemoteConnectionChannel.writeMessage(RemoteConnectionChannel.java:296)

          at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver$ClusterTopologyUpdateListener.sendClusterNodesRemoved(VersionOneProtocolChannelReceiver.java:343)

          at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver$ClusterTopologyUpdateListener.removedEntries(VersionOneProtocolChannelReceiver.java:336)

          at org.jboss.as.clustering.registry.RegistryService.removed(RegistryService.java:244)

          at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [rt.jar:1.6.0_27]

          at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) [rt.jar:1.6.0_27]

          at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) [rt.jar:1.6.0_27]

          at java.lang.reflect.Method.invoke(Method.java:597) [rt.jar:1.6.0_27]

          at org.jboss.as.clustering.infinispan.DefaultEmbeddedCacheManager$ClassLoaderAwareListener.event(DefaultEmbeddedCacheManager.java:367) [jboss-as-clustering-infinispan-7.1.1.Final-SNAPSHOT.jar:7.1.1.Final-SNAPSHOT]

          at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [rt.jar:1.6.0_27]

          at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) [rt.jar:1.6.0_27]

          at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) [rt.jar:1.6.0_27]

          at java.lang.reflect.Method.invoke(Method.java:597) [rt.jar:1.6.0_27]

          at org.infinispan.notifications.AbstractListenerImpl$ListenerInvocation$1.run(AbstractListenerImpl.java:199) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:41) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.notifications.AbstractListenerImpl$ListenerInvocation.invoke(AbstractListenerImpl.java:220) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.notifications.cachelistener.CacheNotifierImpl.notifyCacheEntryRemoved(CacheNotifierImpl.java:175) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.write.RemoveCommand.notify(RemoveCommand.java:114) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.write.RemoveCommand.perform(RemoveCommand.java:95) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.CallInterceptor.handleDefault(CallInterceptor.java:83) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitRemoveCommand(AbstractVisitor.java:65) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.write.RemoveCommand.acceptVisitor(RemoveCommand.java:71) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.ReplicationInterceptor.handleCrudMethod(ReplicationInterceptor.java:178) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.ReplicationInterceptor.visitRemoveCommand(ReplicationInterceptor.java:159) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.write.RemoveCommand.acceptVisitor(RemoveCommand.java:71) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.EntryWrappingInterceptor$EntryWrappingVisitor.visitRemoveCommand(EntryWrappingInterceptor.java:236) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.write.RemoveCommand.acceptVisitor(RemoveCommand.java:71) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:77) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:120) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:105) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:207) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:84) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:70) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:127) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.InboundInvocationHandlerImpl.handleInternal(InboundInvocationHandlerImpl.java:127) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithWaitForBlocks(InboundInvocationHandlerImpl.java:136) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithRetry(InboundInvocationHandlerImpl.java:162) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:114) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:161) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:141) [infinispan-core-5.1.1.FINAL.jar:5.1.1.FINAL]

          at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:447) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:354) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:230) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:543) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jboss.as.clustering.jgroups.ClassLoaderAwareUpHandler.up(ClassLoaderAwareUpHandler.java:56) [jboss-as-clustering-jgroups-7.1.1.Final-SNAPSHOT.jar:7.1.1.Final-SNAPSHOT]

          at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jboss.as.clustering.jgroups.MuxChannel$ClassLoaderAwareMuxUpHandler.up(MuxChannel.java:64) [jboss-as-clustering-jgroups-7.1.1.Final-SNAPSHOT.jar:7.1.1.Final-SNAPSHOT]

          at org.jgroups.JChannel.up(JChannel.java:716) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.pbcast.GMS.up(GMS.java:881) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:383) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:697) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:559) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.BARRIER.up(BARRIER.java:126) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:140) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.FD.up(FD.java:273) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:282) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.Discovery.up(Discovery.java:355) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.stack.Protocol.up(Protocol.java:358) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.TP.passMessageUp(TP.java:1174) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1722) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1704) [jgroups-3.0.5.Final.jar:3.0.5.Final]

          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_27]

          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_27]

          at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_27]