DIST_SYNC exception bringing back a node
I am encountering a problem with infinispan 4.2.0 BETA1 using the following configuration on 3 nodes:
{code:xml}
<?xml version="1.0" encoding="UTF-8"?>
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:4.1 http://www.infinispan.org/schemas/infinispan-config-4.1.xsd"
xmlns="urn:infinispan:config:4.1">
<global>
<transport clusterName="scaleupCluster"/>
</global>
<default>
<locking concurrencyLevel="1000" lockAcquisitionTimeout="60000" />
<lazyDeserialization enabled="false" />
<expiration lifespan="6000000" maxIdle="3000000"/>
<eviction wakeUpInterval="500" maxEntries="16777216" strategy="LRU" threadPolicy="PIGGYBACK" />
</default>
<namedCache name="scaleupCache">
<clustering mode="distribution">
<hash numOwners="2" rehashRpcTimeout="120000"/>
<sync replTimeout="60000" />
<l1 enabled="false" />
</clustering>
</namedCache>
</infinispan>
{code}
I do the following:
# start infinispan on node 1
# start infinispan on node 2
# start infinispan on node 3
# insert 1000 elements in node 1
# observe there are now 891, 210 and 899 entries in nodes 1, 2 and 3 respectively.
# quit node 3
# observe there are now 1000 entries in each nodes 1 and 2
# bring back infinispan on node 3 and I get
{quote}
Exception in thread "main" java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.eclipse.jdt.internal.jarinjarloader.JarRsrcLoader.main(JarRsrcLoader.java:58)
Caused by: org.infinispan.CacheException: Unable to invoke method public void org.infinispan.distribution.DistributionManagerImpl.waitForJoinToComplete() throws java.lang.Throwable on object
at org.infinispan.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:173)
at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:861)
at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:680)
at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:582)
at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:148)
at org.infinispan.CacheDelegate.start(CacheDelegate.java:317)
at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:493)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:437)
at com.van.bwo.scm.test.scaleup.main(scaleup.java:139)
... 5 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.infinispan.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:170)
... 13 more
Caused by: org.infinispan.CacheException: Unexpected exception
at org.infinispan.distribution.JoinTask.performRehash(JoinTask.java:142)
at org.infinispan.distribution.RehashTask.call(RehashTask.java:53)
at org.infinispan.distribution.RehashTask.call(RehashTask.java:33)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.util.concurrent.ExecutionException: org.infinispan.CacheException: java.lang.UnsupportedOperationException: Not supported. Use getKeys() instead.
at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:222)
at java.util.concurrent.FutureTask.get(FutureTask.java:83)
at org.infinispan.util.concurrent.NotifyingFutureImpl.get(NotifyingFutureImpl.java:51)
at org.infinispan.distribution.RehashTask.invalidateInvalidHolders(RehashTask.java:93)
at org.infinispan.distribution.RehashTask.invalidateInvalidHolders(RehashTask.java:97)
at org.infinispan.distribution.JoinTask.performRehash(JoinTask.java:136)
... 7 more
Caused by: org.infinispan.CacheException: java.lang.UnsupportedOperationException: Not supported. Use getKeys() instead.
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:114)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:125)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:230)
at org.infinispan.remoting.rpc.RpcManagerImpl$1.call(RpcManagerImpl.java:249)
... 5 more
Caused by: java.lang.UnsupportedOperationException: Not supported. Use getKeys() instead.
at org.infinispan.commands.write.InvalidateCommand.getKey(InvalidateCommand.java:137)
at com.van.bwo.scm.test.CustomInterceptor.visitInvalidateCommand(CustomInterceptor.java:75)
at org.infinispan.commands.write.InvalidateCommand.acceptVisitor(InvalidateCommand.java:132)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:273)
at org.infinispan.commands.remote.BaseRpcInvokingCommand.processVisitableCommand(BaseRpcInvokingCommand.java:42)
at org.infinispan.commands.remote.SingleRpcCommand.perform(SingleRpcCommand.java:44)
at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:94)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:176)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:148)
at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:578)
at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:489)
at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:365)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:771)
at org.jgroups.JChannel.up(JChannel.java:1465)
at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:954)
at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:430)
at org.jgroups.protocols.pbcast.STREAMING_STATE_TRANSFER.up(STREAMING_STATE_TRANSFER.java:265)
at org.jgroups.protocols.FRAG2.up(FRAG2.java:190)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:419)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:419)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:888)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234)
at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:577)
at org.jgroups.protocols.UNICAST.up(UNICAST.java:295)
at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:707)
at org.jgroups.protocols.BARRIER.up(BARRIER.java:120)
at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:169)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:269)
at org.jgroups.protocols.MERGE2.up(MERGE2.java:210)
at org.jgroups.protocols.Discovery.up(Discovery.java:292)
at org.jgroups.protocols.PING.up(PING.java:67)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1093)
at org.jgroups.protocols.TP.access$100(TP.java:56)
at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1633)
at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1615)
...
{quote}
Any idea?
BTW I also observe this with 4.1 FINAL.
Thanks for your help!
- Edouard
