Uneven data distribution across the cluster with virtual nodes
agnihere Mar 21, 2012 1:12 AMHi,
We are using Infinispan5.1.2.Final and we are still experiencing significantly un-even distribution of data across the cluster even after using virtual nodes as specified in the documentation - https://docs.jboss.org/author/display/ISPN/Clustering+modes#Clusteringmodes-VirtualNodesImprovingthedistributionofdata
We have about 320 odd keys and we went upto 500 virtual nodes but we still saw keys distributed as (160, 80, 80) on a 3-node cluster.
Any pointers on how we can further improve distibution would be greatly appreciated.
Thanks.
Following is how our configuration looks:
tns:namedCacheConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<tns:locking writeSkewCheck="false" useLockStriping="false" lockAcquisitionTimeout="300000" isolationLevel="READ_COMMITTED" concurrencyLevel="5000"/>
<tns:loaders shared="false" preload="false" passivation="false"/>
<tns:transaction useSynchronization="false" useEagerLocking="false" use1PcForAutoCommitTransactions="false" transactionMode="TRANSACTIONAL" syncRollbackPhase="true" syncCommitPhase="true" lockingMode="OPTIMISTIC" eagerLockSingleNode="false" cacheStopTimeout="30000" autoCommit="true">
<tns:recovery recoveryInfoCacheName="__recoveryInfoCacheName__" enabled="false"/>
</tns:transaction>
<tns:customInterceptors/>
<tns:dataContainer class="org.infinispan.container.DefaultDataContainer">
<tns:properties/>
</tns:dataContainer>
<tns:eviction wakeUpInterval="-9223372036854775808" threadPolicy="DEFAULT" strategy="NONE" maxEntries="-1"/>
<tns:expiration wakeUpInterval="-1" maxIdle="-1" lifespan="-1" reaperEnabled="true"/>
<tns:unsafe unreliableReturnValues="false"/>
<tns:clustering>
<tns:sync replTimeout="300000"/>
<tns:stateRetrieval timeout="300000" retryWaitTimeIncreaseFactor="2" numRetries="10" maxNonProgressingLogWrites="100" logFlushTimeout="60000" initialRetryWaitTime="500" fetchInMemoryState="false" alwaysProvideInMemoryState="false">
<tns:chunkSize>1000</tns:chunkSize>
</tns:stateRetrieval>
<tns:l1 onRehash="true" lifespan="60000" invalidationThreshold="0" enabled="true"/>
<tns:async useReplQueue="false" replQueueMaxElements="1000" replQueueInterval="5000" replQueueClass="org.infinispan.remoting.ReplicationQueueImpl" asyncMarshalling="false"/>
<tns:hash rehashWait="60000" rehashRpcTimeout="600000" rehashEnabled="true" numVirtualNodes="500" numOwners="1" hashFunctionClass="org.infinispan.commons.hash.MurmurHash3">
<tns:groups enabled="true"/>
</tns:hash>
</tns:clustering>
<tns:jmxStatistics enabled="true"/>
<tns:storeAsBinary storeValuesAsBinary="true" storeKeysAsBinary="true" enabled="false"/>
<tns:lazyDeserialization enabled="false"/>
<tns:deadlockDetection spinDuration="100" enabled="false"/>
<tns:indexing indexLocalOnly="false" enabled="false">
<tns:properties/>
</tns:indexing>
<tns:versioning versioningScheme="NONE" enabled="false"/>
<tns:invocationBatching enabled="true"/>
</tns:namedCacheConfiguration>
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<tns:globalConfiguration xmlns:tns="urn:infinispan:config:5.1" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<tns:asyncListenerExecutor factory="org.infinispan.executors.DefaultExecutorFactory">
<tns:properties/>
</tns:asyncListenerExecutor>
<tns:asyncTransportExecutor factory="org.infinispan.executors.DefaultExecutorFactory">
<tns:properties>
<tns:property value="asyncTransportThread" name="threadNamePrefix"/>
</tns:properties>
</tns:asyncTransportExecutor>
<tns:evictionScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">
<tns:properties/>
</tns:evictionScheduledExecutor>
<tns:replicationQueueScheduledExecutor factory="org.infinispan.executors.DefaultScheduledExecutorFactory">
<tns:properties/>
</tns:replicationQueueScheduledExecutor>
<tns:globalJmxStatistics mBeanServerLookup="org.infinispan.jmx.PlatformMBeanServerLookup" jmxDomain="org.infinispan" enabled="true" cacheManagerName="DefaultCacheManager" allowDuplicateDomains="true">
<tns:properties/>
</tns:globalJmxStatistics>
<tns:transport transportClass="org.infinispan.remoting.transport.jgroups.JGroupsTransport" strictPeerToPeer="false" distributedSyncTimeout="60000" clusterName="PERFpivotCluster">
<tns:properties>
<tns:property value="10000" name="jgroups.merge2.min_interval"/>
<tns:property value="true" name="jgroups.tcp.thread_pool.rejection_policy"/>
<tns:property value="discard" name="jgroups.tcp.oob_thread_pool.rejection_policy"/>
<tns:property value="discard" name="jgroups.tcp.discard_incompatible_packets"/>
<tns:property value="15000" name="jgroups.fd_all.timeout"/>
<tns:property value="500" name="jgroups.pbcast.gms.max_bundling_time"/>
<tns:property value="10" name="jgroups.tcp.thread_pool.min_threads"/>
<tns:property value="8" name="jgroups.tcp.oob_thread_pool.min_threads"/>
<tns:property value="25" name="listener.maxThreads"/>
<tns:property value="true" name="jgroups.barrier"/>
<tns:property value="600,1200" name="jgroups.pbcast.nakack.retransmit_timeout"/>
<tns:property value="300,600,1200" name="jgroups.pbcast.unicast.timeout"/>
<tns:property value="true" name="jgroups.pbcast.nakack.discard_delivered_msgs"/>
<tns:property value="false" name="jgroups.tcp.oob_thread_pool.queue_enabled"/>
<tns:property value="true" name="jgroups.pbcast.NAKACK.use_mcast_xmit"/>
<tns:property value="3000" name="jgroups.pbcast.gms.join_timeout"/>
<tns:property value="500000" name="jgroups.fc.max_credits"/>
<tns:property value="1000000" name="jgroups.pbcast.stable.max_bytes"/>
<tns:property value="jgroupsBigDataTcpStatic.xml" name="configurationFile"/>
<tns:property value="640000" name="jgroups.tcp.send_buf_size"/>
<tns:property value="true" name="jgroups.tcp.thread_pool.enabled"/>
<tns:property value="5000" name="jgroups.ping.timeout"/>
<tns:property value="50000" name="jgroups.pbcast.stable.desired_avg_gossip"/>
<tns:property value="30" name="jgroups.tcp.port_range"/>
<tns:property value="true" name="jgroups.tcp.thread_pool.queue_enabled"/>
<tns:property value="1000" name="jgroups.pbcast.stable.stability_delay"/>
<tns:property value="true" name="jgroups.pbcast.gms.view_bundling"/>
<tns:property value="30000" name="jgroups.merge2.max_interval"/>
<tns:property value="100" name="jgroups.tcp.oob_thread_pool.queue_max_size"/>
<tns:property value="20000000" name="jgroups.tcp.recv_buf_size"/>
<tns:property value="false" name="jgroups.tcp.loopback"/>
<tns:property value="5000" name="jgroups.tcp.oob_thread_pool.keep_alive_time"/>
<tns:property value="60000" name="jgroups.frag2.frag_size"/>
<tns:property value="0" name="jgroups.pbcast.NAKACK.exponential_backoff"/>
<tns:property value="true" name="jgroups.fdsock"/>
<tns:property value="400" name="jgroups.tcp.oob_thread_pool.max_threads"/>
<tns:property value="300" name="jgroups.tcp.thread_pool.max_threads"/>
<tns:property value="10000" name="jgroups.tcp.thread_pool.queue_max_size"/>
<tns:property value="5000" name="jgroups.fd_all.interval"/>
<tns:property value="true" name="jgroups.pbcast.gms.print_local_addr"/>
<tns:property value="true" name="jgroups.pbcast.streaming_state_transfer"/>
<tns:property value="PERFpivotCluster" name="clusterName"/>
<tns:property value="15000" name="jgroups.verify_suspect.timeout"/>
<tns:property value="0" name="jgroups.pbcast.nakack.gc_lag"/>
<tns:property value="10" name="jgroups.udp.tos"/>
<tns:property value="4" name="jgroups.ping.num_initial_members"/>
<tns:property value="25" name="transport.maxThreads"/>
<tns:property value="0.20" name="jgroups.fc.min_threshold"/>
<tns:property value="5000" name="jgroups.tcp.thread_pool.keep_alive_time"/>
</tns:properties>
</tns:transport>
<tns:serialization version="5.1" marshallerClass="org.infinispan.marshall.VersionAwareMarshaller">
<tns:advancedExternalizers/>
</tns:serialization>
<tns:shutdown hookBehavior="DEFAULT"/>
</tns:globalConfiguration>