Cluster config: Channel per Cache instance?
dukehoops Mar 19, 2009 1:07 PMLooking for some best practices advice on configuring clustering with the following setup:
JBC 3
Hibernate 3.3.1
hib-jbc2 integration
4 Cache instances: entityCache, collectionCache, queryCache, timestampsCache
queryCache is LOCAL, three others are INVALIDATION_ASYNC
all cluster nodes in the same subnet
My main question is this: should I configure a distinct JGroups Channel for each cache instance or have all 3 instances share the same one?
I've read that, on one hand, Channels are heavyweight (use multiple threads) which would argue for latter approach. However with a single channel configured as shown below I do see cross-talking between the 3 caches of the same cluster:
[2/27/09 16:25:45:084] WARN org.jgroups.protocols.UDP - discarded message from different group "dgEntityCacheCluster" (our group is "dgCollectionCacheCluster"). Sender was 172.17.50.103:62010
[2/27/09 16:25:45:613] WARN org.jgroups.protocols.UDP - discarded message from different group "dgTimestampsCacheCluster" (our group is "dgEntityCacheCluster"). Sender was 172.17.50.103:62012
[2/27/09 16:25:45:613] WARN org.jgroups.protocols.UDP - discarded message from different group "dgTimestampsCacheCluster" (our group is "dgCollectionCacheCluster"). Sender was 172.17.50.103:62012
Here's my cache config:
<bean id="jbcRuntimeConfig" class="org.jboss.cache.config.RuntimeConfig" depends-on="transactionManager">
<property name="transactionManager" ref="BitronixTransactionManager"/>
</bean>
<bean id="baseCacheConfig" class="org.jboss.cache.config.Configuration" abstract="true" depends-on="jbcRuntimeConfig">
<!-- TODO: same cluster for all caches or one per cache? -->
<property name="jgroupsConfigFile" value="classpath:dg-jgroups-udp.xml"/>
<property name="runtimeConfig" ref="jbcRuntimeConfig"/>
<property name="cacheMode" value="LOCAL"/>
<property name="nodeLockingScheme" value="MVCC"/>
<property name="isolationLevel" value="READ_COMMITTED"/>
</bean>
<bean id="entityCacheConfig" parent="baseCacheConfig">
<property name="clusterName" value="dgEntityCacheCluster"/>
<property name="cacheMode" value="INVALIDATION_ASYNC"/>
</bean>
<bean id="collectionCacheConfig" parent="baseCacheConfig">
<property name="clusterName" value="dgCollectionCacheCluster"/>
<property name="cacheMode" value="INVALIDATION_ASYNC"/>
</bean>
<bean id="queryCacheConfig" parent="baseCacheConfig">
</bean>
<bean id="timestampsCacheConfig" parent="baseCacheConfig">
<property name="clusterName" value="dgTimestampsCacheCluster"/>
<property name="cacheMode" value="INVALIDATION_ASYNC"/>
</bean>
and jgroups config:
<!--
Default stack using IP multicasting. It is similar to the "udp"
stack in stacks.xml, but doesn't use streaming state transfer and flushing
author: Bela Ban
version: $Id: udp.xml,v 1.29 2008/09/26 15:59:33 belaban Exp $
-->
<!-- mcast_addr="${jgroups.udp.mcast_addr:232.10.10.10}" -->
<config>
<UDP
mcast_addr="${jgroups.udp.mcast_addr:232.10.10.10}"
mcast_port="${jgroups.udp.mcast_port:45588}"
tos="8"
ucast_recv_buf_size="20000000"
ucast_send_buf_size="640000"
mcast_recv_buf_size="25000000"
mcast_send_buf_size="640000"
loopback="false"
discard_incompatible_packets="true"
max_bundle_size="64000"
max_bundle_timeout="30"
ip_ttl="${jgroups.udp.ip_ttl:2}"
enable_bundling="true"
enable_diagnostics="true"
thread_naming_pattern="cl"
thread_pool.enabled="true"
thread_pool.min_threads="2"
thread_pool.max_threads="8"
thread_pool.keep_alive_time="5000"
thread_pool.queue_enabled="true"
thread_pool.queue_max_size="10000"
thread_pool.rejection_policy="discard"
oob_thread_pool.enabled="true"
oob_thread_pool.min_threads="1"
oob_thread_pool.max_threads="8"
oob_thread_pool.keep_alive_time="5000"
oob_thread_pool.queue_enabled="false"
oob_thread_pool.queue_max_size="100"
oob_thread_pool.rejection_policy="Run"/>
<PING timeout="2000"
num_initial_members="3"/>
<MERGE2 max_interval="30000"
min_interval="10000"/>
<FD_SOCK/>
<FD_ALL/>
<VERIFY_SUSPECT timeout="1500" />
<BARRIER />
<pbcast.NAKACK use_stats_for_retransmission="false"
exponential_backoff="150"
use_mcast_xmit="true" gc_lag="0"
retransmit_timeout="50,300,600,1200"
discard_delivered_msgs="true"/>
<UNICAST timeout="300,600,1200"/>
<pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
max_bytes="1000000"/>
<VIEW_SYNC avg_send_interval="60000" />
<pbcast.GMS print_local_addr="true" join_timeout="3000"
shun="false"
view_bundling="true"/>
<FC max_credits="500000"
min_threshold="0.20"/>
<FRAG2 frag_size="60000" />
<!--pbcast.STREAMING_STATE_TRANSFER /-->
<pbcast.STATE_TRANSFER />
<!-- pbcast.FLUSH /-->
</config>