TreeCache Initial state transfer failed!
We have upgraded from JBossCache 1.2.3 to 1.3.0SP1 running on JBoss version 4.0.3SP1. There are two servers running part of the same cluster. First server startup without any error. Second server throws the following exception when it is trying to initialize cache. Just to give additional info I gave Tomcat Session cache and TreeCache configured. I am using a partion name that is not default. You should be able to see my config files below as well:
[08-05-2006 19:36:06.226] [43073] [org.jboss.system.ServiceController] [WARN ] [main] Problem starting service jboss.cache:service
=TreeCache
org.jboss.cache.CacheException: Initial state transfer failed: Channel.getState() returned false
at org.jboss.cache.TreeCache.fetchStateOnStartup(TreeCache.java:2749)
at org.jboss.cache.TreeCache.startService(TreeCache.java:1330)
at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:274)
at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:230)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:72)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:943)
at $Proxy0.start(Unknown Source)
at org.jboss.system.ServiceController.start(ServiceController.java:428)
at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:72)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
at $Proxy4.start(Unknown Source)
at org.jboss.deployment.SARDeployer.start(SARDeployer.java:285)
at org.jboss.deployment.MainDeployer.start(MainDeployer.java:989)
at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:790)
at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:753)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:118)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:74)
at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:127)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:74)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
at $Proxy9.deploy(Unknown Source)
at org.jboss.deployment.scanner.URLDeploymentScanner.deploy(URLDeploymentScanner.java:319)
at org.jboss.deployment.scanner.URLDeploymentScanner.scan(URLDeploymentScanner.java:507)
at org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.doScan(AbstractDeploymentScanner.java:192)
at org.jboss.deployment.scanner.AbstractDeploymentScanner.startService(AbstractDeploymentScanner.java:265)
at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:274)
at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:230)
at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:72)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:943)
at $Proxy0.start(Unknown Source)
at org.jboss.system.ServiceController.start(ServiceController.java:428)
at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:72)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
at $Proxy4.start(Unknown Source)
at org.jboss.deployment.SARDeployer.start(SARDeployer.java:285)
at org.jboss.deployment.MainDeployer.start(MainDeployer.java:989)
at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:790)
at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:753)
at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:737)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:141)
at org.jboss.mx.server.Invocation.dispatch(Invocation.java:80)
at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:118)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:74)
at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:127)
at org.jboss.mx.server.Invocation.invoke(Invocation.java:74)
at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:245)
at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:644)
at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
at $Proxy5.deploy(Unknown Source)
at org.jboss.system.server.ServerImpl.doStart(ServerImpl.java:453)
at org.jboss.system.server.ServerImpl.start(ServerImpl.java:330)
at org.jboss.Main.boot(Main.java:187)
at org.jboss.Main$1.run(Main.java:438)
at java.lang.Thread.run(Thread.java:595)
[08-05-2006 19:36:07.988] [44835] [org.jboss.ha.framework.interfaces.HAPartition.BLKviewserver] [INFO ] [main] Initializing
[08-05-2006 19:36:08.177] [45024] [STDOUT] [INFO ] [UpHandler (GMS)]
-------------------------------------------------------
GMS: address is 172.16.52.34:36183 (additional data: 18 bytes)
-------------------------------------------------------
[08-05-2006 19:36:10.195] [47042] [org.jboss.ha.framework.interfaces.HAPartition.BLKviewserver] [INFO ] [main] Number of cluster m
embers: 2
[08-05-2006 19:36:10.195] [47042] [org.jboss.ha.framework.interfaces.HAPartition.BLKviewserver] [INFO ] [main] Other members: 1
[08-05-2006 19:36:10.196] [47043] [org.jboss.ha.framework.interfaces.HAPartition.BLKviewserver] [INFO ] [main] Fetching state (wil
l wait for 30000 milliseconds):
[08-05-2006 19:36:10.196] [47043] [org.jboss.ha.framework.interfaces.HAPartition.BLKviewserver] [INFO ] [Thread-31] New cluster vi
ew for partition BLKviewserver: 1 ([172.16.52.31:10159, 172.16.52.34:10159] delta: 0)
[08-05-2006 19:36:10.206] [47053] [org.jboss.ha.framework.server.DistributedReplicantManagerImpl.BLKviewserver] [INFO ] [Thread-31
] I am (null) received membershipChanged event:
[08-05-2006 19:36:10.206] [47053] [org.jboss.ha.framework.server.DistributedReplicantManagerImpl.BLKviewserver] [INFO ] [Thread-31
] Dead members: 0 ([])
[08-05-2006 19:36:10.207] [47054] [org.jboss.ha.framework.server.DistributedReplicantManagerImpl.BLKviewserver] [INFO ] [Thread-31
] New Members : 0 ([])
[08-05-2006 19:36:10.207] [47054] [org.jboss.ha.framework.server.DistributedReplicantManagerImpl.BLKviewserver] [INFO ] [Thread-31
] All Members : 2 ([172.16.52.31:10159, 172.16.52.34
Cluster-service.xml:
<?xml version="1.0" encoding="UTF-8"?>
<!-- -->
<!-- Sample Clustering Service Configuration -->
<!-- -->
<!-- ===================================================================== -->
<!-- ==================================================================== -->
<!-- Cluster Partition: defines cluster -->
<!-- ==================================================================== -->
<!-- Name of the partition being built -->
${jboss.partition.name}
<!-- The address used to determine the node name -->
${jboss.bind.address}
<!-- Determine if deadlock detection is enabled -->
False
<!-- Max time (in ms) to wait for state transfer to complete. Increase for large states -->
30000
<!-- The JGroups protocol configuration -->
<!--
The default UDP stack:
- If you have a multihomed machine, set the UDP protocol's bind_addr attribute to the
appropriate NIC IP address, e.g bind_addr="192.168.0.2".
- On Windows machines, because of the media sense feature being broken with multicast
(even after disabling media sense) set the UDP protocol's loopback attribute to true
-->
<UDP mcast_addr="228.1.2.3" mcast_port="45566"
ip_ttl="8" ip_mcast="true"
mcast_send_buf_size="800000" mcast_recv_buf_size="150000"
ucast_send_buf_size="800000" ucast_recv_buf_size="150000"
loopback="false"/>
<PING timeout="2000" num_initial_members="3"
up_thread="true" down_thread="true"/>
<MERGE2 min_interval="10000" max_interval="20000"/>
<FD shun="true" up_thread="true" down_thread="true"
timeout="2500" max_tries="5"/>
<VERIFY_SUSPECT timeout="3000" num_msgs="3"
up_thread="true" down_thread="true"/>
<pbcast.NAKACK gc_lag="50" retransmit_timeout="300,600,1200,2400,4800"
max_xmit_size="8192"
up_thread="true" down_thread="true"/>
<UNICAST timeout="300,600,1200,2400,4800" window_size="100" min_threshold="10"
down_thread="true"/>
<pbcast.STABLE desired_avg_gossip="20000"
up_thread="true" down_thread="true"/>
<FRAG frag_size="8192"
down_thread="true" up_thread="true"/>
<pbcast.GMS join_timeout="5000" join_retry_timeout="2000"
shun="true" print_local_addr="true"/>
<pbcast.STATE_TRANSFER up_thread="true" down_thread="true"/>
<!-- Alternate TCP stack: customize it for your environment, change bind_addr and initial_hosts -->
<!--
<TCP bind_addr="thishost" start_port="7800" loopback="true"/>
<TCPPING initial_hosts="thishost[7800],otherhost[7800]" port_range="3" timeout="3500"
num_initial_members="3" up_thread="true" down_thread="true"/>
<MERGE2 min_interval="5000" max_interval="10000"/>
<FD shun="true" timeout="2500" max_tries="5" up_thread="true" down_thread="true" />
<VERIFY_SUSPECT timeout="1500" down_thread="false" up_thread="false" />
<pbcast.NAKACK down_thread="true" up_thread="true" gc_lag="100"
retransmit_timeout="3000"/>
<pbcast.STABLE desired_avg_gossip="20000" down_thread="false" up_thread="false" />
<pbcast.GMS join_timeout="5000" join_retry_timeout="2000" shun="false"
print_local_addr="true" down_thread="true" up_thread="true"/>
<pbcast.STATE_TRANSFER up_thread="true" down_thread="true"/>
-->
<!-- ==================================================================== -->
<!-- HA Session State Service for SFSB -->
<!-- ==================================================================== -->
jboss:service=${jboss.partition.name}
<!-- Name of the partition to which the service is linked -->
${jboss.partition.name}
<!-- JNDI name under which the service is bound -->
/HASessionState/Default
<!-- Max delay before cleaning unreclaimed state.
Defaults to 30*60*1000 => 30 minutes -->
0
<!-- ==================================================================== -->
<!-- HA JNDI -->
<!-- ==================================================================== -->
jboss:service=${jboss.partition.name}
<!-- Name of the partition to which the service is linked -->
${jboss.partition.name}
<!-- Bind address of bootstrap and HA-JNDI RMI endpoints -->
${jboss.bind.address}
<!-- Port on which the HA-JNDI stub is made available -->
${myserver.service.HAJNDI.port}
<!-- Accept backlog of the bootstrap socket -->
50
<!-- The thread pool service used to control the bootstrap and
auto discovery lookups -->
<depends optional-attribute-name="LookupPool"
proxy-type="attribute">jboss.system:service=ThreadPool
<!-- A flag to disable the auto discovery via multicast -->
false
<!-- Set the auto-discovery bootstrap multicast bind address. If not
specified and a BindAddress is specified, the BindAddress will be used. -->
${jboss.bind.address}
<!-- Multicast Address and group port used for auto-discovery -->
230.0.0.4
1102
<!-- The TTL (time-to-live) for autodiscovery IP multicast packets -->
16
<!-- RmiPort to be used by the HA-JNDI service once bound. 0 => auto. -->
0
<!-- Client socket factory to be used for client-server
RMI invocations during JNDI queries
custom
-->
<!-- Server socket factory to be used for client-server
RMI invocations during JNDI queries
custom
-->
${jboss.bind.address}
<!-- 0
custom
custom
-->
<!-- the JRMPInvokerHA creates a thread per request. This implementation uses a pool of threads -->
1
300
300
60000
${jboss.bind.address}
${myserver.service.invoker-pooledha.port}
${jboss.bind.address}
0
false
<depends optional-attribute-name="TransactionManagerService">jboss:service=TransactionManager
<!-- ==================================================================== -->
<!-- ==================================================================== -->
<!-- Distributed cache invalidation -->
<!-- ==================================================================== -->
<!--
jboss:service=${jboss.partition.name}
jboss.cache:service=InvalidationManager
jboss.cache:service=InvalidationManager
${jboss.partition.name}
DefaultJGBridge
-->
tc5-cluster-service.xml:
<?xml version="1.0" encoding="UTF-8"?>
<!-- -->
<!-- Customized TreeCache Service Configuration for Tomcat 5 Clustering -->
<!-- -->
<!-- ===================================================================== -->
<!-- ==================================================================== -->
<!-- Defines TreeCache configuration -->
<!-- ==================================================================== -->
jboss:service=Naming
jboss:service=TransactionManager
<!-- Configure the TransactionManager -->
org.jboss.cache.JBossTransactionManagerLookup
<!--
Isolation level : SERIALIZABLE
REPEATABLE_READ (default)
READ_COMMITTED
READ_UNCOMMITTED
NONE
-->
REPEATABLE_READ
<!--
Valid modes are LOCAL, REPL_ASYNC and REPL_SYNC
-->
REPL_ASYNC
<!-- Name of cluster. Needs to be the same for all clusters, in order
to find each other
-->
${jboss.partition.name}-tomcat-session
<!-- JGroups protocol stack properties. Can also be a URL,
e.g. file:/home/bela/default.xml
-->
<!--
The default UDP stack:
- If you have a multihomed machine, set the UDP protocol's bind_addr attribute to the
appropriate NIC IP address, e.g bind_addr="192.168.0.2".
- On Windows machines, because of the media sense feature being broken with multicast
(even after disabling media sense) set the UDP protocol's loopback attribute to true
-->
<UDP mcast_addr="230.1.2.7" mcast_port="45577"
ip_ttl="8" ip_mcast="true"
mcast_send_buf_size="150000" mcast_recv_buf_size="80000"
ucast_send_buf_size="150000" ucast_recv_buf_size="80000"
loopback="true"/>
<PING timeout="2000" num_initial_members="3"
up_thread="false" down_thread="false"/>
<MERGE2 min_interval="10000" max_interval="20000"/>
<FD_SOCK/>
<VERIFY_SUSPECT timeout="1500"
up_thread="false" down_thread="false"/>
<pbcast.NAKACK gc_lag="50" retransmit_timeout="600,1200,2400,4800"
max_xmit_size="8192" up_thread="false" down_thread="false"/>
<UNICAST timeout="600,1200,2400" window_size="100" min_threshold="10"
down_thread="false"/>
<pbcast.STABLE desired_avg_gossip="20000"
up_thread="false" down_thread="false"/>
<FRAG frag_size="8192"
down_thread="false" up_thread="false"/>
<pbcast.GMS join_timeout="5000" join_retry_timeout="2000"
shun="true" print_local_addr="true"/>
<pbcast.STATE_TRANSFER up_thread="true" down_thread="true"/>
<!-- Alternate TCP stack: customize it for your environment, change bind_addr and initial_hosts -->
<!--
<TCP bind_addr="thishost" start_port="7810" loopback="true"/>
<TCPPING initial_hosts="thishost[7810],otherhost[7810]" port_range="3" timeout="3500"
num_initial_members="3" up_thread="true" down_thread="true"/>
<MERGE2 min_interval="5000" max_interval="10000"/>
<FD shun="true" timeout="2500" max_tries="5" up_thread="true" down_thread="true" />
<VERIFY_SUSPECT timeout="1500" down_thread="false" up_thread="false" />
<pbcast.NAKACK down_thread="true" up_thread="true" gc_lag="100"
retransmit_timeout="3000"/>
<pbcast.STABLE desired_avg_gossip="20000" down_thread="false" up_thread="false" />
<pbcast.GMS join_timeout="5000" join_retry_timeout="2000" shun="false"
print_local_addr="true" down_thread="true" up_thread="true"/>
<pbcast.STATE_TRANSFER up_thread="true" down_thread="true"/>
-->
<!-- Max number of milliseconds to wait for a lock acquisition -->
15000
TreeCache-service.xml:
<?xml version="1.0" encoding="UTF-8"?>
<!-- -->
<!-- Customized TreeCache Service Configuration for Tomcat 5 Clustering -->
<!-- -->
<!-- ===================================================================== -->
jboss:service=invoker,type=jrmp
jboss.cache:service=TreeCache
TreeCache true org.jboss.cache.TreeCacheMBean
org.jboss.proxy.ClientMethodInterceptor
org.jboss.proxy.SecurityInterceptor
org.jboss.invocation.InvokerInterceptor
jboss:service=invoker,type=jrmp
jboss.cache:service=TreeCache
<!-- ==================================================================== -->
<!-- Defines TreeCache configuration -->
<!-- ==================================================================== -->
jboss:service=Naming
jboss:service=TransactionManager
<!-- Configure the TransactionManager -->
org.jboss.cache.JBossTransactionManagerLookup
<!--
Node locking level : SERIALIZABLE
REPEATABLE_READ (default)
READ_COMMITTED
READ_UNCOMMITTED
NONE
-->
REPEATABLE_READ
<!--
Valid modes are LOCAL
REPL_ASYNC
REPL_SYNC
-->
REPL_ASYNC
<!-- Name of cluster. Needs to be the same for all clusters, in order
to find each other
-->
${jboss.partition.name}
<!-- JGroups protocol stack properties. Can also be a URL,
e.g. file:/home/bela/default.xml
-->
<!-- UDP: if you have a multihomed machine,
set the bind_addr attribute to the appropriate NIC IP address -->
<!-- UDP: On Windows machines, because of the media sense feature
being broken with multicast (even after disabling media sense)
set the loopback attribute to true -->
<UDP mcast_addr="228.1.2.3" mcast_port="48866"
ip_ttl="64" ip_mcast="true"
mcast_send_buf_size="150000" mcast_recv_buf_size="80000"
ucast_send_buf_size="150000" ucast_recv_buf_size="80000"
loopback="false"/>
<PING timeout="2000" num_initial_members="3"
up_thread="false" down_thread="false"/>
<MERGE2 min_interval="10000" max_interval="20000"/>
<FD shun="true" up_thread="true" down_thread="true"/>
<VERIFY_SUSPECT timeout="1500"
up_thread="false" down_thread="false"/>
<pbcast.NAKACK gc_lag="50" retransmit_timeout="600,1200,2400,4800"
max_xmit_size="8192" up_thread="false" down_thread="false"/>
<UNICAST timeout="600,1200,2400" window_size="100" min_threshold="10"
down_thread="false"/>
<pbcast.STABLE desired_avg_gossip="20000"
up_thread="false" down_thread="false"/>
<FRAG frag_size="8192"
down_thread="false" up_thread="false"/>
<pbcast.GMS join_timeout="5000" join_retry_timeout="2000"
shun="true" print_local_addr="true"/>
<pbcast.STATE_TRANSFER up_thread="false" down_thread="false"/>
<!--
The max amount of time (in milliseconds) we wait until the
initial state (ie. the contents of the cache) are retrieved from
existing members in a clustered environment
-->
5000
<!--
Number of milliseconds to wait until all responses for a
synchronous call have been received.
-->
10000
<!-- Max number of milliseconds to wait for a lock acquisition -->
15000
<!-- Name of the eviction policy class. -->
org.jboss.cache.eviction.LRUPolicy
<!-- Specific eviction policy configurations. This is LRU -->
5
<!-- Cache wide default -->
1000
1000
5000
1000
5
4