hung JBoss context lookup
I have JMS Client inside weblogic 8.1 listening to a queue hosted in JBoss3.2.5. I have an ExceptionListener which tries to reconnect the queue if connection is broken and restart the listener.
Our client is rebooting JBOSS every morning, and my jms client would detect the broken connection and would try to reconnect every 60 seconds till it is finally connected. But sometimes, once in a 10 days, it would get into hung mode and after few tries, it would give up. Once it get into this condition, I need to manually stop the weblogic process and restart it.
I took the thread dump today, where it seems that while doing the context lookup, it is reading socket and get hung there. Below is the thread dump and re-try code for connection. Please help me if anyone has seen this issue and solved it.
ThreadDump of the hung thread
"UIL2.SocketManager.ReadTask#3 client=172.17.100.14:8093" daemon prio=5 tid=0x0494ddb0 nid=0x1704 runnable [0x072cf000..0x072cfd94]
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:183)
at java.io.BufferedInputStream.read1(BufferedInputStream.java:222)
at java.io.BufferedInputStream.read(BufferedInputStream.java:277)
- locked <0x2a7a3c50> (a java.io.BufferedInputStream)
at java.io.ObjectInputStream$PeekInputStream.read(ObjectInputStream.java:2140)
at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2153)
at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:2621)
at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:734)
at java.io.ObjectInputStream.(ObjectInputStream.java:251)
at org.jnp.interfaces.NamingContext.getServer(NamingContext.java:194)
at org.jnp.interfaces.NamingContext.checkRef(NamingContext.java:1185)
at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:516)
at org.jnp.interfaces.NamingContext.lookup(NamingContext.java:509)
at javax.naming.InitialContext.lookup(InitialContext.java:347)
at com.bluemartini.nextor.jms.JBossMQManager.initConnection(JBossMQManager.java:124)
at com.bluemartini.nextor.jms.JBossMQManager.initialize(JBossMQManager.java:97)
at com.bluemartini.nextor.jms.JBossMQManager.reinitialize(JBossMQManager.java:245)
at com.bluemartini.nextor.jms.JBossMQManager$_ExceptionListener.onException(JBossMQManager.java:359)
- locked <0x1a41af38> (a com.bluemartini.nextor.jms.JBossMQManager$_ExceptionListener)
at org.jboss.mq.Connection.asynchFailure(Connection.java:439)
- locked <0x1a41af38> (a com.bluemartini.nextor.jms.JBossMQManager$_ExceptionListener)
at org.jboss.mq.il.uil2.UILClientILService.asynchFailure(UILClientILService.java:145)
at org.jboss.mq.il.uil2.SocketManager$ReadTask.handleStop(SocketManager.java:398)
at org.jboss.mq.il.uil2.SocketManager$ReadTask.run(SocketManager.java:332)
at java.lang.Thread.run(Thread.java:534)
My reconnect Code
while (!bConnected_ && !BMThreadManager.aborted())
{
try {
Hashtable props = new Hashtable();
props.put(Context.INITIAL_CONTEXT_FACTORY, initialContextFactory_);
props.put(Context.PROVIDER_URL, providerURL_);
//props.put("java.naming.rmi.security.manager", "yes");
props.put(Context.URL_PKG_PREFIXES, urlPkgPrefixes_);
//Get the initial context with given properties
ctx_ = new InitialContext(props);
BMLog.log(LOG_COMPONENT, 4, "BM JMS: Preparing for context lookup");
//Get the connection factory
QueueConnectionFactory queueConnectionFactory_ =
(QueueConnectionFactory)ctx_.lookup("UIL2ConnectionFactory");
BMLog.log(LOG_COMPONENT, 4, "BM JMS: QueueConnectionFactory:" + queueConnectionFactory_ );
BMLog.log(LOG_COMPONENT, 0, "BM JMS: Creating Queue Connection to JMS Server " + providerURL_);
// Get a Queue Connection to the JMS server
System.out.println("BM JMS: Creating Queue Connection to JMS Server " + providerURL_);
BMLog.log(LOG_COMPONENT, 0, "BM JMS: Creating Queue Connection to JMS Server " + providerURL_);
queueConnection_ = queueConnectionFactory_.createQueueConnection();
queueConnection_.setExceptionListener(exceptionListener_);
System.out.println("BM JMS: Successfully connected to JMS Server " + providerURL_);
BMLog.log(LOG_COMPONENT, 0, "BM JMS: Successfully connected to JMS Server " + providerURL_);
bConnected_ = true;
// increment connectionID
//connectionID_ = connectionID_ + 1;
}
catch (Throwable e)
{
System.out.println("BM JMS: Connect to JMS Server " + providerURL_ + " failed. Will retry in " + nSleepSecs_ + " seconds");
BMLog.log(LOG_COMPONENT, 0, "BM JMS: Unable to create connection to server: " + providerURL_, e);
if (e instanceof JMSException) {
Exception le = ((JMSException)e).getLinkedException();
if (le != null )
BMLog.log(LOG_COMPONENT, 0, "BM JMS: Linked Exception: ", le);
}
cleanup(true);
try {
Thread.sleep(nSleepSecs_.intValue() * 1000);
} catch (InterruptedException ie) {
BMLog.log(LOG_COMPONENT, 2, "Thread Exception ", ie);
}
}
BMLog.log(LOG_COMPONENT, 2, "bConnected_ :" + bConnected_);
BMLog.log(LOG_COMPONENT, 2, "BMThreadManager.aborted()" + BMThreadManager.aborted());
}