I think that you are seeing the same problem that is described here:
http://www.jboss.org/index.html?module=bb&op=viewtopic&t=151397
I am attaching a version of ConnectionValidator that fixed the problem for us:
/*
* JBoss, Home of Professional Open Source
* Copyright 2005, JBoss Inc., and individual contributors as indicated
* by the @authors tag. See the copyright.txt in the distribution for a
* full listing of individual contributors.
*
* This is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1 of
* the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this software; if not, write to the Free
* Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
* 02110-1301 USA, or see the FSF site: http://www.fsf.org.
*/
package org.jboss.remoting;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import org.jboss.logging.Logger;
import org.jboss.remoting.transport.ClientInvoker;
import org.jboss.remoting.util.StoppableTimerTask;
import org.jboss.remoting.util.TimerUtil;
/**
* @author <a href="mailto:tom.elrod@jboss.com">Tom Elrod</a>
* @author <a href="mailto:ovidiu@jboss.org">Ovidiu Feodorov</a>
* @author <a href="mailto:tlee@redhat.com">Trustin Lee</a>
*/
public class ConnectionValidator extends TimerTask implements StoppableTimerTask
{
// Constants ------------------------------------------------------------------------------------
private static final Logger log = Logger.getLogger(ConnectionValidator.class.getName());
/** Configuration map key for ping period. */
public static final String VALIDATOR_PING_PERIOD = "validatorPingPeriod";
/** Default ping period. Value is 2 seconds. */
public static final long DEFAULT_PING_PERIOD = 2000;
/** Configuration map key for ping timeout. */
public static final String VALIDATOR_PING_TIMEOUT = "validatorPingTimeout";
/** Default ping timeout period. Value is 1 second. */
public static final String DEFAULT_PING_TIMEOUT = "1000";
/**
* Default number of ping retries. Value is 1.
* Currently implemented only on socket transport family.
*/
public static final String DEFAULT_NUMBER_OF_PING_RETRIES = "1";
/**
* Key to determine if ConnectionValidator should tie failure to presence
* of active lease on server side. Default value is "true".
*/
public static final String TIE_TO_LEASE = "tieToLease";
/**
* Key to determine whether to stop ConnectionValidator when PING fails.
* Default value is "true".
*/
public static final String STOP_LEASE_ON_FAILURE = "stopLeaseOnFailure";
// Static ---------------------------------------------------------------------------------------
private static boolean trace = log.isTraceEnabled();
/**
* Will make $PING$ invocation on server. If sucessful, will return true. Otherwise, will throw
* an exception.
*
* @param locator - locator for the server to ping
* @param config - any configuration needed for server
* @return true if alive, false if not
*/
public static boolean checkConnection(InvokerLocator locator, Map config) throws Throwable
{
boolean pingWorked = false;
Map configMap = createPingConfig(config, null);
int pingTimeout = Integer.parseInt((String) configMap.get(ServerInvoker.TIMEOUT));
ClientInvoker innerClientInvoker = null;
try
{
innerClientInvoker = InvokerRegistry.createClientInvoker(locator, configMap);
if (!innerClientInvoker.isConnected())
{
if (trace) { log.trace("inner client invoker not connected, connecting ..."); }
innerClientInvoker.connect();
}
pingWorked = doCheckConnection(innerClientInvoker, pingTimeout);
}
catch (Throwable throwable)
{
log.debug("ConnectionValidator to connect to server " +
innerClientInvoker.getLocator().getProtocol() + "://" +
innerClientInvoker.getLocator().getHost() + ":" +
innerClientInvoker.getLocator().getPort(), throwable);
}
finally
{
if (innerClientInvoker != null)
{
InvokerRegistry.destroyClientInvoker(locator, configMap);
}
}
return pingWorked;
}
private static boolean doCheckConnection(ClientInvoker clientInvoker, int pingTimeout) throws Throwable
{
boolean pingWorked = false;
try
{
// Sending null client id as don't want to trigger lease on server side. This also means
// that client connection validator will NOT impact client lease, so can not depend on it
// to maintain client lease with the server.
InvocationRequest ir;
ir = new InvocationRequest(null, Subsystem.SELF, "$PING$", null, null, null);
ConnectionCheckThread t = new ConnectionCheckThread(clientInvoker, ir);
t.start();
Thread.sleep(pingTimeout);
pingWorked = t.isValid();
}
catch (Throwable t)
{
log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
}
return pingWorked;
}
private static Map createPingConfig(Map config, Map metadata)
{
Map localConfig = new HashMap();
localConfig.put("connection_checker", "true");
if (config != null)
{
Object o = config.get(VALIDATOR_PING_TIMEOUT);
log.trace("config timeout: " + o);
if (o != null)
{
try
{
Integer.parseInt((String) o);
localConfig.put(ServerInvoker.TIMEOUT, o);
}
catch (NumberFormatException e)
{
log.warn("Need integer for value of parameter " + VALIDATOR_PING_TIMEOUT +
". Using default value " + DEFAULT_PING_TIMEOUT);
}
}
o = config.get("NumberOfCallRetries");
if (o != null)
{
localConfig.put("NumberOfCallRetries", o);
}
o = config.get("NumberOfRetries");
if (o != null)
{
localConfig.put("NumberOfRetries", o);
}
}
if (metadata != null)
{
metadata.remove(ServerInvoker.TIMEOUT);
localConfig.putAll(metadata);
Object o = metadata.get(VALIDATOR_PING_TIMEOUT);
if (o != null)
{
try
{
Integer.parseInt((String) o);
localConfig.put(ServerInvoker.TIMEOUT, o);
}
catch (NumberFormatException e)
{
log.warn("Need integer for value of parameter " + VALIDATOR_PING_TIMEOUT +
". Using default value " + DEFAULT_PING_TIMEOUT);
}
}
}
if (localConfig.get(ServerInvoker.TIMEOUT) == null)
{
localConfig.put(ServerInvoker.TIMEOUT, DEFAULT_PING_TIMEOUT);
}
if (localConfig.get("NumberOfCallRetries") == null)
{
localConfig.put("NumberOfCallRetries", DEFAULT_NUMBER_OF_PING_RETRIES);
}
return localConfig;
}
// Attributes -----------------------------------------------------------------------------------
private Client client;
private long pingPeriod;
private Map metadata;
private InvokerLocator locator;
private Map configMap;
private List listeners;
private ClientInvoker clientInvoker;
private Object lock = new Object();
private Object notificationLock = new Object();
private boolean started;
private volatile boolean stopped;
private String invokerSessionId;
private boolean tieToLease = true;
private boolean stopLeaseOnFailure = true;
private int pingTimeout;
private boolean isValid;
private Timer timer;
volatile TimerTask tt;
// Constructors ---------------------------------------------------------------------------------
public ConnectionValidator(Client client)
{
this(client, DEFAULT_PING_PERIOD);
}
public ConnectionValidator(Client client, long pingPeriod)
{
this.client = client;
this.pingPeriod = pingPeriod;
listeners = new ArrayList();
stopped = false;
getParameters(client, new HashMap());
log.debug(this + " created");
}
public ConnectionValidator(Client client, Map metadata)
{
this.client = client;
pingPeriod = DEFAULT_PING_PERIOD;
listeners = new ArrayList();
stopped = false;
this.metadata = new HashMap(metadata);
getParameters(client, metadata);
log.debug(this + " created");
}
// StoppableTimerTask implementation ------------------------------------------------------------
public void stop()
{
if (stopped)
{
return;
}
doStop();
}
// TimerTask overrides --------------------------------------------------------------------------
/**
* The action to be performed by this timer task.
*/
public void run()
{
synchronized (lock) {
if (!started)
{
throw new IllegalStateException(
ConnectionValidator.class.getName() + ".run() should not be " +
"called directly; use " + ConnectionValidator.class.getName() +
".addConnectionListener() instead.");
}
if (stopped)
{
return;
}
while(tt != null) {}
tt = new WaitOnConnectionCheckTimerTask();
try
{
timer.schedule(tt, 0);
}
catch (IllegalStateException e)
{
log.debug("Unable to schedule TimerTask on existing Timer", e);
timer = new Timer(true);
timer.schedule(tt, 0);
}
}
try
{
if(!stopped)
{
isValid = false;
if (tieToLease && client.getLeasePeriod() > 0)
{
if (trace)
{
log.trace(this + " sending PING tied to lease");
}
isValid = doCheckConnectionWithLease();
}
else
{
if (trace) { log.trace(this + " pinging ..."); }
isValid = doCheckConnectionWithoutLease();
}
}
}
catch (Throwable thr)
{
log.debug(this + " got throwable while pinging", thr);
if (stopLeaseOnFailure)
{
log.debug(this + " detected connection failure: stopping");
cancel();
}
}
finally
{
synchronized (notificationLock)
{
notificationLock.notifyAll();
}
}
}
public boolean cancel()
{
return doStop();
}
// Public ---------------------------------------------------------------------------------------
public void addConnectionListener(ConnectionListener listener)
{
if (listener != null)
{
synchronized (lock)
{
if (listeners.size() == 0)
{
start();
}
listeners.add(listener);
}
}
}
public boolean removeConnectionListener(ConnectionListener listener)
{
boolean isRemoved = false;
if (listener != null)
{
synchronized (lock)
{
isRemoved = listeners.remove(listener);
if (listeners.size() == 0)
{
stop();
}
}
}
return isRemoved;
}
public long getPingPeriod()
{
if (stopped)
{
return -1;
}
return pingPeriod;
}
public String toString()
{
return "ConnectionValidator[" + clientInvoker + ", pingPeriod=" + pingPeriod + " ms]";
}
// Package protected ----------------------------------------------------------------------------
// Protected ------------------------------------------------------------------------------------
// Private --------------------------------------------------------------------------------------
private void getParameters(Client client, Map metadata)
{
getParametersFromMap(client.getConfiguration());
getParametersFromMap(metadata);
ClientInvoker clientInvoker = client.getInvoker();
if (clientInvoker instanceof MicroRemoteClientInvoker)
{
invokerSessionId = ((MicroRemoteClientInvoker) clientInvoker).getSessionId();
}
else
{
throw new RuntimeException("creating a ConnectionValidator on a local connection");
}
}
private void getParametersFromMap(Map config)
{
if (config != null)
{
Object o = config.get(VALIDATOR_PING_PERIOD);
if (o != null)
{
if (o instanceof String)
{
try
{
pingPeriod = Long.parseLong((String)o);
}
catch (Exception e)
{
log.warn(this + " could not convert " + VALIDATOR_PING_PERIOD +
" value of " + o + " to a long value");
}
}
else
{
log.warn(this + " could not convert " + VALIDATOR_PING_PERIOD +
" value of " + o + " to a long value: must be a String");
}
}
o = config.get(TIE_TO_LEASE);
if (o != null)
{
if (o instanceof String)
{
try
{
tieToLease = Boolean.valueOf(((String) o)).booleanValue();
}
catch (Exception e)
{
log.warn(this + " could not convert " + TIE_TO_LEASE + " value" +
" to a boolean: " + o);
}
}
else
{
log.warn(this + " could not convert " + TIE_TO_LEASE + " value" +
" to a boolean: must be a String");
}
}
o = config.get(STOP_LEASE_ON_FAILURE);
if (o != null)
{
if (o instanceof String)
{
try
{
stopLeaseOnFailure = Boolean.valueOf(((String) o)).booleanValue();
}
catch (Exception e)
{
log.warn(this + " could not convert " + STOP_LEASE_ON_FAILURE + " value" +
" to a boolean: " + o);
}
}
else
{
log.warn(this + " could not convert " + STOP_LEASE_ON_FAILURE + " value" +
" to a boolean: must be a String");
}
}
}
}
private void start()
{
configMap = createPingConfig(client.getConfiguration(), metadata);
pingTimeout = Integer.parseInt((String) configMap.get(ServerInvoker.TIMEOUT));
log.debug(this + " timeout: " + pingTimeout);
log.debug(this + " ping retries: " + configMap.get("NumberOfCallRetries"));
log.debug(this + " connection retries: " + configMap.get("NumberOfRetries"));
locator = client.getInvoker().getLocator();
try
{
clientInvoker = InvokerRegistry.createClientInvoker(locator, configMap);
}
catch (Exception e)
{
log.debug("Unable to create client invoker for locator: " + locator);
throw new RuntimeException("Unable to create client invoker for locator: " + locator, e);
}
if (!clientInvoker.isConnected())
{
if (trace) { log.trace("inner client invoker not connected, connecting ..."); }
clientInvoker.connect();
}
TimerUtil.schedule(this, pingPeriod);
started = true;
timer = new Timer(true);
log.debug(this + " started");
}
private boolean doCheckConnectionWithLease() throws Throwable
{
boolean pingWorked = false;
try
{
Map metadata = new HashMap();
metadata.put(ServerInvoker.INVOKER_SESSION_ID, invokerSessionId);
InvocationRequest ir =
new InvocationRequest(null, Subsystem.SELF, "$PING$", metadata, null, null);
if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
Object o = clientInvoker.invoke(ir);
if (trace) {
log.trace("Return from server ping is :" + o);
}
if (o instanceof Boolean && !((Boolean) o).booleanValue())
{
// Server indicates lease has stopped.
throw new Exception();
}
if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
pingWorked = true;
}
catch (Throwable t)
{
log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
}
return pingWorked;
}
private boolean doCheckConnectionWithoutLease() throws Throwable
{
boolean pingWorked = false;
try
{
// Sending null client id as don't want to trigger lease on server side. This also means
// that client connection validator will NOT impact client lease, so can not depend on it
// to maintain client lease with the server.
InvocationRequest ir =
new InvocationRequest(null, Subsystem.SELF, "$PING$", null, null, null);
if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
clientInvoker.invoke(ir);
if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
pingWorked = true;
}
catch (Throwable t)
{
log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
}
return pingWorked;
}
private boolean doStop()
{
synchronized(lock)
{
if (stopped)
{
return false;
}
if (!listeners.isEmpty())
{
listeners.clear();
}
stopped = true;
timer = null;
}
if (clientInvoker != null)
{
InvokerRegistry.destroyClientInvoker(locator, configMap);
}
TimerUtil.unschedule(this);
boolean result = super.cancel();
log.debug(this + " stopped, returning " + result);
return result;
}
private void notifyListeners(Throwable thr)
{
final Throwable t = thr;
synchronized (lock)
{
if (stopped)
{
return;
}
ListIterator itr = listeners.listIterator();
while (itr.hasNext())
{
final ConnectionListener listener = (ConnectionListener) itr.next();
new Thread()
{
public void run()
{
listener.handleConnectionException(t, client);
}
}.start();
}
}
stop();
listeners.clear();
}
// Inner classes --------------------------------------------------------------------------------
private class WaitOnConnectionCheckTimerTask extends TimerTask
{
public void run()
{
long start = System.currentTimeMillis();
synchronized (notificationLock)
{
while (true)
{
int elapsed = (int) (System.currentTimeMillis() - start);
int wait = pingTimeout - elapsed;
if (wait <= 0)
{
break;
}
try
{
notificationLock.wait(wait);
break;
}
catch (InterruptedException e)
{
continue;
}
}
}
if (!isValid)
{
log.debug(ConnectionValidator.this + "'s connections is invalid");
notifyListeners(new Exception("Could not connect to server!"));
if (stopLeaseOnFailure)
{
log.debug(this + " detected connection failure: stopping LeasePinger");
MicroRemoteClientInvoker invoker = (MicroRemoteClientInvoker) client.getInvoker();
if (invoker != null)
{
invoker.terminateLease(null, client.getDisconnectTimeout());
log.debug(ConnectionValidator.this + " shut down lease pinger");
}
else
{
log.debug(ConnectionValidator.this + " unable to shut down lease pinger: client must have shut down");
}
cancel();
}
}
tt = null;
}
}
private static class ConnectionCheckThread extends Thread
{
private InvocationRequest ir;
private ClientInvoker clientInvoker;
private boolean isValid;
public ConnectionCheckThread(ClientInvoker clientInvoker, InvocationRequest ir)
{
this.clientInvoker = clientInvoker;
this.ir = ir;
setDaemon(true);
}
public void run()
{
try
{
if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
clientInvoker.invoke(ir);
isValid = true;
if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
}
catch (Throwable t)
{
log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
}
}
public boolean isValid()
{
return isValid;
}
}
}
