Clover coverage report -
Coverage timestamp: Wed Jan 31 2007 15:38:53 EST
file stats: LOC: 277   Methods: 10
NCLOC: 141   Classes: 2
 
 Source file Conditionals Statements Methods TOTAL
ReplicatedTransactionDeadlockTest.java 75% 89.7% 90% 88.2%
coverage coverage
 1    package org.jboss.cache.transaction;
 2   
 3    import junit.framework.Test;
 4    import junit.framework.TestCase;
 5    import junit.framework.TestSuite;
 6    import org.apache.commons.logging.Log;
 7    import org.apache.commons.logging.LogFactory;
 8    import org.jboss.cache.CacheImpl;
 9    import org.jboss.cache.DefaultCacheFactory;
 10    import org.jboss.cache.Fqn;
 11    import org.jboss.cache.config.Configuration;
 12    import org.jboss.cache.config.Configuration.CacheMode;
 13    import org.jboss.cache.factories.UnitTestCacheFactory;
 14   
 15    import javax.naming.Context;
 16    import javax.naming.InitialContext;
 17    import javax.transaction.UserTransaction;
 18    import java.util.Properties;
 19   
 20    /**
 21    * A test created to simulate an unexpected TimeoutException in JBossCache
 22    * 1.3.0SP1 (not relevant for 1.2.4 and earlier releases). The error has been
 23    * initially observed in a production environment, the test has been created to
 24    * simplify the analysis without the complexity of hundreds concurrent
 25    * transactions.<br>
 26    * This test is relevant for REPL_SYNC mode, (default) isolation level
 27    * REPEATABLE_READ and SyncCommitPhase.<br>
 28    * The error scenario is:
 29    * <ul>
 30    * <li>Two caches: SrcCache is the one where we put the modifications, DstCache
 31    * only receives the replicated modifications.</li>
 32    * <li>Two threads (resp. transactions) A and B modifying the same node X
 33    * concurrently on SrcCache.</li>
 34    * <li>Let's assume the transaction A is faster in aquiring the lock on X.</li>
 35    * <li>Transaction A modifies the node X and starts committing: local prepare,
 36    * remote prepare, local commit. The remote prepare will lock the X on DstCache
 37    * and it will stay locked until the remote commit releases it later. Note that
 38    * in JBossCache 1.3.0SP1 the transaction will release the lock on SrcCache
 39    * immediately after the local commit step and before the remote (sync.) commit.
 40    * It seems that this have changed between 1.2.4 and 1.3.0 releases.</li>
 41    * <li>As soon as the lock on SrcCache is released by A, transaction B aquires
 42    * the lock immediately and starts local modifications.<b>Note that in some
 43    * cases B is fast enough to do the local prepare and send a remote prepare
 44    * message before the remote commit message of A. </b>B is able to do this
 45    * because the lock is released by A before its remote commit call.</li>
 46    * <li>Now, we have the X locked by A on DstCache waiting for the remote commit
 47    * of A to release it. The next messages from SrcCache are in the following
 48    * order coming up the JGROUPS stack of the DstCache: remote prepare for B,
 49    * remote commit for A.</li>
 50    * <li>The remote prepare of B blocks on DstCache, trying to acquire the lock
 51    * still held by A.</li>
 52    * <li>The remote commit of A waits in the UP queue of the STATE_TRANSFER,
 53    * waiting for the previous message (which is the remote prepare of B) to be
 54    * processed.</li>
 55    * <li>So A cannot be committed because it's blocked by B, which cannot be
 56    * prepared, because it's blocked by A, which cannot be committed. Of course the
 57    * result is a TimeoutException and too many rolled back transactions.</li>
 58    * </ul>
 59    *
 60    * @author Marian Nikolov
 61    * @author $Author: msurtani $
 62    * @version $Date: 2007/01/11 13:49:06 $
 63    */
 64    public class ReplicatedTransactionDeadlockTest extends TestCase
 65    {
 66   
 67    // Number of worker threads
 68    private static final int NUM_WORKERS = 2;
 69   
 70    // The number of test runs to perform.
 71    private static final int NUM_RUNS = 100;
 72   
 73    // useful to increase this to get thread dumps, etc. Typically should be set to 10,000 though.
 74    private static final long LOCK_ACQUISITION_TIMEOUT = 10000;
 75   
 76    /**
 77    * The initial context factory properties.
 78    */
 79    private static final Properties PROPERTIES;
 80    /**
 81    * The context factory to be used for the test.
 82    */
 83    private static final String CONTEXT_FACTORY =
 84    "org.jboss.cache.transaction.DummyContextFactory";
 85    /**
 86    * The original context factory to be restored after the test.
 87    */
 88    private String m_contextFactory = null;
 89   
 90    /**
 91    * Exception recorded if any of the worker threads fails.
 92    */
 93    private static volatile Exception exception = null;
 94   
 95    /**
 96    * The source cache where we put modifications.
 97    */
 98    private CacheImpl srcCache = null;
 99    /**
 100    * The target cache where we replicate modifications.
 101    */
 102    private CacheImpl dstCache = null;
 103   
 104    private Log log = LogFactory.getLog(ReplicatedTransactionDeadlockTest.class);
 105   
 106    static
 107    {
 108  1 PROPERTIES = new Properties();
 109  1 PROPERTIES.put(Context.INITIAL_CONTEXT_FACTORY,
 110    "org.jboss.cache.transaction.DummyContextFactory");
 111    }
 112   
 113    /**
 114    * Constructor.
 115    *
 116    * @param name The test name.
 117    */
 118  1 public ReplicatedTransactionDeadlockTest(String name)
 119    {
 120  1 super(name);
 121    }
 122   
 123    /**
 124    * {@inheritDoc}
 125    */
 126  1 protected void setUp() throws Exception
 127    {
 128  1 super.setUp();
 129  1 exception = null;
 130  1 m_contextFactory = System.getProperty(Context.INITIAL_CONTEXT_FACTORY);
 131  1 System.setProperty(Context.INITIAL_CONTEXT_FACTORY, CONTEXT_FACTORY);
 132  1 DummyTransactionManager.getInstance();
 133    // setup and start the source cache
 134  1 srcCache = (CacheImpl) DefaultCacheFactory.getInstance().createCache(false);
 135  1 srcCache.setConfiguration(UnitTestCacheFactory.createConfiguration(CacheMode.REPL_SYNC));
 136  1 srcCache.getConfiguration().setTransactionManagerLookupClass(
 137    "org.jboss.cache.DummyTransactionManagerLookup");
 138  1 srcCache.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
 139   
 140  1 srcCache.getConfiguration().setSyncCommitPhase(true);
 141  1 srcCache.getConfiguration().setLockAcquisitionTimeout(LOCK_ACQUISITION_TIMEOUT);
 142  1 srcCache.create();
 143  1 srcCache.start();
 144    // setup and start the destination cache
 145  1 dstCache = (CacheImpl) DefaultCacheFactory.getInstance().createCache(false);
 146  1 dstCache.setConfiguration(UnitTestCacheFactory.createConfiguration(CacheMode.REPL_SYNC));
 147  1 dstCache.getConfiguration().setTransactionManagerLookupClass(
 148    "org.jboss.cache.DummyTransactionManagerLookup");
 149  1 dstCache.getConfiguration().setCacheMode(Configuration.CacheMode.REPL_SYNC);
 150   
 151  1 dstCache.getConfiguration().setSyncCommitPhase(true);
 152  1 dstCache.getConfiguration().setLockAcquisitionTimeout(LOCK_ACQUISITION_TIMEOUT);
 153  1 dstCache.create();
 154  1 dstCache.start();
 155    }
 156   
 157    /**
 158    * {@inheritDoc}
 159    */
 160  1 protected void tearDown() throws Exception
 161    {
 162  1 super.tearDown();
 163  1 DummyTransactionManager.destroy();
 164  1 srcCache.stop();
 165  1 srcCache = null;
 166  1 dstCache.stop();
 167  1 dstCache = null;
 168  1 if (m_contextFactory != null)
 169    {
 170  0 System.setProperty(Context.INITIAL_CONTEXT_FACTORY,
 171    m_contextFactory);
 172  0 m_contextFactory = null;
 173    }
 174    }
 175   
 176    /**
 177    * Test for a synchronously replicated cache with concurrent transactions on
 178    * the same node.<br>
 179    * This test fails very often with a TimeoutException.
 180    *
 181    * @throws Exception Any exception if thrown by the cache.
 182    */
 183  1 public void testConcurrentReplicatedTransaction() throws Exception
 184    {
 185  1 performTest();
 186    }
 187   
 188    /**
 189    * Perform a single test, using the pre-configured cache.
 190    *
 191    * @throws Exception Any exception if thrown by the cache.
 192    */
 193  1 private void performTest() throws Exception
 194    {
 195    // repeat the test several times since it's not always reproducible
 196  1 for (int i = 0; i < NUM_RUNS; i++)
 197    {
 198  100 if (exception != null)
 199    {
 200    // terminate the test on the first failed worker
 201  0 fail("Due to an exception: " + exception);
 202    }
 203    // start several worker threads to work with the same FQN
 204  100 Worker[] t = new Worker[NUM_WORKERS];
 205  100 for (int j = 0; j < t.length; j++)
 206    {
 207  200 t[j] = new Worker("worker " + i + ":" + j);
 208  200 t[j].start();
 209    }
 210    // wait for all workers to complete before repeating the test
 211  200 for (Worker aT : t) aT.join();
 212    }
 213    }
 214   
 215    /**
 216    * Returns a user transaction to be associated with the calling thread.
 217    *
 218    * @return A user transaction.
 219    * @throws Exception Any exception thrown by the context lookup.
 220    */
 221  200 private UserTransaction getTransaction() throws Exception
 222    {
 223  200 return (UserTransaction) new InitialContext(PROPERTIES)
 224    .lookup("UserTransaction");
 225    }
 226   
 227    /**
 228    * A worker thread that applies the concurrent modifications.
 229    *
 230    * @author Marian Nikolov
 231    * @author $Author: msurtani $
 232    * @version $Date: 2007/01/11 13:49:06 $
 233    */
 234    private class Worker extends Thread
 235    {
 236    /**
 237    * Constructor.
 238    */
 239  200 public Worker(String name)
 240    {
 241  200 super(name);
 242    }
 243   
 244    /**
 245    * {@inheritDoc}
 246    */
 247  200 public void run()
 248    {
 249  200 try
 250    {
 251  200 UserTransaction tx = getTransaction();
 252  200 log.warn("begin");
 253  200 tx.begin();
 254  200 log.warn("put");
 255  200 srcCache.put(new Fqn("Node"), Boolean.FALSE, Boolean.TRUE);
 256  200 log.warn("commit");
 257  200 tx.commit();
 258  200 log.warn("leave");
 259    }
 260    catch (Exception e)
 261    {
 262  0 log.error("caught exception " + e, e);
 263  0 exception = e;
 264    }
 265    }
 266    }
 267   
 268  1 public static Test suite()
 269    {
 270  1 return new TestSuite(ReplicatedTransactionDeadlockTest.class);
 271    }
 272   
 273  0 public static void main(String[] args) throws Exception
 274    {
 275  0 junit.textui.TestRunner.run(suite());
 276    }
 277    }