Clover coverage report -
Coverage timestamp: Wed Jan 31 2007 15:38:53 EST
file stats: LOC: 530   Methods: 24
NCLOC: 255   Classes: 2
 
 Source file Conditionals Statements Methods TOTAL
ConcurrentCreationDeadlockTest.java 88.5% 76.5% 54.2% 75%
coverage coverage
 1    package org.jboss.cache.loader.deadlock;
 2   
 3    import junit.framework.Test;
 4    import junit.framework.TestSuite;
 5    import org.jboss.cache.CacheImpl;
 6    import org.jboss.cache.DefaultCacheFactory;
 7    import org.jboss.cache.Fqn;
 8    import org.jboss.cache.config.Configuration;
 9    import org.jboss.cache.config.Configuration.CacheMode;
 10    import org.jboss.cache.factories.UnitTestCacheFactory;
 11    import org.jboss.cache.factories.XmlConfigurationParser;
 12    import org.jboss.cache.loader.AbstractCacheLoaderTestBase;
 13    import org.jboss.cache.transaction.DummyTransactionManager;
 14   
 15    import javax.naming.Context;
 16    import javax.naming.InitialContext;
 17    import javax.transaction.UserTransaction;
 18    import java.util.Properties;
 19    import java.util.concurrent.CountDownLatch;
 20   
 21    /**
 22    * <b>Test based on a contribution by Marian Nokolov/Paul Miodonski at Siemens AG.</b>
 23    * <p/>
 24    * This test has been created to simulate a unexpected TimeoutException that is
 25    * thrown by the JBossCache. The original scenario that has been observed: <br>
 26    * Cache in either LOCAL or REPL_SYNC mode with CacheLoader.
 27    * <ul>
 28    * <li>1. Concurrent threads A, B and C, each associated with a transaction.
 29    * Threads A and B try to modify FQN X, thread C tries to modify FQN Y.</li>
 30    * <li>2. Thread A locks X.</li>
 31    * <li>3. Thread B blocks on X (correct since A is the active writer).</li>
 32    * <li>4. Thread A tries to do multiple modifications and suddenly blocks on X
 33    * (although it is the active writer already) - this is the 1-st problem.</li>
 34    * <li>5. Thread C blocks somewhere as well, although it has nothing to do with
 35    * X and is the only one that works on Y - this is a 2-nd problem.</li>
 36    * <li>6. Thread B fails with TimeoutException and its transaction is rolled
 37    * back - this is correct given that A is still holding the lock on X.</li>
 38    * <li>7. Thread A continues its job and its transaction completes successfully
 39    * (after unexpected locking timeout delay).</li>
 40    * <li>8. Thread C continues its job and successfully commits the transaction
 41    * (with unexpected locking timeout delay).</li>
 42    * </ul>
 43    * <br>
 44    * There are two problems with this:
 45    * <ul>
 46    * <li>1. One or more concurrent transactions fail, although the pessimistic
 47    * locking should sequentialize them and guarantee that all should succeed.</li>
 48    * <li>2. Any other thread that tries to acquire a lock in the cache (even for
 49    * a different FQN!?) is blocked for the duration of the locking timeout, i.e.
 50    * the whole application is locked for few seconds. The active writer for the
 51    * corresponding FQN is blocked as well in the middle of the transaction!</li>
 52    * </ul>
 53    * <br>
 54    * At least until now, the error can be reproduced only if the following is
 55    * true:
 56    * <ul>
 57    * <li>Concurrent transactions forcing creation of the same FQN at the same
 58    * time.</li>
 59    * <li>More than one update per TX per FQN - trying to acquire lock on the same
 60    * FQN multiple times per TX.</li>
 61    * <li>Cache with CacheLoader - maybe it has something to do with the
 62    * CacheLoader/StoreInterceptor's...</li>
 63    * </ul>
 64    */
 65    public class ConcurrentCreationDeadlockTest extends AbstractCacheLoaderTestBase
 66    {
 67    /**
 68    * The number of worker threads to start concurrently.
 69    */
 70    private static final int NUM_WORKERS = 10;
 71    /**
 72    * The number of test runs to perform.
 73    */
 74    private static final int NUM_RUNS = 100;
 75    /**
 76    * The number of FQN's per test run.
 77    */
 78    private static final int NUM_FQNS_PER_RUN = 10;
 79   
 80    /**
 81    * The initial context factory properties.
 82    */
 83    private static final Properties PROPERTIES;
 84    /**
 85    * The context factory to be used for the test.
 86    */
 87    private static final String CONTEXT_FACTORY =
 88    "org.jboss.cache.transaction.DummyContextFactory";
 89    /**
 90    * The original context factory to be restored after the test.
 91    */
 92    private String m_contextFactory = null;
 93   
 94    /**
 95    * Exception recorded if any of the worker threads fails.
 96    */
 97    private static volatile Exception mcl_exception = null;
 98   
 99    /**
 100    * The cache under test.
 101    */
 102    private CacheImpl cache = null;
 103   
 104    static
 105    {
 106  1 PROPERTIES = new Properties();
 107  1 PROPERTIES.put(Context.INITIAL_CONTEXT_FACTORY,
 108    "org.jboss.cache.transaction.DummyContextFactory");
 109    }
 110   
 111    /**
 112    * Constructor.
 113    *
 114    * @param name The test name.
 115    */
 116  2 public ConcurrentCreationDeadlockTest(String name)
 117    {
 118  2 super(name);
 119    }
 120   
 121    /**
 122    * {@inheritDoc}
 123    */
 124  2 public void setUp() throws Exception
 125    {
 126  2 super.setUp();
 127  2 mcl_exception = null;
 128  2 m_contextFactory = System.getProperty(Context.INITIAL_CONTEXT_FACTORY);
 129  2 System.setProperty(Context.INITIAL_CONTEXT_FACTORY, CONTEXT_FACTORY);
 130  2 DummyTransactionManager.getInstance();
 131  2 cache = (CacheImpl) DefaultCacheFactory.getInstance().createCache(false);
 132   
 133  2 XmlConfigurationParser parser = new XmlConfigurationParser();
 134  2 Configuration c = UnitTestCacheFactory.createConfiguration(CacheMode.REPL_SYNC);
 135  2 cache.setConfiguration(c);
 136   
 137  2 c.setTransactionManagerLookupClass("org.jboss.cache.DummyTransactionManagerLookup");
 138    }
 139   
 140    /**
 141    * {@inheritDoc}
 142    */
 143  2 public void tearDown() throws Exception
 144    {
 145  2 super.tearDown();
 146  2 DummyTransactionManager.destroy();
 147  2 cache.stop();
 148  2 cache = null;
 149  2 if (m_contextFactory != null)
 150    {
 151  1 System.setProperty(Context.INITIAL_CONTEXT_FACTORY,
 152    m_contextFactory);
 153  1 m_contextFactory = null;
 154    }
 155    }
 156   
 157    /**
 158    * Initializes and starts the cache.
 159    *
 160    * @param cacheMode The cache mode.
 161    * @param cacheLoaderClass The name of the cache loader class.
 162    * @throws Exception Any exception if thrown by the cache.
 163    */
 164  2 private void startCache(Configuration.CacheMode cacheMode, String cacheLoaderClass)
 165    throws Exception
 166    {
 167  2 cache.getConfiguration().setCacheMode(cacheMode);
 168  2 if (cacheLoaderClass != null)
 169    {
 170  2 cache.getConfiguration().setCacheLoaderConfig(getSingleCacheLoaderConfig("", cacheLoaderClass, "", false, false, false));
 171    }
 172  2 cache.create();
 173  2 cache.start();
 174    }
 175   
 176    /**
 177    * Test for a local cache without cache loader and single modification per
 178    * transaction.<br>
 179    * This test has never failed up to now.
 180    *
 181    * @throws Exception Any exception if thrown by the cache.
 182    */
 183  0 public void disabledtestLocalNoCacheLoader1Modification() throws Exception
 184    {
 185  0 startCache(Configuration.CacheMode.LOCAL, null);
 186  0 performTest(1);
 187    }
 188   
 189    /**
 190    * Test for a local cache without cache loader and two modifications per
 191    * transaction.<br>
 192    * This test has never failed up to now.
 193    *
 194    * @throws Exception Any exception if thrown by the cache.
 195    */
 196  0 public void disabledtestLocalNoCacheLoader2Modifications()
 197    throws Exception
 198    {
 199  0 startCache(Configuration.CacheMode.LOCAL, null);
 200  0 performTest(2);
 201    }
 202   
 203    /**
 204    * Test for a local cache with cache loader and single modification per
 205    * transaction.<br>
 206    * This test has never failed up to now.
 207    *
 208    * @throws Exception Any exception if thrown by the cache.
 209    */
 210  0 public void disabledtestLocalCacheLoader1Modification() throws Exception
 211    {
 212  0 startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
 213  0 performTest(1);
 214    }
 215   
 216    /**
 217    * Test for a local cache with cache loader and two modifications per
 218    * transaction.<br>
 219    * This test does very often fail with a TimeoutException.
 220    *
 221    * @throws Exception Any exception if thrown by the cache.
 222    */
 223  1 public void testLocalCacheLoader2Modifications() throws Exception
 224    {
 225  1 startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
 226  1 performTest(2);
 227    }
 228   
 229    /**
 230    * Test for an asynchronously replicated cache without cache loader and
 231    * single modification per transaction.<br>
 232    * This test has never failed up to now.
 233    *
 234    * @throws Exception Any exception if thrown by the cache.
 235    */
 236  0 public void disabledtestReplAsyncNoCacheLoader1Modification()
 237    throws Exception
 238    {
 239  0 startCache(Configuration.CacheMode.REPL_ASYNC, null);
 240  0 performTest(1);
 241    }
 242   
 243    /**
 244    * Test for an asynchronously replicated cache without cache loader and two
 245    * modifications per transaction.<br>
 246    * This test has never failed up to now.
 247    *
 248    * @throws Exception Any exception if thrown by the cache.
 249    */
 250  0 public void disabledtestReplAsyncNoCacheLoader2Modifications()
 251    throws Exception
 252    {
 253  0 startCache(Configuration.CacheMode.REPL_ASYNC, null);
 254  0 performTest(2);
 255    }
 256   
 257    /**
 258    * Test for an asynchronously replicated cache with cache loader and single
 259    * modification per transaction.<br>
 260    * This test has never failed up to now.
 261    *
 262    * @throws Exception Any exception if thrown by the cache.
 263    */
 264  0 public void disabledtestReplAsyncCacheLoader1Modification()
 265    throws Exception
 266    {
 267  0 startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
 268  0 performTest(1);
 269    }
 270   
 271    /**
 272    * Test for an asynchronously replicated cache with cache loader and two
 273    * modification per transaction.<br>
 274    * This test mysteriously never fails, although it should as the LOCAL
 275    * and REPL_SYNC do.
 276    *
 277    * @throws Exception Any exception if thrown by the cache.
 278    */
 279  0 public void disabledtestReplAsyncCacheLoader2Modifications()
 280    throws Exception
 281    {
 282  0 startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
 283  0 performTest(2);
 284    }
 285   
 286    /**
 287    * Test for a synchronously replicated cache without cache loader and single
 288    * modification per transaction.<br>
 289    * This test has never failed up to now.
 290    *
 291    * @throws Exception Any exception if thrown by the cache.
 292    */
 293  0 public void disabledtestReplSyncNoCacheLoader1Modification()
 294    throws Exception
 295    {
 296  0 startCache(Configuration.CacheMode.REPL_SYNC, null);
 297  0 performTest(1);
 298    }
 299   
 300    /**
 301    * Test for a synchronously replicated cache without cache loader and two
 302    * modification per transaction.<br>
 303    * This test has never failed up to now.
 304    *
 305    * @throws Exception Any exception if thrown by the cache.
 306    */
 307  0 public void disabledtestReplSyncNoCacheLoader2Modifications()
 308    throws Exception
 309    {
 310  0 startCache(Configuration.CacheMode.REPL_SYNC, null);
 311  0 performTest(2);
 312    }
 313   
 314    /**
 315    * Test for a synchronously replicated cache with cache loader and single
 316    * modification per transaction.<br>
 317    * This test has never failed up to now.
 318    *
 319    * @throws Exception Any exception if thrown by the cache.
 320    */
 321  0 public void disabledtestReplSyncCacheLoader1Modification() throws Exception
 322    {
 323  0 startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
 324  0 performTest(1);
 325    }
 326   
 327    /**
 328    * Test for a synchronously replicated cache with cache loader and two
 329    * modifications per transaction.<br>
 330    * This test fails very often with a TimeoutException.
 331    *
 332    * @throws Exception Any exception if thrown by the cache.
 333    */
 334  1 public void testReplSyncCacheLoader2Modifications()
 335    throws Exception
 336    {
 337  1 startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
 338  1 performTest(2);
 339    }
 340   
 341    /**
 342    * Perform a single test, using the pre-configured cache.
 343    *
 344    * @param modificationsPerTx The number of modifications per transaction.
 345    * @throws Exception Any exception if thrown by the cache.
 346    */
 347  2 private void performTest(int modificationsPerTx) throws Exception
 348    {
 349  2 for (int i = 0; i < NUM_RUNS; i++)
 350    {
 351  105 if (mcl_exception != null)
 352    {
 353    // terminate the test on the first failed worker
 354  1 fail("Due to an exception: " + mcl_exception);
 355    }
 356    // start several worker threads to work with the same set of FQN's
 357  104 Worker[] t = new Worker[NUM_WORKERS];
 358  104 CountDownLatch latch = new CountDownLatch(1);
 359  104 for (int j = 0; j < t.length; j++)
 360    {
 361  1040 t[j] = new Worker(latch, NUM_FQNS_PER_RUN * i,
 362    NUM_FQNS_PER_RUN, modificationsPerTx);
 363  1040 t[j].start();
 364    }
 365    // fire the workers to start processing
 366  104 latch.countDown();
 367    // wait for all workers to complete
 368  104 for (int j = 0; j < t.length; j++)
 369    {
 370  1040 t[j].join();
 371    }
 372    }
 373    }
 374   
 375    /**
 376    * Returns a user transaction to be associated with the calling thread.
 377    *
 378    * @return A user transaction.
 379    * @throws Exception Any exception thrown by the context lookup.
 380    */
 381  10363 private UserTransaction getTransaction() throws Exception
 382    {
 383  10363 return (UserTransaction) new InitialContext(PROPERTIES)
 384    .lookup("UserTransaction");
 385    }
 386   
 387    /**
 388    * Log a message.
 389    *
 390    * @param msg The meessage to be logged.
 391    */
 392  20728 private void log(String msg)
 393    {
 394  20728 System.out.println(System.currentTimeMillis() + " "
 395    + Thread.currentThread() + " " + msg);
 396    }
 397   
 398    /**
 399    * A worker thread that applies the concurrent modifications.
 400    *
 401    * @author Marian Nikolov
 402    * @author $Author: msurtani $
 403    * @version $Date: 2007/01/31 02:40:21 $
 404    */
 405    private class Worker extends Thread
 406    {
 407    /**
 408    * Used to fire all workers at the same time.
 409    */
 410    private final CountDownLatch m_latch;
 411    /**
 412    * The start id, used as part of the node FQN.
 413    */
 414    private final int m_start;
 415    /**
 416    * The number of nodes to create in a single run.
 417    */
 418    private final int m_count;
 419    /**
 420    * The number of modifications per single transaction.
 421    */
 422    private final int m_modificationsPerTx;
 423   
 424    /**
 425    * The state of the thread, used for logging.
 426    */
 427    private int m_state;
 428   
 429    /**
 430    * Constructor.
 431    *
 432    * @param latch Used to synchronize the startup of all worker threads.
 433    * @param start The start id.
 434    * @param count The number of nodes to create in a single run.
 435    * @param modificationsPerTx The number of modifications per
 436    * transaction.
 437    */
 438  1040 public Worker(CountDownLatch latch, int start, int count, int modificationsPerTx)
 439    {
 440  1040 m_latch = latch;
 441  1040 m_start = start;
 442  1040 m_count = count;
 443  1040 m_state = -1;
 444  1040 m_modificationsPerTx = modificationsPerTx;
 445    }
 446   
 447    /**
 448    * {@inheritDoc}
 449    */
 450  1040 public void run()
 451    {
 452  1040 try
 453    {
 454    // the latch shall fire all workers at the same time
 455  1040 m_latch.await();
 456  1040 for (int i = m_start; i < m_start + m_count; i++)
 457    {
 458  10364 m_state = -1;
 459  10364 log("enter " + i);
 460  10364 if (checkAndSetState())
 461    {
 462  1 return;
 463    }
 464  10363 long time = System.currentTimeMillis();
 465  10363 UserTransaction tx = getTransaction();
 466  10363 tx.begin();
 467  10363 if (checkAndSetState())
 468    {
 469  0 return;
 470    }
 471    // the first worker would create a new node for the FQN
 472    // all the others would update the same node
 473  10363 Fqn fqn = new Fqn(new Fqn("NODE"), i);
 474  10363 for (int m = 0; m < m_modificationsPerTx; m++)
 475    {
 476  20719 cache.put(fqn, new Integer(m), new Integer(i));
 477  20714 if (checkAndSetState())
 478    {
 479  2 return;
 480    }
 481    }
 482  10356 tx.commit();
 483  10356 if (checkAndSetState())
 484    {
 485  0 return;
 486    }
 487  10356 time = System.currentTimeMillis() - time;
 488  10356 log("leave " + i + " took " + time + " msec");
 489    }
 490    }
 491    catch (Exception e)
 492    {
 493  5 log("caught exception in state " + m_state + ": " + e);
 494  5 mcl_exception = e;
 495    }
 496    }
 497   
 498    /**
 499    * Checks the current thread and sets it state.
 500    *
 501    * @return True if the worker has to terminate, false otherwise.
 502    */
 503  51797 private boolean checkAndSetState()
 504    {
 505  51797 if (mcl_exception != null)
 506    {
 507    // another worker failed, terminate
 508  3 log("detected failure in state " + m_state);
 509  3 return true;
 510    }
 511  51794 m_state++;
 512  51794 return false;
 513    }
 514    }
 515   
 516  1 public static Test suite()
 517    {
 518   
 519  1 return new TestSuite(ConcurrentCreationDeadlockTest.class);
 520   
 521    }
 522   
 523   
 524  0 public static void main(String[] args) throws Exception
 525    {
 526   
 527  0 junit.textui.TestRunner.run(suite());
 528   
 529    }
 530    }