Clover coverage report -
Coverage timestamp: Thu Jul 5 2007 20:02:32 EDT
file stats: LOC: 540   Methods: 24
NCLOC: 267   Classes: 2
 
 Source file Conditionals Statements Methods TOTAL
ConcurrentCreationDeadlockTest.java 70.8% 65.3% 54.2% 64.4%
coverage coverage
 1    package org.jboss.cache.loader.deadlock;
 2   
 3    import junit.framework.Test;
 4    import junit.framework.TestSuite;
 5    import org.jboss.cache.CacheImpl;
 6    import org.jboss.cache.DefaultCacheFactory;
 7    import org.jboss.cache.Fqn;
 8    import org.jboss.cache.config.Configuration;
 9    import org.jboss.cache.config.Configuration.CacheMode;
 10    import org.jboss.cache.factories.UnitTestCacheConfigurationFactory;
 11    import org.jboss.cache.loader.AbstractCacheLoaderTestBase;
 12    import org.jboss.cache.transaction.DummyTransactionManager;
 13   
 14    import javax.naming.Context;
 15    import javax.naming.InitialContext;
 16    import javax.transaction.UserTransaction;
 17    import java.util.Properties;
 18    import java.util.concurrent.CountDownLatch;
 19   
 20    /**
 21    * <b>Test based on a contribution by Marian Nokolov/Paul Miodonski at Siemens AG.</b>
 22    * <p/>
 23    * This test has been created to simulate a unexpected TimeoutException that is
 24    * thrown by the JBossCache. The original scenario that has been observed: <br>
 25    * Cache in either LOCAL or REPL_SYNC mode with CacheLoader.
 26    * <ul>
 27    * <li>1. Concurrent threads A, B and C, each associated with a transaction.
 28    * Threads A and B try to modify FQN X, thread C tries to modify FQN Y.</li>
 29    * <li>2. Thread A locks X.</li>
 30    * <li>3. Thread B blocks on X (correct since A is the active writer).</li>
 31    * <li>4. Thread A tries to do multiple modifications and suddenly blocks on X
 32    * (although it is the active writer already) - this is the 1-st problem.</li>
 33    * <li>5. Thread C blocks somewhere as well, although it has nothing to do with
 34    * X and is the only one that works on Y - this is a 2-nd problem.</li>
 35    * <li>6. Thread B fails with TimeoutException and its transaction is rolled
 36    * back - this is correct given that A is still holding the lock on X.</li>
 37    * <li>7. Thread A continues its job and its transaction completes successfully
 38    * (after unexpected locking timeout delay).</li>
 39    * <li>8. Thread C continues its job and successfully commits the transaction
 40    * (with unexpected locking timeout delay).</li>
 41    * </ul>
 42    * <br>
 43    * There are two problems with this:
 44    * <ul>
 45    * <li>1. One or more concurrent transactions fail, although the pessimistic
 46    * locking should sequentialize them and guarantee that all should succeed.</li>
 47    * <li>2. Any other thread that tries to acquire a lock in the cache (even for
 48    * a different FQN!?) is blocked for the duration of the locking timeout, i.e.
 49    * the whole application is locked for few seconds. The active writer for the
 50    * corresponding FQN is blocked as well in the middle of the transaction!</li>
 51    * </ul>
 52    * <br>
 53    * At least until now, the error can be reproduced only if the following is
 54    * true:
 55    * <ul>
 56    * <li>Concurrent transactions forcing creation of the same FQN at the same
 57    * time.</li>
 58    * <li>More than one update per TX per FQN - trying to acquire lock on the same
 59    * FQN multiple times per TX.</li>
 60    * <li>Cache with CacheLoader - maybe it has something to do with the
 61    * CacheLoader/StoreInterceptor's...</li>
 62    * </ul>
 63    */
 64    public class ConcurrentCreationDeadlockTest extends AbstractCacheLoaderTestBase
 65    {
 66    /**
 67    * The number of worker threads to start concurrently.
 68    */
 69    private static final int NUM_WORKERS = 10;
 70    /**
 71    * The number of test runs to perform.
 72    */
 73    private static final int NUM_RUNS = 100;
 74    /**
 75    * The number of FQN's per test run.
 76    */
 77    private static final int NUM_FQNS_PER_RUN = 10;
 78   
 79    /**
 80    * The initial context factory properties.
 81    */
 82    private static final Properties PROPERTIES;
 83    /**
 84    * The context factory to be used for the test.
 85    */
 86    private static final String CONTEXT_FACTORY =
 87    "org.jboss.cache.transaction.DummyContextFactory";
 88    /**
 89    * The original context factory to be restored after the test.
 90    */
 91    private String m_contextFactory = null;
 92   
 93    /**
 94    * Exception recorded if any of the worker threads fails.
 95    */
 96    private static volatile Exception mcl_exception = null;
 97   
 98    /**
 99    * The cache under test.
 100    */
 101    private CacheImpl cache = null;
 102   
 103    static
 104    {
 105  1 PROPERTIES = new Properties();
 106  1 PROPERTIES.put(Context.INITIAL_CONTEXT_FACTORY,
 107    "org.jboss.cache.transaction.DummyContextFactory");
 108    }
 109   
 110    /**
 111    * Constructor.
 112    *
 113    * @param name The test name.
 114    */
 115  2 public ConcurrentCreationDeadlockTest(String name)
 116    {
 117  2 super(name);
 118    }
 119   
 120    /**
 121    * {@inheritDoc}
 122    */
 123  2 public void setUp() throws Exception
 124    {
 125  2 super.setUp();
 126  2 mcl_exception = null;
 127  2 m_contextFactory = System.getProperty(Context.INITIAL_CONTEXT_FACTORY);
 128  2 System.setProperty(Context.INITIAL_CONTEXT_FACTORY, CONTEXT_FACTORY);
 129  2 cache = (CacheImpl) DefaultCacheFactory.getInstance().createCache(false);
 130  2 Configuration c = UnitTestCacheConfigurationFactory.createConfiguration(CacheMode.REPL_SYNC);
 131  2 c.setTransactionManagerLookupClass("org.jboss.cache.transaction.DummyTransactionManagerLookup");
 132  2 cache.setConfiguration(c);
 133    }
 134   
 135    /**
 136    * {@inheritDoc}
 137    */
 138  2 public void tearDown() throws Exception
 139    {
 140  2 super.tearDown();
 141  2 DummyTransactionManager.destroy();
 142  2 cache.stop();
 143  2 cache = null;
 144  2 if (m_contextFactory != null)
 145    {
 146  1 System.setProperty(Context.INITIAL_CONTEXT_FACTORY,
 147    m_contextFactory);
 148  1 m_contextFactory = null;
 149    }
 150    }
 151   
 152    /**
 153    * Initializes and starts the cache.
 154    *
 155    * @param cacheMode The cache mode.
 156    * @param cacheLoaderClass The name of the cache loader class.
 157    * @throws Exception Any exception if thrown by the cache.
 158    */
 159  2 private void startCache(Configuration.CacheMode cacheMode, String cacheLoaderClass)
 160    throws Exception
 161    {
 162  2 cache.getConfiguration().setCacheMode(cacheMode);
 163  2 if (cacheLoaderClass != null)
 164    {
 165  2 cache.getConfiguration().setCacheLoaderConfig(getSingleCacheLoaderConfig("", cacheLoaderClass, "", false, false, false));
 166    }
 167  2 cache.getConfiguration().setLockAcquisitionTimeout(600000);
 168  2 cache.start();
 169    }
 170   
 171    /**
 172    * Test for a local cache without cache loader and single modification per
 173    * transaction.<br>
 174    * This test has never failed up to now.
 175    *
 176    * @throws Exception Any exception if thrown by the cache.
 177    */
 178  0 public void disabledtestLocalNoCacheLoader1Modification() throws Exception
 179    {
 180  0 startCache(Configuration.CacheMode.LOCAL, null);
 181  0 performTest(1);
 182    }
 183   
 184    /**
 185    * Test for a local cache without cache loader and two modifications per
 186    * transaction.<br>
 187    * This test has never failed up to now.
 188    *
 189    * @throws Exception Any exception if thrown by the cache.
 190    */
 191  0 public void disabledtestLocalNoCacheLoader2Modifications()
 192    throws Exception
 193    {
 194  0 startCache(Configuration.CacheMode.LOCAL, null);
 195  0 performTest(2);
 196    }
 197   
 198    /**
 199    * Test for a local cache with cache loader and single modification per
 200    * transaction.<br>
 201    * This test has never failed up to now.
 202    *
 203    * @throws Exception Any exception if thrown by the cache.
 204    */
 205  0 public void disabledtestLocalCacheLoader1Modification() throws Exception
 206    {
 207  0 startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
 208  0 performTest(1);
 209    }
 210   
 211    /**
 212    * Test for a local cache with cache loader and two modifications per
 213    * transaction.<br>
 214    * This test does very often fail with a TimeoutException.
 215    *
 216    * @throws Exception Any exception if thrown by the cache.
 217    */
 218  1 public void testLocalCacheLoader2Modifications() throws Exception
 219    {
 220  1 startCache(Configuration.CacheMode.LOCAL, "org.jboss.cache.loader.DummyCacheLoader");
 221  1 performTest(2);
 222    }
 223   
 224    /**
 225    * Test for an asynchronously replicated cache without cache loader and
 226    * single modification per transaction.<br>
 227    * This test has never failed up to now.
 228    *
 229    * @throws Exception Any exception if thrown by the cache.
 230    */
 231  0 public void disabledtestReplAsyncNoCacheLoader1Modification()
 232    throws Exception
 233    {
 234  0 startCache(Configuration.CacheMode.REPL_ASYNC, null);
 235  0 performTest(1);
 236    }
 237   
 238    /**
 239    * Test for an asynchronously replicated cache without cache loader and two
 240    * modifications per transaction.<br>
 241    * This test has never failed up to now.
 242    *
 243    * @throws Exception Any exception if thrown by the cache.
 244    */
 245  0 public void disabledtestReplAsyncNoCacheLoader2Modifications()
 246    throws Exception
 247    {
 248  0 startCache(Configuration.CacheMode.REPL_ASYNC, null);
 249  0 performTest(2);
 250    }
 251   
 252    /**
 253    * Test for an asynchronously replicated cache with cache loader and single
 254    * modification per transaction.<br>
 255    * This test has never failed up to now.
 256    *
 257    * @throws Exception Any exception if thrown by the cache.
 258    */
 259  0 public void disabledtestReplAsyncCacheLoader1Modification()
 260    throws Exception
 261    {
 262  0 startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
 263  0 performTest(1);
 264    }
 265   
 266    /**
 267    * Test for an asynchronously replicated cache with cache loader and two
 268    * modification per transaction.<br>
 269    * This test mysteriously never fails, although it should as the LOCAL
 270    * and REPL_SYNC do.
 271    *
 272    * @throws Exception Any exception if thrown by the cache.
 273    */
 274  0 public void disabledtestReplAsyncCacheLoader2Modifications()
 275    throws Exception
 276    {
 277  0 startCache(Configuration.CacheMode.REPL_ASYNC, "org.jboss.cache.loader.DummyCacheLoader");
 278  0 performTest(2);
 279    }
 280   
 281    /**
 282    * Test for a synchronously replicated cache without cache loader and single
 283    * modification per transaction.<br>
 284    * This test has never failed up to now.
 285    *
 286    * @throws Exception Any exception if thrown by the cache.
 287    */
 288  0 public void disabledtestReplSyncNoCacheLoader1Modification()
 289    throws Exception
 290    {
 291  0 startCache(Configuration.CacheMode.REPL_SYNC, null);
 292  0 performTest(1);
 293    }
 294   
 295    /**
 296    * Test for a synchronously replicated cache without cache loader and two
 297    * modification per transaction.<br>
 298    * This test has never failed up to now.
 299    *
 300    * @throws Exception Any exception if thrown by the cache.
 301    */
 302  0 public void disabledtestReplSyncNoCacheLoader2Modifications()
 303    throws Exception
 304    {
 305  0 startCache(Configuration.CacheMode.REPL_SYNC, null);
 306  0 performTest(2);
 307    }
 308   
 309    /**
 310    * Test for a synchronously replicated cache with cache loader and single
 311    * modification per transaction.<br>
 312    * This test has never failed up to now.
 313    *
 314    * @throws Exception Any exception if thrown by the cache.
 315    */
 316  0 public void disabledtestReplSyncCacheLoader1Modification() throws Exception
 317    {
 318  0 startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
 319  0 performTest(1);
 320    }
 321   
 322    /**
 323    * Test for a synchronously replicated cache with cache loader and two
 324    * modifications per transaction.<br>
 325    * This test fails very often with a TimeoutException.
 326    *
 327    * @throws Exception Any exception if thrown by the cache.
 328    */
 329  1 public void testReplSyncCacheLoader2Modifications()
 330    throws Exception
 331    {
 332  1 startCache(Configuration.CacheMode.REPL_SYNC, "org.jboss.cache.loader.DummyCacheLoader");
 333  1 performTest(2);
 334    }
 335   
 336    /**
 337    * Perform a single test, using the pre-configured cache.
 338    *
 339    * @param modificationsPerTx The number of modifications per transaction.
 340    * @throws Exception Any exception if thrown by the cache.
 341    */
 342  2 private void performTest(int modificationsPerTx) throws Exception
 343    {
 344  2 for (int i = 0; i < NUM_RUNS; i++)
 345    {
 346  200 if (mcl_exception != null)
 347    {
 348    // terminate the test on the first failed worker
 349  0 throw mcl_exception;
 350    }
 351    // start several worker threads to work with the same set of FQN's
 352  200 Worker[] t = new Worker[NUM_WORKERS];
 353  200 CountDownLatch latch = new CountDownLatch(1);
 354  200 for (int j = 0; j < t.length; j++)
 355    {
 356  2000 t[j] = new Worker(latch, NUM_FQNS_PER_RUN * i,
 357    NUM_FQNS_PER_RUN, modificationsPerTx);
 358  2000 t[j].start();
 359    }
 360    // fire the workers to start processing
 361  200 latch.countDown();
 362    // wait for all workers to complete
 363  200 for (Worker worker : t)
 364    {
 365  2000 worker.join();
 366    }
 367    }
 368    }
 369   
 370    /**
 371    * Returns a user transaction to be associated with the calling thread.
 372    *
 373    * @return A user transaction.
 374    * @throws Exception Any exception thrown by the context lookup.
 375    */
 376  20000 private UserTransaction getTransaction() throws Exception
 377    {
 378  20000 return (UserTransaction) new InitialContext(PROPERTIES)
 379    .lookup("UserTransaction");
 380    }
 381   
 382    /**
 383    * Log a message.
 384    *
 385    * @param msg The meessage to be logged.
 386    */
 387  40000 private void log(String msg)
 388    {
 389  40000 System.out.println(System.currentTimeMillis() + " "
 390    + Thread.currentThread() + " " + msg);
 391    }
 392   
 393    /**
 394    * A worker thread that applies the concurrent modifications.
 395    *
 396    * @author Marian Nikolov
 397    * @author $Author: msurtani $
 398    * @version $Date: 2007/06/18 22:44:26 $
 399    */
 400    private class Worker extends Thread
 401    {
 402    /**
 403    * Used to fire all workers at the same time.
 404    */
 405    private final CountDownLatch m_latch;
 406    /**
 407    * The start id, used as part of the node FQN.
 408    */
 409    private final int m_start;
 410    /**
 411    * The number of nodes to create in a single run.
 412    */
 413    private final int m_count;
 414    /**
 415    * The number of modifications per single transaction.
 416    */
 417    private final int m_modificationsPerTx;
 418   
 419    /**
 420    * The state of the thread, used for logging.
 421    */
 422    private int m_state;
 423   
 424    /**
 425    * Constructor.
 426    *
 427    * @param latch Used to synchronize the startup of all worker threads.
 428    * @param start The start id.
 429    * @param count The number of nodes to create in a single run.
 430    * @param modificationsPerTx The number of modifications per
 431    * transaction.
 432    */
 433  2000 public Worker(CountDownLatch latch, int start, int count, int modificationsPerTx)
 434    {
 435  2000 m_latch = latch;
 436  2000 m_start = start;
 437  2000 m_count = count;
 438  2000 m_state = -1;
 439  2000 m_modificationsPerTx = modificationsPerTx;
 440    }
 441   
 442    /**
 443    * {@inheritDoc}
 444    */
 445  2000 public void run()
 446    {
 447  2000 try
 448    {
 449    // the latch shall fire all workers at the same time
 450  2000 m_latch.await();
 451  2000 for (int i = m_start; i < m_start + m_count; i++)
 452    {
 453  20000 m_state = -1;
 454  20000 log("enter " + i);
 455  20000 if (checkAndSetState())
 456    {
 457  0 return;
 458    }
 459  20000 long time = System.currentTimeMillis();
 460  20000 UserTransaction tx = getTransaction();
 461  20000 tx.begin();
 462  20000 if (checkAndSetState())
 463    {
 464  0 try
 465    {
 466  0 tx.rollback();
 467    }
 468    catch (Exception e)
 469    {
 470    }
 471  0 return;
 472    }
 473    // the first worker would create a new node for the FQN
 474    // all the others would update the same node
 475  20000 Fqn fqn = Fqn.fromString("/NODE/" + i);
 476  20000 for (int m = 0; m < m_modificationsPerTx; m++)
 477    {
 478  40000 cache.put(fqn, m, i);
 479  40000 if (checkAndSetState())
 480    {
 481  0 try
 482    {
 483  0 tx.rollback();
 484    }
 485    catch (Exception e)
 486    {
 487    }
 488  0 return;
 489    }
 490    }
 491  20000 tx.commit();
 492  20000 if (checkAndSetState())
 493    {
 494  0 return;
 495    }
 496  20000 time = System.currentTimeMillis() - time;
 497  20000 log("leave " + i + " took " + time + " msec");
 498    }
 499    }
 500    catch (Exception e)
 501    {
 502  0 log("caught exception in state " + m_state + ": " + e);
 503  0 e.printStackTrace();
 504  0 mcl_exception = e;
 505    }
 506    }
 507   
 508    /**
 509    * Checks the current thread and sets it state.
 510    *
 511    * @return True if the worker has to terminate, false otherwise.
 512    */
 513  100000 private boolean checkAndSetState()
 514    {
 515  100000 if (mcl_exception != null)
 516    {
 517    // another worker failed, terminate
 518  0 log("detected failure in state " + m_state);
 519  0 return true;
 520    }
 521  100000 m_state++;
 522  100000 return false;
 523    }
 524    }
 525   
 526  1 public static Test suite()
 527    {
 528   
 529  1 return new TestSuite(ConcurrentCreationDeadlockTest.class);
 530   
 531    }
 532   
 533   
 534  0 public static void main(String[] args) throws Exception
 535    {
 536   
 537  0 junit.textui.TestRunner.run(suite());
 538   
 539    }
 540    }