0 Replies Latest reply on Dec 9, 2009 4:56 PM by Clebert Suconic

    Stress tests hang today

    Clebert Suconic Master

      There was this message first:

      [junit] java.lang.IllegalStateException: Cannot send a packet while channel is doing failover
       [junit] at org.hornetq.core.remoting.impl.ChannelImpl.send(ChannelImpl.java:179)
       [junit] at org.hornetq.core.remoting.impl.ChannelImpl.send(ChannelImpl.java:143)
       [junit] at org.hornetq.core.server.impl.ServerConsumerImpl$1.run(ServerConsumerImpl.java:361)
       [junit] at org.hornetq.utils.OrderedExecutorFactory$OrderedExecutor$1.run(OrderedExecutorFactory.java:96)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit] [Timer-4] 16:32:34,714 INFO [org.hornetq.tests.integration.cluster.reattach.RandomReattachTest] ** Fail complete
      



      And then I logged on the QA lab and forced a kill -3:
       [junit] Full thread dump Java HotSpot(TM) Server VM (1.5.0_15-b04 mixed mode):
       [junit]
       [junit] "Thread-1 (group:HornetQ-server-threads12368552-8551522)" prio=1 tid=0x0836c770 nid=0x3695 waiting on condition [0xa8265000..0xa8265ea0]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireNanos(AbstractQueuedSynchronizer.java:807)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:1122)
       [junit] at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:291)
       [junit] at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:443)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:475)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "hornetq-failure-check-thread" prio=1 tid=0xab3e2670 nid=0x3693 in Object.wait() [0xa8b77000..0xa8b77fa0]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] - waiting on <0xf0b84060> (a org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread)
       [junit] at org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread.run(RemotingServiceImpl.java:548)
       [junit] - locked <0xf0b84060> (a org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread)
       [junit]
       [junit] "Thread-0 (group:HornetQ-scheduled-threads-11807256)" prio=1 tid=0xab6500c0 nid=0x3692 waiting on condition [0xaad1b000..0xaad1c120]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "hornetq-expiry-reaper-thread" prio=1 tid=0xab562170 nid=0x3691 in Object.wait() [0xa997b000..0xa997c0a0]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] - waiting on <0xf0b841d8> (a org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper)
       [junit] at org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper.run(PostOfficeImpl.java:1061)
       [junit] - locked <0xf0b841d8> (a org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Timer-4" daemon prio=1 tid=0xaa4081f0 nid=0x364a in Object.wait() [0xab297000..0xab298120]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] - waiting on <0xb398d5d8> (a java.util.TaskQueue)
       [junit] at java.lang.Object.wait(Object.java:474)
       [junit] at java.util.TimerThread.mainLoop(Timer.java:483)
       [junit] - locked <0xb398d5d8> (a java.util.TaskQueue)
       [junit] at java.util.TimerThread.run(Timer.java:462)
       [junit]
       [junit] "Thread-4 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0xab5b2ad8 nid=0x1e23 waiting on condition [0xab0a2000..0xab0a2da0]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Thread-3 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x08365868 nid=0x1e1d waiting on condition [0xaa811000..0xaa8120a0]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Thread-2 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0xabcaea08 nid=0x1e1a waiting on condition [0xab8dc000..0xab8dcf20]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Thread-1 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x0835d828 nid=0x1e14 waiting on condition [0xaa913000..0xaa913e20]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Thread-8 (group:HornetQ-client-global-threads-2569862)" daemon prio=1 tid=0x083576f8 nid=0x1e11 waiting on condition [0xaaa96000..0xaaa96ea0]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireNanos(AbstractQueuedSynchronizer.java:807)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:1122)
       [junit] at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:291)
       [junit] at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:443)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:475)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Thread-0 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x082cbdf8 nid=0x1e06 waiting on condition [0xab021000..0xab022120]
       [junit] at sun.misc.Unsafe.park(Native Method)
       [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
       [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879)
       [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504)
       [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497)
       [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470)
       [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674)
       [junit] at java.lang.Thread.run(Thread.java:595)
       [junit]
       [junit] "Low Memory Detector" daemon prio=1 tid=0xacba7af0 nid=0x1e00 runnable [0x00000000..0x00000000]
       [junit]
       [junit] "CompilerThread1" daemon prio=1 tid=0xacba6710 nid=0x1dff waiting on condition [0x00000000..0xac6791c8]
       [junit]
       [junit] "CompilerThread0" daemon prio=1 tid=0xacba5790 nid=0x1dfe waiting on condition [0x00000000..0xac6fa348]
       [junit]
       [junit] "AdapterThread" daemon prio=1 tid=0xacba47e0 nid=0x1dfd waiting on condition [0x00000000..0x00000000]
       [junit]
       [junit] "Signal Dispatcher" daemon prio=1 tid=0xacb9c3c8 nid=0x1dfc waiting on condition [0x00000000..0x00000000]
       [junit]
       [junit] "Finalizer" daemon prio=1 tid=0xacb9a8f8 nid=0x1dfb in Object.wait() [0xaca7d000..0xaca7dda0]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:120)
       [junit] - locked <0xb13bd5f0> (a java.lang.ref.ReferenceQueue$Lock)
       [junit] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:136)
       [junit] at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
       [junit]
       [junit] "Reference Handler" daemon prio=1 tid=0xacb9a3b0 nid=0x1dfa in Object.wait() [0xacafe000..0xacafef20]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] at java.lang.Object.wait(Object.java:474)
       [junit] at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
       [junit] - locked <0xb13bd5f8> (a java.lang.ref.Reference$Lock)
       [junit]
       [junit] "main" prio=1 tid=0x08060078 nid=0x1df1 in Object.wait() [0xffff9000..0xffff9fc8]
       [junit] at java.lang.Object.wait(Native Method)
       [junit] at org.hornetq.core.client.impl.ClientConsumerImpl.receive(ClientConsumerImpl.java:234)
       [junit] - locked <0xec35f7c8> (a org.hornetq.core.client.impl.ClientConsumerImpl)
       [junit] at org.hornetq.core.client.impl.ClientConsumerImpl.receiveImmediate(ClientConsumerImpl.java:328)
       [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.doTestE(RandomReattachTest.java:864)
       [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest$5.run(RandomReattachTest.java:127)
       [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.runTest(RandomReattachTest.java:248)
       [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.testE(RandomReattachTest.java:122)
       [junit] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
       [junit] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
       [junit] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
       [junit] at java.lang.reflect.Method.invoke(Method.java:585)
       [junit] at junit.framework.TestCase.runTest(TestCase.java:164)
       [junit] at junit.framework.TestCase.runBare(TestCase.java:130)
       [junit] at junit.framework.TestResult$1.protect(TestResult.java:106)
       [junit] at junit.framework.TestResult.runProtected(TestResult.java:124)
       [junit] at junit.framework.TestResult.run(TestResult.java:109)
       [junit] at junit.framework.TestCase.run(TestCase.java:120)
       [junit] at junit.framework.TestSuite.runTest(TestSuite.java:230)
       [junit] at junit.framework.TestSuite.run(TestSuite.java:225)
       [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420)
       [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911)
       [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768)
       [junit]
       [junit] "VM Thread" prio=1 tid=0xacb97fb8 nid=0x1df9 runnable
       [junit]
       [junit] "GC task thread#0 (ParallelGC)" prio=1 tid=0x080ca550 nid=0x1df5 runnable
       [junit]
       [junit] "GC task thread#1 (ParallelGC)" prio=1 tid=0x080cb5a8 nid=0x1df6 runnable
       [junit]
       [junit] "GC task thread#2 (ParallelGC)" prio=1 tid=0x080cc1c0 nid=0x1df7 runnable
       [junit]
       [junit] "GC task thread#3 (ParallelGC)" prio=1 tid=0x080ccde0 nid=0x1df8 runnable
       [junit]
       [junit] "VM Periodic Task Thread" prio=1 tid=0xacba9698 nid=0x1e01 waiting on condition
       [junit]