Stress tests hang today
There was this message first:
[junit] java.lang.IllegalStateException: Cannot send a packet while channel is doing failover [junit] at org.hornetq.core.remoting.impl.ChannelImpl.send(ChannelImpl.java:179) [junit] at org.hornetq.core.remoting.impl.ChannelImpl.send(ChannelImpl.java:143) [junit] at org.hornetq.core.server.impl.ServerConsumerImpl$1.run(ServerConsumerImpl.java:361) [junit] at org.hornetq.utils.OrderedExecutorFactory$OrderedExecutor$1.run(OrderedExecutorFactory.java:96) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [Timer-4] 16:32:34,714 INFO [org.hornetq.tests.integration.cluster.reattach.RandomReattachTest] ** Fail complete
And then I logged on the QA lab and forced a kill -3:
[junit] Full thread dump Java HotSpot(TM) Server VM (1.5.0_15-b04 mixed mode): [junit] [junit] "Thread-1 (group:HornetQ-server-threads12368552-8551522)" prio=1 tid=0x0836c770 nid=0x3695 waiting on condition [0xa8265000..0xa8265ea0] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireNanos(AbstractQueuedSynchronizer.java:807) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:1122) [junit] at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:291) [junit] at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:443) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:475) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "hornetq-failure-check-thread" prio=1 tid=0xab3e2670 nid=0x3693 in Object.wait() [0xa8b77000..0xa8b77fa0] [junit] at java.lang.Object.wait(Native Method) [junit] - waiting on <0xf0b84060> (a org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread) [junit] at org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread.run(RemotingServiceImpl.java:548) [junit] - locked <0xf0b84060> (a org.hornetq.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread) [junit] [junit] "Thread-0 (group:HornetQ-scheduled-threads-11807256)" prio=1 tid=0xab6500c0 nid=0x3692 waiting on condition [0xaad1b000..0xaad1c120] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "hornetq-expiry-reaper-thread" prio=1 tid=0xab562170 nid=0x3691 in Object.wait() [0xa997b000..0xa997c0a0] [junit] at java.lang.Object.wait(Native Method) [junit] - waiting on <0xf0b841d8> (a org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper) [junit] at org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper.run(PostOfficeImpl.java:1061) [junit] - locked <0xf0b841d8> (a org.hornetq.core.postoffice.impl.PostOfficeImpl$Reaper) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Timer-4" daemon prio=1 tid=0xaa4081f0 nid=0x364a in Object.wait() [0xab297000..0xab298120] [junit] at java.lang.Object.wait(Native Method) [junit] - waiting on <0xb398d5d8> (a java.util.TaskQueue) [junit] at java.lang.Object.wait(Object.java:474) [junit] at java.util.TimerThread.mainLoop(Timer.java:483) [junit] - locked <0xb398d5d8> (a java.util.TaskQueue) [junit] at java.util.TimerThread.run(Timer.java:462) [junit] [junit] "Thread-4 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0xab5b2ad8 nid=0x1e23 waiting on condition [0xab0a2000..0xab0a2da0] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Thread-3 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x08365868 nid=0x1e1d waiting on condition [0xaa811000..0xaa8120a0] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Thread-2 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0xabcaea08 nid=0x1e1a waiting on condition [0xab8dc000..0xab8dcf20] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Thread-1 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x0835d828 nid=0x1e14 waiting on condition [0xaa913000..0xaa913e20] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Thread-8 (group:HornetQ-client-global-threads-2569862)" daemon prio=1 tid=0x083576f8 nid=0x1e11 waiting on condition [0xaaa96000..0xaaa96ea0] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireNanos(AbstractQueuedSynchronizer.java:807) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:1122) [junit] at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:291) [junit] at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:443) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:475) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Thread-0 (group:HornetQ-client-global-scheduled-threads-7122710)" daemon prio=1 tid=0x082cbdf8 nid=0x1e06 waiting on condition [0xab021000..0xab022120] [junit] at sun.misc.Unsafe.park(Native Method) [junit] at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) [junit] at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1879) [junit] at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) [junit] at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) [junit] at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) [junit] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) [junit] at java.lang.Thread.run(Thread.java:595) [junit] [junit] "Low Memory Detector" daemon prio=1 tid=0xacba7af0 nid=0x1e00 runnable [0x00000000..0x00000000] [junit] [junit] "CompilerThread1" daemon prio=1 tid=0xacba6710 nid=0x1dff waiting on condition [0x00000000..0xac6791c8] [junit] [junit] "CompilerThread0" daemon prio=1 tid=0xacba5790 nid=0x1dfe waiting on condition [0x00000000..0xac6fa348] [junit] [junit] "AdapterThread" daemon prio=1 tid=0xacba47e0 nid=0x1dfd waiting on condition [0x00000000..0x00000000] [junit] [junit] "Signal Dispatcher" daemon prio=1 tid=0xacb9c3c8 nid=0x1dfc waiting on condition [0x00000000..0x00000000] [junit] [junit] "Finalizer" daemon prio=1 tid=0xacb9a8f8 nid=0x1dfb in Object.wait() [0xaca7d000..0xaca7dda0] [junit] at java.lang.Object.wait(Native Method) [junit] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:120) [junit] - locked <0xb13bd5f0> (a java.lang.ref.ReferenceQueue$Lock) [junit] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:136) [junit] at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) [junit] [junit] "Reference Handler" daemon prio=1 tid=0xacb9a3b0 nid=0x1dfa in Object.wait() [0xacafe000..0xacafef20] [junit] at java.lang.Object.wait(Native Method) [junit] at java.lang.Object.wait(Object.java:474) [junit] at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) [junit] - locked <0xb13bd5f8> (a java.lang.ref.Reference$Lock) [junit] [junit] "main" prio=1 tid=0x08060078 nid=0x1df1 in Object.wait() [0xffff9000..0xffff9fc8] [junit] at java.lang.Object.wait(Native Method) [junit] at org.hornetq.core.client.impl.ClientConsumerImpl.receive(ClientConsumerImpl.java:234) [junit] - locked <0xec35f7c8> (a org.hornetq.core.client.impl.ClientConsumerImpl) [junit] at org.hornetq.core.client.impl.ClientConsumerImpl.receiveImmediate(ClientConsumerImpl.java:328) [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.doTestE(RandomReattachTest.java:864) [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest$5.run(RandomReattachTest.java:127) [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.runTest(RandomReattachTest.java:248) [junit] at org.hornetq.tests.integration.cluster.reattach.RandomReattachTest.testE(RandomReattachTest.java:122) [junit] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) [junit] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) [junit] at java.lang.reflect.Method.invoke(Method.java:585) [junit] at junit.framework.TestCase.runTest(TestCase.java:164) [junit] at junit.framework.TestCase.runBare(TestCase.java:130) [junit] at junit.framework.TestResult$1.protect(TestResult.java:106) [junit] at junit.framework.TestResult.runProtected(TestResult.java:124) [junit] at junit.framework.TestResult.run(TestResult.java:109) [junit] at junit.framework.TestCase.run(TestCase.java:120) [junit] at junit.framework.TestSuite.runTest(TestSuite.java:230) [junit] at junit.framework.TestSuite.run(TestSuite.java:225) [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420) [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911) [junit] at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768) [junit] [junit] "VM Thread" prio=1 tid=0xacb97fb8 nid=0x1df9 runnable [junit] [junit] "GC task thread#0 (ParallelGC)" prio=1 tid=0x080ca550 nid=0x1df5 runnable [junit] [junit] "GC task thread#1 (ParallelGC)" prio=1 tid=0x080cb5a8 nid=0x1df6 runnable [junit] [junit] "GC task thread#2 (ParallelGC)" prio=1 tid=0x080cc1c0 nid=0x1df7 runnable [junit] [junit] "GC task thread#3 (ParallelGC)" prio=1 tid=0x080ccde0 nid=0x1df8 runnable [junit] [junit] "VM Periodic Task Thread" prio=1 tid=0xacba9698 nid=0x1e01 waiting on condition [junit]