0 Replies Latest reply on Feb 19, 2009 5:12 PM by Clebert Suconic

    Another deadlock...

    Clebert Suconic Master

      Just to make a record of another deadlock I saw today....


      While running:

      [junit] Running org.jboss.messaging.tests.integration.http.NettyHttpTest
       [junit] Thread-1 (group:JBM-scheduled-threads-5514070) 15:33:33,169 INFO [QueueImpl] Timed out waiting for all consumers to reconnect to queue FailoverTestAddress so queue will be activated now
       [junit] Timer-1137 15:36:14,736 WARN [RemotingConnectionImpl] Connection failed, client false 875070127 Did not receive ping on connection. It is likely a client has exited or crashed without closing its connection, or the network between the server and client has failed. The connection will now be closed.
      






       [junit] Found one Java-level deadlock:
       [junit] =============================
       [junit] "Thread-5741":
       [junit] waiting to lock monitor 0x00007ffe67662e38 (object 0x00007ffe721d8520, a org.jboss.messaging.core.remoting.impl.invm.InVMConnection),
       [junit] which is held by "Thread-5754"
       [junit] "Thread-5754":
       [junit] waiting to lock monitor 0x00007ffe67662d48 (object 0x00007ffe721dade0, a java.lang.Object),
       [junit] which is held by "Thread-5741"
       [junit]
       [junit] Java stack information for the threads listed above:
       [junit] ===================================================
       [junit] "Thread-5741":
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:87)
       [junit] - waiting to lock <0x00007ffe721d8520> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe721dade0> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe722213f8> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe721dade0> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe72201f50> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe721dade0> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe72200f48> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit] "Thread-5754":
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:868)
       [junit] - waiting to lock <0x00007ffe721dade0> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe721d8520> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit]
       [junit] Found one Java-level deadlock:
       [junit] =============================
       [junit] "Thread-5760":
       [junit] waiting to lock monitor 0x00000000407959a0 (object 0x00007ffe720d2fa0, a org.jboss.messaging.core.remoting.impl.invm.InVMConnection),
       [junit] which is held by "Thread-5761"
       [junit] "Thread-5761":
       [junit] waiting to lock monitor 0x00007ffe67663540 (object 0x00007ffe720d1178, a java.lang.Object),
       [junit] which is held by "Thread-5760"
       [junit]
       [junit] Java stack information for the threads listed above:
       [junit] ===================================================
       [junit] "Thread-5760":
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:87)
       [junit] - waiting to lock <0x00007ffe720d2fa0> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe720d1178> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe720ce8b8> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe720d1178> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe720da9f8> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe720d1178> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe720d1fb8> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit] "Thread-5761":
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:868)
       [junit] - waiting to lock <0x00007ffe720d1178> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe720d2fa0> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit]
       [junit] Found one Java-level deadlock:
       [junit] =============================
       [junit] "Thread-5764":
       [junit] waiting to lock monitor 0x0000000040795d60 (object 0x00007ffe72154df8, a java.lang.Object),
       [junit] which is held by "Thread-5755"
       [junit] "Thread-5755":
       [junit] waiting to lock monitor 0x0000000040795ec8 (object 0x00007ffe7218e978, a org.jboss.messaging.core.remoting.impl.invm.InVMConnection),
       [junit] which is held by "Thread-5764"
       [junit]
       [junit] Java stack information for the threads listed above:
       [junit] ===================================================
       [junit] "Thread-5764":
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:868)
       [junit] - waiting to lock <0x00007ffe72154df8> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe7218e978> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit] "Thread-5755":
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:87)
       [junit] - waiting to lock <0x00007ffe7218e978> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe72154df8> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe72192188> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe72154df8> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe72156650> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe72154df8> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe7218d958> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe72154df8> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe721911f8> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit]
       [junit] Found one Java-level deadlock:
       [junit] =============================
       [junit] "Thread-1911":
       [junit] waiting to lock monitor 0x000000004038fa78 (object 0x00007ffe7204c990, a java.lang.Object),
       [junit] which is held by "Thread-1910"
       [junit] "Thread-1910":
       [junit] waiting to lock monitor 0x000000004038fc58 (object 0x00007ffe7204c9a0, a org.jboss.messaging.core.remoting.impl.invm.InVMConnection),
       [junit] which is held by "Thread-1911"
       [junit]
       [junit] Java stack information for the threads listed above:
       [junit] ===================================================
       [junit] "Thread-1911":
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:868)
       [junit] - waiting to lock <0x00007ffe7204c990> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe7204c9a0> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit] "Thread-1910":
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:87)
       [junit] - waiting to lock <0x00007ffe7204c9a0> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe7204c990> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe7219eb30> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit]
       [junit] Found one Java-level deadlock:
       [junit] =============================
       [junit] "Thread-471":
       [junit] waiting to lock monitor 0x000000004038f208 (object 0x00007ffe71df5158, a java.lang.Object),
       [junit] which is held by "Thread-470"
       [junit] "Thread-470":
       [junit] waiting to lock monitor 0x000000004038f3e8 (object 0x00007ffe71e12718, a org.jboss.messaging.core.remoting.impl.invm.InVMConnection),
       [junit] which is held by "Thread-471"
       [junit]
       [junit] Java stack information for the threads listed above:
       [junit] ===================================================
       [junit] "Thread-471":
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:868)
       [junit] - waiting to lock <0x00007ffe71df5158> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe71e12718> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit] "Thread-470":
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:87)
       [junit] - waiting to lock <0x00007ffe71e12718> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.internalClose(RemotingConnectionImpl.java:603)
       [junit] at org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl.fail(RemotingConnectionImpl.java:457)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.failConnection(ConnectionManagerImpl.java:872)
       [junit] - locked <0x00007ffe71df5158> (a java.lang.Object)
       [junit] at org.jboss.messaging.core.client.impl.ConnectionManagerImpl.connectionDestroyed(ConnectionManagerImpl.java:198)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector$Listener.connectionDestroyed(InVMConnector.java:187)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnection.close(InVMConnection.java:97)
       [junit] - locked <0x00007ffe71e2d408> (a org.jboss.messaging.core.remoting.impl.invm.InVMConnection)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMConnector.disconnect(InVMConnector.java:161)
       [junit] at org.jboss.messaging.core.remoting.impl.invm.InVMAcceptor$Listener$1.run(InVMAcceptor.java:167)
       [junit]
       [junit] Found 5 deadlocks.
       [junit]