jboss remotting3 threads BLOCKED
vp889x Aug 22, 2016 3:06 PMGreetings ..
We are using jboss EAP 6.4 as ... we have client and ejb server both hosted on jboss EAP 6.4 ..and use remote ejb invocation ...
Issue we are seeing is all the servlet threads except 1 seem to be waiting on a monitor and only 1 thread is acquiring the lock from remotting read thread ... causing high response times ...looks like its using only 1 connection/channel for processing all the requests
NOT SURE ... how we configure to get it working parallel .....
**** We really appreciate your time and help ...
We have upgraded remotting related jars on both client and server jboss servers as below ....
modules\system\layers\base\org\jboss\xnio\nio\main
http://central.maven.org/maven2/org/jboss/xnio/xnio-nio/3.4.0.Final/
----
modules\system\layers\base\org\jboss\xnio\main
https://mvnrepository.com/artifact/org.jboss.xnio/xnio-api/3.4.0.Final
---
modules\system\layers\base\org\jboss\remote-naming\main
https://mvnrepository.com/artifact/org.jboss/jboss-remote-naming/2.0.4.Final
---
modules\system\layers\base\org\jboss\remoting3\main
https://mvnrepository.com/artifact/org.jboss.remoting/jboss-remoting/4.0.21.Final
---
modules\system\layers\base\org\jboss\ejb-client\main
https://mvnrepository.com/artifact/org.jboss/jboss-ejb-client/2.1.6.Final
standalone.xml @ client
<subsystem xmlns="urn:jboss:domain:remoting:1.2">
<connector name="remoting-connector" socket-binding="remoting" security-realm="ApplicationRealm">
<properties>
<property name="org.jboss.remoting3.RemotingOptions.SEND_BUFFER_SIZE" value="25000"/>
<property name="org.jboss.remoting3.RemotingOptions.RECEIVE_BUFFER_SIZE" value="25000"/>
<property name="org.jboss.remoting3.RemotingOptions.TRANSMIT_WINDOW_SIZE" value="104857600"/>
<property name="org.jboss.remoting3.RemotingOptions.RECEIVE_WINDOW_SIZE" value="104857600"/>
<property name="org.jboss.remoting3.RemotingOptions.MAX_OUTBOUND_CHANNELS" value="160"/>
<property name="org.jboss.remoting3.RemotingOptions.MAX_INBOUND_CHANNELS" value="160"/>
<property name="org.jboss.remoting3.RemotingOptions.MAX_OUTBOUND_MESSAGES" value="250"/>
<property name="org.jboss.remoting3.RemotingOptions.MAX_INBOUND_MESSAGES" value="250"/>
<property name="org.jboss.remoting3.RemotingOptions.HEARTBEAT_INTERVAL" value="10000"/>
</properties>
</connector>
</subsystem>
---------------------------------
"Remoting "config-based-naming-client-endpoint" task-1" #153 daemon prio=5 os_prio=0 tid=0x00007f98a4704630 nid=0xabcd waiting on condition [0x00007f97b1420000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000069e4b8f10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"Remoting "config-based-naming-client-endpoint" write-1" #151 daemon prio=5 os_prio=0 tid=0x00007f97f8090e60 nid=0xabc7 runnable [0x00007f97b1199000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x000000069e3ae508> (a sun.nio.ch.Util$2)
- locked <0x000000069e3ae4f0> (a java.util.Collections$UnmodifiableSet)
- locked <0x000000069e393cb8> (a sun.nio.ch.EPollSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.xnio.nio.WorkerThread.run(WorkerThread.java:168)
Remoting "config-based-naming-client-endpoint" task-2" #154 daemon prio=5 os_prio=0 tid=0x00007f98a48715e0 nid=0xabce waiting for monitor entry [0x00007f97b13df000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.jboss.remoting3.remote.RemoteConnectionChannel.receiveMessage(RemoteConnectionChannel.java:360)
- waiting to lock <0x000000069e3ae3e8> (a java.util.ArrayDeque)
at org.jboss.ejb.client.remoting.ChannelAssociation$ResponseReceiver.handleMessage(ChannelAssociation.java:466)
at org.jboss.remoting3.remote.RemoteConnectionChannel$5.run(RemoteConnectionChannel.java:451)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"Remoting "config-based-naming-client-endpoint" read-1" #150 daemon prio=5 os_prio=0 tid=0x00007f97f80905f0 nid=0xabc6 waiting for monitor entry [0x00007f97b1b4f000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.jboss.remoting3.remote.InboundMessage.handleIncoming(InboundMessage.java:200)
- waiting to lock <0x00000006630d96c8> (a org.xnio.streams.BufferPipeInputStream)
at org.jboss.remoting3.remote.RemoteConnectionChannel.handleMessageData(RemoteConnectionChannel.java:474)
at org.jboss.remoting3.remote.RemoteReadListener.handleEvent(RemoteReadListener.java:288)
at org.jboss.remoting3.remote.RemoteReadListener.handleEvent(RemoteReadListener.java:45)
at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189)
at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103)
at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72)
at org.xnio.nio.NioHandle.run(NioHandle.java:90)
at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)
"thread-ajp-threads - 8" #197 prio=5 os_prio=0 tid=0x00007f9840075f60 nid=0xb1c9 in Object.wait() [0x00007f97b1258000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.jboss.ejb.client.EJBClientInvocationContext.awaitResponse(EJBClientInvocationContext.java:413)
- locked <0x00000006636487a8> (a java.lang.Object)
at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:204)
at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:183)
at org.jboss.ejb.client.EJBInvocationHandler.invoke(EJBInvocationHandler.java:146)
at com.sun.proxy.$Proxy47.processXailMessage(Unknown Source)
at com.xanboo.core.xail.web.controller.XailHandlerServlet.processXailMessages(XailHandlerServlet.java:767)
at com.xanboo.core.xail.web.controller.XailHandlerServlet.processRequest(XailHandlerServlet.java:598)
at com.xanboo.core.xail.web.controller.XailHandlerServlet.doPost(XailHandlerServlet.java:173)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:754)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:295)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:231)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:149)
at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:150)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:344)
at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:490)
at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:420)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:511)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at org.jboss.threads.JBossThread.run(JBossThread.java:122)
"thread-ajp-threads - 201" #393 prio=5 os_prio=0 tid=0x00007f9840157ff0 nid=0xb369 runnable [0x00007f97a96eb000]
java.lang.Thread.State: RUNNABLE
at com.wily.introscope.appmap.agent.trace.hc2.AppMapReplayer.searchPreviousTracerAndExecute(AppMapReplayer.java:237)
at com.wily.introscope.appmap.agent.trace.hc2.AppMapReplayer.replayAppMapOnStack(AppMapReplayer.java:118)
at com.wily.introscope.appmap.agent.trace.hc2.AppMapMarkerTracer.ITracer_finishTrace(AppMapMarkerTracer.java:245)
at com.wily.introscope.agent.trace.InvocationData.IMethodTracer_finishTrace(InvocationData.java:1490)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:483)
at org.xnio.nio.AbstractNioStreamChannel.write(AbstractNioStreamChannel.java:303)
at org.xnio.channels.FramedMessageChannel.doFlushBuffer(FramedMessageChannel.java:302)
at org.xnio.channels.FramedMessageChannel.doFlush(FramedMessageChannel.java:316)
at org.xnio.channels.FramedMessageChannel.send(FramedMessageChannel.java:243)
- locked <0x000000069e448c20> (a java.lang.Object)
at org.jboss.remoting3.remote.RemoteConnection$RemoteWriteListener.send(RemoteConnection.java:312)
- locked <0x000000069e3ae3e8> (a java.util.ArrayDeque)
at org.jboss.remoting3.remote.RemoteConnection.send(RemoteConnection.java:122)
at org.jboss.remoting3.remote.InboundMessage.doAcknowledge(InboundMessage.java:133)
at org.jboss.remoting3.remote.InboundMessage.access$000(InboundMessage.java:40)
at org.jboss.remoting3.remote.InboundMessage$2.acknowledge(InboundMessage.java:69)
at org.xnio.streams.BufferPipeInputStream.read(BufferPipeInputStream.java:195)
- locked <0x00000006630d96c8> (a org.xnio.streams.BufferPipeInputStream)
at org.jboss.remoting3.remote.InboundMessage$3.read(InboundMessage.java:155)
- locked <0x00000006630d96c8> (a org.xnio.streams.BufferPipeInputStream)
at java.io.DataInputStream.read(DataInputStream.java:149)
at org.jboss.ejb.client.remoting.ProtocolMessageHandler$1.read(ProtocolMessageHandler.java:105)
at java.io.FilterInputStream.read(FilterInputStream.java:133)
at java.io.FilterInputStream.read(FilterInputStream.java:107)
at org.jboss.marshalling.SimpleDataInput.readUnsignedByteDirect(SimpleDataInput.java:293)
at org.jboss.marshalling.SimpleDataInput.readUnsignedByte(SimpleDataInput.java:249)
at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:284)
at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:224)
at org.jboss.marshalling.river.RiverUnmarshaller.readFields(RiverUnmarshaller.java:1745)
at org.jboss.marshalling.river.RiverUnmarshaller.doInitSerializable(RiverUnmarshaller.java:1658)
at org.jboss.marshalling.river.RiverUnmarshaller.doReadNewObject(RiverUnmarshaller.java:1285)
at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:276)
at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)
at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)
at org.jboss.ejb.client.remoting.MethodInvocationResponseHandler$MethodInvocationResultProducer.getResult(MethodInvocationResponseHandler.java:108)
at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:276)
at org.jboss.ejb.client.EJBObjectInterceptor.handleInvocationResult(EJBObjectInterceptor.java:64)
at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290)
at org.jboss.ejb.client.EJBHomeInterceptor.handleInvocationResult(EJBHomeInterceptor.java:88)
at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290)
at org.jboss.ejb.client.TransactionInterceptor.handleInvocationResult(TransactionInterceptor.java:46)
at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290)
at org.jboss.ejb.client.ReceiverInterceptor.handleInvocationResult(ReceiverInterceptor.java:129)
at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:265)
at org.jboss.ejb.client.EJBClientInvocationContext.awaitResponse(EJBClientInvocationContext.java:453)
at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:204)
at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:183)
at org.jboss.ejb.client.EJBInvocationHandler.invoke(EJBInvocationHandler.java:146)
at com.sun.proxy.$Proxy25.authenticateGateway(Unknown Source)
at com.xanboo.core.xail.web.controller.Authenticator.authenticateFromDatabase(Authenticator.java:192)
at com.xanboo.core.xail.web.controller.Authenticator.authenticateFromDataStore(Authenticator.java:176)
at com.xanboo.core.xail.web.controller.Authenticator.authenticate(Authenticator.java:84)
at com.xanboo.core.xail.web.controller.XailHandlerServlet.processRequest(XailHandlerServlet.java:419)
at com.xanboo.core.xail.web.controller.XailHandlerServlet.doPost(XailHandlerServlet.java:173)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:754)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:295)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:214)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:231)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:149)
at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:169)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:150)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:97)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:102)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:344)
at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:490)
at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:420)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:511)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at org.jboss.threads.JBossThread.run(JBossThread.java:122)
"thread-ajp-threads - 200" #392 prio=5 os_prio=0 tid=0x00007f9840156870 nid=0xb368 waiting on condition [0x00007f97a972d000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x000000069df97cc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
at org.jboss.threads.JBossThread.run(JBossThread.java:122)