JBoss-4.0.2 server hangs with many threads blocked
JBoss-4.0.2 server hangs with many threads blocked. Since morning it was working properly, at around 2pm our application started running slowly. Checked the jboss server log, but found no such issue which may cause slowness of the system. With the help of VisualVM tool, we came to know about thread blocking and found that due to some threads all other multiple threads were not able to execute. Following is the sample threaddump, full threaddump attached herewith.
****************************************************************************************************************
"http-0.0.0.0-8282-32" - Thread t@63
java.lang.Thread.State: BLOCKED
at java.util.Vector.size(Vector.java:268)
- waiting to lock <3e742ab6> (a java.util.Stack) owned by "http-0.0.0.0-8282-132" t@369
at org.apache.tomcat.util.net.PoolTcpEndpoint.getCurrentThreadsBusy(PoolTcpEndpoint.java:280)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:787)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
at java.lang.Thread.run(Thread.java:619)
Locked ownable synchronizers:
- None
"http-0.0.0.0-8282-31" - Thread t@62
java.lang.Thread.State: BLOCKED
at java.util.Vector.size(Vector.java:268)
- waiting to lock <3e742ab6> (a java.util.Stack) owned by "http-0.0.0.0-8282-132" t@369
at org.apache.tomcat.util.net.PoolTcpEndpoint.getCurrentThreadsBusy(PoolTcpEndpoint.java:280)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:787)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
at java.lang.Thread.run(Thread.java:619)
Locked ownable synchronizers:
- None
"http-0.0.0.0-8282-30" - Thread t@61
java.lang.Thread.State: BLOCKED
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:293)
- waiting to lock <4aad3ba4> (a sun.misc.Launcher$AppClassLoader) owned by "http-0.0.0.0-8282-10" t@41
at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1247)
at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1181)
at sun.org.mozilla.javascript.internal.Kit.classOrNull(Kit.java:74)
at sun.org.mozilla.javascript.internal.NativeJavaPackage.getPkgProperty(NativeJavaPackage.java:140)
- locked <7cd3770c> (a sun.org.mozilla.javascript.internal.NativeJavaTopPackage)
at sun.org.mozilla.javascript.internal.NativeJavaPackage.get(NativeJavaPackage.java:91)
at sun.org.mozilla.javascript.internal.ScriptableObject.getProperty(ScriptableObject.java:1301)
at sun.org.mozilla.javascript.internal.ScriptRuntime.getObjectProp(ScriptRuntime.java:1326)
at sun.org.mozilla.javascript.internal.ScriptRuntime.getObjectProp(ScriptRuntime.java:1315)
at sun.org.mozilla.javascript.internal.Interpreter.interpretLoop(Interpreter.java:2804)
at sun.org.mozilla.javascript.internal.Interpreter.interpret(Interpreter.java:2239)
at sun.org.mozilla.javascript.internal.InterpretedFunction.call(InterpretedFunction.java:138)
at sun.org.mozilla.javascript.internal.ContextFactory.doTopCall(ContextFactory.java:323)
at sun.org.mozilla.javascript.internal.ScriptRuntime.doTopCall(ScriptRuntime.java:2747)
at sun.org.mozilla.javascript.internal.InterpretedFunction.exec(InterpretedFunction.java:149)
at sun.org.mozilla.javascript.internal.Context.evaluateString(Context.java:1138)
at com.sun.script.javascript.RhinoTopLevel.<init>(RhinoTopLevel.java:55)
at com.sun.script.javascript.RhinoScriptEngine.<init>(RhinoScriptEngine.java:75)
at com.sun.script.javascript.RhinoScriptEngineFactory.getScriptEngine(RhinoScriptEngineFactory.java:57)
at javax.script.ScriptEngineManager.getEngineByName(ScriptEngineManager.java:225)
at org.apache.jsp.Framework.Frame_jsp._jspService(org.apache.jsp.Framework.Frame_jsp:578)
at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:97)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:810)
at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:322)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:314)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:264)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:810)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:81)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.jboss.web.tomcat.security.CustomPrincipalValve.invoke(CustomPrincipalValve.java:39)
at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:153)
at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:59)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.jboss.web.tomcat.tc5.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:138)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:856)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
at java.lang.Thread.run(Thread.java:619)
Locked ownable synchronizers:
- None
"http-0.0.0.0-8282-29" - Thread t@60
java.lang.Thread.State: BLOCKED
at java.util.Vector.size(Vector.java:268)
- waiting to lock <3e742ab6> (a java.util.Stack) owned by "http-0.0.0.0-8282-132" t@369
at org.apache.tomcat.util.net.PoolTcpEndpoint.getCurrentThreadsBusy(PoolTcpEndpoint.java:280)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:787)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWorkerThread.java:112)
at java.lang.Thread.run(Thread.java:619)
Locked ownable synchronizers:
- None
****************************************************************************************************************
Performed GC activity, cleared connection pool, but still the application system was hanged. At last, i was required to restart the jboss application server processes. Application server restarted properly and was working properly since then for the day.
Does anyone know of something that might work for this problem.
-
threaddump.tdump.zip 21.5 KB