We are running a load test on a web application using JMeter. When scaling the application to 100 parallel threads, we are facing a lot of blocked threads waiting to lock java.util.jar.JarFile.

The application is running on JBoss AS 7.1.1 Final. richfaces 4.3.2 jars bundled along with the application.

We have upgraded the JSF to 2.2 by following this post: https://community.jboss.org/thread/203257

The thread which locks the resource from the thread dumps is as follows:

"https-executor-threads - 161" prio=10 tid=0x000000005e3d5000 nid=0x2a3d waiting for monitor entry [0x000000004c77a000]
  java.lang.Thread.State: BLOCKED (on object monitor)
  at java.util.zip.ZipFile.getEntry(ZipFile.java:159)
   - locked <0x00000007619d21e8> (a java.util.jar.JarFile)
  at java.util.jar.JarFile.getEntry(JarFile.java:208)
  at java.util.jar.JarFile.getJarEntry(JarFile.java:191)
  at org.jboss.modules.JarFileResourceLoader.getJarEntry(JarFileResourceLoader.java:157)
  at org.jboss.modules.JarFileResourceLoader.getResource(JarFileResourceLoader.java:198)
  at org.jboss.modules.ModuleClassLoader.loadResourceLocal(ModuleClassLoader.java:317)
  at org.jboss.modules.ModuleClassLoader$1.loadResourceLocal(ModuleClassLoader.java:84)
  at org.jboss.modules.Module.getResource(Module.java:550)
  at org.jboss.modules.ModuleClassLoader.findResource(ModuleClassLoader.java:483)
  at org.jboss.modules.ModuleClassLoader.findResourceAsStream(ModuleClassLoader.java:496)
  at org.jboss.modules.ConcurrentClassLoader.getResourceAsStream(ConcurrentClassLoader.java:371)
  at javax.el.FactoryFinder.find(FactoryFinder.java:143)
  at javax.el.ExpressionFactory.newInstance(ExpressionFactory.java:185)
  at javax.el.ExpressionFactory.newInstance(ExpressionFactory.java:156)
  at javax.el.BeanELResolver.invokeMethod(BeanELResolver.java:723)
  at javax.el.BeanELResolver.invoke(BeanELResolver.java:467)
  at javax.el.CompositeELResolver.invoke(CompositeELResolver.java:246)
  at org.apache.el.parser.AstValue.getValue(AstValue.java:159)
  at org.apache.el.ValueExpressionImpl.getValue(ValueExpressionImpl.java:189)
  at com.sun.faces.facelets.el.TagValueExpression.getValue(TagValueExpression.java:109)
  at javax.faces.component.ComponentStateHelper.eval(ComponentStateHelper.java:194)
  at javax.faces.component.UIComponentBase.isRendered(UIComponentBase.java:457)
  at com.sun.faces.renderkit.html_basic.HtmlBasicRenderer.encodeRecursive(HtmlBasicRenderer.java:297)
  at com.sun.faces.renderkit.html_basic.GridRenderer.renderRow(GridRenderer.java:185)
  at com.sun.faces.renderkit.html_basic.GridRenderer.encodeChildren(GridRenderer.java:129)
  at javax.faces.component.UIComponentBase.encodeChildren(UIComponentBase.java:889)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1896)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1899)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.PanelRenderer.doEncodeEnd(PanelRenderer.java:183)
  at org.richfaces.renderkit.RendererBase.encodeEnd(RendererBase.java:175)
  at javax.faces.component.UIComponentBase.encodeEnd(UIComponentBase.java:919)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1903)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.PopupPanelRenderer.doEncodeEnd(PopupPanelRenderer.java:545)
  at org.richfaces.renderkit.RendererBase.encodeEnd(RendererBase.java:175)
  at javax.faces.component.UIComponentBase.encodeEnd(UIComponentBase.java:919)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1903)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.PanelRenderer.doEncodeEnd(PanelRenderer.java:183)
  at org.richfaces.renderkit.RendererBase.encodeEnd(RendererBase.java:175)
  at javax.faces.component.UIComponentBase.encodeEnd(UIComponentBase.java:919)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1903)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.TogglePanelItemRenderer.doEncodeChildren(TogglePanelItemRenderer.java:136)
  at org.richfaces.renderkit.html.TabRenderer.doEncodeChildren(TabRenderer.java:96)
  at org.richfaces.renderkit.RendererBase.encodeChildren(RendererBase.java:157)
  at javax.faces.component.UIComponentBase.encodeChildren(UIComponentBase.java:889)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1896)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.TogglePanelRenderer.doEncodeChildren(TogglePanelRenderer.java:124)
  at org.richfaces.renderkit.RendererBase.encodeChildren(RendererBase.java:157)
  at javax.faces.component.UIComponentBase.encodeChildren(UIComponentBase.java:889)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1896)
  at org.richfaces.renderkit.RendererBase.renderChildren(RendererBase.java:276)
  at org.richfaces.renderkit.html.PanelRenderer.doEncodeEnd(PanelRenderer.java:183)
  at org.richfaces.renderkit.RendererBase.encodeEnd(RendererBase.java:175)
  at javax.faces.component.UIComponentBase.encodeEnd(UIComponentBase.java:919)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1903)
  at org.richfaces.context.ExtendedPartialViewContextImpl$RenderVisitCallback.visit(ExtendedPartialViewContextImpl.java:557)
  at org.richfaces.context.BaseExtendedVisitContext.invokeVisitCallback(BaseExtendedVisitContext.java:321)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1729)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1740)
  at javax.faces.component.UIForm.visitTree(UIForm.java:362)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1740)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1740)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1740)
  at javax.faces.component.UIComponent.visitTree(UIComponent.java:1740)
  at org.richfaces.context.ExtendedPartialViewContextImpl.processPartialRenderPhase(ExtendedPartialViewContextImpl.java:310)
  at org.richfaces.context.ExtendedPartialViewContextImpl.processPartial(ExtendedPartialViewContextImpl.java:213)
  at javax.faces.component.UIViewRoot.encodeChildren(UIViewRoot.java:1004)
  at javax.faces.component.UIComponent.encodeAll(UIComponent.java:1896)
  at com.sun.faces.application.view.FaceletViewHandlingStrategy.renderView(FaceletViewHandlingStrategy.java:425)
  at com.sun.faces.application.view.MultiViewHandler.renderView(MultiViewHandler.java:131)
  at javax.faces.application.ViewHandlerWrapper.renderView(ViewHandlerWrapper.java:337)
  at com.sun.faces.lifecycle.RenderResponsePhase.execute(RenderResponsePhase.java:120)
  at com.sun.faces.lifecycle.Phase.doPhase(Phase.java:101)
  at com.sun.faces.lifecycle.LifecycleImpl.render(LifecycleImpl.java:219)
  at javax.faces.webapp.FacesServlet.service(FacesServlet.java:647)
  at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:329)
  at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:248)
  at edu.yale.its.tp.cas.client.filter.CASFilter.doFilter(CASFilter.java:317)
  at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:280)
  at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:248)
  at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:275)
  at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:161)
  at org.jboss.as.jpa.interceptor.WebNonTxEmCloserValve.invoke(WebNonTxEmCloserValve.java:50)
  at org.jboss.as.web.security.SecurityContextAssociationValve.invoke(SecurityContextAssociationValve.java:153)
  at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:155)
  at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
  at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
  at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:368)
  at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:877)
  at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:671)
  at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:518)
  at org.jboss.threads.SimpleDirectExecutor.execute(SimpleDirectExecutor.java:33)
  at org.jboss.threads.QueueExecutor.runTask(QueueExecutor.java:801)
  at org.jboss.threads.QueueExecutor.access$100(QueueExecutor.java:45)
  at org.jboss.threads.QueueExecutor$Worker.run(QueueExecutor.java:821)
  at java.lang.Thread.run(Thread.java:662)
  at org.jboss.threads.JBossThread.run(JBossThread.java:122

The thread dump of the same is attached herewith.

Please help us to resolce this scalability problem.