loadbalance is unbalanced in 20node, many threads was waiting on get session
xuxiankun88 Jan 20, 2011 9:46 AMour performance testing env is :
2 physical server each one has 10 node.
1 webserver apache
During 400 users stress testing.
one server cpu is busy average 80%. another is less busy average 30%.
we use mod_cluster default setting. just change ping to 60s.
dump thread in idle server we found many threads was waiting on get session. like:
Thread: ajp-0.0.0.0-8309-33 : priority:5, demon:true, threadId:1309, threadState:TIMED_WAITING
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1969)
org.jgroups.blocks.GroupRequest.collectResponses(GroupRequest.java:518)
org.jgroups.blocks.GroupRequest.execute(GroupRequest.java:232)
org.jgroups.blocks.MessageDispatcher.castMessage(MessageDispatcher.java:468)
org.jboss.cache.marshall.CommandAwareRpcDispatcher$ReplicationTask.call(CommandAwareRpcDispatcher.java:383)
org.jboss.cache.marshall.CommandAwareRpcDispatcher$ReplicationTask.call(CommandAwareRpcDispatcher.java:341)
org.jboss.cache.util.concurrent.WithinThreadExecutor.submit(WithinThreadExecutor.java:82)
org.jboss.cache.marshall.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:206)
org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:748)
org.jboss.cache.interceptors.LegacyDataGravitatorInterceptor.gravitateData(LegacyDataGravitatorInterceptor.java:358)
org.jboss.cache.interceptors.LegacyDataGravitatorInterceptor.remoteBackupGet(LegacyDataGravitatorInterceptor.java:295)
org.jboss.cache.interceptors.LegacyDataGravitatorInterceptor.handleGetMethod(LegacyDataGravitatorInterceptor.java:223)
org.jboss.cache.interceptors.LegacyDataGravitatorInterceptor.visitGetDataMapCommand(LegacyDataGravitatorInterceptor.java:115)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.LegacyCacheLoaderInterceptor.visitGetDataMapCommand(LegacyCacheLoaderInterceptor.java:209)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
org.jboss.cache.interceptors.base.PrePostProcessingCommandInterceptor.handleGetDataMapCommand(PrePostProcessingCommandInterceptor.java:249)
org.jboss.cache.interceptors.base.PrePostProcessingCommandInterceptor.visitGetDataMapCommand(PrePostProcessingCommandInterceptor.java:239)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.TxInterceptor.attachGtxAndPassUpChain(TxInterceptor.java:301)
org.jboss.cache.interceptors.TxInterceptor.handleDefault(TxInterceptor.java:283)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
org.jboss.cache.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:178)
org.jboss.cache.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:130)
org.jboss.cache.commands.AbstractVisitor.visitGetDataMapCommand(AbstractVisitor.java:100)
org.jboss.cache.commands.read.GetDataMapCommand.acceptVisitor(GetDataMapCommand.java:66)
org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:287)
org.jboss.cache.invocation.CacheInvocationDelegate.getData(CacheInvocationDelegate.java:570)
org.jboss.ha.cachemanager.CacheManagerManagedCache.getData(CacheManagerManagedCache.java:212)
org.jboss.web.tomcat.service.session.distributedcache.impl.jbc.JBossCacheWrapper.getData(JBossCacheWrapper.java:55)
org.jboss.web.tomcat.service.session.distributedcache.impl.jbc.AbstractJBossCacheService.getSessionData(AbstractJBossCacheService.java:348)
org.jboss.web.tomcat.service.session.JBossCacheManager.loadSession(JBossCacheManager.java:1832)
org.jboss.web.tomcat.service.session.JBossCacheManager.findSession(JBossCacheManager.java:489)
org.apache.catalina.connector.Request.doGetSession(Request.java:2351)
org.apache.catalina.connector.Request.getSession(Request.java:2094)
org.apache.catalina.connector.RequestFacade.getSession(RequestFacade.java:833)
org.apache.catalina.connector.RequestFacade.getSession(RequestFacade.java:844)
org.jboss.seam.web.IdentityRequestWrapper.(IdentityRequestWrapper.java:23)
org.jboss.seam.web.IdentityFilter.doFilter(IdentityFilter.java:40)
org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69)
org.jboss.seam.web.RedirectFilter.doFilter(RedirectFilter.java:45)
org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69)
org.jboss.seam.servlet.SeamFilter.doFilter(SeamFilter.java:158)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:190)
org.jboss.web.tomcat.service.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:135)
org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:94)
org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:88)
org.jboss.web.tomcat.service.session.LockingValve.invoke(LockingValve.java:62)
org.jboss.modcluster.catalina.CatalinaContext$RequestListenerValve.event(CatalinaContext.java:285)
org.jboss.modcluster.catalina.CatalinaContext$RequestListenerValve.invoke(CatalinaContext.java:261)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:433)
org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92)
org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126)
org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:158)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330)
org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:436)
org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:384)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)
we set sticksession to true.
i think it's like failover. but why we got failover. i didn't get any log said any jboss node is dead.
any suggestion?