StackOverflow after OutOfMemory
mihirp Mar 9, 2010 12:36 AMHello everyone,
We are using ja-sig's CAS (Central Authentication Service) version 3.3.5 which has support for JBossCache and we are using JBoss Cache 3.2.1 with JGroups 2.6.13 on tomcat 6.0.20.
All tickets generated by CAS application are stored as attributes under single node "ticket" so the Cache structure looks something like below:
\ <root node>
\ticket {key1=value1, key2=value2} (here key1 is ticket and value1 is one object representing that ticket with id key1)
so, as you can see the cache will continue growing horizontally, with single node "ticket" and everything else would be added as attributes to that node.
We are using READ_COMMITTED isolation level with SYNC replication using TCP protocol for replication through JGroups. We are not using any inbuilt eviction policy, instead there is a thread running which goes through all the key/value pair and calls "isExpire" method on the object (value) and removes any expired ticket attributes from the cache. We also have state transferred enable. Please see attached config.
Here is the stack trace for OutOfMemory:
2010-02-26 12:39:23,518 ERROR [org.jasig.cas.ticket.registry.JBossCacheTicketRegistry] (http-8443-183) org.jboss.cache.CacheException: java.lang.OutOfMemoryError: GC overhead li
mit exceeded
org.jboss.cache.CacheException: java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:304)
at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:555)
at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:560)
at org.jasig.cas.ticket.registry.JBossCacheTicketRegistry.addTicket(JBossCacheTicketRegistry.java:51)
at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:224)
at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:241)
at org.jasig.cas.web.flow.GenerateServiceTicketAction.doExecute(GenerateServiceTicketAction.java:40)
at org.springframework.webflow.action.AbstractAction.execute(AbstractAction.java:192)
at org.springframework.webflow.engine.AnnotatedAction.execute(AnnotatedAction.java:146)
at org.springframework.webflow.engine.ActionExecutor.execute(ActionExecutor.java:59)
at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:156)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.TransitionableState.onEvent(TransitionableState.java:107)
at org.springframework.webflow.engine.Flow.onEvent(Flow.java:534)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.signalEvent(RequestControlContextImpl.java:205)
at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:161)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Flow.start(Flow.java:521)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.start(RequestControlContextImpl.java:193)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:177)
at org.springframework.webflow.executor.FlowExecutorImpl.launch(FlowExecutorImpl.java:187)
at org.springframework.webflow.executor.support.FlowRequestHandler.handleFlowRequest(FlowRequestHandler.java:125)
at org.springframework.webflow.executor.mvc.FlowController.handleRequestInternal(FlowController.java:165)
at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:153)
at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:48)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:875)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:807)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:571)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:501)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:115)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.inspektr.common.web.ClientInfoThreadLocalFilter.doFilterInternal(ClientInfoThreadLocalFilter.java:48)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.jamonapi.JAMonFilter.doFilter(JAMonFilter.java:57)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:567)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:849)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:454)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.jboss.cache.io.ExposedByteArrayOutputStream.write(ExposedByteArrayOutputStream.java:113)
at java.io.ObjectOutputStream$BlockDataOutputStream.drain(ObjectOutputStream.java:1838)
at java.io.ObjectOutputStream$BlockDataOutputStream.setBlockDataMode(ObjectOutputStream.java:1747)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1161)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
at java.util.HashMap.writeObject(HashMap.java:1000)
at sun.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:460)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:191)
at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:136)
2010-02-26 12:39:25,562 ERROR [org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/cas].[cas]] (http-8443-183) Servlet.service() for servlet cas threw exception
java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.jboss.cache.io.ExposedByteArrayOutputStream.write(ExposedByteArrayOutputStream.java:113)
at java.io.ObjectOutputStream$BlockDataOutputStream.drain(ObjectOutputStream.java:1838)
at java.io.ObjectOutputStream$BlockDataOutputStream.setBlockDataMode(ObjectOutputStream.java:1747)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1161)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
at java.util.HashMap.writeObject(HashMap.java:1000)
at sun.reflect.GeneratedMethodAccessor166.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:460)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:191)
at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:136)
2010-02-26 12:39:35,140 ERROR [org.jgroups.protocols.TCP] (http-8443-117) failed handling data from 10.10.22.18:33500
java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.jgroups.protocols.TP.dispatchToThreadPool(TP.java:1352)
at org.jgroups.protocols.TP.receive(TP.java:1325)
at org.jgroups.protocols.BasicTCP.receive(BasicTCP.java:258)
at org.jgroups.blocks.BasicConnectionTable.receive(BasicConnectionTable.java:239)
at org.jgroups.blocks.BasicConnectionTable.send(BasicConnectionTable.java:307)
at org.jgroups.protocols.TCP.send(TCP.java:55)
at org.jgroups.protocols.BasicTCP.sendToSingleMember(BasicTCP.java:219)
at org.jgroups.protocols.BasicTCP.sendToAllMembers(BasicTCP.java:204)
at org.jgroups.protocols.TP.doSend(TP.java:1474)
at org.jgroups.protocols.TP.send(TP.java:1464)
at org.jgroups.protocols.TP.down(TP.java:1185)
at org.jgroups.protocols.Discovery.down(Discovery.java:374)
at org.jgroups.protocols.MERGE2.down(MERGE2.java:175)
at org.jgroups.protocols.FD_SOCK.down(FD_SOCK.java:360)
at org.jgroups.protocols.FD.down(FD.java:315)
at org.jgroups.protocols.VERIFY_SUSPECT.down(VERIFY_SUSPECT.java:95)
at org.jgroups.protocols.pbcast.NAKACK.send(NAKACK.java:803)
at org.jgroups.protocols.pbcast.NAKACK.down(NAKACK.java:604)
at org.jgroups.protocols.pbcast.STABLE.down(STABLE.java:316)
at org.jgroups.protocols.pbcast.GMS.down(GMS.java:901)
at org.jgroups.protocols.FRAG2.fragment(FRAG2.java:256)
at org.jgroups.protocols.FRAG2.down(FRAG2.java:138)
at org.jgroups.protocols.pbcast.STREAMING_STATE_TRANSFER.down(STREAMING_STATE_TRANSFER.java:377)
at org.jgroups.protocols.pbcast.FLUSH.down(FLUSH.java:291)
at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:461)
at org.jgroups.JChannel.downcall(JChannel.java:1540)
at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.down(MessageDispatcher.java:791)
at org.jgroups.blocks.RequestCorrelator.sendRequest(RequestCorrelator.java:304)
at org.jgroups.blocks.GroupRequest.sendRequest(GroupRequest.java:531)
at org.jgroups.blocks.GroupRequest.execute(GroupRequest.java:227)
at org.jgroups.blocks.MessageDispatcher.castMessage(MessageDispatcher.java:468)
at org.jboss.cache.marshall.CommandAwareRpcDispatcher$ReplicationTask.call(CommandAwareRpcDispatcher.java:397)
We start seeing StackOverflow after the above mentioned OutOfMemory:
2010-02-26 12:41:02,865 ERROR [org.jasig.cas.ticket.registry.JBossCacheTicketRegistry] (http-8443-147) org.jboss.cache.CacheException: java.lang.StackOverflowError
org.jboss.cache.CacheException: java.lang.StackOverflowError
at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:304)
at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:555)
at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:560)
at org.jasig.cas.ticket.registry.JBossCacheTicketRegistry.addTicket(JBossCacheTicketRegistry.java:51)
at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:224)
at org.jasig.cas.CentralAuthenticationServiceImpl.grantServiceTicket(CentralAuthenticationServiceImpl.java:241)
at org.jasig.cas.web.flow.GenerateServiceTicketAction.doExecute(GenerateServiceTicketAction.java:40)
at org.springframework.webflow.action.AbstractAction.execute(AbstractAction.java:192)
at org.springframework.webflow.engine.AnnotatedAction.execute(AnnotatedAction.java:146)
at org.springframework.webflow.engine.ActionExecutor.execute(ActionExecutor.java:59)
at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:156)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.DecisionState.doEnter(DecisionState.java:54)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Transition.execute(Transition.java:212)
at org.springframework.webflow.engine.TransitionableState.onEvent(TransitionableState.java:107)
at org.springframework.webflow.engine.Flow.onEvent(Flow.java:534)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.signalEvent(RequestControlContextImpl.java:205)
at org.springframework.webflow.engine.ActionState.doEnter(ActionState.java:161)
at org.springframework.webflow.engine.State.enter(State.java:191)
at org.springframework.webflow.engine.Flow.start(Flow.java:521)
at org.springframework.webflow.engine.impl.RequestControlContextImpl.start(RequestControlContextImpl.java:193)
at org.springframework.webflow.engine.impl.FlowExecutionImpl.start(FlowExecutionImpl.java:177)
at org.springframework.webflow.executor.FlowExecutorImpl.launch(FlowExecutorImpl.java:187)
at org.springframework.webflow.executor.support.FlowRequestHandler.handleFlowRequest(FlowRequestHandler.java:125)
at org.springframework.webflow.executor.mvc.FlowController.handleRequestInternal(FlowController.java:165)
at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:153)
at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:48)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:875)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:807)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:571)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:501)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.jasig.cas.web.init.SafeDispatcherServlet.service(SafeDispatcherServlet.java:115)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.inspektr.common.web.ClientInfoThreadLocalFilter.doFilterInternal(ClientInfoThreadLocalFilter.java:48)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at com.jamonapi.JAMonFilter.doFilter(JAMonFilter.java:57)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:128)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:567)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:849)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:454)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.StackOverflowError
at org.jboss.cache.mvcc.NodeReference.getFqn(NodeReference.java:257)
at org.jboss.cache.mvcc.NodeReference.getFqn(NodeReference.java:257)