3 Replies Latest reply on Jun 16, 2010 6:49 PM by deanhiller2000

    how to debug cluster serialization issue...

    deanhiller2000
    deanhiller2000 May 5, 2010 4:29 PM

      On a clustered environment(jboss 5.1) we are getting java.lang.IllegalStateException: Cannot serialize a session while connected


      What could cause this and how do we fix it?  We are getting this in production now and then.


      The detailed error is as follows...


      2010-05-04 19:54:32,494 ERROR [org.jboss.cache.marshall.CacheMarshaller300] (http-0.0.0.0-8080-6) N/A Error while marshalling object: ReplicateCommand{cmds=PutDataMapCommand{fqn=/JSESSION/scripts_localhost/XBTopxSiP+iyAWIUhd6M9g__, dataVersion=null, data={0=248, 1=1273020872409, 3=org.jboss.ha.framework.server.SimpleCachableMarshalledValue{raw={org.jboss.seam.CONVERSATION#103$loadScript=null[i:13], org.jboss.seam.CONVERSATION#103$custInfo=org.jboss.seam.contexts.EntityBean@691d85, org.jboss.seam.CONVERSATION#103$org.jboss.seam.faces.redirect=org.jboss.seam.faces.Redirect@60bc44, org.jboss.seam.CONVERSATION#103$errorMessage=net.voicelog.agent.validator.ErrorMessage@b0b032, development=net.voicelog.seam.general.DevelopmentBean@1ffd643, org.jboss.seam.CONVERSATION#103$org.jboss.seam.core.conversation=org.jboss.seam.core.Conversation@1ec1f4a, org.jboss.seam.CONVERSATION#103$ansQuestion=[q:13,Agent:id:8825] , org.jboss.seam.CONVERSATION#103$entityManager=ManagedPersistenceContext(java:/xcoreNonJtaEntityManagerFactory), org.jboss.seam.CONVERSATION#103$org.jboss.seam.ui.EntityConverter=org.jboss.seam.ui.EntityConverter@14bcedc, org.jboss.seam.web.session=org.jboss.seam.web.Session@daa28d, vRememberMe=net.voicelog.seam.overrides.RememberMe@1c46063, org.jboss.seam.security.rememberMe=org.jboss.seam.security.RememberMe@19439d4, org.ajax4jsf.application.AjaxStateHolder=org.ajax4jsf.application.AjaxStateHolder@1f9f31c, org.jboss.seam.international.localeSelector=org.jboss.seam.international.LocaleSelector@e845ca, org.jboss.seam.core.conversationEntries=ConversationEntries([ConversationEntry(103)]), org.jboss.seam.CONVERSATION#103$loadScript.comments=org.jboss.seam.contexts.EntityBeanList@1110b13, org.ajax4jsf.application.AjaxStateManager.view_sequence=13, org.jboss.seam.CONVERSATION#103$scriptVersion=org.jboss.seam.contexts.EntityBean@1ee2382, org.jboss.seam.security.identity=org.jboss.seam.security.Identity@fa3a6a, org.jboss.seam.CONVERSATION#103$org.jboss.seam.international.statusMessages=org.jboss.seam.faces.FacesMessages@1109c0, javax.faces.request.charset=UTF-8, org.jboss.seam.security.credentials=org.jboss.seam.security.Credentials@1e94988, org.jboss.seam.CONVERSATION#103$org.jboss.seam.persistence.persistenceContexts=org.jboss.seam.persistence.PersistenceContexts@1e5db85, user=[OP1125] }serialized=false}}, globalTransaction=null, erase=false}}
java.lang.IllegalStateException: Cannot serialize a session while connected
        at org.hibernate.impl.SessionImpl.writeObject(SessionImpl.java:1965)
        at sun.reflect.GeneratedMethodAccessor3876.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
        at java.util.HashMap.writeObject(HashMap.java:1001)
        at sun.reflect.GeneratedMethodAccessor289.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
        at org.jboss.ha.framework.server.SimpleCachableMarshalledValue.serialize(SimpleCachableMarshalledValue.java:271)
        at org.jboss.ha.framework.server.SimpleCachableMarshalledValue.writeExternal(SimpleCachableMarshalledValue.java:252)
        at java.io.ObjectOutputStream.writeExternalData(ObjectOutputStream.java:1421)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1390)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:460)
        at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallMap(CacheMarshaller200.java:569)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:370)
        at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
        at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallCommand(CacheMarshaller200.java:519)
        at org.jboss.cache.marshall.CacheMarshaller200.marshallObject(CacheMarshaller200.java:314)
        at org.jboss.cache.marshall.CacheMarshaller300.marshallObject(CacheMarshaller300.java:47)
        at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:191)
        at org.jboss.cache.marshall.CacheMarshaller200.objectToObjectStream(CacheMarshaller200.java:136)
        at org.jboss.cache.marshall.VersionAwareMarshaller.objectToBuffer(VersionAwareMarshaller.java:182)
        at org.jboss.cache.marshall.VersionAwareMarshaller.objectToBuffer(VersionAwareMarshaller.java:52)
        at org.jboss.cache.marshall.CommandAwareRpcDispatcher$ReplicationTask.call(CommandAwareRpcDispatcher.java:369)
        at org.jboss.cache.marshall.CommandAwareRpcDispatcher$ReplicationTask.call(CommandAwareRpcDispatcher.java:341)
        at org.jboss.cache.util.concurrent.WithinThreadExecutor.submit(WithinThreadExecutor.java:82)
        at org.jboss.cache.marshall.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:206)
        at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:748)
        at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:716)
        at org.jboss.cache.RPCManagerImpl.callRemoteMethods(RPCManagerImpl.java:721)
        at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:161)
        at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:135)
        at org.jboss.cache.interceptors.BaseRpcInterceptor.replicateCall(BaseRpcInterceptor.java:107)
        at org.jboss.cache.interceptors.ReplicationInterceptor.handleCrudMethod(ReplicationInterceptor.java:160)
        at org.jboss.cache.interceptors.ReplicationInterceptor.visitPutDataMapCommand(ReplicationInterceptor.java:113)
        at org.jboss.cache.commands.write.PutDataMapCommand.acceptVisitor(PutDataMapCommand.java:104)
        at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
        at org.jboss.cache.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:131)
        at org.jboss.cache.commands.AbstractVisitor.visitPutDataMapCommand(AbstractVisitor.java:60)
        at org.jboss.cache.commands.write.PutDataMapCommand.acceptVisitor(PutDataMapCommand.java:104)
        at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
        at org.jboss.cache.interceptors.TxInterceptor.attachGtxAndPassUpChain(TxInterceptor.java:301)
        at org.jboss.cache.interceptors.TxInterceptor.handleDefault(TxInterceptor.java:283)
        at org.jboss.cache.commands.AbstractVisitor.visitPutDataMapCommand(AbstractVisitor.java:60)
        at org.jboss.cache.commands.write.PutDataMapCommand.acceptVisitor(PutDataMapCommand.java:104)
        at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
        at org.jboss.cache.interceptors.CacheMgmtInterceptor.visitPutDataMapCommand(CacheMgmtInterceptor.java:97)
        at org.jboss.cache.commands.write.PutDataMapCommand.acceptVisitor(PutDataMapCommand.java:104)
        at org.jboss.cache.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116)
        at org.jboss.cache.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:178)
        at org.jboss.cache.interceptors.InvocationContextInterceptor.visitPutDataMapCommand(InvocationContextInterceptor.java:64)
        at org.jboss.cache.commands.write.PutDataMapCommand.acceptVisitor(PutDataMapCommand.java:104)
        at org.jboss.cache.interceptors.InterceptorChain.invoke(InterceptorChain.java:287)
        at org.jboss.cache.invocation.CacheInvocationDelegate.invokePut(CacheInvocationDelegate.java:705)
        at org.jboss.cache.invocation.CacheInvocationDelegate.put(CacheInvocationDelegate.java:519)
        at org.jboss.ha.cachemanager.CacheManagerManagedCache.put(CacheManagerManagedCache.java:277)
        at org.jboss.web.tomcat.service.session.distributedcache.impl.jbc.JBossCacheWrapper.put(JBossCacheWrapper.java:148)
        at org.jboss.web.tomcat.service.session.distributedcache.impl.jbc.AbstractJBossCacheService.storeSessionData(AbstractJBossCacheService.java:405)
        at org.jboss.web.tomcat.service.session.ClusteredSession.processSessionReplication(ClusteredSession.java:1166)
        at org.jboss.web.tomcat.service.session.JBossCacheManager.processSessionRepl(JBossCacheManager.java:1937)
        at org.jboss.web.tomcat.service.session.JBossCacheManager.storeSession(JBossCacheManager.java:309)
        at org.jboss.web.tomcat.service.session.InstantSnapshotManager.snapshot(InstantSnapshotManager.java:51)
        at org.jboss.web.tomcat.service.session.ClusteredSessionValve.handleRequest(ClusteredSessionValve.java:147)
        at org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:94)
        at org.jboss.web.tomcat.service.session.LockingValve.invoke(LockingValve.java:62)
        at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92)
        at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126)
        at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
        at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:158)
        at org.jboss.web.tomcat.service.sso.ClusteredSingleSignOn.invoke(ClusteredSingleSignOn.java:711)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330)
        at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:829)
        at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:598)
        at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)
        at java.lang.Thread.run(Thread.java:619)



      • 915 Views
      • Tags:
        • 1. Re: how to debug cluster serialization issue...
          sherkan777
          sherkan777 May 6, 2010 12:10 PM (in response to deanhiller2000)

          First result  from google




          https://forum.hibernate.org/viewtopic.php?t=968578%29,



          Searching doesn't hurt.


            • Actions
          • 2. Re: how to debug cluster serialization issue...
            deanhiller2000
            deanhiller2000 May 6, 2010 10:23 PM (in response to deanhiller2000)

            I had already read that post, and I don't have that setting that he had set at all, but still get this issue.

              • Actions
            • 3. Re: how to debug cluster serialization issue...
              deanhiller2000
              deanhiller2000 Jun 16, 2010 6:49 PM (in response to deanhiller2000)

              posting to put back at the top.


              Any ideas how to debug this?  I just checked production and it is happening about 1 time per day on each node in the cluster though it happened 3 times yesterday. 


              Is there any logs I can turn up.  Here is the new stack trace with the full session logged out from seam/jboss/hiberante as well.....


              I formatted the PutDataMapCommand best I could(not sure why I missed pasting that last time)....too bad they don't format that in the logs so I don't have too :(.


              2010-06-14 16:22:32,390 ERROR [org.jboss.cache.marshall.CacheMarshaller300] (http-0.0.0.0-8080-12) N/A  Error while marshalling object: ReplicateCommand{
cmds=PutDataMapCommand{
   fqn=/JSESSION/scripts_localhost/W3dYyCmk6m8fwQ-OMJhRrg__, 
   dataVersion=null, 
   data={
       0=418, 
       1=1276550552278, 
       3=org.jboss.ha.framework.server.SimpleCachableMarshalledValue{
          raw={
             org.jboss.seam.CONVERSATION#273$loadScript=[i:8],            
             development=net.voicelog.seam.general.DevelopmentBean@bb2c7b, 
             org.jboss.seam.CONVERSATION#273$ansQuestion=[q:8,Green:id:8821] , 
             org.jboss.seam.CONVERSATION#273$org.jboss.seam.international.statusMessages=org.jboss.seam.faces.FacesMessages@1664dbd, 
             org.jboss.seam.CONVERSATION#273$org.jboss.seam.persistence.persistenceContexts=org.jboss.seam.persistence.PersistenceContexts@d4fc66, 
             org.jboss.seam.CONVERSATION#273$entityManager=ManagedPersistenceContext(java:/xcoreNonJtaEntityManagerFactory), 
             org.jboss.seam.web.session=org.jboss.seam.web.Session@7652d7,
             vRememberMe=net.voicelog.seam.overrides.RememberMe@ba6270, 
             org.jboss.seam.CONVERSATION#273$custInfo=org.jboss.seam.contexts.EntityBean@e6d41, 
             org.jboss.seam.security.rememberMe=org.jboss.seam.security.RememberMe@94b3be, 
             org.jboss.seam.CONVERSATION#273$scriptVersion=org.jboss.seam.contexts.EntityBean@126ed69, 
             org.ajax4jsf.application.AjaxStateHolder=org.ajax4jsf.application.AjaxStateHolder@16df607, 
             org.jboss.seam.international.localeSelector=org.jboss.seam.international.LocaleSelector@a4babd, 
             org.jboss.seam.core.conversationEntries=ConversationEntries([ConversationEntry(273)]), 
             net.voicelog.RequestNum=net.voicelog.web.http.RequestCounter@1b8d4e3, 
             org.ajax4jsf.application.AjaxStateManager.view_sequence=16, 
             org.jboss.seam.security.identity=org.jboss.seam.security.Identity@136fdfd, 
             javax.faces.request.charset=UTF-8, 
             org.jboss.seam.CONVERSATION#273$loadScript.comments=org.jboss.seam.contexts.EntityBeanList@acacfd, 
             org.jboss.seam.CONVERSATION#273$org.jboss.seam.ui.EntityConverter=org.jboss.seam.ui.EntityConverter@1dfa153, 
             org.jboss.seam.security.credentials=org.jboss.seam.security.Credentials@1f6886b, 
             org.jboss.seam.CONVERSATION#273$errorMessage=net.voicelog.agent.validator.ErrorMessage@d8835e, 
             user=[OP1465] , 
             org.jboss.seam.CONVERSATION#273$org.jboss.seam.core.conversation=org.jboss.seam.core.Conversation@1a39e10
          }
          serialized=false
        }
      }, 
   globalTransaction=null, 
   erase=false}}

java.lang.IllegalStateException: Cannot serialize a session while connected
        at org.hibernate.impl.SessionImpl.writeObject(SessionImpl.java:1965)
        at sun.reflect.GeneratedMethodAccessor5260.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)
        at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150)
        at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509)
        at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474)
        at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392)




                • Actions