My JBOSS application frequently getting hanged...
Hi There, I am using JBOSS Version 4.2.3.GA.
Architecture - Client --> Apache --> JBOSS --> Apache ---> Client.
Every request goes to JBOSS via Apache only. For the past 4 days, application is frequently getting hanged and i had to restart the JBOSS services to start working again. Pls look at the logs below and suggest what is going wrong. Kindly help ! I have attached the full logs as well. Pls help urgently.
2013-09-25 09:28:38,132 INFO [STDOUT] [DEBUG] 09:28:38 SymbiosysIVRAuthProcessFilter - Authentication request failed: org.springframework.security.BadCredentialsException: Invalid User id and/or Password. Please try again
2013-09-25 09:28:38,163 INFO [STDOUT] [ INFO] 09:28:38 FormTag - Total time taken to paint jsp:16
2013-09-25 09:28:38,180 INFO [STDOUT] [DEBUG] 09:28:38 AbstractDAO - get successful, instance found
2013-09-25 09:28:38,180 INFO [STDOUT] [DEBUG] 09:28:38 BaseBO - Retrieved
2013-09-25 09:28:38,211 INFO [STDOUT] [DEBUG] 09:28:38 LoggerAspect - End of method: com.c2lbiz.symbiosys.systemcommon.service.GenericService.load
2013-09-25 09:28:38,211 ERROR [org.apache.catalina.core.ContainerBase.[jboss.web].[localhost].[/insurance].[jsp]] Servlet.service() for servlet jsp threw exception
java.lang.IllegalStateException: getOutputStream() has already been called for this response
at org.apache.catalina.connector.Response.getWriter(Response.java:604)
at org.apache.catalina.connector.ResponseFacade.getWriter(ResponseFacade.java:198)
at javax.servlet.ServletResponseWrapper.getWriter(ServletResponseWrapper.java:112)
at org.apache.jasper.runtime.JspWriterImpl.initOut(JspWriterImpl.java:125)
at org.apache.jasper.runtime.JspWriterImpl.flushBuffer(JspWriterImpl.java:118)
at org.apache.jasper.runtime.PageContextImpl.release(PageContextImpl.java:186)
at org.apache.jasper.runtime.JspFactoryImpl.internalReleasePageContext(JspFactoryImpl.java:117)
at org.apache.jasper.runtime.JspFactoryImpl.releasePageContext(JspFactoryImpl.java:76)
at org.apache.jsp.jsp.bqrm.bqrmfiledownload_jsp._jspService(bqrmfiledownload_jsp.java:78)
at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:373)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:336)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:265)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:654)
at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:445)
at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:379)
at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:292)
at org.springframework.web.servlet.view.InternalResourceView.renderMergedOutputModel(InternalResourceView.java:240)
at org.springframework.web.servlet.view.AbstractView.render(AbstractView.java:258)
at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1174)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:901)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:809)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:571)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:501)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:690)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:359)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:109)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:83)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.ExceptionTranslationFilter.doFilterHttp(ExceptionTranslationFilter.java:101)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.AbstractProcessingFilter.doFilterHttp(AbstractProcessingFilter.java:271)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:89)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:89)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.context.HttpSessionContextIntegrationFilter.doFilterHttp(HttpSessionContextIntegrationFilter.java:235)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.SessionFixationProtectionFilter.doFilterHttp(SessionFixationProtectionFilter.java:67)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.concurrent.ConcurrentSessionFilter.doFilterHttp(ConcurrentSessionFilter.java:99)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.util.FilterChainProxy.doFilter(FilterChainProxy.java:174)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:236)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:182)
at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:262)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:446)
at java.lang.Thread.run(Thread.java:595)
2013-09-25 09:28:38,211 ERROR [org.apache.catalina.core.ContainerBase.[jboss.web].[localhost].[/insurance].[insurance]] Servlet.service() for servlet insurance threw exception
java.lang.IllegalStateException: getOutputStream() has already been called for this response
at org.apache.catalina.connector.Response.getWriter(Response.java:604)
at org.apache.catalina.connector.ResponseFacade.getWriter(ResponseFacade.java:198)
at javax.servlet.ServletResponseWrapper.getWriter(ServletResponseWrapper.java:112)
at org.apache.jasper.runtime.JspWriterImpl.initOut(JspWriterImpl.java:125)
at org.apache.jasper.runtime.JspWriterImpl.flushBuffer(JspWriterImpl.java:118)
at org.apache.jasper.runtime.PageContextImpl.release(PageContextImpl.java:186)
at org.apache.jasper.runtime.JspFactoryImpl.internalReleasePageContext(JspFactoryImpl.java:117)
at org.apache.jasper.runtime.JspFactoryImpl.releasePageContext(JspFactoryImpl.java:76)
at org.apache.jsp.jsp.bqrm.bqrmfiledownload_jsp._jspService(bqrmfiledownload_jsp.java:78)
at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:373)
at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:336)
at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:265)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:654)
at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:445)
at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:379)
at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:292)
at org.springframework.web.servlet.view.InternalResourceView.renderMergedOutputModel(InternalResourceView.java:240)
at org.springframework.web.servlet.view.AbstractView.render(AbstractView.java:258)
at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1174)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:901)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:809)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:571)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:501)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:690)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:359)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:109)
at org.springframework.security.intercept.web.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:83)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.ExceptionTranslationFilter.doFilterHttp(ExceptionTranslationFilter.java:101)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.AbstractProcessingFilter.doFilterHttp(AbstractProcessingFilter.java:271)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:89)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.logout.LogoutFilter.doFilterHttp(LogoutFilter.java:89)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.context.HttpSessionContextIntegrationFilter.doFilterHttp(HttpSessionContextIntegrationFilter.java:235)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.ui.SessionFixationProtectionFilter.doFilterHttp(SessionFixationProtectionFilter.java:67)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.concurrent.ConcurrentSessionFilter.doFilterHttp(ConcurrentSessionFilter.java:99)
at org.springframework.security.ui.SpringSecurityFilter.doFilter(SpringSecurityFilter.java:53)
at org.springframework.security.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:371)
at org.springframework.security.util.FilterChainProxy.doFilter(FilterChainProxy.java:174)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:236)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:230)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:182)
at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:84)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:157)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:262)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:844)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:583)
at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:446)
at java.lang.Thread.run(Thread.java:595)
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 BqrmNotesOpenController - Log inside NotesOpenController process()
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 LoggerAspect - Start of method: com.c2lbiz.symbiosys.systemcommon.service.GenericService.load
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 BaseBO - Start of Method
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 AbstractDAO - getting instance of com.c2lbiz.symbiosys.insurance.businesscommon.dataobjects.bqrm.BqrmNotesAttachmentDO with id: 18844
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 AbstractDAO - get successful, instance found
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 BaseBO - Retrieved
2013-09-25 09:28:38,367 INFO [STDOUT] [DEBUG] 09:28:38 PartyFacadeImpl - Party service find start loading
Also, when the slowness getting start, I have observed below in my application logs...
2013-09-25 09:29:32,626 INFO [STDOUT] [DEBUG] 09:29:32 ParameterDAOImpl - Time taken to execute query for StaffDO st,StaffDO st1 : 12870
2013-09-25 09:29:32,626 INFO [STDOUT] [DEBUG] 09:29:32 ParameterDAOImpl - Time taken to execute query for StaffDO st,StaffDO st1 : 12870
2013-09-25 09:29:32,626 INFO [STDOUT] [DEBUG] 09:29:32 ParameterDAOImpl - Executing HQL Query for Combo : (CHANNEL)>>>>>>>>select lkp.channelId ,lkp.channelDescription from ChannelDO lkp where channelStatusCd='APPROVED'
2013-09-25 09:29:32,626 ERROR [STDERR] java.lang.IllegalStateException: getAttribute: Session already invalidated
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.catalina.session.StandardSession.getAttribute(StandardSession.java:1032)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.catalina.session.StandardSessionFacade.getAttribute(StandardSessionFacade.java:110)
2013-09-25 09:29:32,626 ERROR [STDERR] at com.c2lbiz.symbiosys.systemcommon.ui.util.RequestUtil.getSelectedLanguage(RequestUtil.java:200)
2013-09-25 09:29:32,626 ERROR [STDERR] at com.c2lbiz.symbiosys.systemcommon.taglib.BaseControlTag.doStartTag(BaseControlTag.java:57)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jsp.jsp.policy.RELCompleteProposalEntry_jsp._jspx_meth_ui_005fLabelTag_005f16(RELCompleteProposalEntry_jsp.java:3897)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jsp.jsp.policy.RELCompleteProposalEntry_jsp._jspx_meth_ui_005fPanelTag_005f1(RELCompleteProposalEntry_jsp.java:3677)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jsp.jsp.policy.RELCompleteProposalEntry_jsp._jspService(RELCompleteProposalEntry_jsp.java:752)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jasper.runtime.HttpJspBase.service(HttpJspBase.java:70)
2013-09-25 09:29:32,626 ERROR [STDERR] at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:373)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:336)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:265)
2013-09-25 09:29:32,626 ERROR [STDERR] at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
2013-09-25 09:29:32,626 ERROR [STDERR] at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:654)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:445)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:379)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:292)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.springframework.web.servlet.view.InternalResourceView.renderMergedOutputModel(InternalResourceView.java:240)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.springframework.web.servlet.view.AbstractView.render(AbstractView.java:258)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1174)
2013-09-25 09:29:32,642 ERROR [STDERR] at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.j
-
server_25.log.zip 376.5 KB
-
server.log.zip 557.0 KB