1 Reply Latest reply on Aug 2, 2012 4:55 AM by Radoslav Husar

    mod_cluster time out

    Willem Noorduin Newbie

      JBoss 5.1.1 JBoss-EWS 1.0.2 mod-cluster Oracle DB

       

      We have an application that has the global structure: Website --> JBoss-AS Cluster --> Database and the communication between website and JBoss-AS is running via mod_cluster. Once in a month we have to generate a report in the JBoss application on basis of a query. On a higher load on the cluster, the JBoss-AS cluster times out with:

       

      Tracelog (generated by log4j by the application):

       

      SPP WARN  tenceContext.ProxyWarnLog 20120730 08:39:25,771 Narrowing proxy to class com.tpgmail.spp.model.ContractProduct - this operation breaks ==

      SPP WARN  tenceContext.ProxyWarnLog 20120730 08:39:25,772 Narrowing proxy to class com.tpgmail.spp.model.ContractProduct - this operation breaks ==

      SPP WARN  tenceContext.ProxyWarnLog 20120730 08:39:25,773 Narrowing proxy to class com.tpgmail.spp.model.ContractProduct - this operation breaks ==

      SPP WARN  tenceContext.ProxyWarnLog 20120730 08:39:25,774 Narrowing proxy to class com.tpgmail.spp.model.ContractProduct - this operation breaks ==

      SPP ERROR org.apache.axis.Message 20120730 09:06:21,015 java.io.IOException:

      ClientAbortException:  java.net.SocketException: Broken pipe

          at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:401)

          at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:449)

          at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:349)

          at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:424)

          at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:413)

          at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:89)

          at java.util.zip.DeflaterOutputStream.deflate(DeflaterOutputStream.java:161)

          at java.util.zip.DeflaterOutputStream.write(DeflaterOutputStream.java:118)

          at java.util.zip.GZIPOutputStream.write(GZIPOutputStream.java:72)

          at com.tpgmail.spp.web.filter.CompressionResponseStream.writeToGZip(CompressionResponseStream.java:352)

          at com.tpgmail.spp.web.filter.CompressionResponseStream.write(CompressionResponseStream.java:329)

          at org.apache.axis.utils.ByteArray.writeTo(ByteArray.java:375)

          at org.apache.axis.SOAPPart.writeTo(SOAPPart.java:265)

          at org.apache.axis.Message.writeTo(Message.java:528)

          at org.apache.axis.transport.http.AxisServlet.sendResponse(AxisServlet.java:902)

          at org.apache.axis.transport.http.AxisServlet.doPost(AxisServlet.java:777)

          at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)

          at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327)

          at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)

          at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)

          at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

          at com.tpgmail.spp.web.filter.CompressionFilter.doFilter(CompressionFilter.java:245)

          at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

          at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

          at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)

          at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)

          at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)

          at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235)

          at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)

          at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:183)

          at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:95)

          at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126)

          at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70)

          at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)

          at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)

          at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:158)

          at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:567)

          at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)

          at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330)

          at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:436)

          at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:385)

          at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:451)

          at java.lang.Thread.run(Thread.java:662)

      Caused by: java.net.SocketException: Broken pipe

          at java.net.SocketOutputStream.socketWrite0(Native Method)

          at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92)

          at java.net.SocketOutputStream.write(SocketOutputStream.java:136)

          at org.apache.coyote.ajp.AjpProcessor$SocketOutputBuffer.doWrite(AjpProcessor.java:1236)

          at org.apache.coyote.Response.doWrite(Response.java:567)

          at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:396)

          ... 42 more

       

      While it is certainly possible that the stress on the server wasn't incorporated in the database / application design (with lower stress and the same database this application doesn't give this error and we get our monthly report), I want to know what can be tweaked in mod-cluster, jdbc and apache to let mod-cluster wait longer for an answer of the database (or the application)

       

      ==================================================================================

       

      JDBC Connection:

       

      <?xml version="1.0" encoding="UTF-8" standalone="yes"?>

      <datasources>

          <local-tx-datasource>

              <jndi-name>jdbc/spp</jndi-name>

              <rar-name>jboss-local-jdbc.rar</rar-name>

              <use-java-context>true</use-java-context>

              <connection-definition>javax.sql.DataSource</connection-definition>

              <jmx-invoker-name>jboss:service=invoker,type=jrmp</jmx-invoker-name>

              <min-pool-size>0</min-pool-size>

              <max-pool-size>64</max-pool-size>

              <blocking-timeout-millis>30000</blocking-timeout-millis>

              <idle-timeout-minutes>30</idle-timeout-minutes>

              <prefill>false</prefill>

              <background-validation>false</background-validation>

              <background-validation-millis>0</background-validation-millis>

              <validate-on-match>true</validate-on-match>

              <statistics-formatter>org.jboss.resource.statistic.pool.JBossDefaultSubPoolStatisticFormatter</statistics-formatter>

              <isSameRM-override-value>false</isSameRM-override-value>

              <allocation-retry>0</allocation-retry>

              <allocation-retry-wait-millis>5000</allocation-retry-wait-millis>

          <application-managed-security xsi:type="securityMetaData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instantie"/>

              <metadata/>

              <local-transaction/>

              <user-name>????</user-name>

              <password>####</password>

              <prepared-statement-cache-size>0</prepared-statement-cache-size>

              <share-prepared-statements>false</share-prepared-statements>

              <set-tx-query-timeout>false</set-tx-query-timeout>

              <query-timeout>0</query-timeout>

              <use-try-lock>0</use-try-lock>

              <url-delimiter>|</url-delimiter>

              <driver-class>oracle.jdbc.pool.OracleDataSource</driver-class>

              <connection-url>jdbc:oracle:thin:@server:port:database</connection-url>

          </local-tx-datasource>

      </datasources>

       

      Virtual Host:

       

      <VirtualHost 1.2.3.4:80>

       

          ServerName spp.tpgpost.nl:80

          ServerAdmin unix@tntpost.nl

          DocumentRoot /var/www/spp.tpgpost.nl/data

          ErrorLog logs/spp.tpgpost.nl-error_log

          CustomLog logs/spp.tpgpost.nl-access_log combined

          LogLevel warn

       

      <Directory />

                      Order deny,allow

                      Deny from all

                      Allow from all

      </Directory>

       

              KeepAliveTimeout 60

              MaxKeepAliveRequests 0

              ManagerBalancerName fbewsppprod

              Createbalancers 1

       

              <Location /mcm>

                      SetHandler mod_cluster-manager

                      Order deny,allow

                      Deny from all

                      Allow from 1.2

              </Location>

       

              <Location /TPGApps/spp>

                       ProxyPass balancer://fbewsppprod/TPGApps/spp stickysession=JSESSIONID|jsessionid nofailover=On

                       ProxyPassReverse  balancer://fbewsppprod/TPGApps/spp

              </Location>

       

      </VirtualHost>