6 Replies Latest reply on Aug 19, 2013 5:17 AM by jfclere

    503 errors with mod_cluster

    bcouillard

      We have been struggling with this setup for a while now. Any help greatly appreciated...

      Our configuration works fine with low activity but fails under heavier load.

      When we load the system (50 parallel clients, each posting 20 requests in JMeter) a first group of transactions are processed correctly but then we get 503 errors.

      Test done on Apache A directly (without load balancer) 

       

      In the Apache log we have the following (for each 503 error) :

      ==================================================================

      [Mon Apr 22 08:14:13 2013] [debug] mod_proxy_cluster.c(1543): proxy: byrequests balancer FAILED

      [Mon Apr 22 08:14:13 2013] [error] proxy: CLUSTER: (balancer://cluster-int). All workers are in error state

      ==================================================================

       

        

      Our setup is the following :

      all servers are CentOS 6.2 (Final)

      Apache 2.2.15 servers A and B behind a hrdwr Load Balancer

      with mod_cluster 1.2.0

      JBOSS 7.1.1 servers A and B in cluster for Apache A and B.

      Cannot do Multicast -> Unicast

      POstgresql DB

       

      Here are fictive IP adresses (for config below) :

      Apache server A  1.2.3.11

      Apache server B  1.2.3.41

      JBOSS A 1.2.3.12

      JBOSS B 1.2.3.40

      POSTGRESQL 1.2.3.13

       

      Relevant config :

      ==================================================================

      Apache A mod_cluster.conf

      ==================================================================

      # This is mod_cluster module configuration file. Please refer to README

      # for more information on how to enable mod_cluster.

       

       

      LoadModule proxy_cluster_module modules/mod_proxy_cluster.so

      LoadModule slotmem_module       modules/mod_slotmem.so

      LoadModule advertise_module     modules/mod_advertise.so

      LoadModule manager_module       modules/mod_manager.so

      #LoadModule proxy_balancer_module modules/mod_proxy_balancer.so

      #LoadModule proxy_module modules/mod_proxy.so

      SetEnv force-proxy-request-1.0 1

      SetEnv proxy-nokeepalive 1

       

       

      <Location /mod_cluster_manager>

          SetHandler mod_cluster-manager

      </Location>

       

       

      Listen 1.2.3.11:6666

      <VirtualHost 1.2.3.11:6666>

          ErrorLog logs/cluster-error_log

          CustomLog logs/cluster-access_log combined

       

       

          # SecRuleEngine Off

       

       

         CreateBalancers 1

       

       

         <Directory />

            Order deny,allow

            Allow from all

         </Directory>

       

       

         KeepAlive On

       

       

         KeepAliveTimeout 5

         MaxKeepAliveRequests 0

       

       

         EnableMCPMReceive On

         ManagerBalancerName cluster-int

         ServerAdvertise Off

         #AdvertiseFrequency 5

       

       

      </VirtualHost>

       

       

      ==================================================================

      Apache A app.conf

      ==================================================================

       

       

      <VirtualHost *:443>

          ServerAdmin webmaster@ZZZ.com

          DocumentRoot /var/www/html

          ErrorLog logs/https-error_log

          CustomLog logs/https-access_log combined

       

       

      LogLevel debug

       

       

      SSLEngine on

       

       

      SSLProtocol all -SSLv2

       

       

      SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM:+LOW

       

       

      SSLCertificateFile /etc/httpd/ssl/server.crt

       

       

      SSLCertificateKeyFile /etc/httpd/ssl/server.key

       

       

      <IfModule mod_security.c>

      SecFilterEngine Off

      SecFilterScanPOST Off

      </IfModule>

       

       

       

       

      # These configuration allows Apache to store the content of the static

      # files that were retrieved from the application server

      CacheEnable disk /

      CacheRoot /tmp/cache

      CacheDirLevels 5

      CacheDirLength 2

      CacheIgnoreHeaders Set-Cookie

       

       

      # We need to override the default Apache Error pages

      ErrorDocument 500 /PGateway/app/error/2

      ErrorDocument 404 /PGateway/app/error/3

      ErrorDocument 403 /PGateway/app/error/4

       

       

      ProxyTimeout 300

      ProxyPass  *.html !

       

       

      <Location /PGateway>

         ProxyPass          balancer://cluster-int/PGateway stickysession=JSESSIONID|jsessionid  nofailover=on

         ProxyPassReverse   balancer://cluster-int/PGateway

      </Location>

       

       

      </VirtualHost>

       

       

      ==================================================================

      JBOSS A partial standalone.xml

      ==================================================================

              <subsystem xmlns="urn:jboss:domain:jgroups:1.1" default-stack="tcp">

                  <stack name="tcp">

                      <transport type="TCP" socket-binding="jgroups-tcp" diagnostics-socket-binding="jgroups-diagnostics"/>

                      <protocol type="TCPPING">

                          <property name="initial_hosts">

                              1.2.3.12[7800],1.2.3.40[7801]

                          </property>

                          <property name="num_initial_members">

                              2

                          </property>

                          <property name="port_range">

                              0

                          </property>

                          <property name="timeout">

                              2000

                          </property>

                      </protocol>

                  </stack>

              </subsystem>

       

       

      ...

       

              <subsystem xmlns="urn:jboss:domain:modcluster:1.0">

                  <mod-cluster-config advertise-socket="modcluster" advertise-security-key="secret"  proxy-list="1.2.3.11:6666,1.2.3.41:6666" balancer="cluster-int" advertise="false" excluded-contexts="default-host:ROOT,MDemo,invoker,jbossws,juddi,console" sticky-session="true">

                      <dynamic-load-provider history="9" decay="2">

                          <load-metric type="cpu"/>

                      </dynamic-load-provider>

                  </mod-cluster-config>

              </subsystem>

       

       

      ...

       

       

          <socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">

              <socket-binding name="http" port="8080"/>

              <socket-binding name="https" port="8443"/>

              <socket-binding name="jgroups-diagnostics" port="0" multicast-address="224.0.75.75" multicast-port="7500"/>

              <socket-binding name="jgroups-tcp" port="7600"/>

              <socket-binding name="jgroups-tcp-fd" port="57600"/>

              <socket-binding name="management-native" interface="management" port="${jboss.management.native.port:9999}"/>

              <socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>

              <socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9443}"/>

              <socket-binding name="modcluster" port="0" multicast-address="224.0.1.105" multicast-port="23364"/>

              <socket-binding name="osgi-http" interface="management" port="8090"/>

              <socket-binding name="remoting" port="4447"/>

              <socket-binding name="txn-recovery-environment" port="4712"/>

              <socket-binding name="txn-status-manager" port="4713"/>

              <outbound-socket-binding name="mail-smtp">

                  <remote-destination host="localhost" port="25"/>

              </outbound-socket-binding>

          </socket-binding-group>

       

      ==================================================================

      JBOSS B partial standalone.xml

      ==================================================================

              <subsystem xmlns="urn:jboss:domain:jgroups:1.1" default-stack="tcp">

                  <stack name="tcp">

                      <transport type="TCP" socket-binding="jgroups-tcp" diagnostics-socket-binding="jgroups-diagnostics"/>

                      <protocol type="TCPPING">

                          <property name="initial_hosts">

                             1.2.3.12[7800],1.2.3.40[7801]

                          </property>

                          <property name="num_initial_members">

                              2

                          </property>

                          <property name="port_range">

                              0

                          </property>

                          <property name="timeout">

                              2000

                          </property>

                      </protocol>

                  </stack>

              </subsystem>

       

       

      ...

              <subsystem xmlns="urn:jboss:domain:modcluster:1.0">

                  <mod-cluster-config advertise-socket="modcluster" advertise-security-key="secret"  proxy-list="1.2.3.11:6666,1.2.3.41:6666" balancer="cluster-int" advertise="false" excluded-contexts="default-host:ROOT,MDemo,invoker,jbossws,juddi,console" sticky-session="true">

                      <dynamic-load-provider history="9" decay="2">

                          <load-metric type="cpu"/>

                      </dynamic-load-provider>

                  </mod-cluster-config>

              </subsystem>

      ...

          <socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">

              <socket-binding name="http" port="8080"/>

              <socket-binding name="https" port="8443"/>

              <socket-binding name="jgroups-diagnostics" port="0" multicast-address="224.0.75.75" multicast-port="7500"/>

              <socket-binding name="jgroups-tcp" port="7600"/>

              <socket-binding name="jgroups-tcp-fd" port="57600"/>

              <socket-binding name="management-native" interface="management" port="${jboss.management.native.port:9999}"/>

              <socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>

              <socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9443}"/>

              <socket-binding name="modcluster" port="0" multicast-address="224.0.1.105" multicast-port="23364"/>

              <socket-binding name="osgi-http" interface="management" port="8090"/>

              <socket-binding name="remoting" port="4447"/>

              <socket-binding name="txn-recovery-environment" port="4712"/>

              <socket-binding name="txn-status-manager" port="4713"/>

              <outbound-socket-binding name="mail-smtp">

                  <remote-destination host="localhost" port="25"/>

              </outbound-socket-binding>

          </socket-binding-group>