503 errors with mod_cluster
bcouillard Apr 22, 2013 11:05 AMWe have been struggling with this setup for a while now. Any help greatly appreciated...
Our configuration works fine with low activity but fails under heavier load.
When we load the system (50 parallel clients, each posting 20 requests in JMeter) a first group of transactions are processed correctly but then we get 503 errors.
Test done on Apache A directly (without load balancer)
In the Apache log we have the following (for each 503 error) :
==================================================================
[Mon Apr 22 08:14:13 2013] [debug] mod_proxy_cluster.c(1543): proxy: byrequests balancer FAILED
[Mon Apr 22 08:14:13 2013] [error] proxy: CLUSTER: (balancer://cluster-int). All workers are in error state
==================================================================
Our setup is the following :
all servers are CentOS 6.2 (Final)
Apache 2.2.15 servers A and B behind a hrdwr Load Balancer
with mod_cluster 1.2.0
JBOSS 7.1.1 servers A and B in cluster for Apache A and B.
Cannot do Multicast -> Unicast
POstgresql DB
Here are fictive IP adresses (for config below) :
Apache server A 1.2.3.11
Apache server B 1.2.3.41
JBOSS A 1.2.3.12
JBOSS B 1.2.3.40
POSTGRESQL 1.2.3.13
Relevant config :
==================================================================
Apache A mod_cluster.conf
==================================================================
# This is mod_cluster module configuration file. Please refer to README
# for more information on how to enable mod_cluster.
LoadModule proxy_cluster_module modules/mod_proxy_cluster.so
LoadModule slotmem_module modules/mod_slotmem.so
LoadModule advertise_module modules/mod_advertise.so
LoadModule manager_module modules/mod_manager.so
#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
#LoadModule proxy_module modules/mod_proxy.so
SetEnv force-proxy-request-1.0 1
SetEnv proxy-nokeepalive 1
<Location /mod_cluster_manager>
SetHandler mod_cluster-manager
</Location>
Listen 1.2.3.11:6666
<VirtualHost 1.2.3.11:6666>
ErrorLog logs/cluster-error_log
CustomLog logs/cluster-access_log combined
# SecRuleEngine Off
CreateBalancers 1
<Directory />
Order deny,allow
Allow from all
</Directory>
KeepAlive On
KeepAliveTimeout 5
MaxKeepAliveRequests 0
EnableMCPMReceive On
ManagerBalancerName cluster-int
ServerAdvertise Off
#AdvertiseFrequency 5
</VirtualHost>
==================================================================
Apache A app.conf
==================================================================
<VirtualHost *:443>
ServerAdmin webmaster@ZZZ.com
DocumentRoot /var/www/html
ErrorLog logs/https-error_log
CustomLog logs/https-access_log combined
LogLevel debug
SSLEngine on
SSLProtocol all -SSLv2
SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM:+LOW
SSLCertificateFile /etc/httpd/ssl/server.crt
SSLCertificateKeyFile /etc/httpd/ssl/server.key
<IfModule mod_security.c>
SecFilterEngine Off
SecFilterScanPOST Off
</IfModule>
# These configuration allows Apache to store the content of the static
# files that were retrieved from the application server
CacheEnable disk /
CacheRoot /tmp/cache
CacheDirLevels 5
CacheDirLength 2
CacheIgnoreHeaders Set-Cookie
# We need to override the default Apache Error pages
ErrorDocument 500 /PGateway/app/error/2
ErrorDocument 404 /PGateway/app/error/3
ErrorDocument 403 /PGateway/app/error/4
ProxyTimeout 300
ProxyPass *.html !
<Location /PGateway>
ProxyPass balancer://cluster-int/PGateway stickysession=JSESSIONID|jsessionid nofailover=on
ProxyPassReverse balancer://cluster-int/PGateway
</Location>
</VirtualHost>
==================================================================
JBOSS A partial standalone.xml
==================================================================
<subsystem xmlns="urn:jboss:domain:jgroups:1.1" default-stack="tcp">
<stack name="tcp">
<transport type="TCP" socket-binding="jgroups-tcp" diagnostics-socket-binding="jgroups-diagnostics"/>
<protocol type="TCPPING">
<property name="initial_hosts">
1.2.3.12[7800],1.2.3.40[7801]
</property>
<property name="num_initial_members">
2
</property>
<property name="port_range">
0
</property>
<property name="timeout">
2000
</property>
</protocol>
</stack>
</subsystem>
...
<subsystem xmlns="urn:jboss:domain:modcluster:1.0">
<mod-cluster-config advertise-socket="modcluster" advertise-security-key="secret" proxy-list="1.2.3.11:6666,1.2.3.41:6666" balancer="cluster-int" advertise="false" excluded-contexts="default-host:ROOT,MDemo,invoker,jbossws,juddi,console" sticky-session="true">
<dynamic-load-provider history="9" decay="2">
<load-metric type="cpu"/>
</dynamic-load-provider>
</mod-cluster-config>
</subsystem>
...
<socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
<socket-binding name="http" port="8080"/>
<socket-binding name="https" port="8443"/>
<socket-binding name="jgroups-diagnostics" port="0" multicast-address="224.0.75.75" multicast-port="7500"/>
<socket-binding name="jgroups-tcp" port="7600"/>
<socket-binding name="jgroups-tcp-fd" port="57600"/>
<socket-binding name="management-native" interface="management" port="${jboss.management.native.port:9999}"/>
<socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
<socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9443}"/>
<socket-binding name="modcluster" port="0" multicast-address="224.0.1.105" multicast-port="23364"/>
<socket-binding name="osgi-http" interface="management" port="8090"/>
<socket-binding name="remoting" port="4447"/>
<socket-binding name="txn-recovery-environment" port="4712"/>
<socket-binding name="txn-status-manager" port="4713"/>
<outbound-socket-binding name="mail-smtp">
<remote-destination host="localhost" port="25"/>
</outbound-socket-binding>
</socket-binding-group>
==================================================================
JBOSS B partial standalone.xml
==================================================================
<subsystem xmlns="urn:jboss:domain:jgroups:1.1" default-stack="tcp">
<stack name="tcp">
<transport type="TCP" socket-binding="jgroups-tcp" diagnostics-socket-binding="jgroups-diagnostics"/>
<protocol type="TCPPING">
<property name="initial_hosts">
1.2.3.12[7800],1.2.3.40[7801]
</property>
<property name="num_initial_members">
2
</property>
<property name="port_range">
0
</property>
<property name="timeout">
2000
</property>
</protocol>
</stack>
</subsystem>
...
<subsystem xmlns="urn:jboss:domain:modcluster:1.0">
<mod-cluster-config advertise-socket="modcluster" advertise-security-key="secret" proxy-list="1.2.3.11:6666,1.2.3.41:6666" balancer="cluster-int" advertise="false" excluded-contexts="default-host:ROOT,MDemo,invoker,jbossws,juddi,console" sticky-session="true">
<dynamic-load-provider history="9" decay="2">
<load-metric type="cpu"/>
</dynamic-load-provider>
</mod-cluster-config>
</subsystem>
...
<socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
<socket-binding name="http" port="8080"/>
<socket-binding name="https" port="8443"/>
<socket-binding name="jgroups-diagnostics" port="0" multicast-address="224.0.75.75" multicast-port="7500"/>
<socket-binding name="jgroups-tcp" port="7600"/>
<socket-binding name="jgroups-tcp-fd" port="57600"/>
<socket-binding name="management-native" interface="management" port="${jboss.management.native.port:9999}"/>
<socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
<socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9443}"/>
<socket-binding name="modcluster" port="0" multicast-address="224.0.1.105" multicast-port="23364"/>
<socket-binding name="osgi-http" interface="management" port="8090"/>
<socket-binding name="remoting" port="4447"/>
<socket-binding name="txn-recovery-environment" port="4712"/>
<socket-binding name="txn-status-manager" port="4713"/>
<outbound-socket-binding name="mail-smtp">
<remote-destination host="localhost" port="25"/>
</outbound-socket-binding>
</socket-binding-group>