MOD_JK hates my 3rd node
zyadamec Feb 2, 2010 2:51 PMI managed to get 2 nodes load balancing rather nicely using mod_jk. Last week we hit our limit and I added a 3rd node. It seems as if this node is being almost completely ignored by mod_jk. every once in a while I will have my session sent to node3 but none of my users have ever seen node3. My setup is very simple, its 3 nodes on the same windoes 2003 server, Apache/2.0.63 (Win32) mod_jk/1.2.28, and jboss 4.2.1 GA with JDK 1.5.0_16-b02. All 3 jboss instances are running as a windows service and have the heap max/min set to 1274M.
THe JKSTATUS page is showing the very little data going to the node(please note that the usage is low on node 1 because I had to bounce node 1 due to an out of memory error due to node 3 not taking any of the load)
Server: name=localhost port=80 Time: datetime=20100202133948 tz=Central Standard Time unix=1265139588 Software: web_server="Apache/2.0.63 (Win32) mod_jk/1.2.28" jk_version=mod_jk/1.2.28 Balancer Workers: count=1 Balancer Worker: name=loadbalancer type=lb sticky_session=True sticky_session_force=False retries=2 recover_time=60 error_escalation_time=30 max_reply_timeouts=0 method=Sessions lock=Optimistic member_count=3 good=3 degraded=0 bad=0 busy=132 max_busy=169 map_count=12 time_to_maintenance_min=44 time_to_maintenance_max=106 last_reset_at=1265113731 last_reset_ago=25857 Member: name=node1 type=ajp13 host=10.16.72.67 port=8109 address=10.16.72.67:8109 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node1" redirect="" domain="" distance=0 state=OK lbmult=1 lbvalue=0 elected=662 errors=0 client_errors=6 reply_timeouts=0 transferred=384578 read=4878199 busy=3 max_busy=18 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137797 last_reset_ago=1791 Member: name=node2 type=ajp13 host=10.16.72.67 port=8209 address=10.16.72.67:8209 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node2" redirect="" domain="" distance=0 state=OK lbmult=1 lbvalue=0 elected=6552 errors=0 client_errors=18 reply_timeouts=0 transferred=4102522 read=53048967 busy=60 max_busy=103 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137455 last_reset_ago=2133 Member: name=node3 type=ajp13 host=10.16.72.67 port=8309 address=10.16.72.67:8309 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node3" redirect="" domain="" distance=0 state=OK/IDLE lbmult=1 lbvalue=0 elected=18 errors=0 client_errors=0 reply_timeouts=0 transferred=5519 read=5040 busy=51 max_busy=52 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137461 last_reset_ago=2127 Map: id=1 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=2 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=3 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=4 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=5 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=6 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=7 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=8 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=9 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=10 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=11 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" Map: id=12 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0" AJP Workers: count=3 AJP Worker: name=node1 type=ajp13 host=10.16.72.67 port=8109 address=10.16.72.67:8109 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857 AJP Worker: name=node2 type=ajp13 host=10.16.72.67 port=8209 address=10.16.72.67:8209 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857 AJP Worker: name=node3 type=ajp13 host=10.16.72.67 port=8309 address=10.16.72.67:8309 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857 Result: type=OK message="Action finished"
Here are my configs as they stand now:
workers.properties:
# Define list of workers that will be used
# for mapping requests
worker.list=node1,node2,node3,loadbalancer,status
# Define Node1
# modify the host as your host IP or DNS name.
worker.node1.port=8109
worker.node1.host=10.16.72.67
worker.node1.type=ajp13
worker.node1.lbfactor=1
worker.node1.cachesize=100
# Define Node2
# modify the host as your host IP or DNS name.
worker.node2.port=8209
worker.node2.host=10.16.72.67
worker.node2.type=ajp13
worker.node2.lbfactor=1
worker.node2.cachesize=100
# Define Node3
# modify the host as your host IP or DNS name.
worker.node3.port=8309
worker.node3.host=10.16.72.67
worker.node3.type=ajp13
worker.node3.lbfactor=1
worker.node3.cachesize=100
# Load-balancing behaviour
worker.loadbalancer.type=lb
worker.loadbalancer.balanced_workers=node1,node2,node3
worker.loadbalancer.sticky_session=1
#worker.list=loadbalancer
# Status worker for managing load balancer
worker.status.type=status
uriworkermap.properties:
# Simple worker configuration file
# Mount the Servlet context to the ajp13 worker
/getpaid/zeroclient*=loadbalancer
/getpaid/zeroclient/*=loadbalancer
/getpaid/zeroclientAdmin*=loadbalancer
/getpaid/zeroclientAdmin/*=loadbalancer
/getpaid/*=loadbalancer
/getpaid*=loadbalancer
mod_jk.conf
# Load mod_jk module
# Specify the filename of the mod_jk lib
LoadModule jk_module modules/mod_jk.so
JkWorkersFile conf/workers.properties
JkLogFile logs/mod_jk.log
JkLogLevel info
JkLogStampFormat "[%a %b %d %H:%M:%S %Y]"
JkOptions +ForwardKeySize +ForwardURICompat -ForwardDirectories
JkRequestLogFormat "%w %V %T"
JkMount /getpaid/zeroclient* loadbalancer
JkMount /getpaid/zeroclient/* loadbalancer
JkMount /getpaid/zeroclientAdmin* loadbalancer
JkMount /getpaid/zeroclientAdmin/* loadbalancer
JkMount /getpaid/* loadbalancer
JkMount /getpaid* loadbalancer
JkMountFile conf/uriworkermap.properties
JkShmFile logs/jk.shm
<Location /jkstatus/>
JkMount status
Order deny,allow
Deny from all
Allow from 127.0.0.1
</Location>
changes only from the httpd.conf:
UseCanonicalName Off
ServerSignature On
Win32DisableAcceptEx
Include conf/mod-jk.conf