JBoss restart script

[davidrabinowitz]

When we restart the script there are two problems under heavy load:

It takes JBoss long time to come down, or it won't shut down at all
After restarting there are still sockets connected to the AJP port, preventing the JBoss to bind to this port (my guess, as waiting for these sockets to close solves the problem.

I've written the following script in order to restart the jboss. My problem is that even if the scripts waits enough time to have all the sockets to be closed by the OS the start fails. If I kill the script and run it again, it starts the jboss perfectly.

I'd appreciate any fixes to the script.

#!/bin/sh

KILLWAIT=15
JBOSSPORT=8009
JBOSS_DIR=$(grep jbossas /etc/passwd | tr ':' ' ' | awk '{print $6}')/server/default
HYPERIC_TIMEOUT=180s

jboss_pids() {
ps aux | grep java|grep jboss.Main | grep Xmx | awk '{print $2}'| tr '\n' ' '
}

jbossctl() {
sudo /sbin/service jbossas $1
}

kill_jboss() {
jboss_pids | sudo xargs kill $1
}

count_open_sockets() {
sudo netstat -alnp 2>/dev/null|grep tcp | awk '{print $4,$5}' | grep :$1 | wc -l
}

get_process_listening_on() {
sudo netstat -alnp | grep $1 | grep LISTEN|awk '{print $7}' | sed -r 's/([0-9]+)\/.*/\1/g'
}

message() {
echo ""
echo "=========================================================="
echo "$1"
echo "=========================================================="
echo ""
}

wait_for_death_of_sockets_of_port() {
echo "waiting for port $1 to die"
SEC=1
SOCKNUM=$(count_open_sockets $1)
while [ $SOCKNUM -ne 0 ];
do
 sleep 1
 echo "waiting for ${SEC} seconds on port $1, number of sockets is ${SOCKNUM}"
 SEC=`expr $SEC + 1`
 SOCKNUM=$(count_open_sockets $1)
done
echo "No more sockets for port $1"
echo ""
}

reset_jboss_env() {
echo "Copying log files"
for LOG in $(echo ${JBOSS_DIR}/log/*.log)
do
 sudo mv ${LOG} ${LOG}.${DATETIME}
done

# delete jboss work tmp data folders
echo "Deleting jboss work tmp data folders"
sudo rm -fr ${JBOSS_DIR}/work
sudo rm -fr ${JBOSS_DIR}/tmp
sudo rm -fr ${JBOSS_DIR}/data
}

restart_hyperic_if_needed() {
sleep ${HYPERIC_TIMEOUT}
HYPERICPID=$(ps aux | grep hyperic | grep java | grep Xmx | awk '{print $2}' | tr '\n' ' ')
if [ "x$HYPERICPID" = "x" ]; then
 # No hyperic is running
 sudo /opt/hyperic/hyperic-hq-agent/bin/hq-agent.sh start
fi
}

#
# Main
#

# activat sudo
sudo whoami > /dev/null



message "stopping jboss"
jbossctl stop &
sleep 1
echo ""

# waiting
JBOSSPID=$(jboss_pids)
SEC=1
while [ "x$JBOSSPID" != "x" ] && [ $SEC -le $KILLWAIT ];
do
 sleep 1
 echo "waiting ($SEC)"
 SEC=`expr $SEC + 1`
 JBOSSPID=$(jboss_pids)
done

# killing jboss
JBOSSPID=$(jboss_pids)
if [ "x$JBOSSPID" != "x" ];
then
 message "Jboss is still alive, trying to kill it"
 kill_jboss -9
fi

# wait for sockets
wait_for_death_of_sockets_of_port $JBOSSPORT

# check if hyperic listens on 8080
HYPERICPID=$(get_process_listening_on 8080)
if [ "x$HYPERICPID" != "x" ]; then
 messgae "killing Hyperic"
 sudo kill -9 $HYPERICPID
fi

wait_for_death_of_sockets_of_port 8080

reset_jboss_env

sudo netstat -alnp | grep LISTEN | grep tcp

message "Stating Jboss"
sleep 5s
jbossctl start &

sleep 10s
message "please follow log, run 'tail -f /var/log/jbossas/default/server-$(hostname).log'"

restart_hyperic_if_needed &