From 10853b857b55a9a707da90ce6ddedc20980b3b71 Mon Sep 17 00:00:00 2001 From: Hariharan Iyer Date: Mon, 6 May 2019 14:13:32 +0530 Subject: [PATCH 1/5] Start services only if they are monitored by monit --- common/utils.sh | 13 ++++++++ hadoop/util.sh | 86 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 75 insertions(+), 24 deletions(-) create mode 100644 common/utils.sh diff --git a/common/utils.sh b/common/utils.sh new file mode 100644 index 0000000..f493245 --- /dev/null +++ b/common/utils.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +function get_running_services() { + running_svcs=() + for s in "$@"; do + monitored=$(wget -q -O - localhost:2812/_status?format=xml | xmlstarlet sel -t -v "/monit/service[name='$s']/monitor") + if [[ "$monitored" == "0" ]]; then + running_svcs=(${running_svcs[@]} "$s") + fi + done + + echo "${running_svcs[@]}" +} \ No newline at end of file diff --git a/hadoop/util.sh b/hadoop/util.sh index 67ffb16..9f6a767 100644 --- a/hadoop/util.sh +++ b/hadoop/util.sh @@ -1,8 +1,68 @@ #!/bin/bash source /usr/lib/hustler/bin/qubole-bash-lib.sh +source /usr/lib/qubole/bootstrap-functions/common/utils.sh export PROFILE_FILE=${PROFILE_FILE:-/etc/profile} export HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-/usr/lib/hadoop2/etc/hadoop} +declare -A SVC_USERS=([namenode]=hdfs [timelineserver]=yarn [historyserver]=mapred [resourcemanager]=yarn [datanode]=hdfs) + +function start_daemon() { + daemon=shift; + case "SVC_USERS[$daemon]" in + yarn) + /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh start $daemon" yarn + ;; + hdfs) + /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/hadoop-daemon.sh start $daemon" hdfs + ;; + mapred) + /bin/su -s /bin/bash -c "HADOOP_LIBEXEC_DIR=/usr/lib/hadoop2/libexec /usr/lib/hadoop2/sbin/mr-jobhistory-daemon.sh start $daemon" mapred + ;; + *) + echo "Invalid daemon $daemon" + ;; + esac +} + +function stop_daemon() { + daemon=shift; + case "${SVC_USERS[$daemon]}" in + yarn) + /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh stop $daemon" yarn + ;; + hdfs) + /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/hadoop-daemon.sh stop $daemon" hdfs + ;; + mapred) + /bin/su -s /bin/bash -c "HADOOP_LIBEXEC_DIR=/usr/lib/hadoop2/libexec /usr/lib/hadoop2/sbin/mr-jobhistory-daemon.sh stop $daemon" mapred + ;; + *) + echo "Invalid daemon $daemon" + ;; + esac +} + +function restart_services() { + svcs="$@" + running_svcs=find_running_services "${svcs[@]}" + for s in "${running_svcs[@]}"; do + monit unmonitor "$s" + done + + for s in "${running_svcs[@]}"; do + stop_daemon "$s" + done + + last=${#running_svcs[@]} + + for (( i=0; i Date: Mon, 6 May 2019 14:53:01 +0530 Subject: [PATCH 2/5] fixes --- common/utils.sh | 4 ++-- hadoop/util.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/utils.sh b/common/utils.sh index f493245..db968fd 100644 --- a/common/utils.sh +++ b/common/utils.sh @@ -3,8 +3,8 @@ function get_running_services() { running_svcs=() for s in "$@"; do - monitored=$(wget -q -O - localhost:2812/_status?format=xml | xmlstarlet sel -t -v "/monit/service[name='$s']/monitor") - if [[ "$monitored" == "0" ]]; then + monitored=$(wget -q -O - localhost:2812/_status?format=xml | xmlstarlet sel -t -v "/monit/service[name='${s}']/monitor") + if [[ "$monitored" == "1" ]]; then running_svcs=(${running_svcs[@]} "$s") fi done diff --git a/hadoop/util.sh b/hadoop/util.sh index 9f6a767..1b8be41 100644 --- a/hadoop/util.sh +++ b/hadoop/util.sh @@ -7,8 +7,8 @@ export HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-/usr/lib/hadoop2/etc/hadoop} declare -A SVC_USERS=([namenode]=hdfs [timelineserver]=yarn [historyserver]=mapred [resourcemanager]=yarn [datanode]=hdfs) function start_daemon() { - daemon=shift; - case "SVC_USERS[$daemon]" in + daemon=$1; + case "${SVC_USERS[$daemon]}" in yarn) /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh start $daemon" yarn ;; @@ -25,7 +25,7 @@ function start_daemon() { } function stop_daemon() { - daemon=shift; + daemon=$1; case "${SVC_USERS[$daemon]}" in yarn) /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh stop $daemon" yarn @@ -43,8 +43,8 @@ function stop_daemon() { } function restart_services() { - svcs="$@" - running_svcs=find_running_services "${svcs[@]}" + svcs=("$@") + running_svcs=($(find_running_services "${svcs[@]}")) for s in "${running_svcs[@]}"; do monit unmonitor "$s" done From d4fb671c54bfc6b0ae49d6ee44f9459890710855 Mon Sep 17 00:00:00 2001 From: Hariharan Iyer Date: Mon, 6 May 2019 15:01:30 +0530 Subject: [PATCH 3/5] Add comments --- common/utils.sh | 1 + hadoop/util.sh | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/common/utils.sh b/common/utils.sh index db968fd..4dc6b10 100644 --- a/common/utils.sh +++ b/common/utils.sh @@ -5,6 +5,7 @@ function get_running_services() { for s in "$@"; do monitored=$(wget -q -O - localhost:2812/_status?format=xml | xmlstarlet sel -t -v "/monit/service[name='${s}']/monitor") if [[ "$monitored" == "1" ]]; then + # Order of services is retained when returning running_svcs=(${running_svcs[@]} "$s") fi done diff --git a/hadoop/util.sh b/hadoop/util.sh index 1b8be41..0af96f5 100644 --- a/hadoop/util.sh +++ b/hadoop/util.sh @@ -44,7 +44,7 @@ function stop_daemon() { function restart_services() { svcs=("$@") - running_svcs=($(find_running_services "${svcs[@]}")) + running_svcs=($(get_running_services "${svcs[@]}")) for s in "${running_svcs[@]}"; do monit unmonitor "$s" done @@ -55,10 +55,13 @@ function restart_services() { last=${#running_svcs[@]} + # Restart services in reverse order of how + # they were stopped for (( i=0; i Date: Tue, 7 May 2019 11:54:23 +0530 Subject: [PATCH 4/5] Rename few functions --- hadoop/util.sh | 61 +++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/hadoop/util.sh b/hadoop/util.sh index 0af96f5..d3dc86d 100644 --- a/hadoop/util.sh +++ b/hadoop/util.sh @@ -6,8 +6,8 @@ export PROFILE_FILE=${PROFILE_FILE:-/etc/profile} export HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-/usr/lib/hadoop2/etc/hadoop} declare -A SVC_USERS=([namenode]=hdfs [timelineserver]=yarn [historyserver]=mapred [resourcemanager]=yarn [datanode]=hdfs) -function start_daemon() { - daemon=$1; +function _start_daemon() { + local daemon=$1; case "${SVC_USERS[$daemon]}" in yarn) /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh start $daemon" yarn @@ -24,8 +24,8 @@ function start_daemon() { esac } -function stop_daemon() { - daemon=$1; +function _stop_daemon() { + local daemon=$1; case "${SVC_USERS[$daemon]}" in yarn) /bin/su -s /bin/bash -c "/usr/lib/hadoop2/sbin/yarn-daemon.sh stop $daemon" yarn @@ -42,23 +42,23 @@ function stop_daemon() { esac } -function restart_services() { - svcs=("$@") - running_svcs=($(get_running_services "${svcs[@]}")) +function _restart_services() { + local svcs=("$@") + local running_svcs=($(get_running_services "${svcs[@]}")) for s in "${running_svcs[@]}"; do monit unmonitor "$s" done for s in "${running_svcs[@]}"; do - stop_daemon "$s" + _stop_daemon "$s" done - last=${#running_svcs[@]} + local last=${#running_svcs[@]} # Restart services in reverse order of how # they were stopped - for (( i=0; i =0; i-- )); do + _start_daemon "${running_svcs[i]}" done # Order doesn't matter for (un)monitor @@ -67,29 +67,40 @@ function restart_services() { done } -## -# Restart hadoop services on the cluster master # -# This may be used if you're using a different version -# of Java, for example +# Restart hadoop services on the cluster master # -function restart_master_services() { - restart_services timelineserver historyserver resourcemanager namenode +function _restart_master_services() { + _restart_services timelineserver historyserver resourcemanager namenode } - -## +# # Restart hadoop services on cluster workers # # This only restarts the datanode service since the # nodemanager is started after the bootstrap is run # -function restart_worker_services() { - restart_services datanode +function _restart_worker_services() { + _restart_services datanode # No need to restart nodemanager since it starts only # after thhe bootstrap is finished } +## +# Restart hadoop services +# +# This may be used if you're using a different version +# of Java, for example +# +function restart_hadoop_services() { + local is_master=$(nodeinfo is_master) + if [[ "$is_master" == "1" ]]; then + _restart_master_services + else + _restart_worker_services + fi +} + ## # Use Java 8 for hadoop daemons and jobs # @@ -106,11 +117,5 @@ function use_java8() { echo "export PATH=$JAVA_HOME/bin:$PATH" >> "$PROFILE_FILE" sed -i 's/java-1.7.0/java-1.8.0/' "$HADOOP_ETC_DIR/hadoop-env.sh" - - is_master=$(nodeinfo is_master) - if [[ "$is_master" == "1" ]]; then - restart_master_services - else - restart_worker_services - fi + restart_hadoop_services } From 72f9975faf717e70b10d95faecd9d075c4d10df9 Mon Sep 17 00:00:00 2001 From: Hariharan Iyer Date: Tue, 7 May 2019 12:00:08 +0530 Subject: [PATCH 5/5] Make variable local --- common/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/utils.sh b/common/utils.sh index 4dc6b10..cf1440f 100644 --- a/common/utils.sh +++ b/common/utils.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash function get_running_services() { - running_svcs=() + local running_svcs=() for s in "$@"; do monitored=$(wget -q -O - localhost:2812/_status?format=xml | xmlstarlet sel -t -v "/monit/service[name='${s}']/monitor") if [[ "$monitored" == "1" ]]; then