HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -141,6 +141,8 @@ Trunk (Unreleased)
     HADOOP-11352 Clean up test-patch.sh to disable "+1 contrib tests"
     (Akira AJISAKA via stevel)
 
+    HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)
+
   BUG FIXES
 
     HADOOP-9451. Fault single-layer config if node group topology is enabled.

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -666,6 +666,25 @@ function hadoop_finalize_classpath
   hadoop_add_to_classpath_userpath
 }
 
+function hadoop_finalize_catalina_opts
+{
+
+  local prefix=${HADOOP_CATALINA_PREFIX}
+
+  hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}"
+  if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
+    hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}"
+  fi
+  hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_PREFIX}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.admin.port" "-D${prefix}.admin.port=${HADOOP_CATALINA_ADMIN_PORT}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.http.port" "-D${prefix}.http.port=${HADOOP_CATALINA_HTTP_PORT}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.max.threads" "-D${prefix}.max.threads=${HADOOP_CATALINA_MAX_THREADS}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.ssl.keystore.file" "-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}"
+}
+
 function hadoop_finalize
 {
   # user classpath gets added at the last minute. this allows

hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh

@@ -14,19 +14,17 @@
 #
 
 # Set kms specific environment variables here.
-
-# Settings for the Embedded Tomcat that runs KMS
-# Java System properties for KMS should be specified in this variable
 #
-# export CATALINA_OPTS=
+# hadoop-env.sh is read prior to this file.
+#
 
 # KMS logs directory
 #
-# export KMS_LOG=${KMS_HOME}/logs
+# export KMS_LOG=${HADOOP_LOG_DIR}
 
 # KMS temporary directory
 #
-# export KMS_TEMP=${KMS_HOME}/temp
+# export KMS_TEMP=${HADOOP_PREFIX}/temp
 
 # The HTTP port used by KMS
 #
@@ -34,7 +32,7 @@
 
 # The Admin port used by KMS
 #
-# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+# export KMS_ADMIN_PORT=$((KMS_HTTP_PORT + 1))
 
 # The maximum number of Tomcat handler threads
 #
@@ -44,12 +42,37 @@
 #
 # export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
 
+#
 # The password of the SSL keystore if using SSL
 #
 # export KMS_SSL_KEYSTORE_PASS=password
 
-# The full path to any native libraries that need to be loaded
-# (For eg. location of natively compiled tomcat Apache portable
-# runtime (APR) libraries
 #
-# export JAVA_LIBRARY_PATH=${HOME}/lib/native
+# The password of the truststore
+#
+# export KMS_SSL_TRUSTSTORE_PASS=
+
+
+##
+## Tomcat specific settings
+##
+#
+# Location of tomcat
+#
+# export KMS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/kms/tomcat
+
+# Java System properties for KMS should be specified in this variable.
+# The java.library.path and hadoop.home.dir properties are automatically
+# configured.  In order to supplement java.library.path,
+# one should add to the JAVA_LIBRARY_PATH env var.
+#
+# export CATALINA_OPTS=
+
+# PID file
+#
+# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid
+
+# Output file
+#
+# export CATALINA_OUT=${KMS_LOG}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out
+

hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh

@@ -13,182 +13,63 @@
 #  limitations under the License.
 #
 
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
-  ls=`ls -ld "${PRG}"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '/.*' > /dev/null; then
-    PRG="$link"
-  else
-    PRG=`dirname "${PRG}"`/"$link"
+function hadoop_subproject_init
+{
+  local this
+  local binparent
+  local varlist
+
+  if [[ -z "${HADOOP_KMS_ENV_PROCESSED}" ]]; then
+    if [[ -e "${HADOOP_CONF_DIR}/kms-env.sh" ]]; then
+      . "${HADOOP_CONF_DIR}/kms-env.sh"
+      export HADOOP_KMS_ENV_PROCESSED=true
+    fi
   fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
-
-
-function print() {
-  if [ "${KMS_SILENT}" != "true" ]; then
-    echo "$@"
-  fi
-}
-
-# if KMS_HOME is already set warn it will be ignored
-#
-if [ "${KMS_HOME}" != "" ]; then
-  echo "WARNING: current setting of KMS_HOME ignored"
-fi
 
-print
+  export HADOOP_CATALINA_PREFIX=kms
 
-# setting KMS_HOME to the installation dir, it cannot be changed
-#
-export KMS_HOME=${BASEDIR}
-kms_home=${KMS_HOME}
-print "Setting KMS_HOME:          ${KMS_HOME}"
+  export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_PREFIX}/temp}"
 
-# if the installation has a env file, source it
-# this is for native packages installations
-#
-if [ -e "${KMS_HOME}/bin/kms-env.sh" ]; then
-  print "Sourcing:                    ${KMS_HOME}/bin/kms-env.sh"
-  source ${KMS_HOME}/bin/kms-env.sh
-  grep "^ *export " ${KMS_HOME}/bin/kms-env.sh | sed 's/ *export/  setting/'
-fi
-
-# verify that the sourced env file didn't change KMS_HOME
-# if so, warn and revert
-#
-if [ "${KMS_HOME}" != "${kms_home}" ]; then
-  print "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored"
-  export KMS_HOME=${kms_home}
-  print "  using KMS_HOME:        ${KMS_HOME}"
-fi
+  export HADOOP_CONF_DIR="${KMS_CONFIG:-${HADOOP_CONF_DIR}}"
+  export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}"
 
-if [ "${KMS_CONFIG}" = "" ]; then
-  export KMS_CONFIG=${KMS_HOME}/etc/hadoop
-  print "Setting KMS_CONFIG:        ${KMS_CONFIG}"
-else
-  print "Using   KMS_CONFIG:        ${KMS_CONFIG}"
-fi
-kms_config=${KMS_CONFIG}
+  export HADOOP_LOG_DIR="${KMS_LOG:-${HADOOP_LOG_DIR}}"
+  export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}"
 
-# if the configuration dir has a env file, source it
-#
-if [ -e "${KMS_CONFIG}/kms-env.sh" ]; then
-  print "Sourcing:                    ${KMS_CONFIG}/kms-env.sh"
-  source ${KMS_CONFIG}/kms-env.sh
-  grep "^ *export " ${KMS_CONFIG}/kms-env.sh | sed 's/ *export/  setting/'
-fi
+  export HADOOP_CATALINA_HTTP_PORT="${KMS_HTTP_PORT:-16000}"
+  export HADOOP_CATALINA_ADMIN_PORT="${KMS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}"
+  export HADOOP_CATALINA_MAX_THREADS="${KMS_MAX_THREADS:-1000}"
 
-# verify that the sourced env file didn't change KMS_HOME
-# if so, warn and revert
-#
-if [ "${KMS_HOME}" != "${kms_home}" ]; then
-  echo "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored"
-  export KMS_HOME=${kms_home}
-fi
+  export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${KMS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
 
-# verify that the sourced env file didn't change KMS_CONFIG
-# if so, warn and revert
-#
-if [ "${KMS_CONFIG}" != "${kms_config}" ]; then
-  echo "WARN: KMS_CONFIG resetting to ''${KMS_CONFIG}'' ignored"
-  export KMS_CONFIG=${kms_config}
-fi
+  # this is undocumented, but older versions would rip the TRUSTSTORE_PASS out of the
+  # CATALINA_OPTS
+  # shellcheck disable=SC2086
+  export KMS_SSL_TRUSTSTORE_PASS=${KMS_SSL_TRUSTSTORE_PASS:-"$(echo ${CATALINA_OPTS} | grep -o 'trustStorePassword=[^ ]*' | cut -f2 -d= )"}
 
-if [ "${KMS_LOG}" = "" ]; then
-  export KMS_LOG=${KMS_HOME}/logs
-  print "Setting KMS_LOG:           ${KMS_LOG}"
-else
-  print "Using   KMS_LOG:           ${KMS_LOG}"
-fi
+  export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/kms/tomcat}"
+  export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}"
 
-if [ ! -f ${KMS_LOG} ]; then
-  mkdir -p ${KMS_LOG}
-fi
+  export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}"
 
-if [ "${KMS_TEMP}" = "" ]; then
-  export KMS_TEMP=${KMS_HOME}/temp
-  print "Setting KMS_TEMP:           ${KMS_TEMP}"
-else
-  print "Using   KMS_TEMP:           ${KMS_TEMP}"
-fi
+  export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid}"
 
-if [ ! -f ${KMS_TEMP} ]; then
-  mkdir -p ${KMS_TEMP}
-fi
-
-if [ "${KMS_HTTP_PORT}" = "" ]; then
-  export KMS_HTTP_PORT=16000
-  print "Setting KMS_HTTP_PORT:     ${KMS_HTTP_PORT}"
-else
-  print "Using   KMS_HTTP_PORT:     ${KMS_HTTP_PORT}"
-fi
-
-if [ "${KMS_ADMIN_PORT}" = "" ]; then
-  export KMS_ADMIN_PORT=`expr $KMS_HTTP_PORT +  1`
-  print "Setting KMS_ADMIN_PORT:     ${KMS_ADMIN_PORT}"
-else
-  print "Using   KMS_ADMIN_PORT:     ${KMS_ADMIN_PORT}"
-fi
-
-if [ "${KMS_MAX_THREADS}" = "" ]; then
-  export KMS_MAX_THREADS=1000
-  print "Setting KMS_MAX_THREADS:     ${KMS_MAX_THREADS}"
-else
-  print "Using   KMS_MAX_THREADS:     ${KMS_MAX_THREADS}"
-fi
-
-if [ "${KMS_SSL_KEYSTORE_FILE}" = "" ]; then
-  export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-  print "Setting KMS_SSL_KEYSTORE_FILE:     ${KMS_SSL_KEYSTORE_FILE}"
-else
-  print "Using   KMS_SSL_KEYSTORE_FILE:     ${KMS_SSL_KEYSTORE_FILE}"
-fi
-
-# If KMS_SSL_KEYSTORE_PASS is explicitly set to ""
-# then reset to "password". DO NOT set to "password" if
-# variable is NOT defined.
-if [ "${KMS_SSL_KEYSTORE_PASS}" = "" ]; then
-  if [ -n "${KMS_SSL_KEYSTORE_PASS+1}" ]; then
-    export KMS_SSL_KEYSTORE_PASS=password
-    print "Setting KMS_SSL_KEYSTORE_PASS:     ********"
+  if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
+    varlist=$(env | egrep '(^KMS|^CATALINA)' | cut -f1 -d= | grep -v _PASS)
+    for i in ${varlist}; do
+      hadoop_debug "Setting ${i} to ${!i}"
+    done
   fi
-else
-  KMS_SSL_KEYSTORE_PASS_DISP=`echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'`
-  print "Using   KMS_SSL_KEYSTORE_PASS:     ${KMS_SSL_KEYSTORE_PASS_DISP}"
-fi
-
-if [ "${CATALINA_BASE}" = "" ]; then
-  export CATALINA_BASE=${KMS_HOME}/share/hadoop/kms/tomcat
-  print "Setting CATALINA_BASE:       ${CATALINA_BASE}"
-else
-  print "Using   CATALINA_BASE:       ${CATALINA_BASE}"
-fi
-
-if [ "${KMS_CATALINA_HOME}" = "" ]; then
-  export KMS_CATALINA_HOME=${CATALINA_BASE}
-  print "Setting KMS_CATALINA_HOME:       ${KMS_CATALINA_HOME}"
-else
-  print "Using   KMS_CATALINA_HOME:       ${KMS_CATALINA_HOME}"
-fi
-
-if [ "${CATALINA_OUT}" = "" ]; then
-  export CATALINA_OUT=${KMS_LOG}/kms-catalina.out
-  print "Setting CATALINA_OUT:        ${CATALINA_OUT}"
-else
-  print "Using   CATALINA_OUT:        ${CATALINA_OUT}"
-fi
+}
 
-if [ "${CATALINA_PID}" = "" ]; then
-  export CATALINA_PID=/tmp/kms.pid
-  print "Setting CATALINA_PID:        ${CATALINA_PID}"
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+   [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
 else
-  print "Using   CATALINA_PID:        ${CATALINA_PID}"
+  echo "ERROR: Hadoop common not found." 2>&1
+  exit 1
 fi
-
-print

hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh

@@ -13,32 +13,39 @@
 #  limitations under the License.
 #
 
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
-  ls=`ls -ld "${PRG}"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '/.*' > /dev/null; then
-    PRG="$link"
-  else
-    PRG=`dirname "${PRG}"`/"$link"
-  fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
-
-KMS_SILENT=${KMS_SILENT:-true}
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}"
-source ${HADOOP_LIBEXEC_DIR}/kms-config.sh
+function hadoop_usage()
+{
+  echo "Usage: kms.sh [--config confdir] [--debug] --daemon start|status|stop"
+  echo "       kms.sh [--config confdir] [--debug] COMMAND"
+  echo "            where COMMAND is one of:"
+  echo "  run               Start kms in the current window"
+  echo "  run -security     Start in the current window with security manager"
+  echo "  start             Start kms in a separate window"
+  echo "  start -security   Start in a separate window with security manager"
+  echo "  status            Return the LSB compliant status"
+  echo "  stop              Stop kms, waiting up to 5 seconds for the process to end"
+  echo "  stop n            Stop kms, waiting up to n seconds for the process to end"
+  echo "  stop -force       Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
+  echo "  stop n -force     Stop kms, wait up to n seconds and then use kill -KILL if still running"
+}
 
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
 
-if [ "x$JAVA_LIBRARY_PATH" = "x" ]; then
-  JAVA_LIBRARY_PATH="${HADOOP_LIBEXEC_DIR}/../lib/native/"
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/kms-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/kms-config.sh"
 else
-  JAVA_LIBRARY_PATH="${HADOOP_LIBEXEC_DIR}/../lib/native/:${JAVA_LIBRARY_PATH}"
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/kms-config.sh." 2>&1
+  exit 1
 fi
 
 # The Java System property 'kms.http.port' it is not used by Kms,
@@ -46,38 +53,56 @@ fi
 #
 
 # Mask the trustStorePassword
-KMS_SSL_TRUSTSTORE_PASS=`echo $CATALINA_OPTS | grep -o 'trustStorePassword=[^ ]*' | awk -F'=' '{print $2}'`
-CATALINA_OPTS_DISP=`echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/'`
-print "Using   CATALINA_OPTS:       ${CATALINA_OPTS_DISP}"
+# shellcheck disable=SC2086
+CATALINA_OPTS_DISP="$(echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/')"
 
-catalina_opts="-Dkms.home.dir=${KMS_HOME}";
-catalina_opts="${catalina_opts} -Dkms.config.dir=${KMS_CONFIG}";
-catalina_opts="${catalina_opts} -Dkms.log.dir=${KMS_LOG}";
-catalina_opts="${catalina_opts} -Dkms.temp.dir=${KMS_TEMP}";
-catalina_opts="${catalina_opts} -Dkms.admin.port=${KMS_ADMIN_PORT}";
-catalina_opts="${catalina_opts} -Dkms.http.port=${KMS_HTTP_PORT}";
-catalina_opts="${catalina_opts} -Dkms.max.threads=${KMS_MAX_THREADS}";
-catalina_opts="${catalina_opts} -Dkms.ssl.keystore.file=${KMS_SSL_KEYSTORE_FILE}";
-catalina_opts="${catalina_opts} -Djava.library.path=${JAVA_LIBRARY_PATH}";
+hadoop_debug "Using   CATALINA_OPTS:       ${CATALINA_OPTS_DISP}"
 
-print "Adding to CATALINA_OPTS:     ${catalina_opts}"
-print "Found KMS_SSL_KEYSTORE_PASS:     `echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'`"
+# We're using hadoop-common, so set up some stuff it might need:
+hadoop_finalize
 
-export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
+hadoop_verify_logdir
+
+if [[ $# = 0 ]]; then
+  case "${HADOOP_DAEMON_MODE}" in
+    status)
+      hadoop_status_daemon "${CATALINA_PID}"
+      exit
+    ;;
+    start)
+      set -- "start"
+    ;;
+    stop)
+      set -- "stop"
+    ;;
+  esac
+fi
+
+hadoop_finalize_catalina_opts
+export CATALINA_OPTS
 
 # A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
 #
-if [ "${1}" = "stop" ]; then
+if [[ "${1}" = "stop" ]]; then
   export JAVA_OPTS=${CATALINA_OPTS}
 fi
 
 # If ssl, the populate the passwords into ssl-server.xml before starting tomcat
-if [ ! "${KMS_SSL_KEYSTORE_PASS}" = "" ] || [ ! "${KMS_SSL_TRUSTSTORE_PASS}" = "" ]; then
-  # Set a KEYSTORE_PASS if not already set
-  KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password}
-  cat ${CATALINA_BASE}/conf/ssl-server.xml.conf \
-    | sed 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \
-    | sed 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' > ${CATALINA_BASE}/conf/ssl-server.xml
-fi 
+#
+# KMS_SSL_KEYSTORE_PASS is a bit odd.
+# if undefined, then the if test will not enable ssl on its own
+# if "", set it to "password".
+# if custom, use provided password
+#
+if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then
+  if [[ -n "${KMS_SSL_KEYSTORE_PASS+x}" ]] || [[ -n "${KMS_SSL_TRUSTSTORE_PASS}" ]]; then
+      export KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password}
+      sed -e 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \
+          -e 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' \
+        "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \
+        > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml"
+      chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1
+  fi
+fi
 
-exec ${KMS_CATALINA_HOME}/bin/catalina.sh "$@"
+exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"