HADOOP-12857. rework hadoop-tools (aw)

dev-support/bin/dist-tools-hooks-maker

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+VERSION=${1:-3.0.0-SNAPSHOT}
+TARGETDIR=${2:-/tmp/target}
+TOOLSDIR=${3:-/tmp/tools}
+
+function getfilename
+{
+  declare module=$1
+  declare modtype=$2
+
+  if [[ ${modtype} = builtin ]]; then
+    echo "${TARGETDIR}/hadoop-${VERSION}/libexec/tools/${module}.sh"
+  else
+    echo "${TARGETDIR}/hadoop-${VERSION}/libexec/shellprofile.d/${module}.sh"
+  fi
+}
+
+function header
+{
+  declare fn=$1
+
+  cat >>"${fn}" <<-'TOKEN'
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# IMPORTANT: This file is automatically generated by hadoop-dist at
+#            -Pdist time.
+#
+#
+TOKEN
+
+}
+
+function optional_prologue
+{
+  declare fn=$1
+  declare module=$2
+
+  if [[ -z "${OPTMODS}" ]]; then
+    OPTMODS=${module}
+  else
+    OPTMODS="${OPTMODS},${module}"
+  fi
+
+  {
+    echo "if hadoop_verify_entry HADOOP_TOOLS_OPTIONS \"${module}\"; then"
+    echo "  hadoop_add_profile \"${module}\""
+    echo "fi"
+    echo ""
+    echo "function _${module}_hadoop_classpath"
+    echo "{"
+  } >> "${fn}"
+}
+
+function builtin_prologue
+{
+  declare fn=$1
+  declare module=$2
+
+  {
+    echo ""
+    echo "function hadoop_classpath_tools_${module}"
+    echo "{"
+  } >> "${fn}"
+}
+
+function dependencywork
+{
+  declare fn=$1
+  declare module=$2
+  declare depfn=$3
+
+  declare depline
+  declare jarname
+
+  while read -r depline; do
+    jarname=$(echo "${depline}" | awk -F: '{print $2"-"$4".jar"}')
+
+    if [[ -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/tools/lib/${jarname}" ]]; then
+      {
+        echo "  if [[ -f \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${jarname}\" ]]; then"
+        echo "    hadoop_add_classpath \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${jarname}\""
+        echo "  fi"
+      } >> "${fn}"
+
+    elif [[ -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/common/${jarname}"
+         || -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/common/lib/${jarname}" ]]; then
+      true
+    else
+      echo "ERROR: ${module} has missing dependencies: ${jarname}"
+    fi
+  done < <(grep compile "${depfn}")
+
+  {
+    echo "  hadoop_add_classpath \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${module}-${VERSION}.jar\""
+    echo "}"
+    echo ""
+  } >> "${fn}"
+}
+
+function document_optionals
+{
+  echo "Rewriting ${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh"
+  sed -e "s^@@@HADOOP_OPTIONAL_TOOLS@@@^${OPTMODS}^" \
+    "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh" \
+    > "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh.new"
+  mv "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh.new" \
+    "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh"
+}
+
+function process
+{
+  declare fn
+  declare basefn
+  declare modtype
+  declare module
+  declare newfile
+  declare newdir
+
+  while read -r fn; do
+    basefn=${fn##*/}
+    module=$(echo "${basefn}" | cut -f1 -d.)
+    modtype=$(echo "${basefn}" | cut -f2 -d.)
+    modtype=${modtype##tools-}
+
+    newfile=$(getfilename "${module}" "${modtype}")
+    newdir=$(dirname "${newfile}")
+    mkdir -p "${newdir}"
+
+    if [[ -f "${newfile}" ]]; then
+      rm "${newfile}"
+    fi
+
+    touch "${newfile}"
+
+    header "${newfile}" "${module}"
+
+    "${modtype}_prologue" "${newfile}" "${module}"
+
+    dependencywork "${newfile}" "${module}" "${fn}"
+
+    chmod a+rx "${newfile}"
+
+  done < <(find "${TOOLSDIR}" -name '*.tools-builtin.txt' -o -name '*.tools-optional.txt')
+
+  document_optionals
+}
+
+process

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -114,7 +114,7 @@ case ${COMMAND} in
   ;;
   archive)
     CLASS=org.apache.hadoop.tools.HadoopArchives
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-archives
   ;;
   checknative)
     CLASS=org.apache.hadoop.util.NativeLibraryChecker
@@ -133,11 +133,11 @@ case ${COMMAND} in
   ;;
   distch)
     CLASS=org.apache.hadoop.tools.DistCh
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-extras
   ;;
   distcp)
     CLASS=org.apache.hadoop.tools.DistCp
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-distcp
   ;;
   envvars)
     echo "JAVA_HOME='${JAVA_HOME}'"
@@ -146,7 +146,9 @@ case ${COMMAND} in
     echo "HADOOP_COMMON_LIB_JARS_DIR='${HADOOP_COMMON_LIB_JARS_DIR}'"
     echo "HADOOP_COMMON_LIB_NATIVE_DIR='${HADOOP_COMMON_LIB_NATIVE_DIR}'"
     echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-    echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'"
+    echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+    echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+    echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
     exit 0
   ;;
   fs)

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -41,6 +41,44 @@ function hadoop_debug
   fi
 }
 
+## @description  Given variable $1 delete $2 from it
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_delete_entry
+{
+  if [[ ${!1} =~ \ ${2}\  ]] ; then
+    hadoop_debug "Removing ${2} from ${1}"
+    eval "${1}"=\""${!1// ${2} }"\"
+  fi
+}
+
+## @description  Given variable $1 add $2 to it
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_add_entry
+{
+  if [[ ! ${!1} =~ \ ${2}\  ]] ; then
+    hadoop_debug "Adding ${2} to ${1}"
+    #shellcheck disable=SC2140
+    eval "${1}"=\""${!1} ${2} "\"
+  fi
+}
+
+## @description  Given variable $1 determine if $2 is in it
+## @audience     public
+## @stability    stable
+## @replaceable  no
+## @return       0 = yes, 1 = no
+function hadoop_verify_entry
+{
+  # this unfortunately can't really be tested by bats. :(
+  # so if this changes, be aware that unit tests effectively
+  # do this function in them
+  [[ ${!1} =~ \ ${2}\  ]]
+}
+
 ## @description  Add a subcommand to the usage output
 ## @audience     private
 ## @stability    evolving
@@ -264,10 +302,9 @@ function hadoop_bootstrap
   YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
   MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
   MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
-
-  # setup a default HADOOP_TOOLS_PATH
-  hadoop_deprecate_envvar TOOL_PATH HADOOP_TOOLS_PATH
-  HADOOP_TOOLS_PATH=${HADOOP_TOOLS_PATH:-${HADOOP_PREFIX}/share/hadoop/tools/lib/*}
+  HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_PREFIX}}
+  HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
+  HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
 
   # usage output set to zero
   hadoop_reset_usage
@@ -322,6 +359,7 @@ function hadoop_exec_hadoopenv
   if [[ -z "${HADOOP_ENV_PROCESSED}" ]]; then
     if [[ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]]; then
       export HADOOP_ENV_PROCESSED=true
+      # shellcheck disable=SC1090
       . "${HADOOP_CONF_DIR}/hadoop-env.sh"
     fi
   fi
@@ -334,6 +372,7 @@ function hadoop_exec_hadoopenv
 function hadoop_exec_userfuncs
 {
   if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then
+    # shellcheck disable=SC1090
     . "${HADOOP_CONF_DIR}/hadoop-user-functions.sh"
   fi
 }
@@ -348,6 +387,7 @@ function hadoop_exec_hadooprc
 {
   if [[ -f "${HOME}/.hadooprc" ]]; then
     hadoop_debug "Applying the user's .hadooprc"
+    # shellcheck disable=SC1090
     . "${HOME}/.hadooprc"
   fi
 }
@@ -373,11 +413,22 @@ function hadoop_import_shellprofiles
     files2=(${HADOOP_CONF_DIR}/shellprofile.d/*.sh)
   fi
 
+  # enable bundled shellprofiles that come
+  # from hadoop-tools.  This converts the user-facing HADOOP_OPTIONAL_TOOLS
+  # to the HADOOP_TOOLS_OPTIONS that the shell profiles expect.
+  # See dist-tools-hooks-maker for how the example HADOOP_OPTIONAL_TOOLS
+  # gets populated into hadoop-env.sh
+
+  for i in ${HADOOP_OPTIONAL_TOOLS//,/ }; do
+    hadoop_add_entry HADOOP_TOOLS_OPTIONS "${i}"
+  done
+
   for i in "${files1[@]}" "${files2[@]}"
   do
     if [[ -n "${i}"
       && -f "${i}" ]]; then
       hadoop_debug "Profiles: importing ${i}"
+      # shellcheck disable=SC1090
       . "${i}"
     fi
   done
@@ -945,34 +996,25 @@ function hadoop_add_common_to_classpath
   hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*'
 }
 
-## @description  Add the HADOOP_TOOLS_PATH to the classpath
+## @description  Run libexec/tools/module.sh to add to the classpath
 ## @description  environment
 ## @audience     private
 ## @stability    evolving
 ## @replaceable  yes
-function hadoop_add_to_classpath_toolspath
+## @param        module
+function hadoop_add_to_classpath_tools
 {
-  declare -a array
-  declare -i c=0
-  declare -i j
-  declare -i i
-  declare idx
+  declare module=$1
 
-  if [[ -n "${HADOOP_TOOLS_PATH}" ]]; then
-    hadoop_debug "Adding HADOOP_TOOLS_PATH to CLASSPATH"
-    oldifs=${IFS}
-    IFS=:
-    for idx in ${HADOOP_TOOLS_PATH}; do
-      array[${c}]=${idx}
-      ((c=c+1))
-    done
-    IFS=${oldifs}
-    ((j=c-1)) || ${QATESTMODE}
-
-    for ((i=0; i<=j; i++)); do
-      hadoop_add_classpath "${array[$i]}" after
-    done
+  if [[ -f "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh" ]]; then
+    # shellcheck disable=SC1090
+    . "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh"
+  else
+    hadoop_error "ERROR: Tools helper ${HADOOP_LIBEXEC_DIR}/tools/${module}.sh was not found."
+  fi
 
+  if declare -f hadoop_classpath_tools_${module} >/dev/null 2>&1; then
+    "hadoop_classpath_tools_${module}"
   fi
 }
 

hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # Copyright 2014 The Apache Software Foundation
 #
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -87,7 +88,10 @@
 # Misc paths
 ####
 
-# setup a default HADOOP_TOOLS_PATH, where things like distcp lives
+# This is where things like distcp, S3, and other things live
 # note that this path only gets added for certain commands and not
-# part of the general classpath
-# export HADOOP_TOOLS_PATH="${HADOOP_PREFIX}/share/hadoop/tools/lib/*"
+# part of the general classpath unless HADOOP_OPTIONAL_TOOLS is used
+# to configure them in
+# export HADOOP_TOOLS_HOME=${HADOOP_PREFIX}
+# export HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
+# export HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}

hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh

@@ -49,7 +49,7 @@
 # preferred.  Many sites configure these options outside of Hadoop,
 # such as in /etc/profile.d
 
-# The java implementation to use. By default, this environment 
+# The java implementation to use. By default, this environment
 # variable is REQUIRED on ALL platforms except OS X!
 # export JAVA_HOME=
 
@@ -64,15 +64,15 @@
 # path.
 # export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
 
-# The maximum amount of heap to use (Java -Xmx).  If no unit 
-# is provided, it will be converted to MB.  Daemons will 
+# The maximum amount of heap to use (Java -Xmx).  If no unit
+# is provided, it will be converted to MB.  Daemons will
 # prefer any Xmx setting in their respective _OPT variable.
 # There is no default; the JVM will autoscale based upon machine
 # memory size.
 # export HADOOP_HEAPSIZE_MAX=
 
-# The minimum amount of heap to use (Java -Xms).  If no unit 
-# is provided, it will be converted to MB.  Daemons will 
+# The minimum amount of heap to use (Java -Xms).  If no unit
+# is provided, it will be converted to MB.  Daemons will
 # prefer any Xms setting in their respective _OPT variable.
 # There is no default; the JVM will autoscale based upon machine
 # memory size.
@@ -107,8 +107,8 @@ case ${HADOOP_OS_TYPE} in
 esac
 
 # Extra Java runtime options for some Hadoop commands
-# and clients (i.e., hdfs dfs -blah).  These get appended to HADOOP_OPTS for 
-# such commands.  In most cases, # this should be left empty and 
+# and clients (i.e., hdfs dfs -blah).  These get appended to HADOOP_OPTS for
+# such commands.  In most cases, # this should be left empty and
 # let users supply it on the command line.
 # export HADOOP_CLIENT_OPTS=""
 
@@ -146,6 +146,11 @@ esac
 # names starting with a '-' are treated as negative matches. For example,
 # export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."
 
+# Enable optional, bundled Hadoop features
+# This is a comma delimited list.  It may NOT be overridden via .hadooprc
+# Entries may be added/removed as needed.
+# export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@"
+
 ###
 # Options for remote shell connectivity
 ###
@@ -181,7 +186,7 @@ esac
 # non-secure)
 #
 
-# Where (primarily) daemon log files are stored.  # $HADOOP_PREFIX/logs 
+# Where (primarily) daemon log files are stored.  # $HADOOP_PREFIX/logs
 # by default.
 # Java property: hadoop.log.dir
 # export HADOOP_LOG_DIR=${HADOOP_PREFIX}/logs
@@ -201,7 +206,7 @@ esac
 # Java property: hadoop.root.logger
 # export HADOOP_ROOT_LOGGER=INFO,console
 
-# Default log4j setting for daemons spawned explicitly by 
+# Default log4j setting for daemons spawned explicitly by
 # --daemon option of hadoop, hdfs, mapred and yarn command.
 # Java property: hadoop.root.logger
 # export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_tools.bats

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+toolsetup () {
+  HADOOP_LIBEXEC_DIR="${TMP}/libexec"
+  mkdir -p "${HADOOP_LIBEXEC_DIR}/tools"
+}
+
+@test "hadoop_classpath_tools (load)" {
+  toolsetup
+  echo "unittest=libexec" > "${HADOOP_LIBEXEC_DIR}/tools/test.sh"
+  hadoop_add_to_classpath_tools test
+  [ -n "${unittest}" ]
+}
+
+
+@test "hadoop_classpath_tools (not exist)" {
+  toolsetup
+  hadoop_add_to_classpath_tools test
+  [ -z "${unittest}" ]
+}
+
+@test "hadoop_classpath_tools (function)" {
+  toolsetup
+  {
+    echo "function hadoop_classpath_tools_test {"
+    echo " unittest=libexec"
+    echo " }"
+  } > "${HADOOP_LIBEXEC_DIR}/tools/test.sh"
+  hadoop_add_to_classpath_tools test
+  declare -f
+  [ -n "${unittest}" ]
+}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_toolspath.bats

@@ -1,74 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-load hadoop-functions_test_helper
-
-freetheclasses () {
-  local j
-
-  for j in HADOOP_TOOLS_PATH  \
-      CLASSPATH; do
-      unset ${j}
-  done
-}
-
-createdirs () {
-  local j
-
-  for j in new old foo bar baz; do
-    mkdir -p "${TMP}/${j}"
-  done
-}
-
-@test "hadoop_add_to_classpath_toolspath (nothing)" {
-   freetheclasses
-   hadoop_add_to_classpath_toolspath
-   [ -z "${CLASSPATH}" ]
-}
-
-@test "hadoop_add_to_classpath_toolspath (none)" {
-   freetheclasses
-   CLASSPATH=test
-   hadoop_add_to_classpath_toolspath
-   [ "${CLASSPATH}" = "test" ]
-}
-
-@test "hadoop_add_to_classpath_toolspath (only)" {
-   freetheclasses
-   createdirs
-   HADOOP_TOOLS_PATH="${TMP}/new"
-   hadoop_add_to_classpath_toolspath
-   [ "${CLASSPATH}" = "${TMP}/new" ]
-}
-
-@test "hadoop_add_to_classpath_toolspath (1+1)" {
-   freetheclasses
-   createdirs
-   CLASSPATH=${TMP}/foo
-   HADOOP_TOOLS_PATH=${TMP}/foo
-   hadoop_add_to_classpath_toolspath
-   echo ">${CLASSPATH}<"
-   [ ${CLASSPATH} = "${TMP}/foo" ]
-}
-
-@test "hadoop_add_to_classpath_toolspath (3+2)" {
-   freetheclasses
-   createdirs
-   CLASSPATH=${TMP}/foo:${TMP}/bar:${TMP}/baz
-   HADOOP_TOOLS_PATH=${TMP}/new:${TMP}/old
-   hadoop_add_to_classpath_toolspath
-   echo ">${CLASSPATH}<"
-   [ ${CLASSPATH} = "${TMP}/foo:${TMP}/bar:${TMP}/baz:${TMP}/new:${TMP}/old" ]
-}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats

@@ -38,7 +38,8 @@ basicinitsetup () {
   dirvars="HADOOP_COMMON_HOME \
         HADOOP_MAPRED_HOME \
         HADOOP_HDFS_HOME \
-        HADOOP_YARN_HOME"
+        HADOOP_YARN_HOME \
+        HADOOP_TOOLS_HOME"
 
   for j in ${testvars}; do
     unset ${j}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats

@@ -15,13 +15,13 @@
 
 load hadoop-functions_test_helper
 
-@test "hadoop_deprecate_envvar (no libexec)" {
+@test "hadoop_bootstrap (no libexec)" {
   unset HADOOP_LIBEXEC_DIR
   run hadoop_bootstrap
   [ "${status}" -eq 1 ]
 }
 
-@test "hadoop_deprecate_envvar (libexec)" {
+@test "hadoop_bootstrap (libexec)" {
   unset   HADOOP_PREFIX
   unset   HADOOP_COMMON_DIR
   unset   HADOOP_COMMON_LIB_JARS_DIR
@@ -31,7 +31,9 @@ load hadoop-functions_test_helper
   unset   YARN_LIB_JARS_DIR
   unset   MAPRED_DIR
   unset   MAPRED_LIB_JARS_DIR
-  unset   TOOL_PATH
+  unset   HADOOP_TOOLS_HOME
+  unset   HADOOP_TOOLS_DIR
+  unset   HADOOP_TOOLS_LIB_JARS_DIR
   unset   HADOOP_OS_TYPE
 
   hadoop_bootstrap
@@ -46,6 +48,9 @@ load hadoop-functions_test_helper
   [ -n ${YARN_LIB_JARS_DIR} ]
   [ -n ${MAPRED_DIR} ]
   [ -n ${MAPRED_LIB_JARS_DIR} ]
-  [ -n ${TOOL_PATH} ]
   [ -n ${HADOOP_OS_TYPE} ]
-} 
+  [ -n ${HADOOP_TOOLS_PATH} ]
+  [ -n ${HADOOP_TOOLS_HOME} ]
+  [ -n ${HADOOP_TOOLS_DIR} ]
+  [ -n ${HADOOP_TOOLS_LIB_JARS_DIR} ]
+}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_entry_tests.bats

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_add_entry (positive 1)" {
+  hadoop_add_entry testvar foo
+  echo ">${testvar}<"
+  [ "${testvar}" = " foo " ]
+}
+
+@test "hadoop_add_entry (negative)" {
+  hadoop_add_entry testvar foo
+  hadoop_add_entry testvar foo
+  echo ">${testvar}<"
+  [ "${testvar}" = " foo " ]
+}
+
+@test "hadoop_add_entry (positive 2)" {
+  hadoop_add_entry testvar foo
+  hadoop_add_entry testvar foo
+  hadoop_add_entry testvar bar
+  echo ">${testvar}<"
+  [ "${testvar}" = " foo  bar " ]
+}
+
+@test "hadoop_add_entry (positive 3)" {
+  hadoop_add_entry testvar foo
+  hadoop_add_entry testvar foo
+  hadoop_add_entry testvar bar
+  hadoop_add_entry testvar bar
+  hadoop_add_entry testvar baz
+  hadoop_add_entry testvar baz
+
+  echo ">${testvar}<"
+  [ "${testvar}" = " foo  bar  baz " ]
+}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_shellprofile.bats

@@ -66,6 +66,13 @@ _test_hadoop_finalize () {
   [ -z "${unittest}" ]
 }
 
+@test "hadoop_import_shellprofiles (H_O_T)" {
+  HADOOP_OPTIONAL_TOOLS=1,2
+  shellprofilesetup
+  hadoop_import_shellprofiles
+  [ "${HADOOP_TOOLS_OPTIONS}" == " 1  2 " ]
+}
+
 @test "hadoop_add_profile+hadoop_shellprofiles_init" {
   hadoop_add_profile test
   hadoop_shellprofiles_init

hadoop-dist/pom.xml

@@ -103,6 +103,24 @@
                   </arguments>
                 </configuration>
               </execution>
+              <execution>
+                <id>toolshooks</id>
+                <phase>prepare-package</phase>
+                <goals>
+                    <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>${shell-executable}</executable>
+                  <workingDirectory>${basedir}</workingDirectory>
+                  <requiresOnline>false</requiresOnline>
+                  <arguments>
+                    <argument>${basedir}/../dev-support/bin/dist-tools-hooks-maker</argument>
+                    <argument>${project.version}</argument>
+                    <argument>${project.build.directory}</argument>
+                    <argument>${basedir}/../hadoop-tools</argument>
+                  </arguments>
+                </configuration>
+              </execution>
               <execution>
                 <id>tar</id>
                 <phase>package</phase>

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

@@ -141,7 +141,9 @@ case ${COMMAND} in
     echo "HDFS_DIR='${HDFS_DIR}'"
     echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
     echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-    echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'"
+    echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+    echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+    echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
     exit 0
   ;;
   erasurecode)
@@ -165,7 +167,6 @@ case ${COMMAND} in
   ;;
   haadmin)
     CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
-    hadoop_add_to_classpath_toolspath
     hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
   ;;

hadoop-mapreduce-project/bin/mapred

@@ -69,13 +69,13 @@ case ${COMMAND} in
   ;;
   archive)
     CLASS=org.apache.hadoop.tools.HadoopArchives
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-archives
     hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
   ;;
   archive-logs)
     CLASS=org.apache.hadoop.tools.HadoopArchiveLogs
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-archive-logs
     hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
   ;;
@@ -84,7 +84,7 @@ case ${COMMAND} in
   ;;
   distcp)
     CLASS=org.apache.hadoop.tools.DistCp
-    hadoop_add_to_classpath_toolspath
+    hadoop_add_to_classpath_tools hadoop-distcp
     hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
   ;;
@@ -94,7 +94,9 @@ case ${COMMAND} in
     echo "MAPRED_DIR='${MAPRED_DIR}'"
     echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'"
     echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-    echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'"
+    echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+    echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+    echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
     exit 0
   ;;
   historyserver)

hadoop-tools/hadoop-archive-logs/pom.xml

@@ -172,6 +172,23 @@
          </archive>
         </configuration>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>findbugs-maven-plugin</artifactId>

hadoop-tools/hadoop-archives/pom.xml

@@ -128,6 +128,23 @@
          </archive>
         </configuration>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-tools/hadoop-aws/pom.xml

@@ -90,6 +90,23 @@
           <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- build a shellprofile -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -19,6 +19,9 @@ JAR file, `hadoop-aws.jar` also declares a transitive dependency on all
 external artifacts which are needed for this support —enabling downstream
 applications to easily use this support.
 
+To make it part of Apache Hadoop's default classpath, simply make sure that
+HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aws' in the list.
+
 Features
 
 1. The "classic" `s3:` filesystem for storing objects in Amazon S3 Storage
@@ -30,7 +33,7 @@ higher performance.
 
 The specifics of using these filesystems are documented below.
 
-## Warning: Object Stores are not filesystems.
+## Warning #1: Object Stores are not filesystems.
 
 Amazon S3 is an example of "an object store". In order to achieve scalability
 and especially high availability, S3 has —as many other cloud object stores have

hadoop-tools/hadoop-azure/pom.xml

@@ -82,6 +82,24 @@
         </executions>
       </plugin>
 
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- build a shellprofile -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
     </plugins>
 
     

hadoop-tools/hadoop-azure/src/site/markdown/index.md

@@ -34,6 +34,9 @@ The built jar file, named hadoop-azure.jar, also declares transitive dependencie
 on the additional artifacts it requires, notably the
 [Azure Storage SDK for Java](https://github.com/Azure/azure-storage-java).
 
+To make it part of Apache Hadoop's default classpath, simply make sure that
+HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-azure' in the list.
+
 ## <a name="Features" />Features
 
 * Read and write data stored in an Azure Blob Storage account.

hadoop-tools/hadoop-datajoin/pom.xml

@@ -132,6 +132,22 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.basedir}/../../hadoop-dist/target/hadoop-tools-deps/${project.artifactId}.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-tools/hadoop-distcp/pom.xml

@@ -163,6 +163,17 @@
               <outputDirectory>${project.build.directory}/lib</outputDirectory>
             </configuration>
           </execution>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
       <plugin>

hadoop-tools/hadoop-extras/pom.xml

@@ -137,6 +137,23 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-tools/hadoop-gridmix/pom.xml

@@ -163,6 +163,23 @@
          </archive>
         </configuration>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-tools/hadoop-kafka/pom.xml

@@ -88,6 +88,23 @@
           <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
         </configuration>
       </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- build a shellprofile -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hadoop-tools/hadoop-openstack/pom.xml

@@ -85,6 +85,23 @@
           <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- build a shellprofile -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hadoop-tools/hadoop-openstack/src/site/markdown/index.md

@@ -54,6 +54,9 @@ Introduction
 
 This module enables Apache Hadoop applications -including MapReduce jobs, read and write data to and from instances of the [OpenStack Swift object store](http://www.openstack.org/software/openstack-storage/).
 
+To make it part of Apache Hadoop's default classpath, simply make sure that
+HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-openstack' in the list.
+
 Features
 --------
 
@@ -440,7 +443,9 @@ If the host is declared, the proxy port must be set to a valid integer value.
 
 The `hadoop-openstack` JAR -or any dependencies- may not be on your classpath.
 
-If it is a remote MapReduce job that is failing, make sure that the JAR is installed on the servers in the cluster -or that the job submission process uploads the JAR file to the distributed cache.
+Make sure that the:
+* JAR is installed on the servers in the cluster.
+* 'hadoop-openstack' is on the HADOOP_OPTIONAL_TOOLS entry in hadoop-env.sh or that the job submission process uploads the JAR file to the distributed cache.
 
 #### Failure to Authenticate
 

hadoop-tools/hadoop-rumen/pom.xml

@@ -132,6 +132,23 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

hadoop-tools/hadoop-sls/pom.xml

@@ -177,6 +177,23 @@
               </execution>
             </executions>
           </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
         </plugins>
       </build>
     </profile>

hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh

@@ -55,7 +55,7 @@ function parse_args()
 
 function calculate_classpath()
 {
-  hadoop_add_to_classpath_toolspath
+  hadoop_add_to_classpath_tools hadoop-rumen
 }
 
 function run_sls_generator()

hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh

@@ -70,7 +70,7 @@ function parse_args()
 
 function calculate_classpath
 {
-  hadoop_add_to_classpath_toolspath
+  hadoop_add_to_classpath_tools hadoop-sls
   hadoop_debug "Injecting ${HADOOP_PREFIX}/share/hadoop/tools/sls/html into CLASSPATH"
   hadoop_add_classpath "${HADOOP_PREFIX}/share/hadoop/tools/sls/html"
 }

hadoop-tools/hadoop-streaming/pom.xml

@@ -174,6 +174,23 @@
          </archive>
         </configuration>
        </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
     <testResources>
       <testResource>

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -103,7 +103,9 @@ case "${COMMAND}" in
     echo "YARN_DIR='${YARN_DIR}'"
     echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
     echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
-    echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'"
+    echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+    echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+    echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
     exit 0
   ;;
   jar)