HADOOP-13355. Handle HADOOP_CLIENT_OPTS in a function (aw)

Signed-off-by: Allen Wittenauer <aw@apache.org>

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -161,10 +161,6 @@ function hadoopcmd_case
       fi
     ;;
   esac
-
-  # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 # This script runs the hadoop core commands.
@@ -205,6 +201,8 @@ fi
 
 hadoop_verify_user "${HADOOP_SUBCMD}"
 
+hadoop_add_client_opts
+
 if [[ ${HADOOP_WORKER_MODE} = true ]]; then
   hadoop_common_worker_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
   exit $?

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -306,6 +306,13 @@ function hadoop_bootstrap
   HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
   HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
 
+  # by default, whatever we are about to run doesn't support
+  # daemonization
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
+
+  # shellcheck disable=SC2034
+  HADOOP_SUBCMD_SECURESERVICE=false
+
   # usage output set to zero
   hadoop_reset_usage
 
@@ -1230,6 +1237,20 @@ function hadoop_translate_cygwin_path
   fi
 }
 
+## @description  Adds the HADOOP_CLIENT_OPTS variable to
+## @description  HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
+## @audience     public
+## @stability    stable
+## @replaceable  yes
+function hadoop_add_client_opts
+{
+  if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = false
+     || -z "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" ]]; then
+    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+  fi
+}
+
 ## @description  Finish configuring Hadoop specific system properties
 ## @description  prior to executing Java
 ## @audience     private

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_client_opts.bats

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_subcommand_opts (daemonization false)" {
+  HADOOP_OPTS="1"
+  HADOOP_CLIENT_OPTS="2"
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
+  hadoop_add_client_opts
+  [ "${HADOOP_OPTS}" = "1 2" ]
+}
+
+@test "hadoop_subcommand_opts (daemonization true)" {
+  HADOOP_OPTS="1"
+  HADOOP_CLIENT_OPTS="2"
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+  hadoop_add_client_opts
+  [ "${HADOOP_OPTS}" = "1" ]
+}
+
+@test "hadoop_subcommand_opts (daemonization empty)" {
+  HADOOP_OPTS="1"
+  HADOOP_CLIENT_OPTS="2"
+  unset HADOOP_SUBCMD_SUPPORTDAEMONIZATION
+  hadoop_add_client_opts
+  [ "${HADOOP_OPTS}" = "1 2" ]
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

@@ -118,18 +118,12 @@ function hdfscmd_case
     ;;
     dfs)
       HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     dfsadmin)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     diskbalancer)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancerCLI
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
@@ -144,16 +138,12 @@ function hdfscmd_case
     ;;
     erasurecode)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     fetchdt)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
     ;;
     fsck)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     getconf)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
@@ -163,8 +153,6 @@ function hdfscmd_case
     ;;
     haadmin)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     journalnode)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -293,6 +281,8 @@ fi
 
 hadoop_verify_user "${HADOOP_SUBCMD}"
 
+hadoop_add_client_opts
+
 if [[ ${HADOOP_WORKER_MODE} = true ]]; then
   hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
   exit $?

hadoop-mapreduce-project/bin/mapred

@@ -79,31 +79,21 @@ function mapredcmd_case
     ;;
     hsadmin)
       HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     job)
       HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobClient
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     pipes)
       HADOOP_CLASSNAME=org.apache.hadoop.mapred.pipes.Submitter
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     queue)
       HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobQueueClient
     ;;
     sampler)
       HADOOP_CLASSNAME=org.apache.hadoop.mapred.lib.InputSampler
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     version)
       HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
     ;;
     *)
       HADOOP_CLASSNAME="${subcmd}"
@@ -152,6 +142,8 @@ fi
 
 hadoop_verify_user "${HADOOP_SUBCMD}"
 
+hadoop_add_client_opts
+
 if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
   hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
   exit $?

hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh

@@ -32,8 +32,6 @@ function mapred_subcommand_archive-logs
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
   hadoop_add_to_classpath_tools hadoop-archive-logs
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi

hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh

@@ -32,8 +32,6 @@ function hadoop_subcommand_distcp
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
   hadoop_add_to_classpath_tools hadoop-distcp
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi
@@ -55,8 +53,6 @@ function mapred_subcommand_distcp
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
   hadoop_add_to_classpath_tools hadoop-distcp
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi

hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh

@@ -32,8 +32,6 @@ function hadoop_subcommand_distch
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
   hadoop_add_to_classpath_tools hadoop-extras
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi

hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh

@@ -30,8 +30,6 @@ function hadoop_subcommand_rumenfolder
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.Folder
   hadoop_add_to_classpath_tools hadoop-rumen
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi
@@ -51,8 +49,6 @@ function hadoop_subcommand_rumentrace
   # shellcheck disable=SC2034
   HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.TraceBuilder
   hadoop_add_to_classpath_tools hadoop-rumen
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 fi

hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh

@@ -68,8 +68,7 @@ function run_sls_generator()
   hadoop_add_param args -outputJobs "-outputJobs ${outputdir}/${outputprefix}-jobs.json"
   hadoop_add_param args -outputNodes "-outputNodes ${outputdir}/${outputprefix}-nodes.json"
 
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+  hadoop_add_client_opts
 
   hadoop_finalize
   # shellcheck disable=SC2086

hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh

@@ -96,8 +96,7 @@ function run_simulation() {
     hadoop_add_param args -printsimulation "-printsimulation"
   fi
 
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+  hadoop_add_client_opts
 
   hadoop_finalize
   # shellcheck disable=SC2086

hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh

@@ -46,10 +46,6 @@ function mapred_subcommand_streaming
   done
 
   IFS=${oldifs}
-
-  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-
 }
 
 fi

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -68,8 +68,6 @@ function yarncmd_case
   case ${subcmd} in
     application|applicationattempt|container)
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
       set -- "${subcmd}" "$@"
       HADOOP_SUBCMD_ARGS=("$@")
     ;;
@@ -78,13 +76,9 @@ function yarncmd_case
     ;;
     cluster)
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-      YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     daemonlog)
       HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
@@ -99,8 +93,6 @@ function yarncmd_case
     ;;
     jar)
       HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     historyserver)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -111,13 +103,9 @@ function yarncmd_case
     ;;
     logs)
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     node)
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     nodemanager)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -142,8 +130,6 @@ function yarncmd_case
     ;;
     queue)
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     resourcemanager)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -158,13 +144,9 @@ function yarncmd_case
     ;;
     rmadmin)
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     scmadmin)
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     sharedcachemanager)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -191,8 +173,6 @@ function yarncmd_case
     ;;
     version)
       HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     ;;
     top)
       doNotSetCols=0
@@ -222,8 +202,6 @@ function yarncmd_case
         fi
       fi
       HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
-      hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
       HADOOP_SUBCMD_ARGS=("$@")
     ;;
     *)
@@ -273,6 +251,18 @@ fi
 
 hadoop_verify_user "${HADOOP_SUBCMD}"
 
+# It's unclear if YARN_CLIENT_OPTS is actually a useful
+# thing to have separate from HADOOP_CLIENT_OPTS.  Someone
+# might use it, so let's not deprecate it and just override
+# HADOOP_CLIENT_OPTS instead before we (potentially) add it
+# to the command line
+if [[ -n "${YARN_CLIENT_OPTS}" ]]; then
+  # shellcheck disable=SC2034
+  HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS}
+fi
+
+hadoop_add_client_opts
+
 if [[ ${HADOOP_WORKER_MODE} = true ]]; then
   hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
   exit $?