HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands

hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml

@@ -23,6 +23,38 @@
   </formats>
   <includeBaseDirectory>false</includeBaseDirectory>
   <fileSets>
+    <fileSet>
+      <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-archives/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-distcp/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>../hadoop-extras/src/main/shellprofile.d</directory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
       <includes>

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -30,14 +30,11 @@ function hadoop_usage
   hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
   hadoop_add_option "slaves" "turn on slave mode"
 
-  hadoop_add_subcommand "archive" "create a Hadoop archive"
   hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability"
   hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries"
   hadoop_add_subcommand "conftest" "validate configuration XML files"
   hadoop_add_subcommand "credential" "interact with credential providers"
   hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
-  hadoop_add_subcommand "distch" "distributed metadata changer"
-  hadoop_add_subcommand "distcp" "copy file or directories recursively"
   hadoop_add_subcommand "dtutil" "operations related to delegation tokens"
   hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
   hadoop_add_subcommand "fs" "run a generic filesystem user client"
@@ -102,35 +99,23 @@ function hadoopcmd_case
         exit 1
       fi
     ;;
-    archive)
-      HADOOP_CLASS=org.apache.hadoop.tools.HadoopArchives
-      hadoop_add_to_classpath_tools hadoop-archives
-    ;;
     checknative)
-      HADOOP_CLASS=org.apache.hadoop.util.NativeLibraryChecker
+      HADOOP_CLASSNAME=org.apache.hadoop.util.NativeLibraryChecker
     ;;
     classpath)
-      hadoop_do_classpath_subcommand HADOOP_CLASS "$@"
+      hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;
     conftest)
-      HADOOP_CLASS=org.apache.hadoop.util.ConfTest
+      HADOOP_CLASSNAME=org.apache.hadoop.util.ConfTest
     ;;
     credential)
-      HADOOP_CLASS=org.apache.hadoop.security.alias.CredentialShell
+      HADOOP_CLASSNAME=org.apache.hadoop.security.alias.CredentialShell
     ;;
     daemonlog)
-      HADOOP_CLASS=org.apache.hadoop.log.LogLevel
-    ;;
-    distch)
-      HADOOP_CLASS=org.apache.hadoop.tools.DistCh
-      hadoop_add_to_classpath_tools hadoop-extras
-    ;;
-    distcp)
-      HADOOP_CLASS=org.apache.hadoop.tools.DistCp
-      hadoop_add_to_classpath_tools hadoop-distcp
+      HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
     ;;
     dtutil)
-      HADOOP_CLASS=org.apache.hadoop.security.token.DtUtilShell
+      HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
@@ -145,7 +130,7 @@ function hadoopcmd_case
       exit 0
     ;;
     fs)
-      HADOOP_CLASS=org.apache.hadoop.fs.FsShell
+      HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
     ;;
     gridmix)
       CLASS=org.apache.hadoop.mapred.gridmix.Gridmix
@@ -156,7 +141,7 @@ function hadoopcmd_case
       if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
         hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
       fi
-      HADOOP_CLASS=org.apache.hadoop.util.RunJar
+      HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
     ;;
     jnipath)
       hadoop_finalize
@@ -164,20 +149,20 @@ function hadoopcmd_case
       exit 0
     ;;
     kerbname)
-      HADOOP_CLASS=org.apache.hadoop.security.HadoopKerberosName
+      HADOOP_CLASSNAME=org.apache.hadoop.security.HadoopKerberosName
     ;;
     key)
-      HADOOP_CLASS=org.apache.hadoop.crypto.key.KeyShell
+      HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
     ;;
     trace)
-      HADOOP_CLASS=org.apache.hadoop.tracing.TraceAdmin
+      HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
     ;;
     version)
-      HADOOP_CLASS=org.apache.hadoop.util.VersionInfo
+      HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
     ;;
     *)
-      HADOOP_CLASS="${subcmd}"
-      if ! hadoop_validate_classname "${HADOOP_CLASS}"; then
+      HADOOP_CLASSNAME="${subcmd}"
+      if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
         hadoop_exit_with_usage 1
       fi
     ;;
@@ -229,5 +214,5 @@ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 
 hadoop_finalize
-hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASS}" "$@"
+hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "$@"
 

hadoop-mapreduce-project/bin/mapred

@@ -24,10 +24,7 @@ HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
 ## @replaceable  no
 function hadoop_usage
 {
-  hadoop_add_subcommand "archive" "create a hadoop archive"
-  hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
   hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands"
-  hadoop_add_subcommand "distcp" "copy file or directories recursively"
   hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
   hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon"
   hadoop_add_subcommand "hsadmin" "job history server admin interface"
@@ -55,27 +52,9 @@ function mapredcmd_case
       hadoop_error "You may find similar functionality with the \"yarn\" shell command."
       hadoop_exit_with_usage 1
     ;;
-    archive)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
-      hadoop_add_to_classpath_tools hadoop-archives
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
-    archive-logs)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
-      hadoop_add_to_classpath_tools hadoop-archive-logs
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
     classpath)
       hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;
-    distcp)
-      HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
-      hadoop_add_to_classpath_tools hadoop-distcp
-      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-    ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"
       echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"

hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
+fi
+
+## @description  archive-logs command for mapred
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_archive-logs
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
+  hadoop_add_to_classpath_tools hadoop-archive-logs
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}

hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+   || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "archive" "create a Hadoop archive"
+fi
+
+## @description  archive command for hadoop (and mapred)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_archive
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
+  hadoop_add_to_classpath_tools hadoop-archives
+}
+
+## @description  archive-logs command for mapred (calls hadoop version)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_archive
+{
+  hadoop_subcommand_archive
+}

hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop
+   || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+  hadoop_add_subcommand "distcp" "copy file or directories recursively"
+fi
+
+## @description  distcp command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_distcp
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
+  hadoop_add_to_classpath_tools hadoop-distcp
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+## @description  distcp command for mapred (calls hadoop)
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function mapred_subcommand_distcp
+{
+  hadoop_subcommand_distcp
+}

hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+  hadoop_add_subcommand "distch" "distributed metadata changer"
+fi
+
+## @description  distch command for hadoop
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function hadoop_subcommand_distch
+{
+  # shellcheck disable=SC2034
+  HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
+  hadoop_add_to_classpath_tools hadoop-extras
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}