hadoop/hadoop-mapreduce-project/bin/mapred

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MYNAME="${BASH_SOURCE-$0}"

function hadoop_usage
{
  hadoop_add_subcommand "archive" "create a hadoop archive"
  hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
  hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands"
  hadoop_add_subcommand "distcp" "copy file or directories recursively"
  hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
  hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon"
  hadoop_add_subcommand "hsadmin" "job history server admin interface"
  hadoop_add_subcommand "job" "manipulate MapReduce jobs"
  hadoop_add_subcommand "pipes" "run a Pipes job"
  hadoop_add_subcommand "queue" "get information regarding JobQueues"
  hadoop_add_subcommand "sampler" "sampler"
  hadoop_add_subcommand "version" "print the version"
  hadoop_generate_usage "${MYNAME}" true
}

bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)

# let's locate libexec...
if [[ -n "${HADOOP_PREFIX}" ]]; then
  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
else
  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi

HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/mapred-config.sh" ]]; then
  . "${HADOOP_LIBEXEC_DIR}/mapred-config.sh"
else
  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/mapred-config.sh." 2>&1
  exit 1
fi


if [ $# = 0 ]; then
  hadoop_exit_with_usage 1
fi

COMMAND=$1
shift

case ${COMMAND} in
  mradmin|jobtracker|tasktracker|groups)
    hadoop_error "Sorry, the ${COMMAND} command is no longer supported."
    hadoop_error "You may find similar functionality with the \"yarn\" shell command."
    hadoop_exit_with_usage 1
  ;;
  archive)
    CLASS=org.apache.hadoop.tools.HadoopArchives
    hadoop_add_to_classpath_toolspath
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  archive-logs)
    CLASS=org.apache.hadoop.tools.HadoopArchiveLogs
    hadoop_add_to_classpath_toolspath
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  classpath)
    hadoop_do_classpath_subcommand CLASS "$@"
  ;;
  distcp)
    CLASS=org.apache.hadoop.tools.DistCp
    hadoop_add_to_classpath_toolspath
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  envvars)
    echo "JAVA_HOME='${JAVA_HOME}'"
    echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"
    echo "MAPRED_DIR='${MAPRED_DIR}'"
    echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'"
    echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
    echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'"
    exit 0
  ;;
  historyserver)
    supportdaemonization="true"
    CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
    hadoop_debug "Appending HADOOP_JOB_HISTORYSERVER_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}"
    if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then
      HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}"
    fi
    HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
  ;;
  hsadmin)
    CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  job)
    CLASS=org.apache.hadoop.mapred.JobClient
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  pipes)
    CLASS=org.apache.hadoop.mapred.pipes.Submitter
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  queue)
    CLASS=org.apache.hadoop.mapred.JobQueueClient
  ;;
  sampler)
    CLASS=org.apache.hadoop.mapred.lib.InputSampler
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  version)
    CLASS=org.apache.hadoop.util.VersionInfo
    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  ;;
  *)
    CLASS="${COMMAND}"
    if ! hadoop_validate_classname "${CLASS}"; then
      hadoop_exit_with_usage 1
    fi
  ;;
esac

hadoop_verify_user "${COMMAND}"

if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
  hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
  exit $?
fi

daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"


if [[  "${HADOOP_DAEMON_MODE}" != "default" ]]; then
  # shellcheck disable=SC2034
  HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
  hadoop_add_param HADOOP_OPTS mapred.jobsummary.logger "-Dmapred.jobsummary.logger=${HADOOP_ROOT_LOGGER}"
  # shellcheck disable=SC2034
  HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
fi

hadoop_finalize

if [[ -n "${supportdaemonization}" ]]; then
  if [[ -n "${secure_service}" ]]; then
    hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}"\
    "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
    "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
  else
    hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
    "${daemon_pidfile}" "${daemon_outfile}" "$@"
  fi
  exit $?
else
  hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
fi