hadoop/hadoop-mapreduce-project/hadoop-yarn/bin/yarn

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# The Hadoop command script
#
# Environment Variables
#
#   JAVA_HOME        The java implementation to use.  Overrides JAVA_HOME.
#
#   YARN_CLASSPATH Extra Java CLASSPATH entries.
#
#   YARN_USER_CLASSPATH_FIRST      When defined, the YARN_CLASSPATH is 
#                                    added in the beginning of the global
#                                    classpath. Can be defined, for example,
#                                    by doing 
#                                    export YARN_USER_CLASSPATH_FIRST=true
#
#   YARN_HEAPSIZE  The maximum amount of heap to use, in MB. 
#                    Default is 1000.
#
#   YARN_OPTS      Extra Java runtime options.
#   
#   YARN_NAMENODE_OPTS       These options are added to YARN_OPTS 
#   YARN_CLIENT_OPTS         when the respective command is run.
#   YARN_{COMMAND}_OPTS etc  YARN_JT_OPTS applies to JobTracker 
#                              for e.g.  YARN_CLIENT_OPTS applies to 
#                              more than one command (fs, dfs, fsck, 
#                              dfsadmin etc)  
#
#   YARN_CONF_DIR  Alternate conf dir. Default is ${YARN_HOME}/conf.
#
#   YARN_ROOT_LOGGER The root appender. Default is INFO,console
#

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/yarn-config.sh

cygwin=false
case "`uname`" in
CYGWIN*) cygwin=true;;
esac

# if no args specified, show usage
if [ $# = 0 ]; then
  echo "Usage: hadoop [--config confdir] COMMAND"
  echo "where COMMAND is one of:"
  echo "  resourcemanager      run the ResourceManager" 
  echo "  nodemanager          run a nodemanager on each slave" 
  echo "  historyserver        run job history servers as a standalone daemon"
  echo "  rmadmin              admin tools" 
  echo "  version              print the version"
  echo "  jar <jar>            run a jar file"
  echo "  logs                 dump container logs"
  echo "  classpath            prints the class path needed to get the"
  echo "                       Hadoop jar and the required libraries"
  echo "  daemonlog            get/set the log level for each daemon"
  echo " or"
  echo "  CLASSNAME            run the class named CLASSNAME"
  echo "Most commands print help when invoked w/o parameters."
  exit 1
fi

# get arguments
COMMAND=$1
shift

if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
  . "${YARN_CONF_DIR}/yarn-env.sh"
fi

# some Java parameters
if [ "$JAVA_HOME" != "" ]; then
  #echo "run java in $JAVA_HOME"
  JAVA_HOME=$JAVA_HOME
fi
  
if [ "$JAVA_HOME" = "" ]; then
  echo "Error: JAVA_HOME is not set."
  exit 1
fi

JAVA=$JAVA_HOME/bin/java
JAVA_HEAP_MAX=-Xmx1000m 

# check envvars which might override default args
if [ "$YARN_HEAPSIZE" != "" ]; then
  #echo "run with heapsize $YARN_HEAPSIZE"
  JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
  #echo $JAVA_HEAP_MAX
fi

# CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR
CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
if [ "$YARN_USER_CLASSPATH_FIRST" != "" ] && [ "$YARN_CLASSPATH" != "" ] ; then
  CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH}
fi
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar

# for developers, add Hadoop classes to CLASSPATH
if [ -d "$YARN_HOME/yarn-api/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-api/target/classes
fi
if [ -d "$YARN_HOME/yarn-common/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-common/target/classes
fi
if [ -d "$YARN_HOME/yarn-mapreduce/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-mapreduce/target/classes
fi
if [ -d "$YARN_HOME/yarn-master-worker/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-master-worker/target/classes
fi
if [ -d "$YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes
fi
if [ -d "$YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-common/target/classes
fi
if [ -d "$YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes
fi
if [ -d "$YARN_HOME/build/test/classes" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/target/test/classes
fi
if [ -d "$YARN_HOME/build/tools" ]; then
  CLASSPATH=${CLASSPATH}:$YARN_HOME/build/tools
fi


# so that filenames w/ spaces are handled correctly in loops below
IFS=

# add hadoop-common libs to CLASSPATH

if [ -d "$HADOOP_COMMON_HOME/build/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/classes
fi
if [ -d "$HADOOP_COMMON_HOME/build/webapps" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build
fi
if [ -d "$HADOOP_COMMON_HOME/build/test/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/classes
fi
if [ -d "$HADOOP_COMMON_HOME/build/test/core/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/core/classes
fi

for f in $HADOOP_COMMON_HOME/hadoop-*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_COMMON_HOME/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_COMMON_HOME/share/hadoop/common/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_COMMON_HOME/share/hadoop/common/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_COMMON_HOME/share/hadoop/hdfs/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common" ]; then
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done
fi

if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done
fi

if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common" ]; then
for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done
fi

# add hadoop-hdfs libs to CLASSPATH

for f in $HADOOP_HDFS_HOME/hadoop-*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_HDFS_HOME/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes
fi
if [ -d "$HADOOP_HDFS_HOME/build/webapps" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build
fi
if [ -d "$HADOOP_HDFS_HOME/build/test/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/test/classes
fi
if [ -d "$HADOOP_HDFS_HOME/build/tools" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/tools
fi

# add hadoop-mapred libs to CLASSPATH

for f in $HADOOP_HDFS_HOME/hadoop-*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

for f in $HADOOP_HDFS_HOME/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
fi
if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
fi
if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
fi
if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
fi

# for releases, add core mapred jar & webapps to CLASSPATH
if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
fi

# add libs to CLASSPATH
for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

# add libs to CLASSPATH
for f in $HADOOP_MAPRED_HOME/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

# add libs to CLASSPATH
for f in $YARN_HOME/lib/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

# add yarn libs to CLASSPATH
for f in $YARN_HOME/modules/*.jar; do
  CLASSPATH=${CLASSPATH}:$f;
done

# add user-specified CLASSPATH last
if [ "$YARN_USER_CLASSPATH_FIRST" = "" ] && [ "$YARN_CLASSPATH" != "" ]; then
  CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH}
fi

# default log directory & file
if [ "$YARN_LOG_DIR" = "" ]; then
  YARN_LOG_DIR="$YARN_HOME/logs"
fi
if [ "$YARN_LOGFILE" = "" ]; then
  YARN_LOGFILE='yarn.log'
fi

# restore ordinary behaviour
unset IFS

# figure out which class to run
if [ "$COMMAND" = "classpath" ] ; then
  if $cygwin; then
    CLASSPATH=`cygpath -p -w "$CLASSPATH"`
  fi
  echo $CLASSPATH
  exit
elif [ "$COMMAND" = "rmadmin" ] ; then
  CLASS='org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin'
  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "resourcemanager" ] ; then
  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
  CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
  YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
elif [ "$COMMAND" = "nodemanager" ] ; then
  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
  CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
  if [[ $EUID -eq 0 ]]; then
    YARN_OPTS="$YARN_OPTS -jvm server $YARN_NODEMANAGER_OPTS"
  else
    YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
  fi
elif [ "$COMMAND" = "historyserver" ] ; then
  CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
  YARN_OPTS="$YARN_OPTS $YARN_JOB_HISTORYSERVER_OPTS"
elif [ "$COMMAND" = "version" ] ; then
  CLASS=org.apache.hadoop.util.VersionInfo
  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "jar" ] ; then
  CLASS=org.apache.hadoop.util.RunJar
  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "logs" ] ; then
  CLASS=org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogDumper
  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
elif [ "$COMMAND" = "daemonlog" ] ; then
  CLASS=org.apache.hadoop.log.LogLevel
  YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
else
  CLASS=$COMMAND
fi

# cygwin path translation
if $cygwin; then
  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
  YARN_HOME=`cygpath -w "$YARN_HOME"`
  YARN_LOG_DIR=`cygpath -w "$YARN_LOG_DIR"`
  TOOL_PATH=`cygpath -p -w "$TOOL_PATH"`
fi

# cygwin path translation
if $cygwin; then
  JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"`
fi

YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_HOME"
YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
  YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi  

echo "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@"
exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@"
fi