yarn 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. #!/usr/bin/env bash
  2. # Licensed to the Apache Software Foundation (ASF) under one or more
  3. # contributor license agreements. See the NOTICE file distributed with
  4. # this work for additional information regarding copyright ownership.
  5. # The ASF licenses this file to You under the Apache License, Version 2.0
  6. # (the "License"); you may not use this file except in compliance with
  7. # the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. # The Hadoop command script
  17. #
  18. # Environment Variables
  19. #
  20. # JAVA_HOME The java implementation to use. Overrides JAVA_HOME.
  21. #
  22. # YARN_USER_CLASSPATH Additional user CLASSPATH entries.
  23. #
  24. # YARN_USER_CLASSPATH_FIRST If set to non empty value then the user classpath
  25. # specified in YARN_USER_CLASSPATH will be
  26. # appended at the beginning of YARN's final
  27. # classpath instead of at the end.
  28. #
  29. # YARN_HEAPSIZE The maximum amount of heap to use, in MB.
  30. # Default is 1000.
  31. #
  32. # YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
  33. # eg YARN_NODEMANAGER_HEAPSIZE sets the heap
  34. # size for the NodeManager. If you set the
  35. # heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
  36. # they take precedence.
  37. #
  38. # YARN_OPTS Extra Java runtime options.
  39. #
  40. # YARN_CLIENT_OPTS when the respective command is run.
  41. # YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager
  42. # for e.g. YARN_CLIENT_OPTS applies to
  43. # more than one command (fs, dfs, fsck,
  44. # dfsadmin etc)
  45. #
  46. # YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf.
  47. #
  48. # YARN_ROOT_LOGGER The root appender. Default is INFO,console
  49. #
  50. bin=`dirname "${BASH_SOURCE-$0}"`
  51. bin=`cd "$bin"; pwd`
  52. DEFAULT_LIBEXEC_DIR="$bin"/../libexec
  53. HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
  54. . $HADOOP_LIBEXEC_DIR/yarn-config.sh
  55. function print_usage(){
  56. echo "Usage: yarn [--config confdir] [COMMAND | CLASSNAME]"
  57. echo " CLASSNAME run the class named CLASSNAME"
  58. echo " or"
  59. echo " where COMMAND is one of:"
  60. echo " resourcemanager run the ResourceManager"
  61. echo " Use -format-state-store for deleting the RMStateStore."
  62. echo " Use -remove-application-from-state-store <appId> for "
  63. echo " removing application from RMStateStore."
  64. echo " nodemanager run a nodemanager on each slave"
  65. echo " timelineserver run the timeline server"
  66. echo " rmadmin admin tools"
  67. echo " router run the Router daemon"
  68. echo " sharedcachemanager run the SharedCacheManager daemon"
  69. echo " scmadmin SharedCacheManager admin tools"
  70. echo " version print the version"
  71. echo " jar <jar> run a jar file"
  72. echo " application prints application(s)"
  73. echo " report/kill application"
  74. echo " applicationattempt prints applicationattempt(s)"
  75. echo " report"
  76. echo " container prints container(s) report"
  77. echo " node prints node report(s)"
  78. echo " queue prints queue information"
  79. echo " logs dump container logs"
  80. echo " schedulerconf updates scheduler configuration"
  81. echo " classpath prints the class path needed to"
  82. echo " get the Hadoop jar and the"
  83. echo " required libraries"
  84. echo " cluster prints cluster information"
  85. echo " daemonlog get/set the log level for each"
  86. echo " daemon"
  87. echo " top run cluster usage tool"
  88. echo ""
  89. echo "Most commands print help when invoked w/o parameters."
  90. }
  91. # if no args specified, show usage
  92. if [ $# = 0 ]; then
  93. print_usage
  94. exit 1
  95. fi
  96. # get arguments
  97. COMMAND=$1
  98. shift
  99. case $COMMAND in
  100. # usage flags
  101. --help|-help|-h)
  102. print_usage
  103. exit
  104. ;;
  105. esac
  106. if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
  107. . "${YARN_CONF_DIR}/yarn-env.sh"
  108. fi
  109. # some Java parameters
  110. if [ "$JAVA_HOME" != "" ]; then
  111. #echo "run java in $JAVA_HOME"
  112. JAVA_HOME=$JAVA_HOME
  113. fi
  114. if [ "$JAVA_HOME" = "" ]; then
  115. echo "Error: JAVA_HOME is not set."
  116. exit 1
  117. fi
  118. JAVA=$JAVA_HOME/bin/java
  119. JAVA_HEAP_MAX=-Xmx1000m
  120. # check envvars which might override default args
  121. if [ "$YARN_HEAPSIZE" != "" ]; then
  122. #echo "run with heapsize $YARN_HEAPSIZE"
  123. JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
  124. #echo $JAVA_HEAP_MAX
  125. fi
  126. # CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR
  127. if [ ! -d "$HADOOP_CONF_DIR" ]; then
  128. echo No HADOOP_CONF_DIR set.
  129. echo Please specify it either in yarn-env.sh or in the environment.
  130. exit 1
  131. fi
  132. CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}"
  133. # for developers, add Hadoop classes to CLASSPATH
  134. if [ -d "$HADOOP_YARN_HOME/yarn-api/target/classes" ]; then
  135. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-api/target/classes
  136. fi
  137. if [ -d "$HADOOP_YARN_HOME/yarn-common/target/classes" ]; then
  138. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-common/target/classes
  139. fi
  140. if [ -d "$HADOOP_YARN_HOME/yarn-mapreduce/target/classes" ]; then
  141. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-mapreduce/target/classes
  142. fi
  143. if [ -d "$HADOOP_YARN_HOME/yarn-master-worker/target/classes" ]; then
  144. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-master-worker/target/classes
  145. fi
  146. if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then
  147. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes
  148. fi
  149. if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then
  150. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-common/target/classes
  151. fi
  152. if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then
  153. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes
  154. fi
  155. if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes" ]; then
  156. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-applicationhistoryservice/target/classes
  157. fi
  158. if [ -d "$HADOOP_YARN_HOME/build/test/classes" ]; then
  159. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/target/test/classes
  160. fi
  161. if [ -d "$HADOOP_YARN_HOME/build/tools" ]; then
  162. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/build/tools
  163. fi
  164. if [ -d "$HADOOP_YARN_HOME/yarn-server/yarn-server-router/target/classes" ]; then
  165. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/yarn-server/yarn-server-router/target/classes
  166. fi
  167. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_DIR}/*
  168. CLASSPATH=${CLASSPATH}:$HADOOP_YARN_HOME/${YARN_LIB_JARS_DIR}/*
  169. # Add user defined YARN_USER_CLASSPATH to the class path (if defined)
  170. if [ -n "$YARN_USER_CLASSPATH" ]; then
  171. if [ -n "$YARN_USER_CLASSPATH_FIRST" ]; then
  172. # User requested to add the custom entries at the beginning
  173. CLASSPATH=${YARN_USER_CLASSPATH}:${CLASSPATH}
  174. else
  175. # By default we will just append the extra entries at the end
  176. CLASSPATH=${CLASSPATH}:${YARN_USER_CLASSPATH}
  177. fi
  178. fi
  179. # so that filenames w/ spaces are handled correctly in loops below
  180. IFS=
  181. # default log directory & file
  182. if [ "$YARN_LOG_DIR" = "" ]; then
  183. YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
  184. fi
  185. if [ "$YARN_LOGFILE" = "" ]; then
  186. YARN_LOGFILE='yarn.log'
  187. fi
  188. # restore ordinary behaviour
  189. unset IFS
  190. # figure out which class to run
  191. if [ "$COMMAND" = "classpath" ] ; then
  192. if [ "$#" -gt 0 ]; then
  193. CLASS=org.apache.hadoop.util.Classpath
  194. else
  195. if [ "$cygwin" = true ]; then
  196. CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
  197. fi
  198. echo $CLASSPATH
  199. exit 0
  200. fi
  201. elif [ "$COMMAND" = "rmadmin" ] ; then
  202. CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
  203. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  204. elif [ "$COMMAND" = "router" ] ; then
  205. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rt-config/log4j.properties
  206. CLASS='org.apache.hadoop.yarn.server.router.Router'
  207. YARN_OPTS="$YARN_OPTS $YARN_ROUTER_OPTS"
  208. elif [ "$COMMAND" = "scmadmin" ] ; then
  209. CLASS='org.apache.hadoop.yarn.client.SCMAdmin'
  210. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  211. elif [ "$COMMAND" = "application" ] ||
  212. [ "$COMMAND" = "applicationattempt" ] ||
  213. [ "$COMMAND" = "container" ]; then
  214. CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
  215. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  216. set -- $COMMAND $@
  217. elif [ "$COMMAND" = "node" ] ; then
  218. CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
  219. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  220. elif [ "$COMMAND" = "queue" ] ; then
  221. CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
  222. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  223. elif [ "$COMMAND" = "resourcemanager" ] ; then
  224. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
  225. CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
  226. YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
  227. if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
  228. JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
  229. fi
  230. elif [ "$COMMAND" = "historyserver" ] ; then
  231. echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
  232. echo "Instead use the timelineserver command for it." 1>&2
  233. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/ahs-config/log4j.properties
  234. CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
  235. YARN_OPTS="$YARN_OPTS $YARN_HISTORYSERVER_OPTS"
  236. if [ "$YARN_HISTORYSERVER_HEAPSIZE" != "" ]; then
  237. JAVA_HEAP_MAX="-Xmx""$YARN_HISTORYSERVER_HEAPSIZE""m"
  238. fi
  239. elif [ "$COMMAND" = "timelineserver" ] ; then
  240. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/timelineserver-config/log4j.properties
  241. CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
  242. YARN_OPTS="$YARN_OPTS $YARN_TIMELINESERVER_OPTS"
  243. if [ "$YARN_TIMELINESERVER_HEAPSIZE" != "" ]; then
  244. JAVA_HEAP_MAX="-Xmx""$YARN_TIMELINESERVER_HEAPSIZE""m"
  245. fi
  246. elif [ "$COMMAND" = "sharedcachemanager" ] ; then
  247. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/scm-config/log4j.properties
  248. CLASS='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
  249. YARN_OPTS="$YARN_OPTS $YARN_SHAREDCACHEMANAGER_OPTS"
  250. if [ "$YARN_SHAREDCACHEMANAGER_HEAPSIZE" != "" ]; then
  251. JAVA_HEAP_MAX="-Xmx""$YARN_SHAREDCACHEMANAGER_HEAPSIZE""m"
  252. fi
  253. elif [ "$COMMAND" = "nodemanager" ] ; then
  254. CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
  255. CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
  256. YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
  257. if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
  258. JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
  259. fi
  260. elif [ "$COMMAND" = "proxyserver" ] ; then
  261. CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
  262. YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
  263. if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then
  264. JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m"
  265. fi
  266. elif [ "$COMMAND" = "version" ] ; then
  267. CLASS=org.apache.hadoop.util.VersionInfo
  268. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  269. elif [ "$COMMAND" = "jar" ] ; then
  270. CLASS=org.apache.hadoop.util.RunJar
  271. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  272. elif [ "$COMMAND" = "logs" ] ; then
  273. CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
  274. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  275. elif [ "$COMMAND" = "daemonlog" ] ; then
  276. CLASS=org.apache.hadoop.log.LogLevel
  277. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  278. elif [ "$COMMAND" = "cluster" ] ; then
  279. CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
  280. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  281. elif [ "$COMMAND" = "schedulerconf" ] ; then
  282. CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
  283. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  284. elif [ "$COMMAND" = "top" ]; then
  285. CLASS=org.apache.hadoop.yarn.client.cli.TopCLI
  286. YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
  287. doNotSetCols=0
  288. doNotSetRows=0
  289. for i in "$@"; do
  290. if [[ $i == "-cols" ]]; then
  291. doNotSetCols=1
  292. fi
  293. if [[ $i == "-rows" ]]; then
  294. doNotSetRows=1
  295. fi
  296. done
  297. if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
  298. cols=$(tput cols)
  299. if [ -n "$cols" ]; then
  300. args=( $@ )
  301. args=("${args[@]}" "-cols" "$cols")
  302. set -- "${args[@]}"
  303. fi
  304. fi
  305. if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
  306. rows=$(tput lines)
  307. if [ -n "$rows" ]; then
  308. args=( $@ )
  309. args=("${args[@]}" "-rows" "$rows")
  310. set -- "${args[@]}"
  311. fi
  312. fi
  313. else
  314. CLASS=$COMMAND
  315. fi
  316. # cygwin path translation
  317. if [ "$cygwin" = true ]; then
  318. CLASSPATH=$(cygpath -p -w "$CLASSPATH" 2>/dev/null)
  319. HADOOP_LOG_DIR=$(cygpath -w "$HADOOP_LOG_DIR" 2>/dev/null)
  320. HADOOP_PREFIX=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
  321. HADOOP_CONF_DIR=$(cygpath -w "$HADOOP_CONF_DIR" 2>/dev/null)
  322. HADOOP_COMMON_HOME=$(cygpath -w "$HADOOP_COMMON_HOME" 2>/dev/null)
  323. HADOOP_HDFS_HOME=$(cygpath -w "$HADOOP_HDFS_HOME" 2>/dev/null)
  324. HADOOP_YARN_HOME=$(cygpath -w "$HADOOP_YARN_HOME" 2>/dev/null)
  325. HADOOP_MAPRED_HOME=$(cygpath -w "$HADOOP_MAPRED_HOME" 2>/dev/null)
  326. YARN_LOG_DIR=$(cygpath -w "$YARN_LOG_DIR" 2>/dev/null)
  327. fi
  328. YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
  329. YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
  330. YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
  331. YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
  332. YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
  333. if [ "$cygwin" = true ]; then
  334. HADOOP_HOME=$(cygpath -w "$HADOOP_PREFIX" 2>/dev/null)
  335. YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_HOME"
  336. export HADOOP_HOME
  337. else
  338. YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
  339. fi
  340. YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
  341. YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
  342. if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
  343. if [ "$cygwin" = true ]; then
  344. JAVA_LIBRARY_PATH=$(cygpath -w "$JAVA_LIBRARY_PATH" 2>/dev/null)
  345. fi
  346. YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
  347. fi
  348. exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@"