hdfs 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276
  1. #!/usr/bin/env bash
  2. # Licensed to the Apache Software Foundation (ASF) under one or more
  3. # contributor license agreements. See the NOTICE file distributed with
  4. # this work for additional information regarding copyright ownership.
  5. # The ASF licenses this file to You under the Apache License, Version 2.0
  6. # (the "License"); you may not use this file except in compliance with
  7. # the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. function hadoop_usage
  17. {
  18. echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND"
  19. echo " where COMMAND is one of:"
  20. echo " balancer run a cluster balancing utility"
  21. echo " cacheadmin configure the HDFS cache"
  22. echo " classpath prints the class path needed to get the"
  23. echo " Hadoop jar and the required libraries"
  24. echo " crypto configure HDFS encryption zones"
  25. echo " datanode run a DFS datanode"
  26. echo " dfs run a filesystem command on the file system"
  27. echo " dfsadmin run a DFS admin client"
  28. echo " fetchdt fetch a delegation token from the NameNode"
  29. echo " fsck run a DFS filesystem checking utility"
  30. echo " getconf get config values from configuration"
  31. echo " groups get the groups which users belong to"
  32. echo " haadmin run a DFS HA admin client"
  33. echo " jmxget get JMX exported values from NameNode or DataNode."
  34. echo " journalnode run the DFS journalnode"
  35. echo " lsSnapshottableDir list all snapshottable dirs owned by the current user"
  36. echo " Use -help to see options"
  37. echo " mover run a utility to move block replicas across"
  38. echo " storage types"
  39. echo " namenode run the DFS namenode"
  40. echo " Use -format to initialize the DFS filesystem"
  41. echo " nfs3 run an NFS version 3 gateway"
  42. echo " oev apply the offline edits viewer to an edits file"
  43. echo " oiv apply the offline fsimage viewer to an fsimage"
  44. echo " oiv_legacy apply the offline fsimage viewer to a legacy fsimage"
  45. echo " portmap run a portmap service"
  46. echo " secondarynamenode run the DFS secondary namenode"
  47. echo " snapshotDiff diff two snapshots of a directory or diff the"
  48. echo " current directory contents with a snapshot"
  49. echo " zkfc run the ZK Failover Controller daemon"
  50. echo ""
  51. echo "Most commands print help when invoked w/o parameters."
  52. }
  53. # let's locate libexec...
  54. if [[ -n "${HADOOP_PREFIX}" ]]; then
  55. DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
  56. else
  57. this="${BASH_SOURCE-$0}"
  58. bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
  59. DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
  60. fi
  61. HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
  62. # shellcheck disable=SC2034
  63. HADOOP_NEW_CONFIG=true
  64. if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
  65. . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
  66. else
  67. echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
  68. exit 1
  69. fi
  70. if [[ $# = 0 ]]; then
  71. hadoop_exit_with_usage 1
  72. fi
  73. COMMAND=$1
  74. shift
  75. case ${COMMAND} in
  76. balancer)
  77. CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
  78. hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
  79. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
  80. ;;
  81. cacheadmin)
  82. CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
  83. ;;
  84. classpath)
  85. hadoop_finalize
  86. echo "${CLASSPATH}"
  87. exit
  88. ;;
  89. crypto)
  90. CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
  91. ;;
  92. datanode)
  93. daemon="true"
  94. # Determine if we're starting a secure datanode, and
  95. # if so, redefine appropriate variables
  96. if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
  97. secure_service="true"
  98. secure_user="${HADOOP_SECURE_DN_USER}"
  99. # backward compatiblity
  100. HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
  101. HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
  102. hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
  103. hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
  104. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
  105. CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
  106. else
  107. hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
  108. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
  109. CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
  110. fi
  111. ;;
  112. dfs)
  113. CLASS=org.apache.hadoop.fs.FsShell
  114. hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
  115. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  116. ;;
  117. dfsadmin)
  118. CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
  119. hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
  120. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  121. ;;
  122. fetchdt)
  123. CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
  124. ;;
  125. fsck)
  126. CLASS=org.apache.hadoop.hdfs.tools.DFSck
  127. hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
  128. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  129. ;;
  130. getconf)
  131. CLASS=org.apache.hadoop.hdfs.tools.GetConf
  132. ;;
  133. groups)
  134. CLASS=org.apache.hadoop.hdfs.tools.GetGroups
  135. ;;
  136. haadmin)
  137. CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
  138. hadoop_debug "Injecting TOOL_PATH into CLASSPATH"
  139. hadoop_add_classpath "${TOOL_PATH}"
  140. hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
  141. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
  142. ;;
  143. journalnode)
  144. daemon="true"
  145. CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
  146. hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
  147. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
  148. ;;
  149. jmxget)
  150. CLASS=org.apache.hadoop.hdfs.tools.JMXGet
  151. ;;
  152. lsSnapshottableDir)
  153. CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
  154. ;;
  155. mover)
  156. CLASS=org.apache.hadoop.hdfs.server.mover.Mover
  157. hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
  158. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
  159. ;;
  160. namenode)
  161. daemon="true"
  162. CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
  163. hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
  164. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
  165. ;;
  166. nfs3)
  167. daemon="true"
  168. if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
  169. secure_service="true"
  170. secure_user="${HADOOP_PRIVILEGED_NFS_USER}"
  171. # backward compatiblity
  172. HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
  173. HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
  174. hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
  175. hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
  176. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
  177. CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
  178. else
  179. hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
  180. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
  181. CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
  182. fi
  183. ;;
  184. oev)
  185. CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
  186. ;;
  187. oiv)
  188. CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
  189. ;;
  190. oiv_legacy)
  191. CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
  192. ;;
  193. portmap)
  194. daemon="true"
  195. CLASS=org.apache.hadoop.portmap.Portmap
  196. hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
  197. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
  198. ;;
  199. secondarynamenode)
  200. daemon="true"
  201. CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
  202. hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
  203. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
  204. ;;
  205. snapshotDiff)
  206. CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
  207. ;;
  208. zkfc)
  209. daemon="true"
  210. CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
  211. hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
  212. HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
  213. ;;
  214. -*)
  215. hadoop_exit_with_usage 1
  216. ;;
  217. *)
  218. CLASS="${COMMAND}"
  219. ;;
  220. esac
  221. if [[ -n "${secure_service}" ]]; then
  222. HADOOP_SECURE_USER="${secure_user}"
  223. hadoop_verify_secure_prereq
  224. hadoop_setup_secure_service
  225. priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out"
  226. priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err"
  227. priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid"
  228. daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
  229. daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
  230. else
  231. daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
  232. daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
  233. fi
  234. if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
  235. # shellcheck disable=SC2034
  236. HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
  237. if [[ -n "${secure_service}" ]]; then
  238. # shellcheck disable=SC2034
  239. HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
  240. else
  241. # shellcheck disable=SC2034
  242. HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
  243. fi
  244. fi
  245. hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
  246. hadoop_finalize
  247. if [[ -n "${daemon}" ]]; then
  248. if [[ -n "${secure_service}" ]]; then
  249. hadoop_secure_daemon_handler \
  250. "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
  251. "${daemon_pidfile}" "${daemon_outfile}" \
  252. "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
  253. else
  254. hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
  255. "${daemon_pidfile}" "${daemon_outfile}" "$@"
  256. fi
  257. exit $?
  258. else
  259. # shellcheck disable=SC2086
  260. hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
  261. fi