HADOOP-7655. Provide a small validation script that smoke tests the installed cluster. Contributed by Arpit Gupta.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1183539 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -154,6 +154,9 @@ Release 0.23.0 - Unreleased
 
   IMPROVEMENTS
 
+    HADOOP-7655. Provide a small validation script that smoke tests the installed
+    cluster. (Arpit Gupta via mattf)
+
     HADOOP-7042. Updates to test-patch.sh to include failed test names and
     improve other messaging. (nigel)
 

hadoop-common-project/hadoop-common/src/main/packages/hadoop-validate-setup.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+###############################################################################
+# Run the following jobs to validate a hadoop cluster
+## teragen
+## terasort
+## teravalidate
+# If they all pass 0 will be returned and 1 otherwise
+# The test will work for both secure and unsecure deploys. If the kerberos-realm
+# is passed we will assume that the deploy is secure and proceed with a kinit before
+# running the validation jobs.
+################################################################################
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/../libexec/hadoop-config.sh
+
+usage() {
+  echo "
+usage: $0 <parameters>
+
+  Optional parameters:
+     -h                                                              Display this message
+     --user=hdfs
+     --user_keytab=/home/hdfs/hdfs.keytab
+     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'user:' \
+  -l 'user-keytab:' \
+  -l 'kerberos-realm:' \
+  -o 'h' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "${OPTS}"
+while true ; do
+  case "$1" in
+    --user)
+      TEST_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --user-keytab)
+      USER_KEYTAB_FILE=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --kerberos-realm)
+      KERBEROS_REALM=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --)
+      shift ; break
+      ;;
+    *)
+      echo "Unknown option: $1"
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+#set the hadoop command and the path to the hadoop examples jar
+HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
+
+#find the hadoop examples jar
+HADOOP_EXAMPLES_JAR=''
+
+#find under HADOOP_PREFIX (tar ball install)
+HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-examples-*.jar' | head -n1`
+
+#if its not found look under /usr/share/hadoop (rpm/deb installs)
+if [ "$HADOOP_EXAMPLES_JAR" == '' ]
+then
+  HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-examples-*.jar' | head -n1`
+fi
+
+#if it is still empty then dont run the tests
+if [ "$HADOOP_EXAMPLES_JAR" == '' ]
+then
+  echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
+  exit 1
+fi
+
+# do a kinit if secure
+if [ "${KERBEROS_REALM}" != "" ]; then
+  # Determine kerberos location base on Linux distro.
+  if [ -e /etc/lsb-release ]; then
+    KERBEROS_BIN=/usr/bin
+  else
+    KERBEROS_BIN=/usr/kerberos/bin
+  fi
+  kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
+  echo $kinit_cmd
+  eval $kinit_cmd
+  if [ $? -ne 0 ]
+  then
+    echo "kinit command did not run successfully."
+    exit 1
+  fi
+fi
+
+
+#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
+PARENT_DIR="validate_deploy_`date +%s`"
+TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
+TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
+TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
+#tera gen cmd
+TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
+
+#tera sort cmd
+TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
+
+#tera validate cmd
+TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
+
+echo "Starting teragen...."
+
+#run tera gen
+echo $TERA_GEN_CMD
+eval $TERA_GEN_CMD
+if [ $? -ne 0 ]; then
+  echo "tera gen failed."
+  exit 1
+fi
+
+echo "Teragen passed starting terasort...."
+
+
+#run tera sort
+echo $TERA_SORT_CMD
+eval $TERA_SORT_CMD
+if [ $? -ne 0 ]; then
+  echo "tera sort failed."
+  exit 1
+fi
+
+echo "Terasort passed starting teravalidate...."
+
+#run tera validate
+echo $TERA_VALIDATE_CMD
+eval $TERA_VALIDATE_CMD
+if [ $? -ne 0 ]; then
+  echo "tera validate failed."
+  exit 1
+fi
+
+echo "teragen, terasort, teravalidate passed."
+echo "Cleaning the data created by tests: $PARENT_DIR"
+
+CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
+echo $CLEANUP_CMD
+eval $CLEANUP_CMD
+
+exit 0