hadoop/hadoop-common-project/hadoop-common/src/main/packages/hadoop-validate-setup.sh

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.



###############################################################################
# Run the following jobs to validate a hadoop cluster
## teragen
## terasort
## teravalidate
# If they all pass 0 will be returned and 1 otherwise
# The test will work for both secure and unsecure deploys. If the kerberos-realm
# is passed we will assume that the deploy is secure and proceed with a kinit before
# running the validation jobs.
################################################################################

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/../libexec/hadoop-config.sh

usage() {
  echo "
usage: $0 <parameters>

  Optional parameters:
     -h                                                              Display this message
     --user=hdfs
     --user_keytab=/home/hdfs/hdfs.keytab
     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
  "
  exit 1
}

OPTS=$(getopt \
  -n $0 \
  -o '' \
  -l 'user:' \
  -l 'user-keytab:' \
  -l 'kerberos-realm:' \
  -o 'h' \
  -- "$@")

if [ $? != 0 ] ; then
    usage
fi

eval set -- "${OPTS}"
while true ; do
  case "$1" in
    --user)
      TEST_USER=$2; shift 2
      AUTOMATED=1
      ;;
    --user-keytab)
      USER_KEYTAB_FILE=$2; shift 2
      AUTOMATED=1
      ;;
    --kerberos-realm)
      KERBEROS_REALM=$2; shift 2
      AUTOMATED=1
      ;;
    --)
      shift ; break
      ;;
    *)
      echo "Unknown option: $1"
      usage
      exit 1
      ;;
  esac
done

#set the hadoop command and the path to the hadoop examples jar
HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"

#find the hadoop examples jar
HADOOP_EXAMPLES_JAR=''

#find under HADOOP_PREFIX (tar ball install)
HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-examples-*.jar' | head -n1`

#if its not found look under /usr/share/hadoop (rpm/deb installs)
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
then
  HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-examples-*.jar' | head -n1`
fi

#if it is still empty then dont run the tests
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
then
  echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
  exit 1
fi

# do a kinit if secure
if [ "${KERBEROS_REALM}" != "" ]; then
  # Determine kerberos location base on Linux distro.
  if [ -e /etc/lsb-release ]; then
    KERBEROS_BIN=/usr/bin
  else
    KERBEROS_BIN=/usr/kerberos/bin
  fi
  kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
  echo $kinit_cmd
  eval $kinit_cmd
  if [ $? -ne 0 ]
  then
    echo "kinit command did not run successfully."
    exit 1
  fi
fi


#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
PARENT_DIR="validate_deploy_`date +%s`"
TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
#tera gen cmd
TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"

#tera sort cmd
TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"

#tera validate cmd
TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"

echo "Starting teragen...."

#run tera gen
echo $TERA_GEN_CMD
eval $TERA_GEN_CMD
if [ $? -ne 0 ]; then
  echo "tera gen failed."
  exit 1
fi

echo "Teragen passed starting terasort...."


#run tera sort
echo $TERA_SORT_CMD
eval $TERA_SORT_CMD
if [ $? -ne 0 ]; then
  echo "tera sort failed."
  exit 1
fi

echo "Terasort passed starting teravalidate...."

#run tera validate
echo $TERA_VALIDATE_CMD
eval $TERA_VALIDATE_CMD
if [ $? -ne 0 ]; then
  echo "tera validate failed."
  exit 1
fi

echo "teragen, terasort, teravalidate passed."
echo "Cleaning the data created by tests: $PARENT_DIR"

CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
echo $CLEANUP_CMD
eval $CLEANUP_CMD

exit 0