HADOOP-2411. Add support for larger CPU EC2 instance types. Contributed by Chris K Wensel.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696626 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -344,6 +344,9 @@ Trunk (unreleased changes)
     HADOOP-4117. Improve configurability of Hadoop EC2 instances.
     (tomwhite)
 
+    HADOOP-2411. Add support for larger CPU EC2 instance types.
+    (Chris K Wensel via tomwhite)
+
   OPTIMIZATIONS
 
     HADOOP-3556. Removed lock contention in MD5Hash by changing the 

src/contrib/ec2/bin/destroy-hadoop-cluster

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Launch an EC2 cluster of Hadoop instances.
-
-if [ -z $1 ]; then
-  echo "Cluster name required!"
-  exit -1
-fi
-
-CLUSTER=$1
-
-# Import variables
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-ec2-env.sh
-
-rm -f $MASTER_IP_PATH
-rm -f $MASTER_PRIVATE_IP_PATH
-
-ec2-describe-group | egrep "[[:space:]]$CLUSTER_MASTER[[:space:]]" > /dev/null
-if [ $? -eq 0 ]; then
-  echo "Destroying group $CLUSTER_MASTER"
-  ec2-revoke $CLUSTER_MASTER -o $CLUSTER -u $AWS_ACCOUNT_ID
-fi
-
-ec2-describe-group | egrep "[[:space:]]$CLUSTER[[:space:]]" > /dev/null
-if [ $? -eq 0 ]; then
-  echo "Destroying group $CLUSTER"
-  ec2-revoke $CLUSTER -o $CLUSTER_MASTER -u $AWS_ACCOUNT_ID
-fi
-
-ec2-delete-group $CLUSTER_MASTER
-ec2-delete-group $CLUSTER

src/contrib/ec2/bin/hadoop-ec2

@@ -26,7 +26,7 @@ if [ $# = 0 ]; then
   echo "  launch-cluster <group> <num slaves>  launch a cluster of Hadoop EC2 instances - launch-master then launch-slaves"
   echo "  launch-master  <group>               launch or find a cluster master"
   echo "  launch-slaves  <group> <num slaves>  launch the cluster slaves"
-  echo "  terminate-cluster                    terminate all Hadoop EC2 instances"
+  echo "  terminate-cluster  <group>           terminate all Hadoop EC2 instances"
   echo "  login  <group|instance id>           login to the master node of the Hadoop EC2 cluster"
   echo "  screen <group|instance id>           start or attach 'screen' on the master node of the Hadoop EC2 cluster"
   echo "  proxy  <group|instance id>           start a socks proxy on localhost:6666 (use w/foxyproxy)"

src/contrib/ec2/bin/hadoop-ec2-env.sh.template

@@ -40,13 +40,13 @@ PRIVATE_KEY_PATH=`echo "$EC2_KEYDIR"/"id_rsa-$KEY_NAME"`
 SSH_OPTS=`echo -i "$PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
 
 # The version of Hadoop to use.
-HADOOP_VERSION=0.17.0
+HADOOP_VERSION=0.19.0
 
 # The Amazon S3 bucket where the Hadoop AMI is stored.
 # The default value is for public images, so can be left if you are using running a public image.
 # Change this value only if you are creating your own (private) AMI
 # so you can store it in a bucket you own.
-S3_BUCKET=hadoop-ec2-images
+S3_BUCKET=hadoop-images
 
 # Enable public access to JobTracker and TaskTracker web interfaces
 ENABLE_WEB_PORTS=true
@@ -58,6 +58,8 @@ USER_DATA_FILE=hadoop-ec2-init-remote.sh
 INSTANCE_TYPE="m1.small"
 #INSTANCE_TYPE="m1.large"
 #INSTANCE_TYPE="m1.xlarge"
+#INSTANCE_TYPE="c1.medium"
+#INSTANCE_TYPE="c1.xlarge"
 
 # The EC2 group master name. CLUSTER is set by calling scripts
 CLUSTER_MASTER=$CLUSTER-master
@@ -72,16 +74,28 @@ MASTER_ZONE_PATH=~/.hadooop-zone-$CLUSTER_MASTER
 #
 
 # The version number of the installed JDK.
-JAVA_VERSION=1.6.0_05
+JAVA_VERSION=1.6.0_07
 
 # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
 # The download URL for the Sun JDK. Visit http://java.sun.com/javase/downloads/index.jsp and get the URL for the "Linux self-extracting file".
-if [ "$INSTANCE_TYPE" == "m1.small" ]; then
+if [ "$INSTANCE_TYPE" == "m1.small" -o "$INSTANCE_TYPE" == "c1.medium" ]; then
   ARCH='i386'
-  BASE_AMI_IMAGE="ami-f51aff9c"  # ec2-public-images/fedora-8-i386-base-v1.06.manifest.xml
-  JAVA_BINARY_URL=
+  BASE_AMI_IMAGE="ami-2b5fba42"  # ec2-public-images/fedora-8-i386-base-v1.07.manifest.xml
+  JAVA_BINARY_URL=''
 else
   ARCH='x86_64'
-  BASE_AMI_IMAGE="ami-f21aff9b"  # ec2-public-images/fedora-8-x86_64-base-v1.06.manifest.xml
-  JAVA_BINARY_URL=
+  BASE_AMI_IMAGE="ami-2a5fba43"  # ec2-public-images/fedora-8-x86_64-base-v1.07.manifest.xml
+  JAVA_BINARY_URL=''
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.medium" ]; then
+  AMI_KERNEL=aki-9b00e5f2 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.i386.aki.manifest.xml
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.xlarge" ]; then
+  AMI_KERNEL=aki-9800e5f1 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.x86_64.aki.manifest.xml
+fi
+
+if [ "$AMI_KERNEL" != "" ]; then
+  KERNEL_ARG="--kernel ${AMI_KERNEL}"
 fi

src/contrib/ec2/bin/launch-hadoop-cluster

@@ -28,6 +28,7 @@ fi
 
 if [ -z $2 ]; then
   echo "Must specify the number of slaves to start."
+  exit -1
 fi
 
 if ! "$bin"/launch-hadoop-master $1 ; then

src/contrib/ec2/bin/launch-hadoop-master

@@ -81,7 +81,7 @@ AMI_IMAGE=`ec2-describe-images -a | grep $S3_BUCKET | grep $HADOOP_VERSION | gre
 # Start a master
 echo "Starting master with AMI $AMI_IMAGE"
 USER_DATA="MASTER_HOST=master,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE -t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE -t $INSTANCE_TYPE $KERNEL_ARG | grep INSTANCE | awk '{print $2}'`
 echo "Waiting for instance $INSTANCE to start"
 while true; do
   printf "."

src/contrib/ec2/bin/launch-hadoop-slaves

@@ -24,6 +24,7 @@ fi
 
 if [ -z $2 ]; then
   echo "Must specify the number of slaves to start."
+  exit -1
 fi
 
 CLUSTER=$1
@@ -49,6 +50,6 @@ sed -e "s|%MASTER_HOST%|$MASTER_HOST|" "$bin"/$USER_DATA_FILE > "$bin"/$USER_DAT
 
 # Start slaves
 echo "Adding $1 node(s) to cluster group $CLUSTER with AMI $AMI_IMAGE"
-ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" $KERNEL_ARG | grep INSTANCE | awk '{print $2}'
 
 rm "$bin"/$USER_DATA_FILE.slave

src/contrib/ec2/bin/list-hadoop-clusters

@@ -23,7 +23,7 @@ bin=`cd "$bin"; pwd`
 . "$bin"/hadoop-ec2-env.sh
 
 # Finding Hadoop clusters
-CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 {print $4}' | grep -v -e "-master$" | sort | uniq`
+CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 && $4 ~ /-master$/, "INSTANCE" == $1' | tr '\n' '\t' | grep running | cut -f4 | rev | cut -d'-' -f2- | rev`
 
 [ -z "$CLUSTERS" ] && echo "No running clusters." && exit 0
 

src/contrib/ec2/bin/login-hadoop-cluster

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Login to the master node of a running Hadoop EC2 cluster.
-
-# Import variables
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-. "$bin"/hadoop-ec2-env.sh
-
-echo "Logging in to master $MASTER_HOST."
-ssh $SSH_OPTS "root@$MASTER_HOST"