HADOOP-2852. Update gridmix benchmark to avoid an artifically long tail.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@630058 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -137,6 +137,9 @@ Release 0.16.1 - Unreleased
     HADOOP-2769.  TestNNThroughputBnechmark should not use a fixed port for
     the namenode http port. (omalley)
 
+    HADOOP-2852. Update gridmix benchmark to avoid an artifically long tail.
+    (cdouglas)
+
 Release 0.16.0 - 2008-02-07
 
   INCOMPATIBLE CHANGES

src/test/gridmix/gridmix-env

@@ -30,6 +30,7 @@ export VARINFLTEXT=${GRID_MIX_DATA}/SortUncompressed
 export FIXCOMPTEXT=${GRID_MIX_DATA}/EntropySimulationCompressed
 
 ## Job sizing
+export NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS=5
 export NUM_OF_LARGE_JOBS_PER_CLASS=3
 export NUM_OF_MEDIUM_JOBS_PER_CLASS=20
 export NUM_OF_SMALL_JOBS_PER_CLASS=40
@@ -44,7 +45,7 @@ export INTERVAL_BETWEEN_SUBMITION=20
 ## Hod
 #export HOD_OPTIONS="--ringmaster.hadoop-tar-ball=/path/to/hadoop-0.15.0-dev.tar.gz"
 #export HOD_CONFIG=
-#export ALL_HOD_OPTIONS="$HOD_OPTIONS -c ${HOD_CONFIG}"
+#export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}"
 #export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 5"
 #export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 50"
 #export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -m 100"

src/test/gridmix/maxent/maxent.large

@@ -13,7 +13,7 @@ ${HADOOP_HOME}/bin/hadoop dfs -rmr $OUTDIR
 
 ${HADOOP_HOME}/bin/hadoop jar $APP_JAR loadgen -keepmap 50 -keepred 100 -inFormatIndirect org.apache.hadoop.mapred.TextInputFormat -outFormat org.apache.hadoop.mapred.TextOutputFormat -outKey org.apache.hadoop.io.LongWritable -outValue org.apache.hadoop.io.Text -indir $INDIR -outdir $OUTDIR.1 -r $NUM_OF_REDUCERS
 
-ITER=11
+ITER=7
 for ((i=1; i<$ITER; ++i))
 do
   ${HADOOP_HOME}/bin/hadoop jar $APP_JAR loadgen -keepmap 50 -keepred 100 -inFormatIndirect org.apache.hadoop.mapred.TextInputFormat -outFormat org.apache.hadoop.mapred.TextOutputFormat -outKey org.apache.hadoop.io.LongWritable -outValue org.apache.hadoop.io.Text -indir $INDIR -indir $OUTDIR.$i -outdir $OUTDIR.$(($i+1)) -r $NUM_OF_REDUCERS

src/test/gridmix/submissionScripts/allThroughHod

@@ -4,10 +4,10 @@ GRID_DIR=`dirname "$0"`
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
 
+$GRID_MIX_HOME/submissionScripts/maxentHod 2>&1 >  maxentHod.out & 
 $GRID_MIX_HOME/submissionScripts/textSortHod 2>&1 >  textSortHod.out &
 $GRID_MIX_HOME/submissionScripts/monsterQueriesHod 2>&1 > monsterQueriesHod.out &
 $GRID_MIX_HOME/submissionScripts/webdataScanHod 2>&1 > webdataScanHod.out &
 $GRID_MIX_HOME/submissionScripts/webdataSortHod 2>&1 > webdataSortHod.out &
-$GRID_MIX_HOME/submissionScripts/maxentHod 2>&1 >  maxentHod.out & 
 
 

src/test/gridmix/submissionScripts/allToSameCluster

@@ -4,6 +4,8 @@ GRID_DIR=`dirname "$0"`
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
 
+$GRID_MIX_HOME/submissionScripts/maxentToSameCluster 2>&1 > maxentToSameCluster.out &
+sleep 20
 $GRID_MIX_HOME/submissionScripts/textSortToSameCluster 2>&1 > textSortToSameCluster.out  &
 sleep 20
 $GRID_MIX_HOME/submissionScripts/monsterQueriesToSameCluster 2>&1 > monsterQueriesToSameCluster.out &
@@ -11,6 +13,4 @@ sleep 20
 $GRID_MIX_HOME/submissionScripts/webdataScanToSameCluster 2>&1 > webdataScanToSameCluster.out &
 sleep 20
 $GRID_MIX_HOME/submissionScripts/webdataSortToSameCluster  2>&1 > webdataSortToSameCluster.out &
-sleep 20
-$GRID_MIX_HOME/submissionScripts/maxentToSameCluster 2>&1 > maxentToSameCluster.out &
 

src/test/gridmix/submissionScripts/maxentHod

@@ -3,8 +3,7 @@
 GRID_DIR=`dirname "$0"`
 GRID_DIR=`cd "$GRID_DIR"; pwd`
 source $GRID_DIR/../gridmix-env
-
-for ((i=0; i < $NUM_OF_LARGE_JOBS_PER_CLASS; i++))
+for ((i=0; i < $NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS; i++))
 do
     echo $i
     hod $LARGE_JOB_HOD_OPTIONS --hod.script=$GRID_MIX_HOME/maxent/maxent.large  2>&1 > maxent.large.$i.out &

src/test/gridmix/submissionScripts/sleep_if_too_busy

@@ -1,11 +1,10 @@
 #!/bin/bash
 
 sleep 1
-for ((java_process=$((`ps -ef|grep java|wc|awk '{print $1}'`-1)); \
-      java_process > 60; \
-      java_process=$((`ps -ef|grep java|wc|awk '{print $1}'`-1))))
+for ((java_process=$((`ps -ef|grep java|wc -l`-1)); \
+      java_process > 70; \
+      java_process=$((`ps -ef|grep java|wc -l`-1))))
 do
     sleep 10
     echo $java_process
 done
-