MAPREDUCE-2736. svn merge -c 1189982 from trunk

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1190008 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -386,6 +386,8 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2986. Fixed MiniYARNCluster to support multiple NodeManagers.
     (Anupam Seth via vinodkv)
 
+    MAPREDUCE-2736. Remove unused contrib components dependent on MR1. (eli)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

hadoop-mapreduce-project/bin/mapred

@@ -1,119 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`which $0`
-bin=`dirname ${bin}`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-function print_usage(){
-  echo "Usage: mapred [--config confdir] COMMAND"
-  echo "       where COMMAND is one of:"
-  echo "  mradmin              run a Map-Reduce admin client"
-  echo "  jobtracker           run the MapReduce job Tracker node" 
-  echo "  tasktracker          run a MapReduce task Tracker node" 
-  echo "  pipes                run a Pipes job"
-  echo "  job                  manipulate MapReduce jobs"
-  echo "  queue                get information regarding JobQueues"
-  echo "  classpath            prints the class path needed for running"
-  echo "                       mapreduce subcommands"
-  echo "  groups               get the groups which users belong to"
-  echo ""
-  echo "Most commands print help when invoked w/o parameters."
-}
-
-if [ $# = 0 ]; then
-  print_usage
-  exit
-fi
-
-COMMAND=$1
-shift
-
-if [ "$COMMAND" = "mradmin" ] ; then
-  CLASS=org.apache.hadoop.mapred.tools.MRAdmin
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "jobtracker" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobTracker
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
-elif [ "$COMMAND" = "tasktracker" ] ; then
-  CLASS=org.apache.hadoop.mapred.TaskTracker
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
-elif [ "$COMMAND" = "job" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobClient
-elif [ "$COMMAND" = "queue" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobQueueClient
-elif [ "$COMMAND" = "pipes" ] ; then
-  CLASS=org.apache.hadoop.mapred.pipes.Submitter
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "sampler" ] ; then
-  CLASS=org.apache.hadoop.mapred.lib.InputSampler
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "classpath" ] ; then
-  echo -n 
-elif [ "$COMMAND" = "groups" ] ; then
-  CLASS=org.apache.hadoop.mapred.tools.GetGroups
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-else
-  echo $COMMAND - invalid command
-  print_usage
-  exit
-fi
-
-# for developers, add mapred classes to CLASSPATH
-if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
-fi
-
-# for releases, add core mapred jar & webapps to CLASSPATH
-if [ -d "$HADOOP_PREFIX/share/hadoop/mapreduce/webapps" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/mapreduce
-fi
-for f in $HADOOP_MAPRED_HOME/share/hadoop-mapreduce/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
-done
-
-# add libs to CLASSPATH
-for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
-done
-
-if $cygwin; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
-if [ "$COMMAND" = "classpath" ] ; then
-  echo $CLASSPATH
-  exit
-fi
-
-export CLASSPATH
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"

hadoop-mapreduce-project/bin/mapred-config.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# included in all the mapred scripts with source command
-# should not be executed directly
-
-bin=`which "$0"`
-bin=`dirname "${bin}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
-  . "$bin/../libexec/hadoop-config.sh"
-elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
-  . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
-elif [ -e "${HADOOP_COMMON_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_COMMON_HOME"/bin/hadoop-config.sh
-elif [ -e "${HADOOP_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_HOME"/bin/hadoop-config.sh
-elif [ -e "${HADOOP_MAPRED_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_MAPRED_HOME"/bin/hadoop-config.sh
-else
-  echo "Hadoop common not found."
-  exit
-fi

hadoop-mapreduce-project/bin/start-mapred.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop map reduce daemons.  Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-
-# start mapred daemons
-# start jobtracker first to minimize connection errors at startup
-"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker
-"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker

hadoop-mapreduce-project/bin/stop-mapred.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop map reduce daemons.  Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker
-"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker

hadoop-mapreduce-project/build.xml

@@ -39,7 +39,6 @@
   <property name="examples.final.name" value="${name}-examples-${version}"/>
   <property name="tools.final.name" value="${name}-tools-${version}"/>
   <property name="year" value="2009"/>
-  <property name="package.release" value="1"/>
 
   <property name="src.dir" value="${basedir}/src"/>  	
   <property name="mapred.src.dir" value="${src.dir}/java"/> 
@@ -240,17 +239,6 @@
     <equals arg1="${repo}" arg2="staging"/>
   </condition>
 
-  <!-- packaging properties -->
-  <property name="package.prefix" value="/usr"/>
-  <property name="package.conf.dir" value="/etc/hadoop"/>
-  <property name="package.log.dir" value="/var/log/hadoop/mapred"/>
-  <property name="package.pid.dir" value="/var/run/hadoop"/>
-  <property name="package.var.dir" value="/var/lib/hadoop"/>
-  <property name="package.share.dir" value="/share/hadoop/${module}"/>
-  <!-- Use fixed path to build rpm for avoiding rpmbuild conflict with dash path names -->
-  <property name="package.buildroot" value="/tmp/hadoop_mapred_package_build_${user.name}"/>
-  <property name="package.build.dir" value="/tmp/hadoop_mapred_package_build_${user.name}/BUILD"/>
-
   <!-- the normal classpath -->
   <path id="classpath">
     <pathelement location="${build.classes}"/>
@@ -846,8 +834,7 @@
     </antcall>
   </target>
 
-  <target name="nightly" depends="test, tar">
-  </target>
+  <target name="nightly" depends="test" />
 	
   <!-- ================================================================== -->
   <!-- Run optional third-party tool targets                              --> 
@@ -880,10 +867,8 @@
    <subant target="jar">
       <property name="version" value="${version}"/>
       <property name="dist.dir" value="${dist.dir}"/>
-      <fileset file="${contrib.dir}/capacity-scheduler/build.xml"/>
       <fileset file="${contrib.dir}/streaming/build.xml"/>
       <fileset file="${contrib.dir}/gridmix/build.xml"/>
-      <fileset file="${contrib.dir}/mumak/build.xml"/>
     </subant>
  </target>
 
@@ -912,17 +897,13 @@
       <sourcePath path="${mapred.src.dir}"/>
       <sourcePath path="${examples.dir}" />
       <sourcePath path="${tools.src}" />
-      <sourcePath path="${basedir}/src/contrib/capacity-scheduler/src/java" />
       <sourcePath path="${basedir}/src/contrib/streaming/src/java" />
       <sourcePath path="${basedir}/src/contrib/gridmix/src/java" />
-      <sourcePath path="${basedir}/src/contrib/mumak/src/java" />
       <class location="${basedir}/build/${final.name}.jar" />
       <class location="${basedir}/build/${examples.final.name}.jar" />
       <class location="${basedir}/build/${tools.final.name}.jar" />
-      <class location="${basedir}/build/contrib/capacity-scheduler/hadoop-${version}-capacity-scheduler.jar" />
       <class location="${basedir}/build/contrib/streaming/hadoop-${version}-streaming.jar" />
       <class location="${basedir}/build/contrib/gridmix/hadoop-${version}-gridmix.jar" />
-      <class location="${basedir}/build/contrib/mumak/hadoop-${version}-mumak.jar" />
     </findbugs>
 
         <xslt style="${findbugs.home}/src/xsl/default.xsl"
@@ -981,7 +962,6 @@
        <packageset dir="src/contrib/data_join/src/java"/>
        <packageset dir="src/contrib/gridmix/src/java"/>
        <packageset dir="src/contrib/index/src/java"/>
-       <packageset dir="src/contrib/mumak/src/java"/>
        <packageset dir="src/contrib/streaming/src/java"/>
        <packageset dir="src/contrib/vaidya/src/java"/>
        <packageset dir="src/contrib/vertica/src/java"/>
@@ -1042,8 +1022,7 @@
     	<packageset dir="${examples.dir}"/>
 
        <!-- Don't include contrib modules that use the same packages as core
-       MapReduce. This includes capacity-scheduler, dynamic-scheduler,
-       fairscheduler, mumak. See also the javadoc-dev target. -->
+       MapReduce. See also the javadoc-dev target. -->
        <packageset dir="src/contrib/data_join/src/java"/>
        <packageset dir="src/contrib/gridmix/src/java"/>
        <packageset dir="src/contrib/index/src/java"/>
@@ -1155,7 +1134,6 @@
     <mkdir dir="${dist.dir}"/>
     <mkdir dir="${dist.dir}/lib"/>
     <mkdir dir="${dist.dir}/contrib"/>
-    <mkdir dir="${dist.dir}/bin"/>
     <mkdir dir="${dist.dir}/docs"/>
     <mkdir dir="${dist.dir}/docs/api"/>
     <mkdir dir="${dist.dir}/docs/jdiff"/>
@@ -1196,10 +1174,6 @@
       <fileset file="${build.dir}/${tools.final.name}.jar"/> 
     </copy>
     
-    <copy todir="${dist.dir}/bin">
-      <fileset dir="bin"/>
-    </copy>
-
     <copy todir="${dist.dir}/conf">
       <fileset dir="${conf.dir}" excludes="**/*.template"/>
     </copy>
@@ -1237,7 +1211,6 @@
         <fileset file="${dist.dir}/src/c++/pipes/configure"/>
     </chmod>
     <chmod perm="ugo+x" type="file" parallel="false">
-        <fileset dir="${dist.dir}/bin"/>
         <fileset dir="${dist.dir}/src/contrib/">
           <include name="*/bin/*" />
         </fileset>
@@ -1248,290 +1221,6 @@
 
   </target>
 
-  <!-- ================================================================== -->
-  <!-- Make release tarball                                               -->
-  <!-- ================================================================== -->
-  <target name="tar" depends="package" description="Make release tarball">
-    <macro_tar param.destfile="${build.dir}/${final.name}.tar.gz">
-      <param.listofitems>
-        <tarfileset dir="${build.dir}" mode="664">
-          <exclude name="${final.name}/bin/*" />
-          <exclude name="${final.name}/contrib/*/bin/*" />
-          <exclude name="${final.name}/src/examples/pipes/configure"/>
-          <exclude name="${final.name}/src/c++/utils/configure"/>
-          <exclude name="${final.name}/src/c++/pipes/configure"/>
-          <include name="${final.name}/**" />
-        </tarfileset>
-        <tarfileset dir="${build.dir}" mode="755">
-          <include name="${final.name}/bin/*" />
-          <include name="${final.name}/contrib/*/bin/*" />
-          <include name="${final.name}/src/examples/pipes/configure"/>
-          <include name="${final.name}/src/c++/utils/configure"/>
-          <include name="${final.name}/src/c++/pipes/configure"/>
-        </tarfileset>
-      </param.listofitems>
-    </macro_tar>
-  </target>
-
-  <target name="bin-package" depends="compile, jar, examples, tools, jar-test, package-librecordio" 
-		description="assembles artifacts for binary target">
-    <mkdir dir="${dist.dir}"/>
-    <mkdir dir="${dist.dir}/include"/>
-    <mkdir dir="${dist.dir}/lib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/contrib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/lib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/templates"/>
-    <mkdir dir="${dist.dir}/bin"/>
-    <mkdir dir="${dist.dir}/sbin"/>
-
-    <!-- enable this if there is mapred specific dependencies
-    <copy todir="${dist.dir}/${package.share.dir}/lib" includeEmptyDirs="false" flatten="true">
-      <fileset dir="${mapred.ivy.lib.dir}"/>
-    </copy> -->
-
-    <copy todir="${dist.dir}/include" includeEmptyDirs="false">
-      <fileset dir="${build.dir}/c++/${build.platform}/include" 
-               erroronmissingdir="false">
-        <include name="**"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
-      <fileset dir="${build.dir}/c++/${build.platform}/lib"
-               erroronmissingdir="false">
-        <include name="**"/>
-      </fileset>
-    </copy>
-
-    <subant target="package">
-      <!--Pass down the version in case its needed again and the target
-      distribution directory so contribs know where to install to.-->
-      <property name="version" value="${version}"/>
-      <property name="dist.dir" value="${dist.dir}/${package.share.dir}"/>
-      <fileset file="${contrib.dir}/build.xml"/>
-    </subant>  	
-
-    <copy todir="${dist.dir}/${package.share.dir}"> 
-      <fileset file="${build.dir}/${final.name}*.jar"/>
-      <fileset file="${build.dir}/${test.final.name}.jar"/>
-      <fileset file="${build.dir}/${examples.final.name}.jar"/>
-      <fileset file="${build.dir}/${tools.final.name}.jar"/>
-    </copy>
-
-    <copy todir="${dist.dir}/bin">
-      <fileset dir="bin">
-        <include name="mapred"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/libexec">
-      <fileset dir="bin">
-        <include name="mapred-config.sh"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/sbin">
-      <fileset dir="bin">
-        <include name="start-*.sh"/>
-        <include name="stop-*.sh"/>
-      </fileset>
-    </copy>
-
-    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
-    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.redhat"/>
-    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.redhat"/>
-    <copy file="${basedir}/src/packages/deb/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.debian"/>
-    <copy file="${basedir}/src/packages/deb/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.debian"/>
-    
-    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
-      	
-    <copy todir="${dist.dir}/etc/hadoop">
-      <fileset dir="${conf.dir}" excludes="**/*.template"/>
-    </copy>
-
-    <copy todir="${dist.dir}/${package.share.dir}/templates">
-      <fileset dir="${basedir}/src/packages/templates/conf" includes="*"/>
-    </copy>
-
-    <copy todir="${dist.dir}/${package.share.dir}/webapps">
-      <fileset dir="${build.webapps}"/>
-    </copy>
-
-    <copy todir="${dist.dir}/share/doc/hadoop/${module}">
-      <fileset dir=".">
-        <include name="*.txt" />
-      </fileset>
-    </copy>
-
-    <chmod perm="ugo+x" type="file" parallel="false">
-        <fileset dir="${dist.dir}/bin"/>
-        <fileset dir="${dist.dir}/sbin"/>
-    </chmod>  	
-  </target>
-
-  <target name="binary-system" depends="bin-package, jar-system, jar-test-system"
-     description="make system test package for deployment">
-    <!--TODO!!!!! fix this shit...-->
-    <copy todir="${system-test-build-dir}/${final.name}">
-      <fileset dir="${dist.dir}">
-      </fileset>
-    </copy>
-    <copy todir="${system-test-build-dir}/${final.name}/conf">
-      <fileset dir="${test.src.dir}/system/conf/"/>
-    </copy>
-    <copy todir="${system-test-build-dir}">
-      <fileset dir="${build.dir}">
-        <include name="${test.final.name}.jar"/>
-        <include name="${examples.final.name}.jar"/>
-      </fileset>
-    </copy>
-    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-common-${hadoop-common.version}.jar"
-      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-common-${herriot.suffix}-${hadoop-common.version}.jar"
-      overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-hdfs-${version}.jar"
-      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-hdfs-${herriot.suffix}-${version}.jar"
-      overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}.jar"
-      file="${system-test-build-dir}/${instrumented.final.name}.jar" overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}-sources.jar"
-      file="${system-test-build-dir}/${instrumented.final.name}-sources.jar" overwrite="true"/>
-    <macro_tar 
-      param.destfile="${system-test-build-dir}/${final.name}-bin.${herriot.suffix}.tar.gz">
-        <param.listofitems>
-          <tarfileset dir="${system-test-build-dir}" mode="664">
-            <exclude name="${final.name}/bin/*" />
-            <exclude name="${final.name}/src/**" />
-            <exclude name="${final.name}/docs/**" />
-            <include name="${final.name}/**" />
-          </tarfileset>
-        </param.listofitems>
-      </macro_tar>
-  </target>
-  
-  <target name="binary" depends="bin-package" description="Make tarball without source and documentation">
-    <macro_tar param.destfile="${build.dir}/${final.name}-bin.tar.gz">
-      <param.listofitems>
-        <tarfileset dir="${build.dir}" mode="664">
-          <exclude name="${final.name}/bin/*" />
-          <exclude name="${final.name}/src/**" />
-          <exclude name="${final.name}/docs/**" />
-          <include name="${final.name}/**" />
-        </tarfileset>
-        <tarfileset dir="${build.dir}" mode="755">
-          <include name="${final.name}/bin/*" />
-        </tarfileset>
-      </param.listofitems>
-    </macro_tar>
-  </target>
-
-  <target name="rpm" depends="binary" description="Make rpm package">
-    <mkdir dir="${package.buildroot}/BUILD" />
-    <mkdir dir="${package.buildroot}/RPMS" />
-    <mkdir dir="${package.buildroot}/SRPMS" />
-    <mkdir dir="${package.buildroot}/SOURCES" />
-    <mkdir dir="${package.buildroot}/SPECS" />
-    <copy todir="${package.buildroot}/SOURCES">
-      <fileset dir="${build.dir}">
-        <include name="${final.name}-bin.tar.gz" />
-      </fileset>
-    </copy>
-    <copy file="${src.dir}/packages/rpm/spec/hadoop-mapred.spec" todir="${package.buildroot}/SPECS">
-      <filterchain>
-        <replacetokens>
-          <token key="final.name" value="${final.name}" />
-          <token key="version" value="${_version}" />
-          <token key="package.release" value="${package.release}" />
-          <token key="package.build.dir" value="${package.build.dir}" />
-          <token key="package.prefix" value="${package.prefix}" />
-          <token key="package.conf.dir" value="${package.conf.dir}" />
-          <token key="package.log.dir" value="${package.log.dir}" />
-          <token key="package.pid.dir" value="${package.pid.dir}" />
-          <token key="package.var.dir" value="${package.var.dir}" />
-        </replacetokens>
-      </filterchain>
-    </copy>
-    <rpm specFile="hadoop-mapred.spec" command="-bb --target ${os.arch}" topDir="${package.buildroot}" cleanBuildDir="true" failOnError="true"/>
-    <copy todir="${build.dir}/" flatten="true">
-      <fileset dir="${package.buildroot}/RPMS">
-        <include name="**/*.rpm" />
-      </fileset>
-    </copy>
-    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
-  </target>
-
-  <target name="deb" depends="ivy-retrieve-package, binary" description="Make deb package">
-    <taskdef name="deb"
-           classname="org.vafer.jdeb.ant.DebAntTask">
-      <classpath refid="ivy-package.classpath" />
-    </taskdef>
-
-    <mkdir dir="${package.build.dir}/hadoop.control" />
-    <mkdir dir="${package.buildroot}/${package.prefix}" />
-    <copy todir="${package.buildroot}/${package.prefix}">
-      <fileset dir="${build.dir}/${final.name}">
-        <include name="**" />
-      </fileset>
-    </copy>
-    <copy todir="${package.build.dir}/hadoop.control">
-      <fileset dir="${src.dir}/packages/deb/hadoop.control">
-        <exclude name="control" />
-      </fileset>
-    </copy>
-    <copy file="${src.dir}/packages/deb/hadoop.control/control" todir="${package.build.dir}/hadoop.control">
-      <filterchain>
-        <replacetokens>
-          <token key="final.name" value="${final.name}" />
-          <token key="version" value="${_version}" />
-          <token key="package.release" value="${package.release}" />
-          <token key="package.build.dir" value="${package.build.dir}" />
-          <token key="package.prefix" value="${package.prefix}" />
-          <token key="package.conf.dir" value="${package.conf.dir}" />
-          <token key="package.log.dir" value="${package.log.dir}" />
-          <token key="package.pid.dir" value="${package.pid.dir}" />
-        </replacetokens>
-      </filterchain>
-    </copy>
-    <deb destfile="${package.buildroot}/${name}_${_version}-${package.release}_${os.arch}.deb" control="${package.build.dir}/hadoop.control">
-      <tarfileset dir="${build.dir}/${final.name}" filemode="644" prefix="${package.prefix}">
-        <exclude name="bin/*" />
-        <exclude name="${package.share.dir}/contrib/*/bin/*" />
-        <exclude name="etc" />
-        <exclude name="etc/**" />
-        <exclude name="libexec/*" />
-        <exclude name="sbin/*" />
-        <include name="**" />
-      </tarfileset>
-      <tarfileset dir="${build.dir}/${final.name}" filemode="755" prefix="${package.prefix}">
-        <include name="bin/*" />
-        <include name="sbin/*" />
-        <exclude name="sbin/*.redhat" />
-        <exclude name="sbin/*.debian" />
-        <include name="libexec/*" />
-        <include name="${package.share.dir}/contrib/*/bin/*" />
-      </tarfileset>
-      <tarfileset dir="${src.dir}/packages" filemode="755" prefix="${package.prefix}/sbin">
-        <include name="*.sh" />
-      </tarfileset>
-      <tarfileset dir="${build.dir}/${final.name}/etc/hadoop" filemode="644" prefix="${package.conf.dir}">
-        <exclude name="configuration.xsl" />
-        <exclude name="hadoop-metrics2.properties" />
-        <exclude name="core-site.xml" />
-        <exclude name="hdfs-site.xml" />
-        <exclude name="mapred-site.xml" />
-        <include name="**" />
-      </tarfileset>
-      <tarfileset dir="${basedir}/src/packages/deb/init.d" filemode="755" prefix="/etc/init.d">
-        <include name="**" />
-      </tarfileset>
-    </deb>
-    <copy todir="${build.dir}/" flatten="true">
-      <fileset dir="${package.buildroot}">
-        <include name="**/${name}*.deb" />
-      </fileset>
-    </copy>
-    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
-  </target>
-
   <target name="ant-task-download" description="To download mvn-ant-task">
     <get src="${ant_task_repo_url}" dest="${ant_task.jar}" usetimestamp="true"/>
   </target>
@@ -1742,7 +1431,6 @@
         <exclude name="src/test/mapred/org/apache/hadoop/mapred/test.tgz"/>
         <exclude name="src/test/tools/data/rumen/**/*"/>
         <exclude name="src/test/mapred/org/apache/hadoop/mapred/*.txt"/>
-        <exclude name="src/contrib/mumak/src/test/data/*.json"/>
         <exclude name="src/contrib/index/sample/*.txt"/>
         <exclude name="src/test/mapred/org/apache/hadoop/cli/data60bytes"/>
         <exclude name="src/examples/org/apache/hadoop/examples/dancing/puzzle1.dta"/>
@@ -2090,32 +1778,16 @@
                 output="${build.dir.eclipse-tools-classes}" />
         <source path="${contrib.dir}/block_forensics/src/java"
                 output="${build.dir.eclipse-contrib-classes}/block_forensics/main" />
-        <source path="${contrib.dir}/capacity-scheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/main" />
-        <source path="${contrib.dir}/capacity-scheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/test" />
         <source path="${contrib.dir}/data_join/src/java"
                 output="${build.dir.eclipse-contrib-classes}/data_join/main" />
         <source path="${contrib.dir}/data_join/src/examples"
                 output="${build.dir.eclipse-contrib-classes}/data_join/examples" />
         <source path="${contrib.dir}/data_join/src/test"
                 output="${build.dir.eclipse-contrib-classes}/data_join/test" />
-        <source path="${contrib.dir}/dynamic-scheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/main" />
-        <source path="${contrib.dir}/dynamic-scheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/test" />
-        <source path="${contrib.dir}/fairscheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/fairscheduler/main" />
-        <source path="${contrib.dir}/fairscheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/fairscheduler/test" />
         <source path="${contrib.dir}/gridmix/src/java"
                 output="${build.dir.eclipse-contrib-classes}/gridmix/main" />
         <source path="${contrib.dir}/gridmix/src/test"
                 output="${build.dir.eclipse-contrib-classes}/gridmix/test" />
-        <source path="${contrib.dir}/mumak/src/java"
-                output="${build.dir.eclipse-contrib-classes}/mumak/main" />
-        <source path="${contrib.dir}/mumak/src/test"
-                output="${build.dir.eclipse-contrib-classes}/mumak/test" />
         <source path="${contrib.dir}/raid/src/java"
                 output="${build.dir.eclipse-contrib-classes}/raid/main" />
         <source path="${contrib.dir}/raid/src/test"

hadoop-mapreduce-project/conf/capacity-scheduler.xml.template

@@ -1,82 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- This is one of the configuration files for capacity-scheduler
-     (org.apache.hadoop.mapred.CapacityTaskScheduler), a TaskScheduler
-     for Map/Reduce system. The other configuration file is
-     conf/mapred-queues.xml which it shares with the framework for
-     configuring queues in the system. -->
-
-<!-- This file can be used to configure (1) job-initialization-poller
-     related properties and (2) the default values for various properties
-     for all the queues.-->
-
-<configuration>   
-  <!-- The default configuration settings for the capacity task scheduler --> 
-  <!-- The default values would be applied to all the queues which don't have -->    
-  <!-- the appropriate property for the particular queue configured in the -->       
-  <!-- queue-configuration file conf/mapred-queues.xml -->                           
-
-  <property>
-    <name>mapred.capacity-scheduler.default-supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions by default in a job queue.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
-    <value>100</value>
-    <description>The percentage of the resources limited to a particular user
-      for the job queue at any given point of time by default.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user</name>
-    <value>2</value>
-    <description>The maximum number of jobs to be pre-initialized for a user
-    of the job queue.
-    </description>
-  </property>
-
-
-  <!-- Capacity scheduler Job Initialization configuration parameters -->
-  <property>
-    <name>mapred.capacity-scheduler.init-poll-interval</name>
-    <value>5000</value>
-    <description>The amount of time in miliseconds which is used to poll 
-    the job queues for jobs to initialize.
-    </description>
-  </property>
-  <property>
-    <name>mapred.capacity-scheduler.init-worker-threads</name>
-    <value>5</value>
-    <description>Number of worker threads which would be used by
-    Initialization poller to initialize jobs in a set of queue.
-    If number mentioned in property is equal to number of job queues
-    then a single thread would initialize jobs in a queue. If lesser
-    then a thread would get a set of queues assigned. If the number
-    is greater then number of threads would be equal to number of 
-    job queues.
-    </description>
-  </property>
-
-</configuration>

hadoop-mapreduce-project/conf/fair-scheduler.xml.template

@@ -1,12 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-  This file contains pool and user allocations for the Fair Scheduler.
-  Its format is explained in the Fair Scheduler documentation at
-  http://hadoop.apache.org/core/docs/r0.21.0/fair_scheduler.html.
-  The documentation also includes a sample config file.
--->
-
-<allocations>
-
-</allocations>

hadoop-mapreduce-project/src/contrib/build.xml

@@ -60,13 +60,8 @@
     <subant target="test">
       <property name="continueOnFailure" value="true"/>
       <fileset dir="." includes="streaming/build.xml"/> 
-      <fileset dir="." includes="fairscheduler/build.xml"/> 
-      <fileset dir="." includes="capacity-scheduler/build.xml"/>  
-      <fileset dir="." includes="dynamic-scheduler/build.xml"/>
       <fileset dir="." includes="gridmix/build.xml"/>
       <fileset dir="." includes="vertica/build.xml"/>
-      <!-- mumak tests disabled due to timeouts. See MAPREDUCE-2348
-      <fileset dir="." includes="mumak/build.xml"/> -->
       <fileset dir="." includes="raid/build.xml"/>
     </subant>
     <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
@@ -88,20 +83,12 @@
                value="${hadoop.conf.dir.deployed}"/>
           <fileset dir="." includes="hdfsproxy/build.xml"/>
           <fileset dir="." includes="streaming/build.xml"/>
-          <fileset dir="." includes="fairscheduler/build.xml"/>
-         <fileset dir="." includes="capacity-scheduler/build.xml"/>
          <fileset dir="." includes="gridmix/build.xml"/>
       </subant>
       <available file="${build.contrib.dir}/testsfailed" property="testsfailed"/>
       <fail if="testsfailed">Tests failed!</fail>
   </target>
 
-  <target name="docs">
-    <subant target="docs">
-      <fileset dir="." includes="capacity-scheduler/build.xml"/> 
-    </subant>
-  </target>
-
   <!-- ====================================================== -->
   <!-- Clean all the contribs.                              -->
   <!-- ====================================================== -->

hadoop-mapreduce-project/src/contrib/capacity-scheduler/README

@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-This package implements a scheduler for Map-Reduce jobs, called Capacity
-Task Scheduler (or just Capacity Scheduler), which provides a way to share 
-large clusters. 
-
-The functionality of this scheduler is described in the Forrest documentation
-under src\docs\src\documentation\content\xdocs\capacity_scheduler.xml.

hadoop-mapreduce-project/src/contrib/capacity-scheduler/build.xml

@@ -1,46 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- 
-Before you can run these subtargets directly, you need 
-to call at top-level: ant deploy-contrib compile-core-test
--->
-<project name="capacity-scheduler" default="jar">
-
-  <import file="../build-contrib.xml"/>
-
-  <target name="docs.check">
-    <condition property="generate-capacity-scheduler-docs">
-      <and>
-        <isset property="forrest.home"/>
-        <available file="${conf.dir}/capacity-scheduler.xml.template" property="capacity-scheduler.xml.template.present"/>
-      </and>
-    </condition>
-  </target>
-  <target name="docs" depends="forrest.check,docs.check" if="generate-capacity-scheduler-docs" description="Generate forrest-based documentation. To use, specify -Dforrest.home=&lt;base of Apache Forrest installation&gt; on the command line.">
-  	<!-- The template file may not exist if building from a tarball -->
-  	<copy file="src/java/mapred-queues.xml.template"
-  	      tofile="${build.docs}/mapred-queues-capacity-scheduler.xml"
-  	      failonerror="false"/>
-    <xslt in="${conf.dir}/capacity-scheduler.xml.template"
-    	out="${build.docs}/capacity-scheduler-conf.html"
-    	style="${conf.dir}/configuration.xsl"/>
-  </target>
-
-</project>

hadoop-mapreduce-project/src/contrib/capacity-scheduler/ivy.xml

@@ -1,95 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<ivy-module version="1.0" xmlns:m="http://ant.apache.org/ivy/maven">
-  <info organisation="org.apache.hadoop" module="${ant.project.name}">
-    <license name="Apache 2.0"/>
-    <ivyauthor name="Apache Hadoop Team" url="http://hadoop.apache.org"/>
-    <description>
-        Apache Hadoop
-    </description>
-  </info>
-  <configurations defaultconfmapping="default">
-    <!--these match the Maven configurations-->
-    <conf name="default" extends="master,runtime"/>
-    <conf name="master" description="contains the artifact but no dependencies"/>
-    <conf name="runtime" description="runtime but not the artifact" />
-
-    <conf name="common" visibility="private" 
-      extends="runtime"
-      description="artifacts needed to compile/test the application"/>
-    <conf name="test" visibility="private" extends="runtime"/>
-  </configurations>
-
-  <publications>
-    <!--get the artifact from our module name-->
-    <artifact conf="master"/>
-  </publications>
-  <dependencies>
-   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" 
-                rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" 
-                rev="${hadoop-common.version}" conf="test->default">
-     <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs" 
-                rev="${hadoop-hdfs.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs" 
-                rev="${hadoop-hdfs.version}" conf="test->default">
-     <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
-   </dependency>
-   <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
-               rev="${yarn.version}" conf="common->default"/>
-   <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
-               rev="${yarn.version}" conf="common->default"/>
-    <dependency org="commons-cli" name="commons-cli" 
-                rev="${commons-cli.version}" conf="common->default"/>
-    <dependency org="commons-logging" name="commons-logging" 
-                rev="${commons-logging.version}" conf="common->default"/>
-    <dependency org="junit" name="junit" 
-                rev="${junit.version}" conf="common->default"/>
-    <dependency org="log4j" name="log4j" 
-                rev="${log4j.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="jetty-util" 
-                rev="${jetty-util.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="jetty"
-                rev="${jetty.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="jsp-api-2.1" 
-                rev="${jetty.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="jsp-2.1" 
-	        rev="${jetty.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="servlet-api-2.5" 
-                rev="${servlet-api-2.5.version}" conf="common->master"/>
-    <dependency org="commons-httpclient" name="commons-httpclient" 
-                rev="${commons-httpclient.version}" conf="common->master"/>
-    <dependency org="org.apache.avro" name="avro" 
-                rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
-
-   <!-- Exclusions for transitive dependencies pulled in by log4j -->
-   <exclude org="com.sun.jdmk"/>
-   <exclude org="com.sun.jmx"/>
-   <exclude org="javax.jms"/> 
-   <exclude org="javax.mail"/> 
-
-  </dependencies>
-</ivy-module>

hadoop-mapreduce-project/src/contrib/capacity-scheduler/ivy/libraries.properties

@@ -1,17 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#This properties file lists the versions of the various artifacts used by streaming.
-#It drives ivy and the generation of a maven POM
-
-#Please list the dependencies name with version if they are different from the ones 
-#listed in the global libraries.properties file (in alphabetical order)

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/mapred-queues.xml.template

@@ -1,164 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<!--
-     This is the template for queue configuration when the configured
-     scheduler to use is capacity-scheduler
-     (org.apache.hadoop.mapred.CapacityTaskScheduler). To use this,
-     copy this file into conf directory renaming it to mapred-queues.xml.
--->
-<queues aclsEnabled="false">
-  <queue>
-
-    <name>default</name>
-
-    <state>running</state>
-
-    <acl-submit-job>*</acl-submit-job>
-    <acl-administer-jobs>*</acl-administer-jobs>
-
-    <properties>
-
-      <property key="capacity" value="100">
-		<!--
-        <description>
-        For a root-level container queue, this is the percentage of the
-      	number of slots in the cluster that will be available for all its
-      	immediate children together. For a root-level leaf-queue, this is
-      	the percentage of the number of slots in the cluster that will be
-      	available for all its jobs.	For a non-root level container queue,
-      	this is the percentage of the number of slots in its parent queue
-      	that will be available for all its	children together. For a
-      	non-root-level leaf queue, this	is the percentage of the number of
-      	slots in its parent queue that will be available for jobs in this
-      	queue. The sum of capacities for all children of a container queue
-      	should be less than or equal 100. The sum of capacities of all the
-      	root-level queues should be less than or equal to 100.
-
-          This property can be refreshed.
-        </description>
-        -->    
-      </property>
-
-      <property key="maximum-capacity" value="-1">
-        <!--
-        <description>
-          A limit in percentage beyond which a non-root-level queue cannot use
-          the capacity of its parent queue; for a root-level queue, this is
-          the limit in percentage beyond which it cannot use the
-          cluster-capacity. This property provides a means to limit how much
-          excess capacity a queue can use.  It can be used to prevent queues
-          with long running jobs from occupying more than a certain percentage
-          of the parent-queue or the cluster, which, in the absence of
-          pre-emption, can lead to capacity guarantees of other queues getting
-          affected.
-
-          The maximum-capacity of a queue can only be greater than or equal to
-          its capacity. By default, there is no limit for a queue. For a
-          non-root-level queue this means it can occupy till the
-          maximum-capacity of its parent, for a root-level queue, it means that
-          it can occupy the whole cluster. A value of 100 implies that a queue
-          can use the complete capacity of its parent, or the complete
-          cluster-capacity in case of root-level-queues.
-
-          This property can be refreshed.
-        </description>
-        -->
-      </property>
-
-      <property key="supports-priority" value="false">
-        <!--
-        <description>This is only applicable to leaf queues. If true,
-          priorities of jobs will be taken into account in scheduling
-          decisions.
-
-          This property CANNOT be refreshed.
-        </description>
-        -->
-      </property>
-
-      <property key="minimum-user-limit-percent" value="100">
-        <!--
-        <description>This is only applicable to leaf queues. Each queue
-        enforces a limit on the percentage of resources allocated to a user at
-        any given time, if there is competition for them. This user limit can
-        vary between a minimum and maximum value. The former depends on the
-        number of users who have submitted jobs, and the latter is set to this
-        property value. For example, suppose the value of this property is 25.
-        If two users have submitted jobs to a queue, no single user can use
-        more than 50% of the queue resources. If a third user submits a job,
-        no single user can use more than 33% of the queue resources. With 4 
-        or more users, no user can use more than 25% of the queue's resources.
-        A value of 100 implies no user limits are imposed.
-
-        This property can be refreshed.
-        </description>
-        -->
-      </property>
-
-      <property key="maximum-initialized-jobs-per-user" value="2">
-      <!--
-        <description>This is only applicable to leaf queues. The maximum number
-        of jobs to be pre-initialized for a user of the job queue.
-
-        This property can be refreshed.
-        </description>
-      -->
-      </property>
-
-    </properties>
-  </queue>
-
-  <!-- Here is a sample of a hierarchical queue configuration
-       where q2 and q3 are children of q1 sharing the capacity
-       of q1. In this example, q2 and q3 are leaf level
-       queues as it has no queues configured within it. Currently, ACLs
-       and state are only supported for the leaf level queues.
-  <queue>
-    <name>q1</name>
-    <properties>
-      <property key="capacity" value="100"/>
-    </properties>
-    <queue>
-      <name>q2</name>
-      <state>stopped</state>
-      <acl-submit-job>*</acl-submit-job>
-      <acl-administer-jobs>*</acl-administer-jobs>
-      <properties>
-        <property key="capacity" value="50"/>
-        <property key="maximum-capacity" value="60"/>
-        <property key="supports-priority" value="false"/>
-        <property key="minimum-user-limit-percent" value="100"/>
-        <property key="maximum-initialized-jobs-per-user" value="2"/>
-      </properties>
-    </queue>
-    <queue>
-      <name>q3</name>
-      <state>stopped</state>
-      <acl-submit-job>*</acl-submit-job>
-      <acl-administer-jobs>*</acl-administer-jobs>
-      <properties>
-        <property key="capacity" value="50"/>
-        <property key="maximum-capacity" value="-1"/>
-        <property key="supports-priority" value="false"/>
-        <property key="minimum-user-limit-percent" value="100"/>
-        <property key="maximum-initialized-jobs-per-user" value="2"/>
-      </properties>
-    </queue>
-  </queue>
- -->
-</queues>

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/AbstractQueue.java

@@ -1,254 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Parent class for hierarchy of queues.
- * All queues extend this class.
- * <p/>
- * Even though all the Queue classes extend this class , there are 2 categories
- * of queues define.
- * <p/>
- * 1.ContainerQueue: which are composite of queues.
- * 2.JobQueue: leaf level queues.
- * <p/>
- * Typically ContainerQueue consists of JobQueue. All the SchedulingContext data
- * in ContainerQueue is cummulative of its children.
- * <p/>
- * JobQueue consists of actual job list , i.e, runningJob, WaitingJob etc.
- * <p/>
- * This is done so to make sure that all the job related data is at one place
- * and queues at higher level are typically cummulative data of organization at
- * there children level.
- */
-
-abstract class AbstractQueue {
-
-  static final Log LOG = LogFactory.getLog(AbstractQueue.class);
-
-  protected QueueSchedulingContext qsc;
-  protected AbstractQueue parent;
-
-
-  protected AbstractQueue(AbstractQueue parent, QueueSchedulingContext qsc) {
-    setParent(parent);
-    setQueueSchedulingContext(qsc);
-    //Incase of root this value would be null
-    if (parent != null) {
-      parent.addChild(this);
-    }
-  }
-
-  /**
-   * This involves updating each qC structure.
-   * <p/>
-   * First update QueueSchedulingContext at this level is updated.
-   * then update QueueSchedulingContext of all the children.
-   * <p/>
-   * Children consider parent's capacity as the totalclustercapacity
-   * and do there calculations accordingly.
-   *
-   * @param mapClusterCapacity
-   * @param reduceClusterCapacity
-   */
-
-  public void update(int mapClusterCapacity, int reduceClusterCapacity) {
-    qsc.updateContext(mapClusterCapacity,reduceClusterCapacity);
-  }
-
-  /**
-   * @return qsc
-   */
-  public QueueSchedulingContext getQueueSchedulingContext() {
-    return qsc;
-  }
-
-  /**
-   * Set the {@link QueueSchedulingContext} of this {@link AbstractQueue} to the
-   * passed context.
-   * 
-   * @param qsc
-   */
-  void setQueueSchedulingContext(QueueSchedulingContext qsc) {
-    this.qsc = qsc;
-  }
-
-  String getName() {
-    return qsc.getQueueName();
-  }
-
-  protected AbstractQueue getParent() {
-    return parent;
-  }
-
-  protected void setParent(AbstractQueue queue) {
-    this.parent = queue;
-  }
-
-  /**
-   * Get a list of all the {@link JobQueue}s in the {@link AbstractQueue}
-   * hierarchy rooted by 'this' {@link AbstractQueue}.
-   * 
-   * <p>
-   * The list is returned in a depth-first order in which the children of each
-   * {@link AbstractQueue}s in the hierarchy are already ordered.
-   * 
-   * @return an unordered list containing all the job-queues in the hierarchy.
-   */
-  abstract List<AbstractQueue> getDescendentJobQueues();
-
-  /**
-   * Get a list of all the {@link ContainerQueue}s in the {@link AbstractQueue}
-   * hierarchy rooted by 'this' {@link AbstractQueue}, excluding this queue.
-   * 
-   * <p>
-   * The list is returned in a depth-first order in which the children of each
-   * {@link AbstractQueue}s in the hierarchy are already ordered.
-   * 
-   * @return an unordered list containing all the container queues in the
-   *         hierarchy.
-   */
-  abstract List<AbstractQueue> getDescendantContainerQueues();
-
-  /**
-   * Sorts all levels below current level.
-   *
-   * @param queueComparator
-   */
-  public abstract void sort(Comparator queueComparator);
-
-  /**
-   * returns list of immediate children.
-   * null in case of leaf.
-   *
-   * @return
-   */
-  abstract List<AbstractQueue> getChildren();
-
-  /**
-   * adds children to the current level.
-   * There is no support for adding children at leaf level node.
-   *
-   * @param queue
-   */
-  public abstract void addChild(AbstractQueue queue);
-
-  /**
-   * Distribute the unconfigured capacity % among the queues.
-   *
-   */
-  abstract void distributeUnConfiguredCapacity();
-
-  @Override
-  public String toString() {
-    return this.getName().toString() 
-            + "\n" + getQueueSchedulingContext().toString();
-  }
-
-  /**
-   * Comparator to compare {@link AbstractQueue}s by the natural order of the
-   * corresponding queue names.
-   */
-  static class AbstractQueueComparator implements Comparator<AbstractQueue> {
-    @Override
-    public int compare(AbstractQueue o1, AbstractQueue o2) {
-      return o1.getName().compareTo(o2.getName());
-    }
-  }
-
-  @Override
-  /**
-   * Returns true, if the other object is an AbstractQueue
-   * with the same name.
-   */
-  public boolean equals(Object other) {
-    if (other == null) {
-      return false;
-    }
-    if (!(other instanceof AbstractQueue)) {
-      return false;
-    }
-    
-    AbstractQueue otherQueue = (AbstractQueue)other;
-    return otherQueue.getName().equals(getName());
-  }
-  
-  @Override
-  public int hashCode() {
-    return this.getName().hashCode();
-  }
-
-  /**
-   * Copy the configuration enclosed via {@link QueueSchedulingContext} of the
-   * destinationQueue to the sourceQueue recursively.
-   * 
-   * <p>
-   * This method assumes that the total hierarchy of the passed queues is the
-   * same and that the {@link AbstractQueue#getChildren()} on this queue as well
-   * as the sourceQueue are sorted according to the comparator
-   * {@link AbstractQueueComparator} .
-   * 
-   * @param AbstractQueueComparator
-   * @throws IOException
-   */
-  void validateAndCopyQueueContexts(AbstractQueue sourceQueue)
-      throws IOException {
-
-    // Do some validation before copying.
-    QueueSchedulingContext sourceContext =
-        sourceQueue.getQueueSchedulingContext();
-    if (qsc.supportsPriorities() != sourceContext.supportsPriorities()) {
-      throw new IOException("Changing of priorities is not yet supported. "
-          + "Attempt has been made to change priority of the queue "
-          + this.getName());
-    }
-
-    // First update the children queues recursively.
-    List<AbstractQueue> destChildren = getChildren();
-    if (destChildren != null) {
-      Iterator<AbstractQueue> itr1 = destChildren.iterator();
-      Iterator<AbstractQueue> itr2 = sourceQueue.getChildren().iterator();
-      while (itr1.hasNext()) {
-        itr1.next().validateAndCopyQueueContexts(itr2.next());
-      }
-    }
-
-    // Now, copy the configuration for the root-queue itself
-    sourceContext.setNumJobsByUser(qsc.getNumJobsByUser());
-    sourceContext.setNumOfWaitingJobs(qsc.getNumOfWaitingJobs());
-
-    sourceContext.setMapTSC(qsc.getMapTSC());
-    sourceContext.setReduceTSC(qsc.getReduceTSC());
-    setQueueSchedulingContext(sourceContext);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("New Queue-Context for " + sourceQueue.getName() + ": "
-          + sourceQueue.getQueueSchedulingContext());
-    }
-  }
-
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java

@@ -1,383 +0,0 @@
-/** Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import java.util.Properties;
-import java.util.Map;
-import java.util.HashMap;
-
-/**
- * Class providing access to Capacity scheduler configuration and default values
- * for queue-configuration. Capacity scheduler configuration includes settings
- * for the {@link JobInitializationPoller} and default values for queue
- * configuration. These are read from the file
- * {@link CapacitySchedulerConf#SCHEDULER_CONF_FILE} on the CLASSPATH. The main
- * queue configuration is defined in the file
- * {@link QueueManager#QUEUE_CONF_FILE_NAME} on the CLASSPATH.
- * 
- * <p>
- * 
- * This class also provides APIs to get and set the configuration for the
- * queues.
- */
-class CapacitySchedulerConf {
-
-  static final Log LOG = LogFactory.getLog(CapacitySchedulerConf.class);
-
-  static final String CAPACITY_PROPERTY = "capacity";
-
-  static final String SUPPORTS_PRIORITY_PROPERTY = "supports-priority";
-
-  static final String MAXIMUM_INITIALIZED_JOBS_PER_USER_PROPERTY =
-      "maximum-initialized-jobs-per-user";
-
-  static final String MINIMUM_USER_LIMIT_PERCENT_PROPERTY =
-      "minimum-user-limit-percent";
-
-  /** Default file name from which the capacity scheduler configuration is read. */
-  public static final String SCHEDULER_CONF_FILE = "capacity-scheduler.xml";
-  
-  private int defaultUlimitMinimum;
-  
-  private boolean defaultSupportPriority;
-  
-  private static final String QUEUE_CONF_PROPERTY_NAME_PREFIX = 
-    "mapred.capacity-scheduler.queue.";
-
-  private Map<String, Properties> queueProperties
-    = new HashMap<String,Properties>();
-
-  /**
-   * If {@link JobConf#MAPRED_TASK_MAXPMEM_PROPERTY} is set to
-   * {@link JobConf#DISABLED_MEMORY_LIMIT}, this configuration will be used to
-   * calculate job's physical memory requirements as a percentage of the job's
-   * virtual memory requirements set via
-   * {@link JobConf#setMaxVirtualMemoryForTask()}. This property thus provides
-   * default value of physical memory for job's that don't explicitly specify
-   * physical memory requirements.
-   * <p/>
-   * It defaults to {@link JobConf#DISABLED_MEMORY_LIMIT} and if not explicitly
-   * set to a valid value, scheduler will not consider physical memory for
-   * scheduling even if virtual memory based scheduling is enabled.
-   *
-   * @deprecated
-   */
-  @Deprecated
-  static String DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY =
-    "mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem";
-
-  /**
-   * Configuration that provides an upper limit on the maximum physical memory
-   * that can be specified by a job. The job configuration
-   * {@link JobConf#MAPRED_TASK_MAXPMEM_PROPERTY} should,
-   * by definition, be less than this value. If not, the job will be rejected
-   * by the scheduler. If it is set to {@link JobConf#DISABLED_MEMORY_LIMIT},
-   * scheduler will not consider physical memory for scheduling even if virtual
-   * memory based scheduling is enabled.
-   *
-   * @deprecated
-   */
-  @Deprecated
-  static final String UPPER_LIMIT_ON_TASK_PMEM_PROPERTY =
-    "mapred.capacity-scheduler.task.limit.maxpmem";
-  
-  /**
-   * A maximum capacity defines a limit beyond which a sub-queue
-   * cannot use the capacity of its parent queue.
-   */
-  static final String MAX_CAPACITY_PROPERTY ="maximum-capacity";
-
-  /**
-   * The constant which defines the default initialization thread
-   * polling interval, denoted in milliseconds.
-   */
-  private static final int INITIALIZATION_THREAD_POLLING_INTERVAL = 5000;
-
-  /**
-   * The constant which defines the maximum number of worker threads to be
-   * spawned off for job initialization
-   */
-  private static final int MAX_INITIALIZATION_WORKER_THREADS = 5;
-
-  private Configuration rmConf;
-
-  private int defaultMaxJobsPerUsersToInitialize;
-
-  /**
-   * Create a new CapacitySchedulerConf.
-   * This method reads from the default configuration file mentioned in
-   * {@link SCHEDULER_CONF_FILE}, that must be present in the classpath of the
-   * application.
-   */
-  public CapacitySchedulerConf() {
-    rmConf = new Configuration(false);
-    getCSConf().addResource(SCHEDULER_CONF_FILE);
-    initializeDefaults();
-  }
-
-  /**
-   * Create a new CapacitySchedulerConf reading the specified configuration
-   * file.
-   * 
-   * @param configFile {@link Path} to the configuration file containing
-   * the Capacity scheduler configuration.
-   */
-  public CapacitySchedulerConf(Path configFile) {
-    rmConf = new Configuration(false);
-    getCSConf().addResource(configFile);
-    initializeDefaults();
-  }
-  
-  /*
-   * Method used to initialize the default values and the queue list
-   * which is used by the Capacity Scheduler.
-   */
-  private void initializeDefaults() {
-    defaultUlimitMinimum = getCSConf().getInt(
-        "mapred.capacity-scheduler.default-minimum-user-limit-percent", 100);
-    defaultSupportPriority = getCSConf().getBoolean(
-        "mapred.capacity-scheduler.default-supports-priority", false);
-    defaultMaxJobsPerUsersToInitialize = getCSConf().getInt(
-        "mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user",
-        2);
-  }
-
-  void setProperties(String queueName , Properties properties) {
-    this.queueProperties.put(queueName,properties);
-  }
-  
-  /**
-   * Get the percentage of the cluster for the specified queue.
-   * 
-   * This method defaults to configured default Capacity if
-   * no value is specified in the configuration for this queue. 
-   * If the configured capacity is negative value or greater than 100 an
-   * {@link IllegalArgumentException} is thrown.
-   * 
-   * If default capacity is not configured for a queue, then
-   * system allocates capacity based on what is free at the time of 
-   * capacity scheduler start
-   * 
-   * 
-   * @param queue name of the queue
-   * @return percent of the cluster for the queue.
-   */
-  public float getCapacity(String queue) {
-    //Check done in order to return default capacity which can be negative
-    //In case of both capacity and default capacity not configured.
-    //Last check is if the configuration is specified and is marked as
-    //negative we throw exception
-    String raw = getProperty(queue, CAPACITY_PROPERTY);
-
-    float result = this.getFloat(raw,-1);
-    
-    if (result > 100.0) {
-      throw new IllegalArgumentException("Illegal capacity for queue " + queue +
-                                         " of " + result);
-    }
-    return result;
-  }
-
-  String getProperty(String queue,String property) {
-    if(!queueProperties.containsKey(queue))
-     throw new IllegalArgumentException("Invalid queuename " + queue);
-
-    //This check is still required as sometimes we create queue with null
-    //This is typically happens in case of test.
-    if(queueProperties.get(queue) != null) {
-      return queueProperties.get(queue).getProperty(property);
-    }
-
-    return null;
-  }
-
-  /**
-   * Return the maximum percentage of the cluster capacity that can be
-   * used by the given queue
-   * This percentage defines a limit beyond which a
-   * sub-queue cannot use the capacity of its parent queue.
-   * This provides a means to limit how much excess capacity a
-   * sub-queue can use. By default, there is no limit.
-   *
-   * The maximum-capacity-stretch of a queue can only be
-   * greater than or equal to its minimum capacity.
-   * 
-   * @param queue name of the queue
-   * @return maximum capacity percent of cluster for the queue
-   */
-  public float getMaxCapacity(String queue) {
-    String raw = getProperty(queue, MAX_CAPACITY_PROPERTY);
-    float result = getFloat(raw,-1);
-    result = (result <= 0) ? -1 : result; 
-    if (result > 100.0) {
-      throw new IllegalArgumentException("Illegal maximum-capacity-stretch " +
-        "for queue " + queue +" of " + result);
-    }
-
-    if((result != -1) && (result < getCapacity(queue))) {
-      throw new IllegalArgumentException("maximum-capacity-stretch " +
-        "for a queue should be greater than capacity ");
-    }
-    return result;
-  }
-
-  /**
-   * Get whether priority is supported for this queue.
-   * 
-   * If this value is false, then job priorities will be ignored in 
-   * scheduling decisions. This method defaults to <code>false</code> if 
-   * the property is not configured for this queue. 
-   * @param queue name of the queue
-   * @return Whether this queue supports priority or not.
-   */
-  public boolean isPrioritySupported(String queue) {
-    String raw = getProperty(queue, SUPPORTS_PRIORITY_PROPERTY);
-    return Boolean.parseBoolean(raw);
-  }
-
-  /**
-   * Get the minimum limit of resources for any user submitting jobs in 
-   * this queue, in percentage.
-   * 
-   * This method defaults to default user limit configured if
-   * no value is specified in the configuration for this queue.
-   * 
-   * Throws an {@link IllegalArgumentException} when invalid value is 
-   * configured.
-   * 
-   * @param queue name of the queue
-   * @return minimum limit of resources, in percentage, that will be 
-   * available for a user.
-   * 
-   */
-  public int getMinimumUserLimitPercent(String queue) {
-    String raw = getProperty(queue, MINIMUM_USER_LIMIT_PERCENT_PROPERTY);
-    int userLimit = getInt(raw,defaultUlimitMinimum);
-    if(userLimit <= 0 || userLimit > 100) {
-      throw new IllegalArgumentException("Invalid user limit : "
-          + userLimit + " for queue : " + queue);
-    }
-    return userLimit;
-  }
-  
-  static final String toFullPropertyName(String queue, 
-                                                  String property) {
-      return QUEUE_CONF_PROPERTY_NAME_PREFIX + queue + "." + property;
-  }
-
-  /**
-   * Gets the maximum number of jobs which are allowed to initialize in the
-   * job queue.
-   * 
-   * @param queue queue name.
-   * @return maximum number of jobs allowed to be initialized per user.
-   * @throws IllegalArgumentException if maximum number of users is negative
-   * or zero.
-   */
-  public int getMaxJobsPerUserToInitialize(String queue) {
-    String raw =
-        getProperty(queue, MAXIMUM_INITIALIZED_JOBS_PER_USER_PROPERTY);
-    int maxJobsPerUser = getInt(raw,defaultMaxJobsPerUsersToInitialize);
-    if(maxJobsPerUser <= 0) {
-      throw new IllegalArgumentException(
-          "Invalid maximum jobs per user configuration " + maxJobsPerUser);
-    }
-    return maxJobsPerUser;
-  }
-
-
-  /**
-   * Amount of time in milliseconds which poller thread and initialization
-   * thread would sleep before looking at the queued jobs.
-   * 
-   * The default value if no corresponding configuration is present is
-   * 5000 Milliseconds.
-   *  
-   * @return time in milliseconds.
-   * @throws IllegalArgumentException if time is negative or zero.
-   */
-  public long getSleepInterval() {
-    long sleepInterval = getCSConf().getLong(
-        "mapred.capacity-scheduler.init-poll-interval", 
-        INITIALIZATION_THREAD_POLLING_INTERVAL);
-    
-    if(sleepInterval <= 0) {
-      throw new IllegalArgumentException(
-          "Invalid initializater poller interval " + sleepInterval);
-    }
-    
-    return sleepInterval;
-  }
-
-  /**
-   * Gets maximum number of threads which are spawned to initialize jobs
-   * in job queue in  parallel. The number of threads should be always less than
-   * or equal to number of job queues present.
-   * 
-   * If number of threads is configured to be more than job queues present,
-   * then number of job queues is used as number of threads used for initializing
-   * jobs.
-   * 
-   * So a given thread can have responsibility of initializing jobs from more 
-   * than one queue.
-   * 
-   * The default value is 5
-   * 
-   * @return maximum number of threads spawned to initialize jobs in job queue
-   * in parallel.
-   */
-  public int getMaxWorkerThreads() {
-    int maxWorkerThreads = getCSConf().getInt(
-        "mapred.capacity-scheduler.init-worker-threads", 
-        MAX_INITIALIZATION_WORKER_THREADS);
-    if(maxWorkerThreads <= 0) {
-      throw new IllegalArgumentException(
-          "Invalid initializater worker thread number " + maxWorkerThreads);
-    }
-    return maxWorkerThreads;
-  }
-
-  public Configuration getCSConf() {
-    return rmConf;
-  }
-
-  float getFloat(String valueString,float defaultValue) {
-    if (valueString == null)
-      return defaultValue;
-    try {
-      return Float.parseFloat(valueString);
-    } catch (NumberFormatException e) {
-      return defaultValue;
-    }
-  }
-
-  int getInt(String valueString,int defaultValue) {
-    if (valueString == null)
-      return defaultValue;
-    try {
-      return Integer.parseInt(valueString);
-    } catch (NumberFormatException e) {
-      return defaultValue;
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java

@@ -1,1101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.AbstractQueue.AbstractQueueComparator;
-import org.apache.hadoop.mapred.JobTracker.IllegalStateException;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * A {@link TaskScheduler} that implements the requirements in HADOOP-3421
- * and provides a HOD-less way to share large clusters. This scheduler 
- * provides the following features: 
- *  * support for queues, where a job is submitted to a queue. 
- *  * Queues are assigned a fraction of the capacity of the grid (their
- *  'capacity') in the sense that a certain capacity of resources 
- *  will be at their disposal. All jobs submitted to the queues of an Org 
- *  will have access to the capacity to the Org.
- *  * Free resources can be allocated to any queue beyond its 
- *  capacity.
- *  * Queues optionally support job priorities (disabled by default). 
- *  * Within a queue, jobs with higher priority will have access to the 
- *  queue's resources before jobs with lower priority. However, once a job 
- *  is running, it will not be preempted for a higher priority job.
- *  * In order to prevent one or more users from monopolizing its resources, 
- *  each queue enforces a limit on the percentage of resources allocated to a 
- *  user at any given time, if there is competition for them.
- *  
- */
-class CapacityTaskScheduler extends TaskScheduler {
-
-  /** quick way to get qsc object given a queue name */
-  private Map<String, QueueSchedulingContext> queueInfoMap =
-    new HashMap<String, QueueSchedulingContext>();
-
-  //Root level queue . It has all the
-  //cluster capacity at its disposal.
-  //Queues declared by users would
-  //be children of this queue.
-  //CS would have handle to root.
-  private AbstractQueue root = null;
-
-  /**
-   * This class captures scheduling information we want to display or log.
-   */
-  private static class SchedulingDisplayInfo {
-    private String queueName;
-    CapacityTaskScheduler scheduler;
-    
-    SchedulingDisplayInfo(String queueName, CapacityTaskScheduler scheduler) { 
-      this.queueName = queueName;
-      this.scheduler = scheduler;
-    }
-    
-    @Override
-    public String toString(){
-      // note that we do not call updateContextObjects() here for performance
-      // reasons. This means that the data we print out may be slightly
-      // stale. This data is updated whenever assignTasks() is called
-      // If this doesn't happen, the data gets stale. If we see
-      // this often, we may need to detect this situation and call 
-      // updateContextObjects(), or just call it each time.
-      return scheduler.getDisplayInfo(queueName);
-    }
-  }
-
-
-  // this class encapsulates the result of a task lookup
-  private static class TaskLookupResult {
-
-    static enum LookUpStatus {
-      TASK_FOUND,
-      NO_TASK_FOUND,
-      TASK_FAILING_MEMORY_REQUIREMENT,
-    }
-    // constant TaskLookupResult objects. Should not be accessed directly.
-    private static final TaskLookupResult NoTaskLookupResult = 
-      new TaskLookupResult(null, TaskLookupResult.LookUpStatus.NO_TASK_FOUND);
-    private static final TaskLookupResult MemFailedLookupResult = 
-      new TaskLookupResult(null, 
-          TaskLookupResult.LookUpStatus.TASK_FAILING_MEMORY_REQUIREMENT);
-
-    private LookUpStatus lookUpStatus;
-    private Task task;
-
-    // should not call this constructor directly. use static factory methods.
-    private TaskLookupResult(Task t, LookUpStatus lUStatus) {
-      this.task = t;
-      this.lookUpStatus = lUStatus;
-    }
-    
-    static TaskLookupResult getTaskFoundResult(Task t) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Returning task " + t);
-      }
-      return new TaskLookupResult(t, LookUpStatus.TASK_FOUND);
-    }
-    static TaskLookupResult getNoTaskFoundResult() {
-      return NoTaskLookupResult;
-    }
-    static TaskLookupResult getMemFailedResult() {
-      return MemFailedLookupResult;
-    }
-    
-
-    Task getTask() {
-      return task;
-    }
-
-    LookUpStatus getLookUpStatus() {
-      return lookUpStatus;
-    }
-  }
-
-  /**
-   * This class handles the scheduling algorithms.
-   * The algos are the same for both Map and Reduce tasks.
-   * There may be slight variations later, in which case we can make this
-   * an abstract base class and have derived classes for Map and Reduce.
-   */
-  private static abstract class TaskSchedulingMgr {
-
-    /** our TaskScheduler object */
-    protected CapacityTaskScheduler scheduler;
-    protected TaskType type = null;
-
-    abstract Task obtainNewTask(TaskTrackerStatus taskTracker,
-        JobInProgress job) throws IOException;
-
-    abstract int getClusterCapacity();
-    abstract TaskSchedulingContext getTSC(
-      QueueSchedulingContext qsc);
-    /**
-     * To check if job has a speculative task on the particular tracker.
-     *
-     * @param job job to check for speculative tasks.
-     * @param tts task tracker on which speculative task would run.
-     * @return true if there is a speculative task to run on the tracker.
-     */
-    abstract boolean hasSpeculativeTask(JobInProgress job,
-        TaskTrackerStatus tts);
-
-    /**
-     * Comparator to sort queues.
-     * For maps, we need to sort on QueueSchedulingContext.mapTSC. For
-     * reducers, we use reduceTSC. So we'll need separate comparators.
-     */
-    private static abstract class QueueComparator
-      implements Comparator<AbstractQueue> {
-      abstract TaskSchedulingContext getTSC(
-        QueueSchedulingContext qsi);
-      public int compare(AbstractQueue q1, AbstractQueue q2) {
-        TaskSchedulingContext t1 = getTSC(q1.getQueueSchedulingContext());
-        TaskSchedulingContext t2 = getTSC(q2.getQueueSchedulingContext());
-        // look at how much capacity they've filled. Treat a queue with
-        // capacity=0 equivalent to a queue running at capacity
-        double r1 = (0 == t1.getCapacity())? 1.0f:
-          (double) t1.getNumSlotsOccupied() /(double) t1.getCapacity();
-        double r2 = (0 == t2.getCapacity())? 1.0f:
-          (double) t2.getNumSlotsOccupied() /(double) t2.getCapacity();
-        if (r1<r2) return -1;
-        else if (r1>r2) return 1;
-        else return 0;
-      }
-    }
-    // subclass for map and reduce comparators
-    private static final class MapQueueComparator extends QueueComparator {
-      TaskSchedulingContext getTSC(QueueSchedulingContext qsi) {
-        return qsi.getMapTSC();
-      }
-    }
-    private static final class ReduceQueueComparator extends QueueComparator {
-      TaskSchedulingContext getTSC(QueueSchedulingContext qsi) {
-        return qsi.getReduceTSC();
-      }
-    }
-
-    // these are our comparator instances
-    protected final static MapQueueComparator mapComparator =
-      new MapQueueComparator();
-    protected final static ReduceQueueComparator reduceComparator =
-      new ReduceQueueComparator();
-    // and this is the comparator to use
-    protected QueueComparator queueComparator;
-
-    // Returns queues sorted according to the QueueComparator.
-    // Mainly for testing purposes.
-    String[] getOrderedQueues() {
-      List<AbstractQueue> queueList = getOrderedJobQueues();
-      List<String> queues = new ArrayList<String>(queueList.size());
-      for (AbstractQueue q : queueList) {
-        queues.add(q.getName());
-      }
-      return queues.toArray(new String[queues.size()]);
-    }
-
-    /**
-     * Return an ordered list of {@link JobQueue}s wrapped as
-     * {@link AbstractQueue}s. Ordering is according to {@link QueueComparator}.
-     * To reflect the true ordering of the JobQueues, the complete hierarchy is
-     * sorted such that {@link AbstractQueue}s are ordered according to their
-     * needs at each level in the hierarchy, after which only the leaf level
-     * {@link JobQueue}s are returned.
-     * 
-     * @return a list of {@link JobQueue}s wrapped as {@link AbstractQueue}s
-     *         sorted by their needs.
-     */
-    List<AbstractQueue> getOrderedJobQueues() {
-      scheduler.root.sort(queueComparator);
-      return scheduler.root.getDescendentJobQueues();
-    }
-
-    TaskSchedulingMgr(CapacityTaskScheduler sched) {
-      scheduler = sched;
-    }
-  
-    /**
-     * Ceil of result of dividing two integers.
-     * 
-     * This is *not* a utility method. 
-     * Neither <code>a</code> or <code>b</code> should be negative.
-     *  
-     * @param a
-     * @param b
-     * @return ceil of the result of a/b
-     */
-    private int divideAndCeil(int a, int b) {
-      if (b != 0) {
-        return (a + (b - 1)) / b;
-      }
-      
-      LOG.info("divideAndCeil called with a=" + a + " b=" + b);
-      return 0;
-    }
-
-    private boolean isUserOverLimit(JobInProgress j,
-                                    QueueSchedulingContext qsc) {
-      // what is our current capacity? It is equal to the queue-capacity if
-      // we're running below capacity. If we're running over capacity, then its
-      // #running plus slotPerTask of the job (which is the number of extra
-      // slots we're getting).
-      int currentCapacity;
-      TaskSchedulingContext tsi = getTSC(qsc);
-      if (tsi.getNumSlotsOccupied() < tsi.getCapacity()) {
-        currentCapacity = tsi.getCapacity();
-      }
-      else {
-        currentCapacity =
-          tsi.getNumSlotsOccupied() +
-            TaskDataView.getTaskDataView(type).getSlotsPerTask(j);
-      }
-      int limit = Math.max(divideAndCeil(currentCapacity, qsc.getNumJobsByUser().size()),
-			   divideAndCeil(qsc.getUlMin() * currentCapacity, 100));
-      String user = j.getProfile().getUser();
-      if (tsi.getNumSlotsOccupiedByUser().get(user) >= limit) {
-	if (LOG.isDebugEnabled()) {
-          LOG.debug("User " + user + " is over limit, num slots occupied = " +
-              tsi.getNumSlotsOccupiedByUser().get(user) + ", limit = " + limit);
-	}
-        return true;
-      }
-      else {
-        return false;
-      }
-    }
-
-    /*
-     * This is the central scheduling method.
-     * It tries to get a task from jobs in a single queue.
-     * Always return a TaskLookupResult object. Don't return null.
-     */
-    private TaskLookupResult getTaskFromQueue(TaskTracker taskTracker,
-                                              QueueSchedulingContext qsi)
-    throws IOException {
-      TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
-      // we only look at jobs in the running queues, as these are the ones
-      // who have been potentially initialized
-
-      for (JobInProgress j :
-        scheduler.jobQueuesManager.getJobQueue(qsi.getQueueName())
-          .getRunningJobs()) {
-        // only look at jobs that can be run. We ignore jobs that haven't
-        // initialized, or have completed but haven't been removed from the
-        // running queue.
-
-        //Check queue for maximum capacity .
-        if(areTasksInQueueOverMaxCapacity(qsi,j.getNumSlotsPerTask(type))) {
-          continue;
-        }
-        
-        if (j.getStatus().getRunState() != JobStatus.RUNNING) {
-          continue;
-        }
-        // check if the job's user is over limit
-        if (isUserOverLimit(j, qsi)) {
-          continue;
-        }
-        //If this job meets memory requirements. Ask the JobInProgress for
-        //a task to be scheduled on the task tracker.
-        //if we find a job then we pass it on.
-        if (scheduler.memoryMatcher.matchesMemoryRequirements(j, type,
-                                                              taskTrackerStatus)) {
-          // We found a suitable job. Get task from it.
-          Task t = obtainNewTask(taskTrackerStatus, j);
-          //if there is a task return it immediately.
-          if (t != null) {
-            // we're successful in getting a task
-            return TaskLookupResult.getTaskFoundResult(t);
-          } else {
-            //skip to the next job in the queue.
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Job " + j.getJobID().toString()
-                        + " returned no tasks of type " + type);
-            }
-          }
-        } else {
-          // if memory requirements don't match then we check if the job has
-          // pending tasks and has insufficient number of 'reserved'
-          // tasktrackers to cover all pending tasks. If so we reserve the
-          // current tasktracker for this job so that high memory jobs are not
-          // starved
-          TaskDataView view = TaskDataView.getTaskDataView(type);
-          if ((view.getPendingTasks(j) != 0 && 
-                !view.hasSufficientReservedTaskTrackers(j))) {
-            // Reserve all available slots on this tasktracker
-            LOG.info(j.getJobID() + ": Reserving "
-                + taskTracker.getTrackerName()
-                + " since memory-requirements don't match");
-            taskTracker.reserveSlots(type, j, taskTracker
-                .getAvailableSlots(type));
-
-            // Block
-            return TaskLookupResult.getMemFailedResult();
-          }
-        }//end of memory check block
-        // if we're here, this job has no task to run. Look at the next job.
-      }//end of for loop
-
-      // if we're here, we haven't found any task to run among all jobs in
-      // the queue. This could be because there is nothing to run, or that
-      // the user limit for some user is too strict, i.e., there's at least
-      // one user who doesn't have enough tasks to satisfy his limit. If
-      // it's the latter case, re-look at jobs without considering user
-      // limits, and get a task from the first eligible job; however
-      // we do not 'reserve' slots on tasktrackers anymore since the user is
-      // already over the limit
-      // Note: some of the code from above is repeated here. This is on
-      // purpose as it improves overall readability.
-      // Note: we walk through jobs again. Some of these jobs, which weren't
-      // considered in the first pass, shouldn't be considered here again,
-      // but we still check for their viability to keep the code simple. In
-      // some cases, for high mem jobs that have nothing to run, we call
-      // obtainNewTask() unnecessarily. Should this be a problem, we can
-      // create a list of jobs to look at (those whose users were over
-      // limit) in the first pass and walk through that list only.
-      for (JobInProgress j :
-        scheduler.jobQueuesManager.getJobQueue(qsi.getQueueName())
-          .getRunningJobs()) {
-        if (j.getStatus().getRunState() != JobStatus.RUNNING) {
-          continue;
-        }
-
-        //Check for the maximum-capacity.
-        if(areTasksInQueueOverMaxCapacity(qsi,j.getNumSlotsPerTask(type))) {
-          continue;
-        }
-
-
-        if (scheduler.memoryMatcher.matchesMemoryRequirements(j, type,
-            taskTrackerStatus)) {
-          // We found a suitable job. Get task from it.
-          Task t = obtainNewTask(taskTrackerStatus, j);
-          //if there is a task return it immediately.
-          if (t != null) {
-            // we're successful in getting a task
-            return TaskLookupResult.getTaskFoundResult(t);
-          } else {
-          }
-        } else {
-          //if memory requirements don't match then we check if the
-          //job has either pending or speculative task. If the job
-          //has pending or speculative task we block till this job
-          //tasks get scheduled, so that high memory jobs are not
-          //starved
-          if (TaskDataView.getTaskDataView(type).getPendingTasks(j) != 0 ||
-            hasSpeculativeTask(j, taskTrackerStatus)) {
-            return TaskLookupResult.getMemFailedResult();
-          }
-        }//end of memory check block
-      }//end of for loop
-
-      // found nothing for this queue, look at the next one.
-      if (LOG.isDebugEnabled()) {
-        String msg = "Found no task from the queue " + qsi.getQueueName();
-        LOG.debug(msg);
-      }
-      return TaskLookupResult.getNoTaskFoundResult();
-    }
-
-    // Always return a TaskLookupResult object. Don't return null.
-    // The caller is responsible for ensuring that the QSC objects and the
-    // collections are up-to-date.
-    private TaskLookupResult assignTasks(TaskTracker taskTracker)
-    throws IOException {
-      TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
-
-      printQSCs();
-
-      // Check if this tasktracker has been reserved for a job...
-      JobInProgress job = taskTracker.getJobForFallowSlot(type);
-      if (job != null) {
-        int availableSlots = taskTracker.getAvailableSlots(type);
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(job.getJobID() + ": Checking 'reserved' tasktracker " +
-                    taskTracker.getTrackerName() + " with " + availableSlots +
-                    " '" + type + "' slots");
-        }
-
-        if (availableSlots >= job.getNumSlotsPerTask(type)) {
-          // Unreserve
-          taskTracker.unreserveSlots(type, job);
-
-          // We found a suitable job. Get task from it.
-          Task t = obtainNewTask(taskTrackerStatus, job);
-          //if there is a task return it immediately.
-          if (t != null) {
-            if (LOG.isDebugEnabled()) {
-              LOG.info(job.getJobID() + ": Got " + t.getTaskID() +
-                       " for reserved tasktracker " +
-                       taskTracker.getTrackerName());
-            }
-            // we're successful in getting a task
-            return TaskLookupResult.getTaskFoundResult(t);
-          }
-        } else {
-          // Re-reserve the current tasktracker
-          taskTracker.reserveSlots(type, job, availableSlots);
-
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(job.getJobID() + ": Re-reserving " +
-                      taskTracker.getTrackerName());
-          }
-
-          return TaskLookupResult.getMemFailedResult();
-        }
-      }
-
-      for (AbstractQueue q : getOrderedJobQueues()) {
-        QueueSchedulingContext qsc = q.getQueueSchedulingContext();
-        // we may have queues with capacity=0. We shouldn't look at jobs from
-        // these queues
-        if (0 == getTSC(qsc).getCapacity()) {
-          continue;
-        }
-
-        //This call is important for optimization purposes , if we
-        //have reached the limit already no need for traversing the queue.
-        if(this.areTasksInQueueOverMaxCapacity(qsc,1)) {
-          continue;
-        }
-        
-        TaskLookupResult tlr = getTaskFromQueue(taskTracker, qsc);
-        TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus();
-
-        if (lookUpStatus == TaskLookupResult.LookUpStatus.NO_TASK_FOUND) {
-          continue; // Look in other queues.
-        }
-
-        // if we find a task, return
-        if (lookUpStatus == TaskLookupResult.LookUpStatus.TASK_FOUND) {
-          return tlr;
-        }
-        // if there was a memory mismatch, return
-        else if (lookUpStatus ==
-          TaskLookupResult.LookUpStatus.TASK_FAILING_MEMORY_REQUIREMENT) {
-            return tlr;
-        }
-      }
-
-      // nothing to give
-      return TaskLookupResult.getNoTaskFoundResult();
-    }
-
-
-    /**
-     * Check if maximum-capacity is set  for this queue.
-     * If set and greater than 0 ,
-     * check if numofslotsoccupied+numSlotsPerTask is greater than
-     * maximum-Capacity ,if yes , implies this queue is over limit.
-     *
-     * Incase noOfSlotsOccupied is less than maximum-capacity ,but ,
-     * numOfSlotsOccupied+noSlotsPerTask is more than maximum-capacity we still
-     * dont assign the task . This may lead to under utilization of very small
-     * set of slots. But this is ok ,as we strictly respect the maximum-capacity
-     * @param qsc
-     * @param noOfSlotsPerTask
-     * @return true if queue is over maximum-capacity
-     */
-    private boolean areTasksInQueueOverMaxCapacity(
-      QueueSchedulingContext qsc,int noOfSlotsPerTask) {
-      TaskSchedulingContext tsi = getTSC(qsc);
-      //check for maximum-capacity
-      if(tsi.getMaxCapacity() >= 0) {
-        if ((tsi.getNumSlotsOccupied() + noOfSlotsPerTask) >
-          tsi.getMaxCapacity()) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(
-              "Queue " + qsc.getQueueName() + " " + "has reached its  max " +
-                type + "Capacity");
-            LOG.debug("Current running tasks " + tsi.getCapacity());
-
-          }
-          return true;
-        }
-      }
-      return false;
-    }
-
-
-    // for debugging.
-    private void printQSCs() {
-      if (LOG.isDebugEnabled()) {
-        StringBuffer s = new StringBuffer();
-        for (AbstractQueue aq: getOrderedJobQueues()) {
-          QueueSchedulingContext qsi = aq.getQueueSchedulingContext();
-          TaskSchedulingContext tsi = getTSC(qsi);
-          Collection<JobInProgress> runJobs =
-            scheduler.jobQueuesManager.getJobQueue(qsi.getQueueName())
-              .getRunningJobs();
-          s.append(
-            String.format(
-              " Queue '%s'(%s): runningTasks=%d, "
-                + "occupiedSlots=%d, capacity=%d, runJobs=%d  maximumCapacity=%d ",
-              qsi.getQueueName(),
-              this.type, tsi.getNumRunningTasks(),
-              tsi.getNumSlotsOccupied(), tsi.getCapacity(), (runJobs.size()),
-              tsi.getMaxCapacity()));
-        }
-        LOG.debug(s);
-      }
-    }
-
-    /**
-     * Check if one of the tasks have a speculative task to execute on the
-     * particular task tracker.
-     *
-     * @param tips tasks of a job
-     * @param tts task tracker status for which we are asking speculative tip
-     * @return true if job has a speculative task to run on particular TT.
-     */
-    boolean hasSpeculativeTask(
-      TaskInProgress[] tips,
-      TaskTrackerStatus tts) {
-      long currentTime = System.currentTimeMillis();
-      for(TaskInProgress tip : tips)  {
-        if(tip.isRunning()
-            && !(tip.hasRunOnMachine(tts.getHost(), tts.getTrackerName()))
-            && tip.canBeSpeculated(currentTime)) {
-          return true;
-        }
-      }
-      return false;
-    }
-  }
-
-  /**
-   * The scheduling algorithms for map tasks.
-   */
-  private static class MapSchedulingMgr extends TaskSchedulingMgr {
-
-    MapSchedulingMgr(CapacityTaskScheduler schedulr) {
-      super(schedulr);
-      type = TaskType.MAP;
-      queueComparator = mapComparator;
-    }
-
-    @Override
-    Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
-    throws IOException {
-      synchronized (scheduler) {
-        ClusterStatus clusterStatus = scheduler.taskTrackerManager
-            .getClusterStatus();
-        int numTaskTrackers = clusterStatus.getTaskTrackers();
-        return job.obtainNewMapTask(taskTracker, numTaskTrackers,
-            scheduler.taskTrackerManager.getNumberOfUniqueHosts());
-      }
-    }
-
-    @Override
-    int getClusterCapacity() {
-      synchronized (scheduler) {
-        return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
-      }
-    }
-
-    @Override
-    TaskSchedulingContext getTSC(QueueSchedulingContext qsi) {
-      return qsi.getMapTSC();
-    }
-
-
-    @Override
-    boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
-      //Check if job supports speculative map execution first then
-      //check if job has speculative maps.
-      return (job.getMapSpeculativeExecution()) && (
-          hasSpeculativeTask(job.getTasks(TaskType.MAP),
-                             tts));
-    }
-
-  }
-
-  /**
-   * The scheduling algorithms for reduce tasks.
-   */
-  private static class ReduceSchedulingMgr extends TaskSchedulingMgr {
-
-    ReduceSchedulingMgr(CapacityTaskScheduler schedulr) {
-      super(schedulr);
-      type = TaskType.REDUCE;
-      queueComparator = reduceComparator;
-    }
-
-    @Override
-    Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
-    throws IOException {
-      synchronized (scheduler) {
-        ClusterStatus clusterStatus = scheduler.taskTrackerManager
-            .getClusterStatus();
-        int numTaskTrackers = clusterStatus.getTaskTrackers();
-        return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
-            scheduler.taskTrackerManager.getNumberOfUniqueHosts());
-      }
-    }
-
-    @Override
-    int getClusterCapacity() {
-      synchronized (scheduler) {
-        return scheduler.taskTrackerManager.getClusterStatus()
-            .getMaxReduceTasks();
-      }
-    }
-
-    @Override
-    TaskSchedulingContext getTSC(QueueSchedulingContext qsi) {
-      return qsi.getReduceTSC();
-    }
-
-    @Override
-    boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
-      //check if the job supports reduce speculative execution first then
-      //check if the job has speculative tasks.
-      return (job.getReduceSpeculativeExecution()) && (
-          hasSpeculativeTask(job.getTasks(TaskType.REDUCE),
-                             tts));
-    }
-
-  }
-
-  /** the scheduling mgrs for Map and Reduce tasks */
-  protected TaskSchedulingMgr mapScheduler = new MapSchedulingMgr(this);
-  protected TaskSchedulingMgr reduceScheduler = new ReduceSchedulingMgr(this);
-
-  MemoryMatcher memoryMatcher = new MemoryMatcher();
-
-  static final Log LOG = LogFactory.getLog(CapacityTaskScheduler.class);
-  protected JobQueuesManager jobQueuesManager;
-
-  /** whether scheduler has started or not */
-  private boolean started = false;
-
-  /**
-   * A clock class - can be mocked out for testing.
-   */
-  static class Clock {
-    long getTime() {
-      return System.currentTimeMillis();
-    }
-  }
-
-  private Clock clock;
-  private JobInitializationPoller initializationPoller;
-
-  class CapacitySchedulerQueueRefresher extends QueueRefresher {
-    @Override
-    void refreshQueues(List<JobQueueInfo> newRootQueues)
-        throws Throwable {
-      if (!started) {
-        String msg =
-            "Capacity Scheduler is not in the 'started' state."
-                + " Cannot refresh queues.";
-        LOG.error(msg);
-        throw new IOException(msg);
-      }
-      CapacitySchedulerConf schedConf = new CapacitySchedulerConf();
-      initializeQueues(newRootQueues, schedConf, true);
-      initializationPoller.refreshQueueInfo(schedConf);
-    }
-  }
-
-  public CapacityTaskScheduler() {
-    this(new Clock());
-  }
-  
-  // for testing
-  public CapacityTaskScheduler(Clock clock) {
-    this.jobQueuesManager = new JobQueuesManager();
-    this.clock = clock;
-  }
-
-  @Override
-  QueueRefresher getQueueRefresher() {
-    return new CapacitySchedulerQueueRefresher();
-  }
-
-  /**
-   * Only for testing.
-   * @param type
-   * @return
-   */
-  String[] getOrderedQueues(TaskType type) {
-    if (type == TaskType.MAP) {
-      return mapScheduler.getOrderedQueues();
-    } else if (type == TaskType.REDUCE) {
-      return reduceScheduler.getOrderedQueues();
-    }
-    return null;
-  }
-
-  @Override
-  public synchronized void start() throws IOException {
-    if (started) return;
-    super.start();
-
-    // Initialize MemoryMatcher
-    MemoryMatcher.initializeMemoryRelatedConf(conf);
-
-    // read queue info from config file
-    QueueManager queueManager = taskTrackerManager.getQueueManager();
-
-    // initialize our queues from the config settings
-    CapacitySchedulerConf schedConf = new CapacitySchedulerConf();
-    try {
-      initializeQueues(queueManager.getRoot().getJobQueueInfo().getChildren(),
-          schedConf, false);
-    } catch (Throwable e) {
-      LOG.error("Couldn't initialize queues because of the excecption : "
-          + StringUtils.stringifyException(e));
-      throw new IOException(e);
-    }
-
-    // Queues are ready. Now register jobQueuesManager with the JobTracker so as
-    // to listen to job changes
-    taskTrackerManager.addJobInProgressListener(jobQueuesManager);
-
-    //Start thread for initialization
-    if (initializationPoller == null) {
-      this.initializationPoller = new JobInitializationPoller(
-          jobQueuesManager, taskTrackerManager);
-    }
-    initializationPoller.init(jobQueuesManager.getJobQueueNames(), schedConf);
-    initializationPoller.setDaemon(true);
-    initializationPoller.start();
-
-    started = true;
-
-    LOG.info("Capacity scheduler started successfully");  
-  }
-
-  /**
-   * Read the configuration and initialize the queues. This operation should be
-   * done only when either the scheduler is starting or a request is received
-   * from {@link QueueManager} to refresh the queue configuration.
-   * 
-   * <p>
-   * 
-   * Even in case of refresh, we do not explicitly destroy AbstractQueue items,
-   * or the info maps, they will be automatically garbage-collected.
-   * 
-   * <p>
-   * 
-   * We don't explicitly lock the scheduler completely. This method is called at
-   * two times. 1) When the scheduler is starting. During this time, the lock
-   * sequence is JT->scheduler and so we don't need any more locking here. 2)
-   * When refresh is issued to {@link QueueManager}. When this happens, parallel
-   * refreshes are guarded by {@link QueueManager} itself by taking its lock.
-   * 
-   * @param newRootQueues
-   * @param schedConf
-   * @param refreshingQueues
-   * @throws Throwable
-   */
-  private void initializeQueues(List<JobQueueInfo> newRootQueues,
-      CapacitySchedulerConf schedConf, boolean refreshingQueues)
-      throws Throwable {
-
-    if (newRootQueues == null) {
-      throw new IOException(
-          "Cannot initialize the queues with null root-queues!");
-    }
-
-    // Sanity check: there should be at least one queue.
-    if (0 == newRootQueues.size()) {
-      throw new IllegalStateException("System has no queue configured!");
-    }
-
-    // Create a new queue-hierarchy builder and try loading the complete
-    // hierarchy of queues.
-    AbstractQueue newRootAbstractQueue;
-    try {
-      newRootAbstractQueue =
-          new QueueHierarchyBuilder().createHierarchy(newRootQueues, schedConf);
-
-    } catch (Throwable e) {
-      LOG.error("Exception while tryign to (re)initializing queues : "
-          + StringUtils.stringifyException(e));
-      LOG.info("(Re)initializing the queues with the new configuration "
-          + "failed, so keeping the old configuration.");
-      throw e;
-    }
-
-    // New configuration is successfully validated and applied, set the new
-    // configuration to the current queue-hierarchy.
-
-    if (refreshingQueues) {
-      // Scheduler is being refreshed.
-
-      // Going to commit the changes to the hierarchy. Lock the scheduler.
-      synchronized (this) {
-        AbstractQueueComparator comparator = new AbstractQueueComparator();
-        this.root.sort(comparator);
-        newRootAbstractQueue.sort(comparator);
-        root.validateAndCopyQueueContexts(newRootAbstractQueue);
-      }
-    } else {
-      // Scheduler is just starting.
-
-      this.root = newRootAbstractQueue;
-
-      // JobQueue objects are created. Inform the JobQueuesManager so that it
-      // can track the running/waiting jobs. JobQueuesManager is still not added
-      // as a listener to JobTracker, so no locking needed.
-      addJobQueuesToJobQueuesManager();
-    }
-
-    List<AbstractQueue> allQueues = new ArrayList<AbstractQueue>();
-    allQueues.addAll(getRoot().getDescendantContainerQueues());
-    allQueues.addAll(getRoot().getDescendentJobQueues());
-    for (AbstractQueue queue : allQueues) {
-      if (!refreshingQueues) {
-        // Scheduler is just starting, create the display info also
-        createDisplayInfo(taskTrackerManager.getQueueManager(), queue.getName());
-      }
-
-      // QueueSchedulingContext objects are created/have changed. Put them
-      // (back) in the queue-info so as to be consumed by the UI.
-      addToQueueInfoMap(queue.getQueueSchedulingContext());
-    }
-  }
-
-  /**
-   * Inform the {@link JobQueuesManager} about the newly constructed
-   * {@link JobQueue}s.
-   */
-  private void addJobQueuesToJobQueuesManager() {
-    List<AbstractQueue> allJobQueues = getRoot().getDescendentJobQueues();
-    for (AbstractQueue jobQ : allJobQueues) {
-      jobQueuesManager.addQueue((JobQueue)jobQ);
-    }
-  }
-
-  /** mostly for testing purposes */
-  synchronized void setInitializationPoller(JobInitializationPoller p) {
-    this.initializationPoller = p;
-  }
-  
-  @Override
-  public synchronized void terminate() throws IOException {
-    if (!started) return;
-    if (jobQueuesManager != null) {
-      taskTrackerManager.removeJobInProgressListener(
-          jobQueuesManager);
-    }
-    started = false;
-    initializationPoller.terminate();
-    super.terminate();
-  }
-  
-  @Override
-  public synchronized void setConf(Configuration conf) {
-    super.setConf(conf);
-  }
-
-  /**
-   * provided for the test classes
-   * lets you update the QSI objects and sorted collections
-   */ 
-  synchronized void updateContextInfoForTests() {
-    ClusterStatus c = taskTrackerManager.getClusterStatus();
-    int mapClusterCapacity = c.getMaxMapTasks();
-    int reduceClusterCapacity = c.getMaxReduceTasks();
-    // update the QSI objects
-    updateContextObjects(mapClusterCapacity, reduceClusterCapacity);
-    mapScheduler.scheduler.root.sort(mapScheduler.queueComparator);
-    reduceScheduler.scheduler.root.sort(reduceScheduler.queueComparator);
-  }
-
-  /**
-   * Update individual QSC objects.
-   * We don't need exact information for all variables, just enough for us
-   * to make scheduling decisions. For example, we don't need an exact count
-   * of numRunningTasks. Once we count upto the grid capacity, any
-   * number beyond that will make no difference.
-   *
-   **/
-  private synchronized void updateContextObjects(int mapClusterCapacity,
-      int reduceClusterCapacity) {
-    root.update(mapClusterCapacity,reduceClusterCapacity);
-
-  }
-
-  /*
-   * The grand plan for assigning a task. 
-   * Always assigns 1 reduce and 1 map , if sufficient slots are
-   * available for each of types.
-   * If not , then which ever type of slots are available , that type of task is
-   * assigned.
-   * Next, pick a queue. We only look at queues that need a slot. Among these,
-   * we first look at queues whose (# of running tasks)/capacity is the least.
-   * Next, pick a job in a queue. we pick the job at the front of the queue
-   * unless its user is over the user limit. 
-   * Finally, given a job, pick a task from the job. 
-   *  
-   */
-  @Override
-  public synchronized List<Task> assignTasks(TaskTracker taskTracker)
-  throws IOException {
-    
-    TaskLookupResult tlr;
-    TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
-    List<Task> result = new ArrayList<Task>();
-    
-    /* 
-     * If TT has Map and Reduce slot free, we assign 1 map and 1 reduce
-     * We  base decision on how much is needed
-     * versus how much is used
-     */
-    ClusterStatus c = taskTrackerManager.getClusterStatus();
-    int mapClusterCapacity = c.getMaxMapTasks();
-    int reduceClusterCapacity = c.getMaxReduceTasks();
-    int maxMapSlots = taskTrackerStatus.getMaxMapSlots();
-    int currentMapSlots = taskTrackerStatus.countOccupiedMapSlots();
-    int maxReduceSlots = taskTrackerStatus.getMaxReduceSlots();
-    int currentReduceSlots = taskTrackerStatus.countOccupiedReduceSlots();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("TT asking for task, max maps="
-                + taskTrackerStatus.getMaxMapSlots() + 
-                ", run maps=" + taskTrackerStatus.countMapTasks() + ", max reds=" + 
-                taskTrackerStatus.getMaxReduceSlots() + ", run reds=" + 
-                taskTrackerStatus.countReduceTasks() + ", map cap=" + 
-                mapClusterCapacity + ", red cap = " + 
-                reduceClusterCapacity);
-    }
-
-    /* 
-     * update all our QSC objects.
-     * This involves updating each qsC structure. This operation depends
-     * on the number of running jobs in a queue, and some waiting jobs. If it
-     * becomes expensive, do it once every few heartbeats only.
-     */ 
-    updateContextObjects(mapClusterCapacity, reduceClusterCapacity);
-    // make sure we get our map or reduce scheduling object to update its 
-    // collection of QSC objects too.
-
-    if (maxReduceSlots > currentReduceSlots) {
-      //reduce slot available , try to get a
-      //reduce task
-      tlr = reduceScheduler.assignTasks(taskTracker);
-      if (TaskLookupResult.LookUpStatus.TASK_FOUND == 
-        tlr.getLookUpStatus()) {
-        result.add(tlr.getTask());
-      }
-    }
-
-    if(maxMapSlots > currentMapSlots) {
-      //map slot available , try to get a map task
-      tlr = mapScheduler.assignTasks(taskTracker);
-      if (TaskLookupResult.LookUpStatus.TASK_FOUND == 
-        tlr.getLookUpStatus()) {
-        result.add(tlr.getTask());
-      }
-    }
-    
-    return (result.isEmpty()) ? null : result;
-  }
-
-  
-  @Override
-  public synchronized Collection<JobInProgress> getJobs(String queueName) {
-    Collection<JobInProgress> jobCollection = new ArrayList<JobInProgress>();
-    JobQueue jobQueue = jobQueuesManager.getJobQueue(queueName);
-    if (jobQueue == null) {
-      return jobCollection;
-    }
-    Collection<JobInProgress> runningJobs =
-      jobQueue.getRunningJobs();
-    if (runningJobs != null) {
-      jobCollection.addAll(runningJobs);
-    }
-    Collection<JobInProgress> waitingJobs = 
-      jobQueue.getWaitingJobs();
-    Collection<JobInProgress> tempCollection = new ArrayList<JobInProgress>();
-    if(waitingJobs != null) {
-      tempCollection.addAll(waitingJobs);
-    }
-    tempCollection.removeAll(runningJobs);
-    if(!tempCollection.isEmpty()) {
-      jobCollection.addAll(tempCollection);
-    }
-    return jobCollection;
-  }
-  
-  synchronized JobInitializationPoller getInitializationPoller() {
-    return initializationPoller;
-  }
-
-  private synchronized String getDisplayInfo(String queueName) {
-    QueueSchedulingContext qsi = queueInfoMap.get(queueName);
-    if (null == qsi) {
-      return null;
-    }
-    return qsi.toString();
-  }
-
-  private synchronized void addToQueueInfoMap(QueueSchedulingContext qsc) {
-    queueInfoMap.put(qsc.getQueueName(), qsc);
-  }
-
-  /**
-   * Create the scheduler information and set it in the {@link QueueManager}.
-   * this should be only called when the scheduler is starting.
-   * 
-   * @param queueManager
-   * @param queueName
-   */
-  private void createDisplayInfo(QueueManager queueManager, String queueName) {
-    if (queueManager != null) {
-      SchedulingDisplayInfo schedulingInfo =
-        new SchedulingDisplayInfo(queueName, this);
-      queueManager.setSchedulerInfo(queueName, schedulingInfo);
-    }
-  }
-
-
-  /**
-   * Use for testing purposes.
-   * returns the root
-   * @return
-   */
-  AbstractQueue getRoot() {
-    return this.root;
-  }
-
-
-  /**
-   * This is used for testing purpose only
-   * Dont use this method.
-   * @param rt
-   */
-  void setRoot(AbstractQueue rt) {
-    this.root = rt;
-  }
-
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ContainerQueue.java

@@ -1,193 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Collections;
-
-/**
- * Composite class for Queue hierarchy.
- */
-class ContainerQueue extends AbstractQueue {
-
-  //List of immediate children for this container queue.
-  //Duplicate childrens are not allowed.
-  private  List<AbstractQueue> children;
-  public ContainerQueue(AbstractQueue parent , QueueSchedulingContext qsc) {
-    super(parent,qsc);
-  }
-
-  /**
-   * Update current contexts and update children's contexts
-   * @param mapClusterCapacity
-   * @param reduceClusterCapacity
-   */
-  @Override
-  public void update(int mapClusterCapacity, int reduceClusterCapacity) {
-    super.update(mapClusterCapacity,reduceClusterCapacity);
-    updateChildrenContext();
-  }
-  
-  /**
-   * set normalized capacity values for children.
-   * and update children.
-   */
-  private void updateChildrenContext() {
-    for (AbstractQueue queue : children) {
-      int normalizedMapClusterCapacity = qsc.getMapTSC().getCapacity();
-      int normalizedReduceClusterCapacity = qsc.getReduceTSC().getCapacity();
-
-      //update children context,
-      // normalize mapClusterCapacity,reduceClusterCapacity to the current.
-      queue.update(
-        normalizedMapClusterCapacity, normalizedReduceClusterCapacity);
-
-      //update current TaskSchedulingContext information
-      //At parent level , these information is cumulative of all
-      //children's TSC values.
-      //Typically JobQueue's TSC would change first . so as of now
-      //parental level values would be stale unless we call update , which
-      //happens incase of new heartbeat.
-      //This behaviour shuold be fine , as before assignTask we first update
-      //then sort the whole hierarchy
-      qsc.getMapTSC().update(queue.getQueueSchedulingContext().getMapTSC());
-      qsc.getReduceTSC().update(queue.getQueueSchedulingContext().getReduceTSC());
-    }
-  }
-
-  
-  /**
-   * @param queueComparator
-   */
-  @Override
-  public void sort(Comparator queueComparator) {
-    //sort immediate children
-    Collections.sort(children, queueComparator);
-
-    //recursive sort all children.    
-    for (AbstractQueue child : children) {
-      child.sort(queueComparator);
-    }
-  }
-
-  /**
-   * Returns the sorted order of the leaf level queues.
-   * @return
-   */
-  @Override
-  public List<AbstractQueue> getDescendentJobQueues() {
-    List<AbstractQueue> l = new ArrayList<AbstractQueue>();
-
-    for (AbstractQueue child : children) {
-      l.addAll(child.getDescendentJobQueues());
-    }
-    return l;
-  }
-
-  @Override
-  List<AbstractQueue> getDescendantContainerQueues() {
-    List<AbstractQueue> l = new ArrayList<AbstractQueue>();
-    for (AbstractQueue child : this.getChildren()) {
-      if (child.getChildren() != null && child.getChildren().size() > 0) {
-        l.add(child);
-        l.addAll(child.getDescendantContainerQueues());
-      }
-    }
-    return l;
-  }
-
-  /**
-   * Used for test only.
-   * @return
-   */
-  @Override
-  List<AbstractQueue> getChildren() {
-    return children;
-  }
-
-  @Override
-  public void addChild(AbstractQueue queue) {
-    if (children == null) {
-      children = new ArrayList<AbstractQueue>();
-    }
-    if(children.contains(queue)) {
-      LOG.warn(" The queue " + queue.getName() + " already " +
-        "exists hence ignoring  the current value ");
-      return;
-    }
-    this.children.add(queue);
-  }
-
-
-  /**
-   *
-   */
-  @Override
-  void distributeUnConfiguredCapacity() {
-    List<AbstractQueue> unConfiguredQueues = new ArrayList<AbstractQueue>();
-    float totalCapacity = 0;
-    for (AbstractQueue q : children) {
-      if (q.qsc.getCapacityPercent() == -1) {
-        //Add into unConfigured queue.
-        unConfiguredQueues.add(q);
-      } else {
-        //If capacity is set , then add that to totalCapacity.
-        LOG.info(" the capacity percent of the queue " + q.getName() + "  is " +
-          "" + q.qsc.getCapacityPercent());
-        totalCapacity += q.qsc.getCapacityPercent();
-
-        //As we already know current Capacity percent of this queue
-        //make children distribute unconfigured Capacity.
-        q.distributeUnConfiguredCapacity();
-      }
-    }
-
-    if (!unConfiguredQueues.isEmpty()) {
-      LOG.info("Total capacity to be distributed among the others are  " +
-        "" + (100 - totalCapacity));      
-
-      //We have list of queues at this level which are unconfigured.
-      //100 - totalCapacity is the capacity remaining.
-      //Divide it equally among all the un configured queues.      
-      float capacityShare = (100 - totalCapacity) / unConfiguredQueues.size();
-
-      //We dont have to check for 100 - totalCapacity being -ve , as
-      //we already do it while loading.
-      for (AbstractQueue q : unConfiguredQueues) {
-        if(q.qsc.getMaxCapacityPercent() > 0) {
-          if (q.qsc.getMaxCapacityPercent() < capacityShare) {
-            throw new IllegalStateException(
-              " Capacity share (" + capacityShare + ")for unconfigured queue " +
-                q.getName() +
-                " is greater than its maximum-capacity percentage " +
-                q.qsc.getMaxCapacityPercent());
-          }
-        }
-        q.qsc.setCapacityPercent(capacityShare);
-        LOG.info("Capacity share for un configured queue " + q.getName() + "" +
-          " is " + capacityShare);
-        //we have q's capacity now.
-        //make children also distribute it among themselves.
-        q.distributeUnConfiguredCapacity();
-      }
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java

@@ -1,645 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.Map.Entry;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobQueueJobInProgressListener.JobSchedulingInfo;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * This class asynchronously initializes jobs submitted to the
- * Map/Reduce cluster running with the {@link CapacityTaskScheduler}.
- *
- * <p>
- * The class comprises of a main poller thread, and a set of worker
- * threads that together initialize the jobs. The poller thread periodically
- * looks at jobs submitted to the scheduler, and selects a set of them
- * to be initialized. It passes these to the worker threads for initializing.
- * Each worker thread is configured to look at jobs submitted to a fixed
- * set of queues. It initializes jobs in a round robin manner - selecting
- * the first job in order from each queue ready to be initialized.
- * </p>
- * 
- * <p>
- * An initialized job occupies memory resources on the Job Tracker. Hence,
- * the poller limits the number of jobs initialized at any given time to
- * a configured limit. The limit is specified per user per queue.
- * </p>
- * 
- * <p>
- * However, since a job needs to be initialized before the scheduler can
- * select tasks from it to run, it tries to keep a backlog of jobs 
- * initialized so the scheduler does not need to wait and let empty slots
- * go waste. The core logic of the poller is to pick up the right jobs,
- * which have a good potential to be run next by the scheduler. To do this,
- * it picks up jobs submitted across users and across queues to account
- * both for guaranteed capacities and user limits. It also always initializes
- * high priority jobs, whenever they need to be initialized, even if this
- * means going over the limit for initialized jobs.
- * </p>
- */
-public class JobInitializationPoller extends Thread {
-
-  private static final Log LOG = LogFactory
-      .getLog(JobInitializationPoller.class.getName());
-
-  /*
-   * The poller picks up jobs across users to initialize based on user limits.
-   * Suppose the user limit for a queue is 25%, it means atmost 4 users' jobs
-   * can run together. However, in order to account for jobs from a user that
-   * might complete faster than others, it initializes jobs from an additional
-   * number of users as a backlog. This variable defines the additional
-   * number of users whose jobs can be considered for initializing. 
-   */
-  private static final int MAX_ADDITIONAL_USERS_TO_INIT = 2;
-
-  private JobQueuesManager jobQueueManager;
-  private long sleepInterval;
-  private int poolSize;
-
-  /**
-   * A worker thread that initializes jobs in one or more queues assigned to
-   * it.
-   *
-   * Jobs are initialized in a round robin fashion one from each queue at a
-   * time.
-   */
-  class JobInitializationThread extends Thread {
-
-    private JobInProgress initializingJob;
-
-    private volatile boolean startIniting;
-    private AtomicInteger currentJobCount = new AtomicInteger(0); // number of jobs to initialize
-
-    /**
-     * The hash map which maintains relationship between queue to jobs to
-     * initialize per queue.
-     */
-    private HashMap<String, TreeMap<JobSchedulingInfo, JobInProgress>> jobsPerQueue;
-
-    public JobInitializationThread() {
-      startIniting = true;
-      jobsPerQueue = new HashMap<String, TreeMap<JobSchedulingInfo, JobInProgress>>();
-    }
-
-    @Override
-    public void run() {
-      while (startIniting) {
-        initializeJobs();  
-        try {
-          if (startIniting) {
-            Thread.sleep(sleepInterval);
-          } else {
-            break;
-          }
-        } catch (Throwable t) {
-        }
-      }
-    }
-
-    // The key method that initializes jobs from queues
-    // This method is package-private to allow test cases to call it
-    // synchronously in a controlled manner.
-    void initializeJobs() {
-      // while there are more jobs to initialize...
-      while (currentJobCount.get() > 0) {
-        Set<String> queues = jobsPerQueue.keySet();
-        for (String queue : queues) {
-          JobInProgress job = getFirstJobInQueue(queue);
-          if (job == null) {
-            continue;
-          }
-          LOG.info("Initializing job : " + job.getJobID() + " in AbstractQueue "
-              + job.getProfile().getQueueName() + " For user : "
-              + job.getProfile().getUser());
-          if (startIniting) {
-            setInitializingJob(job);
-            ttm.initJob(job);
-            setInitializingJob(null);
-          } else {
-            break;
-          }
-        }
-      }
-    }
-
-    /**
-     * This method returns the first job in the queue and removes the same.
-     * 
-     * @param queue
-     *          queue name
-     * @return First job in the queue and removes it.
-     */
-    private JobInProgress getFirstJobInQueue(String queue) {
-      TreeMap<JobSchedulingInfo, JobInProgress> jobsList = jobsPerQueue
-          .get(queue);
-      synchronized (jobsList) {
-        if (jobsList.isEmpty()) {
-          return null;
-        }
-        Iterator<JobInProgress> jobIterator = jobsList.values().iterator();
-        JobInProgress job = jobIterator.next();
-        jobIterator.remove();
-        currentJobCount.getAndDecrement();
-        return job;
-      }
-    }
-
-    /*
-     * Test method to check if the thread is currently initialising the job
-     */
-    synchronized JobInProgress getInitializingJob() {
-      return this.initializingJob;
-    }
-    
-    synchronized void setInitializingJob(JobInProgress job) {
-      this.initializingJob  = job;
-    }
-
-    void terminate() {
-      startIniting = false;
-    }
-
-    void addJobsToQueue(String queue, JobInProgress job) {
-      TreeMap<JobSchedulingInfo, JobInProgress> jobs = jobsPerQueue
-          .get(queue);
-      if (jobs == null) {
-        LOG.error("Invalid queue passed to the thread : " + queue
-            + " For job :: " + job.getJobID());
-        return;
-      }
-      synchronized (jobs) {
-        JobSchedulingInfo schedInfo = new JobSchedulingInfo(job);
-        jobs.put(schedInfo, job);
-        currentJobCount.getAndIncrement();
-      }
-    }
-
-    void addQueue(String queue) {
-      TreeMap<JobSchedulingInfo, JobInProgress> jobs = new TreeMap<JobSchedulingInfo, JobInProgress>(
-          jobQueueManager.getComparator(queue));
-      jobsPerQueue.put(queue, jobs);
-    }
-  }
-
-  /**
-   * The queue information class maintains following information per queue:
-   * Maximum users allowed to initialize job in the particular queue. Maximum
-   * jobs allowed to be initialize per user in the queue.
-   * 
-   */
-  private static class QueueInfo {
-    int maxUsersAllowedToInitialize;
-    int maxJobsPerUserToInitialize;
-
-    public QueueInfo(int maxUsersAllowedToInitialize,
-        int maxJobsPerUserToInitialize) {
-      this.maxJobsPerUserToInitialize = maxJobsPerUserToInitialize;
-      this.maxUsersAllowedToInitialize = maxUsersAllowedToInitialize;
-    }
-  }
-
-  /**
-   * Map which contains the configuration used for initializing jobs
-   * in that associated to a particular job queue.
-   */
-  private HashMap<String, QueueInfo> jobQueues;
-
-  /**
-   * Set of jobs which have been passed to Initialization threads.
-   * This is maintained so that we dont call initTasks() for same job twice.
-   */
-  private HashMap<JobID,JobInProgress> initializedJobs;
-
-  private volatile boolean running;
-
-  private TaskTrackerManager ttm;
-  /**
-   * The map which provides information which thread should be used to
-   * initialize jobs for a given job queue.
-   */
-  private HashMap<String, JobInitializationThread> threadsToQueueMap;
-
-  public JobInitializationPoller(
-    JobQueuesManager mgr,
-    TaskTrackerManager ttm) {
-    initializedJobs = new HashMap<JobID,JobInProgress>();
-    jobQueues = new HashMap<String, QueueInfo>();
-    this.jobQueueManager = mgr;
-    threadsToQueueMap = new HashMap<String, JobInitializationThread>();
-    super.setName("JobInitializationPollerThread");
-    running = true;
-    this.ttm = ttm;
-  }
-
-  /*
-   * method to read all configuration values required by the initialisation
-   * poller
-   */
-
-  void init(Set<String> queues, 
-            CapacitySchedulerConf capacityConf) {
-    setupJobInitializerConfiguration(queues, capacityConf);
-    assignThreadsToQueues();
-    Collection<JobInitializationThread> threads = threadsToQueueMap.values();
-    for (JobInitializationThread t : threads) {
-      if (!t.isAlive()) {
-        t.setDaemon(true);
-        t.start();
-      }
-    }
-  }
-
-  /**
-   * Initialize the configuration of the JobInitializer as well as of the specific
-   * queues.
-   * 
-   * @param queues
-   * @param schedulerConf
-   */
-  private void setupJobInitializerConfiguration(Set<String> queues,
-      CapacitySchedulerConf schedulerConf) {
-    for (String queue : queues) {
-      int maxUsersToInitialize = getMaxUsersToInit(schedulerConf, queue);
-      int maxJobsPerUserToInitialize =
-          schedulerConf.getMaxJobsPerUserToInitialize(queue);
-      QueueInfo qi =
-          new QueueInfo(maxUsersToInitialize, maxJobsPerUserToInitialize);
-      jobQueues.put(queue, qi);
-    }
-    sleepInterval = schedulerConf.getSleepInterval();
-    poolSize = schedulerConf.getMaxWorkerThreads();
-    if (poolSize > queues.size()) {
-      poolSize = queues.size();
-    }
-  }
-
-  /**
-   * 
-   * @param schedulerConf
-   * @param queue
-   * @return
-   */
-  private int getMaxUsersToInit(CapacitySchedulerConf schedulerConf,
-      String queue) {
-    int userlimit = schedulerConf.getMinimumUserLimitPercent(queue);
-    return (100 / userlimit) + MAX_ADDITIONAL_USERS_TO_INIT;
-  }
-
-  /**
-   * Refresh the Scheduler configuration cached with the initializer. This
-   * should be called only by
-   * {@link CapacityTaskScheduler.CapacitySchedulerQueueRefresher#refreshQueues()}
-   * . The cached configuration currently is only used by the main thread in the
-   * initializer. So, any updates are picked up automatically by subsequent
-   * iterations of the main thread.
-   */
-  void refreshQueueInfo(CapacitySchedulerConf schedulerConf) {
-    for (String queue : jobQueues.keySet()) {
-      QueueInfo queueInfo = jobQueues.get(queue);
-      synchronized (queueInfo) {
-        queueInfo.maxUsersAllowedToInitialize =
-            getMaxUsersToInit(schedulerConf, queue);
-        queueInfo.maxJobsPerUserToInitialize =
-            schedulerConf.getMaxJobsPerUserToInitialize(queue);
-      }
-    }
-  }
-
-  /**
-   * This is main thread of initialization poller, We essentially do 
-   * following in the main threads:
-   * 
-   * <ol>
-   * <li> Clean up the list of initialized jobs list which poller maintains
-   * </li>
-   * <li> Select jobs to initialize in the polling interval.</li>
-   * </ol>
-   */
-  public void run() {
-    while (running) {
-      try {
-        cleanUpInitializedJobsList();
-        selectJobsToInitialize();
-        if (!this.isInterrupted()) {
-          Thread.sleep(sleepInterval);
-        }
-      } catch (InterruptedException e) {
-        LOG.error("Job Initialization poller interrupted"
-            + StringUtils.stringifyException(e));
-      }
-    }
-  }
-
-  /**
-   * The key method which does selecting jobs to be initalized across 
-   * queues and assign those jobs to their appropriate init-worker threads.
-   * <br/>
-   * This method is overriden in test case which is used to test job
-   * initialization poller.
-   * 
-   */
-  void selectJobsToInitialize() {
-    for (String queue : jobQueues.keySet()) {
-      ArrayList<JobInProgress> jobsToInitialize = getJobsToInitialize(queue);
-      printJobs(jobsToInitialize);
-      JobInitializationThread t = threadsToQueueMap.get(queue);
-      for (JobInProgress job : jobsToInitialize) {
-        t.addJobsToQueue(queue, job);
-      }
-    }
-  }
-
-  /**
-   * Method used to print log statements about which jobs are being
-   * passed to init-threads. 
-   * 
-   * @param jobsToInitialize list of jobs which are passed to be 
-   * init-threads.
-   */
-  private void printJobs(ArrayList<JobInProgress> jobsToInitialize) {
-    for (JobInProgress job : jobsToInitialize) {
-      LOG.info("Passing to Initializer Job Id :" + job.getJobID()
-          + " User: " + job.getProfile().getUser() + " AbstractQueue : "
-          + job.getProfile().getQueueName());
-    }
-  }
-
-  /**
-   * This method exists to be overridden by test cases that wish to
-   * create a test-friendly worker thread which can be controlled
-   * synchronously.
-   * 
-   * @return Instance of worker init-threads.
-   */
-  JobInitializationThread createJobInitializationThread() {
-    return new JobInitializationThread();
-  }
-  
-  /**
-   * Method which is used by the poller to assign appropriate worker thread
-   * to a queue. The number of threads would be always less than or equal
-   * to number of queues in a system. If number of threads is configured to 
-   * be more than number of queues then poller does not create threads more
-   * than number of queues. 
-   * 
-   */
-  private void assignThreadsToQueues() {
-    int countOfQueues = jobQueues.size();
-    String[] queues = (String[]) jobQueues.keySet().toArray(
-        new String[countOfQueues]);
-    int numberOfQueuesPerThread = countOfQueues / poolSize;
-    int numberOfQueuesAssigned = 0;
-    for (int i = 0; i < poolSize; i++) {
-      JobInitializationThread initializer = createJobInitializationThread();
-      int batch = (i * numberOfQueuesPerThread);
-      for (int j = batch; j < (batch + numberOfQueuesPerThread); j++) {
-        initializer.addQueue(queues[j]);
-        threadsToQueueMap.put(queues[j], initializer);
-        numberOfQueuesAssigned++;
-      }
-    }
-
-    if (numberOfQueuesAssigned < countOfQueues) {
-      // Assign remaining queues in round robin fashion to other queues
-      int startIndex = 0;
-      for (int i = numberOfQueuesAssigned; i < countOfQueues; i++) {
-        JobInitializationThread t = threadsToQueueMap
-            .get(queues[startIndex]);
-        t.addQueue(queues[i]);
-        threadsToQueueMap.put(queues[i], t);
-        startIndex++;
-      }
-    }
-  }
-
-  /**
-   * 
-   * Method used to select jobs to be initialized for a given queue. <br/>
-   * 
-   * We want to ensure that enough jobs have been initialized, so that when the
-   * Scheduler wants to consider a new job to run, it's ready. We clearly don't
-   * want to initialize too many jobs as each initialized job has a memory
-   * footprint, sometimes significant.
-   * 
-   * Number of jobs to be initialized is restricted by two values: - Maximum
-   * number of users whose jobs we want to initialize, which is equal to 
-   * the number of concurrent users the queue can support. - Maximum number 
-   * of initialized jobs per user. The product of these two gives us the
-   * total number of initialized jobs.
-   * 
-   * Note that this is a rough number, meant for decreasing extra memory
-   * footprint. It's OK if we go over it once in a while, if we have to.
-   * 
-   * This can happen as follows. Suppose we have initialized 3 jobs for a
-   * user. Now, suppose the user submits a job who's priority is higher than
-   * that of the 3 jobs initialized. This job needs to be initialized, since it
-   * will run earlier than the 3 jobs. We'll now have 4 initialized jobs for the
-   * user. If memory becomes a problem, we should ideally un-initialize one of
-   * the 3 jobs, to keep the count of initialized jobs at 3, but that's
-   * something we don't do for now. This situation can also arise when a new
-   * user submits a high priority job, thus superceeding a user whose jobs have
-   * already been initialized. The latter user's initialized jobs are redundant,
-   * but we'll leave them initialized.
-   * 
-   * @param queue name of the queue to pick the jobs to initialize.
-   * @return list of jobs to be initalized in a queue. An empty queue is
-   *         returned if no jobs are found.
-   */
-  ArrayList<JobInProgress> getJobsToInitialize(String queue) {
-    QueueInfo qi = jobQueues.get(queue);
-    ArrayList<JobInProgress> jobsToInitialize = new ArrayList<JobInProgress>();
-    // use the configuration parameter which is configured for the particular
-    // queue.
-    int maximumUsersAllowedToInitialize;
-    int maxJobsPerUserAllowedToInitialize;
-    synchronized (qi) {
-      maximumUsersAllowedToInitialize = qi.maxUsersAllowedToInitialize;
-      maxJobsPerUserAllowedToInitialize = qi.maxJobsPerUserToInitialize;
-    }
-    int maxJobsPerQueueToInitialize = maximumUsersAllowedToInitialize
-        * maxJobsPerUserAllowedToInitialize;
-    int countOfJobsInitialized = 0;
-    HashMap<String, Integer> userJobsInitialized = new HashMap<String, Integer>();
-    Collection<JobInProgress> jobs = jobQueueManager.getJobQueue(queue).getWaitingJobs();
-    /*
-     * Walk through the collection of waiting jobs.
-     *  We maintain a map of jobs that have already been initialized. If a 
-     *  job exists in that map, increment the count for that job's user 
-     *  and move on to the next job.
-     *   
-     *  If the job doesn't exist, see whether we  want to initialize it. 
-     *  We initialize it if: - at least one job of the user has already 
-     *  been initialized, but the user's total initialized jobs are below 
-     *  the limit, OR - this is a new user, and we haven't reached the limit
-     *  for the number of users whose jobs we want to initialize. We break 
-     *  when we've reached the limit of maximum jobs to initialize.
-     */
-    for (JobInProgress job : jobs) {
-      String user = job.getProfile().getUser();
-      int numberOfJobs = userJobsInitialized.get(user) == null ? 0
-          : userJobsInitialized.get(user);
-      // If the job is already initialized then add the count against user
-      // then continue.
-      if (initializedJobs.containsKey(job.getJobID())) {
-        userJobsInitialized.put(user, Integer.valueOf(numberOfJobs + 1));
-        countOfJobsInitialized++;
-        continue;
-      }
-      boolean isUserPresent = userJobsInitialized.containsKey(user);
-      if (!isUserPresent
-          && userJobsInitialized.size() < maximumUsersAllowedToInitialize) {
-        // this is a new user being considered and the number of users
-        // is within limits.
-        userJobsInitialized.put(user, Integer.valueOf(numberOfJobs + 1));
-        jobsToInitialize.add(job);
-        initializedJobs.put(job.getJobID(),job);
-        countOfJobsInitialized++;
-      } else if (isUserPresent
-          && numberOfJobs < maxJobsPerUserAllowedToInitialize) {
-        userJobsInitialized.put(user, Integer.valueOf(numberOfJobs + 1));
-        jobsToInitialize.add(job);
-        initializedJobs.put(job.getJobID(),job);
-        countOfJobsInitialized++;
-      }
-      /*
-       * if the maximum number of jobs to initalize for a queue is reached
-       * then we stop looking at further jobs. The jobs beyond this number
-       * can be initialized.
-       */
-      if(countOfJobsInitialized > maxJobsPerQueueToInitialize) {
-        break;
-      }
-    }
-    return jobsToInitialize;
-  }
-
-
-  /**
-   * Method which is used internally to clean up the initialized jobs
-   * data structure which the job initialization poller uses to check
-   * if a job is initalized or not.
-   * 
-   * Algorithm for cleaning up task is as follows:
-   * 
-   * <ul>
-   * <li> For jobs in <b>initalizedJobs</b> list </li>
-   * <ul>
-   * <li> If job is running</li>
-   * <ul>
-   * <li> If job is scheduled then remove the job from the waiting queue 
-   * of the scheduler and <b>initalizedJobs</b>.<br/>
-   *  The check for a job is scheduled or not is done by following 
-   *  formulae:<br/> 
-   *  if pending <i>task</i> &lt; desired <i>task</i> then scheduled else
-   *  not scheduled.<br/>
-   *  The formulae would return <i>scheduled</i> if one task has run or failed,
-   *  any cases in which there has been a failure but not enough to mark task 
-   *  as failed, we return <i>not scheduled</i> in formulae.
-   * </li>
-   * </ul>
-   * 
-   * <li> If job is complete, then remove the job from <b>initalizedJobs</b>.
-   * </li>
-   * 
-   * </ul>
-   * </ul>
-   * 
-   */
-  void cleanUpInitializedJobsList() {
-    Iterator<Entry<JobID, JobInProgress>> jobsIterator = 
-      initializedJobs.entrySet().iterator();
-    while(jobsIterator.hasNext()) {
-      Entry<JobID,JobInProgress> entry = jobsIterator.next();
-      JobInProgress job = entry.getValue();
-      if (job.getStatus().getRunState() == JobStatus.RUNNING) {
-        if (isScheduled(job)) {
-          LOG.info("Removing scheduled jobs from waiting queue"
-              + job.getJobID());
-          jobsIterator.remove();
-          jobQueueManager.getJobQueue(job).removeWaitingJob(new JobSchedulingInfo(job));
-          continue;
-        }
-      }
-      if(job.isComplete()) {
-        LOG.info("Removing killed/completed job from initalized jobs " +
-        		"list : "+ job.getJobID());
-        jobsIterator.remove();
-      }
-    }
-  }
-
-  /**
-   * Convenience method to check if job has been scheduled or not.
-   * 
-   * The method may return false in case of job which has failure but
-   * has not failed the tip.
-   * @param job
-   * @return
-   */
-  private boolean isScheduled(JobInProgress job) {
-    return ((job.pendingMaps() < job.desiredMaps()) 
-        || (job.pendingReduces() < job.desiredReduces()));
-  }
-
-  void terminate() {
-    running = false;
-    for (Entry<String, JobInitializationThread> entry : threadsToQueueMap
-        .entrySet()) {
-      JobInitializationThread t = entry.getValue();
-      if (t.isAlive()) {
-        t.terminate();
-        t.interrupt();
-      }
-    }
-  }
-
-  /*
-   * Test method used only for testing purposes.
-   */
-  JobInProgress getInitializingJob(String queue) {
-    JobInitializationThread t = threadsToQueueMap.get(queue);
-    if (t == null) {
-      return null;
-    } else {
-      return t.getInitializingJob();
-    }
-  }
-
-  Set<JobID> getInitializedJobList() {
-    return initializedJobs.keySet();
-  }
-
-  public HashMap<String, JobInitializationThread> getThreadsToQueueMap() {
-    return threadsToQueueMap;
-  }
-  
-  public long getSleepInterval(){
-    return sleepInterval;
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java

@@ -1,458 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapred.JobQueueJobInProgressListener.JobSchedulingInfo;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- *
- */
-class JobQueue extends AbstractQueue {
-
-  static final Log LOG = LogFactory.getLog(JobQueue.class);
-
-  public JobQueue(AbstractQueue parent, QueueSchedulingContext qsc) {
-    super(parent, qsc);
-    if (qsc.supportsPriorities()) {
-      // use the default priority-aware comparator
-      comparator = JobQueueJobInProgressListener.FIFO_JOB_QUEUE_COMPARATOR;
-    } else {
-      comparator = STARTTIME_JOB_COMPARATOR;
-    }
-    waitingJobs =
-      new
-        TreeMap<JobSchedulingInfo, JobInProgress>(
-        comparator);
-    runningJobs =               
-      new
-        TreeMap<JobSchedulingInfo, JobInProgress>(
-        comparator);
-  }
-
-
-  /*
-  * If a queue supports priorities, jobs must be
-  * sorted on priorities, and then on their start times (technically,
-  * their insertion time.
-  * If a queue doesn't support priorities, jobs are
-  * sorted based on their start time.
-  */
-  static final Comparator<JobSchedulingInfo>
-    STARTTIME_JOB_COMPARATOR;
-
-  static {
-    STARTTIME_JOB_COMPARATOR =
-      new Comparator<JobSchedulingInfo>() {
-        // comparator for jobs in queues that don't support priorities
-        public int compare(
-          JobSchedulingInfo o1,
-          JobSchedulingInfo o2) {
-          // the job that started earlier wins
-          if (o1.getStartTime() < o2.getStartTime()) {
-            return -1;
-          } else {
-            return (o1.getStartTime() == o2.getStartTime()
-              ? o1.getJobID().compareTo(o2.getJobID())
-              : 1);
-          }
-        }
-      };
-  }
-
-
-  /**
-   * This involves updating each qC structure.
-   *
-   * @param mapClusterCapacity
-   * @param reduceClusterCapacity
-   */
-  @Override
-  public void update(int mapClusterCapacity, int reduceClusterCapacity) {
-    super.update(mapClusterCapacity, reduceClusterCapacity);
-    for (JobInProgress j :
-      this.getRunningJobs()) {
-      updateStatsOnRunningJob(qsc, j);
-    }
-  }
-
-  private void updateStatsOnRunningJob(
-    QueueSchedulingContext qC, JobInProgress j) {
-    if (j.getStatus().getRunState() != JobStatus.RUNNING) {
-      return;
-    }
-
-    TaskSchedulingContext mapTSI = qC.getMapTSC();
-    TaskSchedulingContext reduceTSI = qC.getReduceTSC();
-
-    int numMapsRunningForThisJob = j.runningMaps();
-    int numReducesRunningForThisJob = j.runningReduces();
-    TaskDataView mapScheduler = TaskDataView.getTaskDataView(TaskType.MAP);
-    TaskDataView reduceScheduler = 
-        TaskDataView.getTaskDataView(TaskType.REDUCE);
-    int numRunningMapSlots =
-      numMapsRunningForThisJob * mapScheduler.getSlotsPerTask(j);
-    int numRunningReduceSlots =
-      numReducesRunningForThisJob * reduceScheduler.getSlotsPerTask(j);
-    int numMapSlotsForThisJob = mapScheduler.getSlotsOccupied(j);
-    int numReduceSlotsForThisJob = reduceScheduler.getSlotsOccupied(j);
-    int numReservedMapSlotsForThisJob =
-      (mapScheduler.getNumReservedTaskTrackers(j) *
-        mapScheduler.getSlotsPerTask(j));
-    int numReservedReduceSlotsForThisJob =
-      (reduceScheduler.getNumReservedTaskTrackers(j) *
-        reduceScheduler.getSlotsPerTask(j));
-
-    j.setSchedulingInfo
-      (getJobQueueSchedInfo(numMapsRunningForThisJob, numRunningMapSlots,
-                            numReservedMapSlotsForThisJob,
-                            numReducesRunningForThisJob,
-                            numRunningReduceSlots,
-                            numReservedReduceSlotsForThisJob));
-
-
-    mapTSI.setNumRunningTasks(
-      mapTSI.getNumRunningTasks() + numMapsRunningForThisJob);
-    reduceTSI.setNumRunningTasks(
-      reduceTSI.getNumRunningTasks() + numReducesRunningForThisJob);
-    mapTSI.setNumSlotsOccupied(
-      mapTSI.getNumSlotsOccupied() + numMapSlotsForThisJob);
-    reduceTSI.setNumSlotsOccupied(
-      reduceTSI.getNumSlotsOccupied() + numReduceSlotsForThisJob);
-    Integer i =
-      mapTSI.getNumSlotsOccupiedByUser().get(
-        j.getProfile().getUser());
-    mapTSI.getNumSlotsOccupiedByUser().put(
-      j.getProfile().getUser(),
-      i.intValue() + numMapSlotsForThisJob);
-    i = reduceTSI.getNumSlotsOccupiedByUser().get(
-      j.getProfile().getUser());
-    reduceTSI.getNumSlotsOccupiedByUser().put(
-      j.getProfile().getUser(),
-      i.intValue() + numReduceSlotsForThisJob);
-    if (LOG.isDebugEnabled()) {
-      synchronized (j) {
-        LOG.debug(String.format("updateQSI: job %s: run(m)=%d, "
-            + "occupied(m)=%d, run(r)=%d, occupied(r)=%d, finished(m)=%d,"
-            + " finished(r)=%d, failed(m)=%d, failed(r)=%d, "
-            + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j.getJobID()
-            .toString(), numMapsRunningForThisJob, numMapSlotsForThisJob,
-            numReducesRunningForThisJob, numReduceSlotsForThisJob, j
-                .finishedMaps(), j.finishedReduces(), j.failedMapTasks,
-            j.failedReduceTasks, j.speculativeMapTasks,
-            j.speculativeReduceTasks, j.numMapTasks, j.numReduceTasks));
-      }
-    }
-
-    /*
-    * it's fine walking down the entire list of running jobs - there
-  * probably will not be many, plus, we may need to go through the
-  * list to compute numSlotsOccupiedByUser. If this is expensive, we
-  * can keep a list of running jobs per user. Then we only need to
-  * consider the first few jobs per user.
-  */
-  }
- 
-  private static final int JOBQUEUE_SCHEDULINGINFO_INITIAL_LENGTH = 175;
-  
-  static String getJobQueueSchedInfo
-    (int numMapsRunningForThisJob, 
-     int numRunningMapSlots, int numReservedMapSlotsForThisJob, 
-     int numReducesRunningForThisJob, int numRunningReduceSlots, 
-     int numReservedReduceSlotsForThisJob) {
-    StringBuilder sb = new StringBuilder(JOBQUEUE_SCHEDULINGINFO_INITIAL_LENGTH);
-    sb.append(numMapsRunningForThisJob).append(" running map tasks using ")
-      .append(numRunningMapSlots).append(" map slots. ")
-      .append(numReservedMapSlotsForThisJob).append(" additional slots reserved. ")
-      .append(numReducesRunningForThisJob).append(" running reduce tasks using ")
-      .append(numRunningReduceSlots).append(" reduce slots. ")
-      .append(numReservedReduceSlotsForThisJob).append(" additional slots reserved.");
-    return sb.toString();
-  }
-
-
-  Map<JobSchedulingInfo, JobInProgress>
-    waitingJobs; // for waiting jobs
-  Map<JobSchedulingInfo, JobInProgress>
-    runningJobs; // for running jobs
-
-  public Comparator<JobSchedulingInfo>
-    comparator;
-
-  Collection<JobInProgress> getWaitingJobs() {
-    synchronized (waitingJobs) {
-      return Collections.unmodifiableCollection(
-        new LinkedList<JobInProgress>(waitingJobs.values()));
-    }
-  }
-
-  Collection<JobInProgress> getRunningJobs() {
-    synchronized (runningJobs) {
-      return Collections.unmodifiableCollection(
-        new LinkedList<JobInProgress>(runningJobs.values()));
-    }
-  }
-
-  private void addRunningJob(JobInProgress job) {
-    synchronized (runningJobs) {
-      runningJobs.put(
-        new JobSchedulingInfo(
-          job), job);
-    }
-  }
-
-  private JobInProgress removeRunningJob(
-    JobSchedulingInfo jobInfo) {
-    synchronized (runningJobs) {
-      return runningJobs.remove(jobInfo);
-    }
-  }
-
-  JobInProgress removeWaitingJob(
-    JobSchedulingInfo schedInfo) {
-    synchronized (waitingJobs) {
-      JobInProgress jip = waitingJobs.remove(schedInfo);
-      this.qsc.setNumOfWaitingJobs(waitingJobs.size());
-      return jip;
-    }
-  }
-
-  private void addWaitingJob(JobInProgress job) {
-    synchronized (waitingJobs) {
-      waitingJobs.put(
-        new JobSchedulingInfo(
-          job), job);
-      this.qsc.setNumOfWaitingJobs(waitingJobs.size());
-    }
-  }
-
-  int getWaitingJobCount() {
-    synchronized (waitingJobs) {
-      return waitingJobs.size();
-    }
-  }
-
-  // called when a job is added
-  synchronized void jobAdded(JobInProgress job) throws IOException {
-    // add job to waiting queue. It will end up in the right place,
-    // based on priority. 
-    addWaitingJob(job);
-    // update user-specific info
-    Integer i = qsc.getNumJobsByUser().get(job.getProfile().getUser());
-    if (null == i) {
-      i = 1;
-      // set the count for running tasks to 0
-      qsc.getMapTSC().getNumSlotsOccupiedByUser().put(
-        job.getProfile().getUser(),
-        0);
-      qsc.getReduceTSC().getNumSlotsOccupiedByUser().
-        put(
-          job.getProfile().getUser(),
-          0);
-    } else {
-      i++;
-    }
-    qsc.getNumJobsByUser().put(job.getProfile().getUser(), i);
-
-    // setup scheduler specific job information
-    preInitializeJob(job);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Job " + job.getJobID().toString() + " is added under user "
-                + job.getProfile().getUser() + ", user now has " + i + " jobs");
-    }
-  }
-
-
-  /**
-   * Setup {@link CapacityTaskScheduler} specific information prior to
-   * job initialization.
-   * <p/>
-   * TO DO: Currently this method uses , CapacityTaskScheduler based variables
-   * need to shift those.
-   */
-  void preInitializeJob(JobInProgress job) {
-    JobConf jobConf = job.getJobConf();
-
-    // Compute number of slots required to run a single map/reduce task
-    int slotsPerMap = 1;
-    int slotsPerReduce = 1;
-    if (MemoryMatcher.isSchedulingBasedOnMemEnabled()) {
-      slotsPerMap = jobConf.computeNumSlotsPerMap(
-        MemoryMatcher.getMemSizeForMapSlot());
-      slotsPerReduce =
-        jobConf.computeNumSlotsPerReduce(
-          MemoryMatcher.getMemSizeForReduceSlot());
-    }
-    job.setNumSlotsPerMap(slotsPerMap);
-    job.setNumSlotsPerReduce(slotsPerReduce);
-  }
-
-  // called when a job completes
-  synchronized void jobCompleted(JobInProgress job) {
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Job to be removed for user " + job.getProfile().getUser());
-    }
-    Integer i = qsc.getNumJobsByUser().get(job.getProfile().getUser());
-    i--;
-    if (0 == i.intValue()) {
-      qsc.getNumJobsByUser().remove(job.getProfile().getUser());
-      // remove job footprint from our TSIs
-      qsc.getMapTSC().getNumSlotsOccupiedByUser().remove(
-        job.getProfile().getUser());
-      qsc.getReduceTSC().getNumSlotsOccupiedByUser().remove(
-        job.getProfile().getUser());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("No more jobs for user, number of users = "
-                  + qsc.getNumJobsByUser().size());
-      }
-    } else {
-      qsc.getNumJobsByUser().put(job.getProfile().getUser(), i);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("User still has " + i + " jobs, number of users = "
-                  + qsc.getNumJobsByUser().size());
-      }
-    }
-  }
-
-  // This is used to reposition a job in the queue. A job can get repositioned
-  // because of the change in the job priority or job start-time.
-  private void reorderJobs(
-    JobInProgress job, JobSchedulingInfo oldInfo
-  ) {
-
-    if (removeWaitingJob(oldInfo) != null) {
-      addWaitingJob(job);
-    }
-    if (removeRunningJob(oldInfo) != null) {
-      addRunningJob(job);
-    }
-  }
-
-  /**
-   * @return
-   */
-  @Override
-  List<AbstractQueue> getDescendentJobQueues() {
-    List<AbstractQueue> l = new ArrayList<AbstractQueue>();
-    l.add(this);
-    return l;
-  }
-
-  @Override
-  List<AbstractQueue> getDescendantContainerQueues() {
-    return new ArrayList<AbstractQueue>();
-  }
-
-  public void jobUpdated(JobChangeEvent event) {
-    // Check if this is the status change
-    if (event instanceof JobStatusChangeEvent) {
-      jobStateChanged((JobStatusChangeEvent) event);
-    }
-  }
-
-  /**
-   * @return
-   */
-  @Override
-  List<AbstractQueue> getChildren() {
-    return null;
-  }
-
-  /**
-   * Dont do anything in sort , this is leaf level queue.
-   *
-   * @param queueComparator
-   */
-  @Override
-  public void sort(Comparator queueComparator) {
-    return;
-  }
-
-  // Update the scheduler as job's state has changed
-  private void jobStateChanged(JobStatusChangeEvent event) {
-    JobInProgress job = event.getJobInProgress();
-    JobSchedulingInfo oldJobStateInfo =
-      new JobSchedulingInfo(event.getOldStatus());
-    // Check if the ordering of the job has changed
-    // For now priority and start-time can change the job ordering
-    if (event.getEventType() == JobStatusChangeEvent.EventType.PRIORITY_CHANGED
-      || event.getEventType() ==
-      JobStatusChangeEvent.EventType.START_TIME_CHANGED) {
-      // Make a priority change
-      reorderJobs(job, oldJobStateInfo);
-    } else if (event.getEventType() ==
-      JobStatusChangeEvent.EventType.RUN_STATE_CHANGED) {
-      // Check if the job is complete
-      int runState = job.getStatus().getRunState();
-      if (runState == JobStatus.SUCCEEDED
-        || runState == JobStatus.FAILED
-        || runState == JobStatus.KILLED) {
-        jobCompleted(job, oldJobStateInfo);
-      } else if (runState == JobStatus.RUNNING) {
-        // Removing of the job from job list is responsibility of the
-        //initialization poller.
-        // Add the job to the running queue
-        addRunningJob(job);
-      }
-    }
-  }
-
-  /*
-  * Method removes the jobs from both running and waiting job queue in
-  * job queue manager.
-  */
-  private void jobCompleted(
-    JobInProgress job, JobSchedulingInfo oldInfo
-  ) {
-    LOG.info(
-      "Job " + job.getJobID().toString() + " submitted to queue "
-        + job.getProfile().getQueueName() + " has completed");
-    //remove jobs from both queue's a job can be in
-    //running and waiting queue at the same time.
-    removeRunningJob(oldInfo);
-    removeWaitingJob(oldInfo);
-    // let scheduler know
-    jobCompleted(job);
-  }
-
-  @Override
-  public void addChild(AbstractQueue queue) {
-    throw new UnsupportedOperationException(
-      "addChildren is not allowed for " +
-        "" + getName());
-  }
-
-  @Override
-  void distributeUnConfiguredCapacity() {
-    return;
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueuesManager.java

@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobQueueJobInProgressListener.JobSchedulingInfo;
-
-/**
- * A {@link JobInProgressListener} that maintains the jobs being managed in
- * one or more queues.
- */
-class JobQueuesManager extends JobInProgressListener {
-
-  // we maintain a hashmap of queue-names to queue info
-  private Map<String, JobQueue> jobQueues =
-    new HashMap<String, JobQueue>();
-  private static final Log LOG = LogFactory.getLog(JobQueuesManager.class);
-
-
-  JobQueuesManager() {
-  }
-
-  /**
-   * Add the given queue to the map of queue name to job-queues.
-   * 
-   * @param queue The job-queue
-   */
-  public void addQueue(JobQueue queue) {
-    jobQueues.put(queue.getName(),queue);
-  }
-
-  @Override
-  public void jobAdded(JobInProgress job) throws IOException {
-    LOG.info("Job " + job.getJobID() + " submitted to queue "
-        + job.getProfile().getQueueName());
-    // add job to the right queue
-    JobQueue qi = getJobQueue(job.getProfile().getQueueName());
-    if (null == qi) {
-      // job was submitted to a queue we're not aware of
-      LOG.warn(
-        "Invalid queue " + job.getProfile().getQueueName() +
-          " specified for job " + job.getProfile().getJobID() +
-          ". Ignoring job.");
-      return;
-    }
-    // let scheduler know. 
-    qi.jobAdded(job);
-  }
-
-  // Note that job is removed when the job completes i.e in jobUpated()
-  @Override
-  public void jobRemoved(JobInProgress job) {
-  }
-
-
-  @Override
-  public void jobUpdated(JobChangeEvent event) {
-    JobInProgress job = event.getJobInProgress();
-    JobQueue qi = getJobQueue(job.getProfile().getQueueName());
-    qi.jobUpdated(event);
-
-  }
-
-  Comparator<JobSchedulingInfo> getComparator(String queue) {
-    return getJobQueue(queue).comparator;
-  }
-
-
-  public JobQueue getJobQueue(JobInProgress jip){
-    return getJobQueue(jip.getProfile().getQueueName());   
-  }
-
-  JobQueue getJobQueue(String name) {
-    return jobQueues.get(name);
-  }
-
-  public Set<String> getJobQueueNames() {
-    return jobQueues.keySet();
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/MemoryMatcher.java

@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.conf.Configuration;
-
-class MemoryMatcher {
-
-  private static final Log LOG = LogFactory.getLog(MemoryMatcher.class);
-  static long memSizeForMapSlotOnJT = JobConf.DISABLED_MEMORY_LIMIT;
-  static long memSizeForReduceSlotOnJT = JobConf.DISABLED_MEMORY_LIMIT;
-  static long limitMaxMemForMapTasks = JobConf.DISABLED_MEMORY_LIMIT;
-  static long limitMaxMemForReduceTasks = JobConf.DISABLED_MEMORY_LIMIT;
-
-
-  public MemoryMatcher() {
-  }
-
-  /**
-   * Find the memory that is already used by all the running tasks
-   * residing on the given TaskTracker.
-   * 
-   * @param taskTracker
-   * @param taskType 
-   * @return amount of memory that is used by the residing tasks,
-   *          null if memory cannot be computed for some reason.
-   */
-  synchronized Long getMemReservedForTasks(
-      TaskTrackerStatus taskTracker, TaskType taskType) {
-    long vmem = 0;
-
-    for (TaskStatus task : taskTracker.getTaskReports()) {
-      // the following task states are one in which the slot is
-      // still occupied and hence memory of the task should be
-      // accounted in used memory.
-      if ((task.getRunState() == TaskStatus.State.RUNNING) ||
-          (task.getRunState() == TaskStatus.State.UNASSIGNED) ||
-          (task.inTaskCleanupPhase())) {
-        // Get the memory "allotted" for this task based on number of slots
-        long myVmem = 0;
-        if (task.getIsMap() && taskType == TaskType.MAP) {
-          long memSizePerMapSlot = getMemSizeForMapSlot();
-          myVmem = 
-            memSizePerMapSlot * task.getNumSlots();
-        } else if (!task.getIsMap()
-            && taskType == TaskType.REDUCE) {
-          long memSizePerReduceSlot = getMemSizeForReduceSlot();
-          myVmem = memSizePerReduceSlot * task.getNumSlots();
-        }
-        vmem += myVmem;
-      }
-    }
-
-    return Long.valueOf(vmem);
-  }
-
-  /**
-   * Check if a TT has enough memory to run of task specified from this job.
-   * @param job
-   * @param taskType 
-   * @param taskTracker
-   * @return true if this TT has enough memory for this job. False otherwise.
-   */
-  boolean matchesMemoryRequirements(JobInProgress job,TaskType taskType, 
-                                    TaskTrackerStatus taskTracker) {
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Matching memory requirements of " + job.getJobID().toString()
-                + " for scheduling on " + taskTracker.trackerName);
-    }
-
-    if (!isSchedulingBasedOnMemEnabled()) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Scheduling based on job's memory requirements is disabled."
-                  + " Ignoring any value set by job.");
-      }
-      return true;
-    }
-
-    Long memUsedOnTT = getMemReservedForTasks(taskTracker, taskType);
-    long totalMemUsableOnTT = 0;
-    long memForThisTask = 0;
-    if (taskType == TaskType.MAP) {
-      memForThisTask = job.getMemoryForMapTask();
-      totalMemUsableOnTT =
-          getMemSizeForMapSlot() * taskTracker.getMaxMapSlots();
-    } else if (taskType == TaskType.REDUCE) {
-      memForThisTask = job.getMemoryForReduceTask();
-      totalMemUsableOnTT =
-          getMemSizeForReduceSlot()
-              * taskTracker.getMaxReduceSlots();
-    }
-
-    long freeMemOnTT = totalMemUsableOnTT - memUsedOnTT.longValue();
-    if (memForThisTask > freeMemOnTT) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("memForThisTask (" + memForThisTask + ") > freeMemOnTT ("
-                  + freeMemOnTT + "). A " + taskType + " task from "
-                  + job.getJobID().toString() + " cannot be scheduled on TT "
-                  + taskTracker.trackerName);
-      }
-      return false;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("memForThisTask = " + memForThisTask + ". freeMemOnTT = "
-                + freeMemOnTT + ". A " + taskType.toString() + " task from "
-                + job.getJobID().toString() + " matches memory requirements "
-                + "on TT "+ taskTracker.trackerName);
-    }
-    return true;
-  }
-
-  static boolean isSchedulingBasedOnMemEnabled() {
-    if (getLimitMaxMemForMapSlot()
-                                  == JobConf.DISABLED_MEMORY_LIMIT
-        || getLimitMaxMemForReduceSlot()
-                                  == JobConf.DISABLED_MEMORY_LIMIT
-        || getMemSizeForMapSlot()
-                                  == JobConf.DISABLED_MEMORY_LIMIT
-        || getMemSizeForReduceSlot()
-                                  == JobConf.DISABLED_MEMORY_LIMIT) {
-      return false;
-    }
-    return true;
-  }
-
-  public static void initializeMemoryRelatedConf(Configuration conf) {
-    //handling @deprecated
-    if (conf.get(
-      CapacitySchedulerConf.DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY) !=
-      null) {
-      LOG.warn(
-        JobConf.deprecatedString(
-          CapacitySchedulerConf.DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY));
-    }
-
-    //handling @deprecated
-    if (conf.get(CapacitySchedulerConf.UPPER_LIMIT_ON_TASK_PMEM_PROPERTY) !=
-      null) {
-      LOG.warn(
-        JobConf.deprecatedString(
-          CapacitySchedulerConf.UPPER_LIMIT_ON_TASK_PMEM_PROPERTY));
-    }
-
-    if (conf.get(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY) != null) {
-      LOG.warn(
-        JobConf.deprecatedString(
-          JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY));
-    }
-
-    memSizeForMapSlotOnJT =
-        JobConf.normalizeMemoryConfigValue(conf.getLong(
-            MRConfig.MAPMEMORY_MB, JobConf.DISABLED_MEMORY_LIMIT));
-    memSizeForReduceSlotOnJT =
-        JobConf.normalizeMemoryConfigValue(conf.getLong(
-            MRConfig.REDUCEMEMORY_MB,
-            JobConf.DISABLED_MEMORY_LIMIT));
-
-    //handling @deprecated values
-    if (conf.get(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY) != null) {
-      LOG.warn(
-        JobConf.deprecatedString(
-          JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY)+
-          " instead use " + JTConfig.JT_MAX_MAPMEMORY_MB +
-          " and " + JTConfig.JT_MAX_REDUCEMEMORY_MB
-      );
-
-      limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
-        JobConf.normalizeMemoryConfigValue(
-          conf.getLong(
-            JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY,
-            JobConf.DISABLED_MEMORY_LIMIT));
-      if (limitMaxMemForMapTasks != JobConf.DISABLED_MEMORY_LIMIT &&
-        limitMaxMemForMapTasks >= 0) {
-        limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
-          limitMaxMemForMapTasks /
-            (1024 * 1024); //Converting old values in bytes to MB
-      }
-    } else {
-      limitMaxMemForMapTasks =
-        JobConf.normalizeMemoryConfigValue(
-          conf.getLong(
-            JTConfig.JT_MAX_MAPMEMORY_MB, JobConf.DISABLED_MEMORY_LIMIT));
-      limitMaxMemForReduceTasks =
-        JobConf.normalizeMemoryConfigValue(
-          conf.getLong(
-            JTConfig.JT_MAX_REDUCEMEMORY_MB, JobConf.DISABLED_MEMORY_LIMIT));
-    }
-    LOG.info(String.format("Scheduler configured with "
-        + "(memSizeForMapSlotOnJT, memSizeForReduceSlotOnJT, "
-        + "limitMaxMemForMapTasks, limitMaxMemForReduceTasks)"
-        + " (%d,%d,%d,%d)", Long.valueOf(memSizeForMapSlotOnJT), Long
-        .valueOf(memSizeForReduceSlotOnJT), Long
-        .valueOf(limitMaxMemForMapTasks), Long
-        .valueOf(limitMaxMemForReduceTasks)));
-  }
-
-  static long  getMemSizeForMapSlot() {
-    return memSizeForMapSlotOnJT;
-  }
-
-  static long getMemSizeForReduceSlot() {
-    return memSizeForReduceSlotOnJT;
-  }
-
-  static long getLimitMaxMemForMapSlot() {
-    return limitMaxMemForMapTasks;
-  }
-
-  static long getLimitMaxMemForReduceSlot() {
-    return limitMaxMemForReduceTasks;
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueHierarchyBuilder.java

@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.List;
-import java.util.Properties;
-
-/**
- * Hierarchy builder for the CapacityScheduler.
- * 
- */
-class QueueHierarchyBuilder {
-
-  static final Log LOG = LogFactory.getLog(QueueHierarchyBuilder.class);
-  
-  QueueHierarchyBuilder() {
-  }
-
-  /**
-   * Create a new {@link AbstractQueue}s-hierarchy and set the new queue
-   * properties in the passed {@link CapacitySchedulerConf}.
-   * 
-   * @param rootChildren
-   * @param schedConf
-   * @return the root {@link AbstractQueue} of the newly created hierarchy.
-   */
-  AbstractQueue createHierarchy(List<JobQueueInfo> rootChildren,
-      CapacitySchedulerConf schedConf) {
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Root queues defined : ");
-      for (JobQueueInfo q : rootChildren) {
-        LOG.debug(q.getQueueName());
-      }
-    }
-
-    // Create the root.
-    AbstractQueue newRootAbstractQueue = createRootAbstractQueue();
-
-    // Create the complete hierarchy rooted at newRootAbstractQueue
-    createHierarchy(newRootAbstractQueue, rootChildren, schedConf);
-
-    // Distribute any un-configured capacities
-    newRootAbstractQueue.distributeUnConfiguredCapacity();
-
-    return newRootAbstractQueue;
-  }
-
-  static final String TOTAL_CAPACITY_OVERFLOWN_MSG =
-      "The cumulative capacity for the queues (%s) at the same level "
-          + "has overflown over 100%% at %f%%";
-
-  /**
-   * Recursively create a complete AbstractQueues-hierarchy. 'Parent' is the
-   * root of the hierarchy. 'Children' is the immediate children of the 'parent'
-   * and may in-turn be the parent of further child queues. Any JobQueueInfo
-   * which doesn't have any more children is used to create a JobQueue in the
-   * AbstractQueues-hierarchy and every other AbstractQueue is used to create a
-   * ContainerQueue.
-   * 
-   * <p>
-   * 
-   * While creating the hierarchy, we make sure at each level that the total
-   * capacity of all the children at that level doesn't cross 100%
-   * 
-   * @param parent the queue that will be the root of the new hierarchy.
-   * @param children the immediate children of the 'parent'
-   * @param schedConfig Configuration object to which the new queue
-   *          properties are set. The new queue properties are set with key
-   *          names obtained by expanding the queue-names to reflect the whole
-   *          hierarchy.
-   */
-  private void createHierarchy(AbstractQueue parent,
-      List<JobQueueInfo> children, CapacitySchedulerConf schedConfig) {
-    //check if children have further childrens.
-    if (children != null && !children.isEmpty()) {
-      float totalCapacity = 0.0f;
-      for (JobQueueInfo qs : children) {
-
-        //Check if this child has any more children.
-        List<JobQueueInfo> childQueues = qs.getChildren();
-
-        if (childQueues != null && childQueues.size() > 0) {
-          //generate a new ContainerQueue and recursively
-          //create hierarchy.
-          AbstractQueue cq =
-              new ContainerQueue(parent, loadContext(qs.getProperties(),
-                  qs.getQueueName(), schedConfig));
-          //update totalCapacity
-          totalCapacity += cq.qsc.getCapacityPercent();
-          LOG.info("Created a ContainerQueue " + qs.getQueueName()
-              + " and added it as a child to " + parent.getName());
-          //create child hiearchy
-          createHierarchy(cq, childQueues, schedConfig);
-        } else {
-          //if not this is a JobQueue.
-
-          //create a JobQueue.
-          AbstractQueue jq =
-              new JobQueue(parent, loadContext(qs.getProperties(),
-                  qs.getQueueName(), schedConfig));
-          totalCapacity += jq.qsc.getCapacityPercent();
-          LOG.info("Created a jobQueue " + qs.getQueueName()
-              + " and added it as a child to " + parent.getName());
-        }
-      }
-
-      //check for totalCapacity at each level , the total for children
-      //shouldn't cross 100.
-
-      if (totalCapacity > 100.0) {
-        StringBuilder childQueueNames = new StringBuilder();
-        for (JobQueueInfo child : children) {
-          childQueueNames.append(child.getQueueName()).append(",");
-        }
-        throw new IllegalArgumentException(String.format(
-            TOTAL_CAPACITY_OVERFLOWN_MSG,
-            childQueueNames.toString().substring(0,
-                childQueueNames.toString().length() - 1),
-            Float.valueOf(totalCapacity)));
-      }
-    }
-  }
-
-  /**
-   * Create a new {@link QueueSchedulingContext} from the given props. Also set
-   * these properties in the passed scheduler configuration object.
-   * 
-   * @param props Properties to be set in the {@link QueueSchedulingContext}
-   * @param queueName Queue name
-   * @param schedConf Scheduler configuration object to set the properties in.
-   * @return the generated {@link QueueSchedulingContext} object
-   */
-  private QueueSchedulingContext loadContext(Properties props,
-      String queueName, CapacitySchedulerConf schedConf) {
-    schedConf.setProperties(queueName,props);
-    float capacity = schedConf.getCapacity(queueName);
-    float stretchCapacity = schedConf.getMaxCapacity(queueName);
-    if (capacity == -1.0) {
-      LOG.info("No capacity specified for queue " + queueName);
-    }
-    int ulMin = schedConf.getMinimumUserLimitPercent(queueName);
-    // create our QSC and add to our hashmap
-    QueueSchedulingContext qsi = new QueueSchedulingContext(
-      queueName, capacity, stretchCapacity, ulMin
-    );
-    qsi.setSupportsPriorities(
-      schedConf.isPrioritySupported(
-        queueName));
-    return qsi;
-  }
-
-  /**
-   * Create an {@link AbstractQueue} with an empty
-   * {@link QueueSchedulingContext}. Can be used to as the root queue to create
-   * {@link AbstractQueue} hierarchies.
-   * 
-   * @return a root {@link AbstractQueue}
-   */
-  static AbstractQueue createRootAbstractQueue() {
-    QueueSchedulingContext rootContext =
-        new QueueSchedulingContext("", 100, -1, -1);
-    AbstractQueue root = new ContainerQueue(null, rootContext);
-    return root;
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java

@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.mapreduce.TaskType;
-
-import java.util.Map;
-import java.util.HashMap;
-
-/**
- * ********************************************************************
- * Keeping track of scheduling information for queues
- * <p/>
- * We need to maintain scheduling information relevant to a queue (its
- * name, capacity, etc), along with information specific to
- * each kind of task, Map or Reduce (num of running tasks, pending
- * tasks etc).
- * <p/>
- * This scheduling information is used to decide how to allocate
- * tasks, redistribute capacity, etc.
- * <p/>
- * A QueueSchedulingContext(qsc) object represents scheduling information for
- * a queue. 
- * ********************************************************************
- */
-public class QueueSchedulingContext {
-
-    //Name of this queue
-    private String queueName;
-
-    //Get the maximum capacity of this queue for running map tasks
-    // in the cluster.
-    private int mapCapacity;
-
-    //Get the maximum capacity of this queue for running reduce tasks 
-    // in the cluster.
-    private int reduceCapacity;
-
-    /**
-     * capacity(%) is set in the config as
-     * mapred.capacity-scheduler.queue.<queue-name>.capacity"
-     * Percentage of the number of slots in the cluster that are
-     * to be available for jobs in this queue.
-     */
-    private float capacityPercent = 0;
-
-  /**
-   * maxCapacityStretch(%) is set in config as
-   * mapred.capacity-scheduler.queue.<queue-name>.maximum-capacity
-   * maximum-capacity-stretch defines a limit beyond which a sub-queue
-   * cannot use the capacity of its parent queue.
-   */
-    private float maxCapacityPercent = -1;
-
-    /**
-     * to handle user limits, we need to know how many users have jobs in
-     * the queue.
-     */
-    private Map<String, Integer> numJobsByUser = new HashMap<String, Integer>();
-
-    /**
-     * min value of user limit (same for all users)
-     */
-    private int ulMin;
-  
-    // whether the queue supports priorities
-    //default is false
-    private boolean supportsPriorities = false;
-
-    //No of waiting jobs.
-    private int numOfWaitingJobs = 0;
-
-    //State of mapCapacity
-    private int prevMapCapacity = 0;
-
-    //State of reduceCapacity
-    private int prevReduceCapacity = 0;
-
-
-    /**
-     * We keep a TaskSchedulingInfo object for each kind of task we support
-     */
-    private TaskSchedulingContext mapTSC;
-    private TaskSchedulingContext reduceTSC;
-
-  QueueSchedulingContext(
-    String queueName, float capacityPercent, float maxCapacityPercent,
-    int ulMin) {
-    this.setQueueName(queueName);
-    this.setCapacityPercent(capacityPercent);
-    this.setMaxCapacityPercent(maxCapacityPercent);
-    this.setUlMin(ulMin);
-    this.setMapTSC(new TaskSchedulingContext());
-    this.setReduceTSC(new TaskSchedulingContext());
-  }
-
-  /**
-     * return information about the queue
-     *
-     * @return a String representing the information about the queue.
-     */
-    @Override
-    public String toString() {
-      // We print out the queue information first, followed by info
-      // on map and reduce tasks and job info
-      StringBuffer sb = new StringBuffer();
-      sb.append("Queue configuration\n");
-      sb.append("Capacity Percentage: ");
-      sb.append(getCapacityPercent());
-      sb.append("%\n");
-      sb.append(String.format("User Limit: %d%s\n", getUlMin(), "%"));
-      sb.append(
-        String.format(
-          "Priority Supported: %s\n",
-          (supportsPriorities()) ?
-            "YES" : "NO"));
-      sb.append("-------------\n");
-
-      sb.append("Map tasks\n");
-      sb.append(getMapTSC().toString());
-      sb.append("-------------\n");
-      sb.append("Reduce tasks\n");
-      sb.append(getReduceTSC().toString());
-      sb.append("-------------\n");
-
-      sb.append("Job info\n");
-      sb.append(
-        String.format(
-          "Number of Waiting Jobs: %d\n",
-          this.getNumOfWaitingJobs()));
-      sb.append(
-        String.format(
-          "Number of users who have submitted jobs: %d\n",
-          getNumJobsByUser().size()));
-      return sb.toString();
-    }
-
-  String getQueueName() {
-    return queueName;
-  }
-
-  void setQueueName(String queueName) {
-    this.queueName = queueName;
-  }
-
-  int getMapCapacity() {
-    return mapCapacity;
-  }
-
-  void setMapCapacity(int mapCapacity) {
-    this.mapCapacity = mapCapacity;
-  }
-
-  int getReduceCapacity() {
-    return reduceCapacity;
-  }
-
-  void setReduceCapacity(int reduceCapacity) {
-    this.reduceCapacity = reduceCapacity;
-  }
-
-  float getCapacityPercent() {
-    return capacityPercent;
-  }
-
-  void setCapacityPercent(float capacityPercent) {
-    this.capacityPercent = capacityPercent;
-  }
-
-  Map<String, Integer> getNumJobsByUser() {
-    return numJobsByUser;
-  }
-
-  void setNumJobsByUser(Map<String, Integer> numJobsByUser) {
-    this.numJobsByUser = numJobsByUser;
-  }
-
-  int getUlMin() {
-    return ulMin;
-  }
-
-  void setUlMin(int ulMin) {
-    this.ulMin = ulMin;
-  }
-
-  TaskSchedulingContext getMapTSC() {
-    return mapTSC;
-  }
-
-  void setMapTSC(TaskSchedulingContext mapTSC) {
-    this.mapTSC = mapTSC;
-  }
-
-  TaskSchedulingContext getReduceTSC() {
-    return reduceTSC;
-  }
-
-  void setReduceTSC(TaskSchedulingContext reduceTSC) {
-    this.reduceTSC = reduceTSC;
-  }
-
-  boolean supportsPriorities() {
-    return supportsPriorities;
-  }
-
-  void setSupportsPriorities(boolean supportsPriorities) {
-    this.supportsPriorities = supportsPriorities;
-  }
-
-  int getNumOfWaitingJobs() {
-    return numOfWaitingJobs;
-  }
-
-  void setNumOfWaitingJobs(int numOfWaitingJobs) {
-    this.numOfWaitingJobs = numOfWaitingJobs;
-  }
-
-  float getMaxCapacityPercent() {
-    return maxCapacityPercent;
-  }
-
-  void setMaxCapacityPercent(float maxCapacityPercent) {
-    this.maxCapacityPercent = maxCapacityPercent;
-  }
-
-  void updateContext(int mapClusterCapacity , int reduceClusterCapacity) {
-    setMapCapacity(mapClusterCapacity);
-    setReduceCapacity(reduceClusterCapacity);
-    // if # of slots have changed since last time, update.
-    // First, compute whether the total number of TT slots have changed
-    // compute new capacities, if TT slots have changed
-    if (getMapCapacity() != prevMapCapacity) {
-      getMapTSC().setCapacity(
-        (int)
-          (getCapacityPercent() * getMapCapacity() / 100));
-
-      //Check if max capacity percent is set for this queue.
-      //if yes then set the maxcapacity for this queue.
-      if (getMaxCapacityPercent() > 0) {
-        getMapTSC().setMaxCapacity(
-          (int) (getMaxCapacityPercent() * getMapCapacity() /
-            100)
-        );
-      }
-    }
-
-    //REDUCES
-    if (getReduceCapacity() != prevReduceCapacity) {
-      getReduceTSC().setCapacity(
-        (int)
-          (getCapacityPercent() * getReduceCapacity() / 100));
-
-      //set stretch capacity for reduce
-      //check if max capacity percent is set for this QueueSchedulingContext.
-      //if yes then set the maxCapacity for this JobQueue.
-      if (getMaxCapacityPercent() > 0) {
-        getReduceTSC().setMaxCapacity(
-          (int) (getMaxCapacityPercent() * getReduceCapacity() /
-            100));
-      }
-    }
-
-    // reset running/pending tasks, tasks per user
-    getMapTSC().resetTaskVars();
-    getReduceTSC().resetTaskVars();
-    // update stats on running jobs
-    prevMapCapacity = getMapCapacity();
-    prevReduceCapacity = getReduceCapacity();
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java

@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-* <p/>
-* http://www.apache.org/licenses/LICENSE-2.0
-* <p/>
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.mapreduce.TaskType;
-
-/**
- * Task view class for the job .
- * returns the running pending and other information for a Job(JobInProgress).
- *
- * has a factory method which provides
- * map and reduce data view based on the type
- *
- */
-abstract class TaskDataView {
-  abstract int getRunningTasks(JobInProgress job);
-
-  abstract int getPendingTasks(JobInProgress job);
-
-  abstract int getSlotsPerTask(JobInProgress job);
-
-  abstract TaskSchedulingContext getTSI(QueueSchedulingContext qsi);
-
-  abstract int getNumReservedTaskTrackers(JobInProgress job);
-
-  int getSlotsOccupied(JobInProgress job) {
-    return (getNumReservedTaskTrackers(job) + getRunningTasks(job)) *
-      getSlotsPerTask(job);
-  }
-
-  /**
-   * Check if the given job has sufficient reserved tasktrackers for all its
-   * pending tasks.
-   *
-   * @param job job to check for sufficient reserved tasktrackers
-   * @return <code>true</code> if the job has reserved tasktrackers,
-   *         else <code>false</code>
-   */
-  boolean hasSufficientReservedTaskTrackers(JobInProgress job) {
-    return getNumReservedTaskTrackers(job) >= getPendingTasks(job);
-  }
-
-  private static TaskDataView mapTaskDataView;
-  private static TaskDataView reduceTaskDataView;
-
-  static TaskDataView getTaskDataView(TaskType type) {
-     if(type == TaskType.MAP) {
-       if(mapTaskDataView == null) {
-         mapTaskDataView = new MapTaskDataView();
-       }
-       return mapTaskDataView;
-     }else if(type == TaskType.REDUCE) {
-       if(reduceTaskDataView == null) {
-         reduceTaskDataView = new ReduceTaskDataView();
-       }
-       return reduceTaskDataView;
-     }
-    return null;
-  }
-
-  /**
-   * The data view for map tasks
-   */
-  static class MapTaskDataView extends TaskDataView {
-    MapTaskDataView() {
-    }
-
-    @Override
-    int getRunningTasks(JobInProgress job) {
-      return job.runningMaps();
-    }
-
-    @Override
-    int getPendingTasks(JobInProgress job) {
-      return job.pendingMaps();
-    }
-
-    @Override
-    int getSlotsPerTask(JobInProgress job) {
-      return job.getNumSlotsPerMap();
-    }
-
-    @Override
-    TaskSchedulingContext getTSI(QueueSchedulingContext qsi) {
-      return qsi.getMapTSC();
-    }
-
-    int getNumReservedTaskTrackers(JobInProgress job) {
-      return job.getNumReservedTaskTrackersForMaps();
-    }
-
-  }
-
-  /**
-   *  The data view for reduce tasks
-   */
-  static class ReduceTaskDataView extends TaskDataView {
-    ReduceTaskDataView() {
-    }
-
-    @Override
-    int getRunningTasks(JobInProgress job) {
-      return job.runningReduces();
-    }
-
-    @Override
-    int getPendingTasks(JobInProgress job) {
-      return job.pendingReduces();
-    }
-
-    @Override
-    int getSlotsPerTask(JobInProgress job) {
-      return job.getNumSlotsPerReduce();
-    }
-
-    @Override
-    TaskSchedulingContext getTSI(QueueSchedulingContext qsi) {
-      return qsi.getReduceTSC();
-    }
-
-    int getNumReservedTaskTrackers(JobInProgress job) {
-      return job.getNumReservedTaskTrackersForReduces();
-    }
-
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java

@@ -1,191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.mapreduce.TaskType;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Set;
-
-/**
- * ********************************************************************
- * Keeping track of scheduling information for queues
- * <p/>
- * Maintain information specific to
- * each kind of task, Map or Reduce (num of running tasks, pending
- * tasks etc).
- * <p/>
- * This scheduling information is used to decide how to allocate
- * tasks, redistribute capacity, etc.
- * <p/>
- * A TaskSchedulingContext (TSI) object represents scheduling
- * information for a particular kind of task (Map or Reduce).
- * <p/>
- * ********************************************************************
- */
-public class TaskSchedulingContext {
-  /**
-   * the actual capacity, which depends on how many slots are available
-   * in the cluster at any given time.
-   */
-  private int capacity = 0;
-  // number of running tasks
-  private int numRunningTasks = 0;
-  // number of slots occupied by running tasks
-  private int numSlotsOccupied = 0;
-
-  //the actual capacity stretch which depends on how many slots are available
-  //in cluster at any given time.
-  private int maxCapacity = -1;
-
-  /**
-   * for each user, we need to keep track of number of slots occupied by
-   * running tasks
-   */
-  private Map<String, Integer> numSlotsOccupiedByUser =
-    new HashMap<String, Integer>();
-
-  /**
-   * reset the variables associated with tasks
-   */
-  void resetTaskVars() {
-    setNumRunningTasks(0);
-    setNumSlotsOccupied(0);
-    for (String s : getNumSlotsOccupiedByUser().keySet()) {
-      getNumSlotsOccupiedByUser().put(s, Integer.valueOf(0));
-    }
-  }
-
-
-  /**
-   * returns the capacity of queue as no of slots.
-   * @return
-   */
-  int getCapacity() {
-    return capacity;
-  }
-
-  /**
-   * Mutator method for capacity
-   *
-   * @param capacity
-   */
-  void setCapacity(int capacity) {
-    this.capacity = capacity;
-  }
-
-
-  /**
-   * return information about the tasks
-   */
-  @Override
-  public String toString() {
-    float occupiedSlotsAsPercent =
-      getCapacity() != 0 ?
-        ((float) getNumSlotsOccupied() * 100 / getCapacity()) : 0;
-    StringBuffer sb = new StringBuffer();
-
-    sb.append("Capacity: " + getCapacity() + " slots\n");
-    if(getMaxCapacity() >= 0) {
-      sb.append("Maximum capacity: " + getMaxCapacity() +" slots\n");
-    }
-    sb.append(
-      String.format(
-        "Used capacity: %d (%.1f%% of Capacity)\n",
-        Integer.valueOf(getNumSlotsOccupied()), Float
-          .valueOf(occupiedSlotsAsPercent)));
-    sb.append(
-      String.format(
-        "Running tasks: %d\n", Integer
-          .valueOf(getNumRunningTasks())));
-    // include info on active users
-    if (getNumSlotsOccupied() != 0) {
-      sb.append("Active users:\n");
-      for (Map.Entry<String, Integer> entry : getNumSlotsOccupiedByUser()
-        .entrySet()) {
-        if ((entry.getValue() == null) ||
-          (entry.getValue().intValue() <= 0)) {
-          // user has no tasks running
-          continue;
-        }
-        sb.append("User '" + entry.getKey() + "': ");
-        int numSlotsOccupiedByThisUser = entry.getValue().intValue();
-        float p =
-          (float) numSlotsOccupiedByThisUser * 100 / getNumSlotsOccupied();
-        sb.append(
-          String.format(
-            "%d (%.1f%% of used capacity)\n", Long
-              .valueOf(numSlotsOccupiedByThisUser), Float.valueOf(p)));
-      }
-    }
-    return sb.toString();
-  }
-
-  int getNumRunningTasks() {
-    return numRunningTasks;
-  }
-
-  void setNumRunningTasks(int numRunningTasks) {
-    this.numRunningTasks = numRunningTasks;
-  }
-
-  int getNumSlotsOccupied() {
-    return numSlotsOccupied;
-  }
-
-  void setNumSlotsOccupied(int numSlotsOccupied) {
-    this.numSlotsOccupied = numSlotsOccupied;
-  }
-
-  Map<String, Integer> getNumSlotsOccupiedByUser() {
-    return numSlotsOccupiedByUser;
-  }
-
-  void setNumSlotsOccupiedByUser(
-    Map<String, Integer> numSlotsOccupiedByUser) {
-    this.numSlotsOccupiedByUser = numSlotsOccupiedByUser;
-  }
-
-  int getMaxCapacity() {
-    return maxCapacity;
-  }
-
-  void setMaxCapacity(int maxCapacity) {
-    this.maxCapacity = maxCapacity;
-  }
-
-  void update(TaskSchedulingContext tc) {
-    this.numSlotsOccupied += tc.numSlotsOccupied;
-    this.numRunningTasks += tc.numRunningTasks;
-    //this.maxTaskLimit += tc.maxTaskLimit;
-    updateNoOfSlotsOccupiedByUser(tc.numSlotsOccupiedByUser);
-  }
-
-  private void updateNoOfSlotsOccupiedByUser(Map<String, Integer> nou) {
-    for (Iterator<Map.Entry<String, Integer>> it = nou.entrySet().iterator(); it.hasNext(); ) {
-      Map.Entry<String, Integer> entry = it.next();
-      String key = entry.getKey();
-      Integer currentVal = numSlotsOccupiedByUser.get(key);
-      if (currentVal != null) {
-        this.numSlotsOccupiedByUser.put(key, currentVal + entry.getValue());
-      }
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java

@@ -1,1009 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.QueueState;
-import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
-import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-import org.apache.hadoop.mapreduce.split.JobSplit;
-import org.apache.hadoop.security.authorize.AccessControlList;
-
-
-public class CapacityTestUtils {
-  static final Log LOG =
-    LogFactory.getLog(org.apache.hadoop.mapred.CapacityTestUtils.class);
-  static final String MAP = "map";
-  static final String REDUCE = "reduce";
-
-
-  /**
-   * Test class that removes the asynchronous nature of job initialization.
-   * <p/>
-   * The run method is a dummy which just waits for completion. It is
-   * expected that test code calls the main method, initializeJobs, directly
-   * to trigger initialization.
-   */
-  static class ControlledJobInitializer extends
-                              JobInitializationPoller.JobInitializationThread {
-
-    boolean stopRunning;
-
-    public ControlledJobInitializer(JobInitializationPoller p) {
-      p.super();
-    }
-
-    @Override
-    public void run() {
-      while (!stopRunning) {
-        try {
-          synchronized (this) {
-            this.wait();
-          }
-        } catch (InterruptedException ie) {
-          break;
-        }
-      }
-    }
-
-    void stopRunning() {
-      stopRunning = true;
-    }
-  }
-
-
-  /**
-   * Test class that removes the asynchronous nature of job initialization.
-   * <p/>
-   * The run method is a dummy which just waits for completion. It is
-   * expected that test code calls the main method, selectJobsToInitialize,
-   * directly to trigger initialization.
-   * <p/>
-   * The class also creates the test worker thread objects of type
-   * ControlledJobInitializer instead of the objects of the actual class
-   */
-  static class ControlledInitializationPoller extends JobInitializationPoller {
-
-    private boolean stopRunning;
-    private ArrayList<ControlledJobInitializer> workers;
-
-    public ControlledInitializationPoller(JobQueuesManager mgr,
-        TaskTrackerManager ttm) {
-      super(mgr, ttm);
-    }
-
-    @Override
-    public void run() {
-      // don't do anything here.
-      while (!stopRunning) {
-        try {
-          synchronized (this) {
-            this.wait();
-          }
-        } catch (InterruptedException ie) {
-          break;
-        }
-      }
-    }
-
-    @Override
-    JobInitializationThread createJobInitializationThread() {
-      ControlledJobInitializer t = new ControlledJobInitializer(this);
-      if (workers == null) {
-        workers = new ArrayList<ControlledJobInitializer>();
-      }
-      workers.add(t);
-      return t;
-    }
-
-    @Override
-    void selectJobsToInitialize() {
-      super.cleanUpInitializedJobsList();
-      super.selectJobsToInitialize();
-      for (ControlledJobInitializer t : workers) {
-        t.initializeJobs();
-      }
-    }
-
-    void stopRunning() {
-      stopRunning = true;
-      for (ControlledJobInitializer t : workers) {
-        t.stopRunning();
-        t.interrupt();
-      }
-    }
-  }
-
-  static class FakeClock extends CapacityTaskScheduler.Clock {
-    private long time = 0;
-
-    public void advance(long millis) {
-      time += millis;
-    }
-
-    @Override
-    long getTime() {
-      return time;
-    }
-  }
-
-  /**
-   * The method accepts a attempt string and checks for validity of
-   * assignTask w.r.t attempt string.
-   * 
-   * @param taskTrackerManager
-   * @param scheduler
-   * @param taskTrackerName
-   * @param expectedTaskString
-   * @return
-   * @throws IOException
-   */
-  static Task checkAssignment(
-    CapacityTestUtils.FakeTaskTrackerManager taskTrackerManager,
-    CapacityTaskScheduler scheduler, String taskTrackerName,
-    String expectedTaskString) throws IOException {
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    if (expectedTaskString.contains("_m_")) {
-      expectedStrings.put(MAP, expectedTaskString);
-    } else if (expectedTaskString.contains("_r_")) {
-      expectedStrings.put(REDUCE, expectedTaskString);
-    }
-    List<Task> tasks = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, taskTrackerName, expectedStrings);
-    for (Task task : tasks) {
-      if (task.toString().equals(expectedTaskString)) {
-        return task;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Checks the validity of tasks assigned by scheduler's assignTasks method
-   * According to JIRA:1030 every assignTasks call in CapacityScheduler
-   * would result in either MAP or REDUCE or BOTH.
-   *
-   * This method accepts a Map<String,String>.
-   * The map should always have <=2 entried in hashMap.
-   *
-   * sample calling code .
-   *
-   *  Map<String, String> expectedStrings = new HashMap<String, String>();
-   * ......
-   * .......
-   * expectedStrings.clear();
-   * expectedStrings.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-   * expectedStrings.put(REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-   * checkMultipleTaskAssignment(
-   *   taskTrackerManager, scheduler, "tt1",
-   *   expectedStrings);
-   * 
-   * @param taskTrackerManager
-   * @param scheduler
-   * @param taskTrackerName
-   * @param expectedTaskStrings
-   * @return
-   * @throws IOException
-   */
-  static List<Task> checkMultipleTaskAssignment(
-    CapacityTestUtils.FakeTaskTrackerManager taskTrackerManager,
-    CapacityTaskScheduler scheduler, String taskTrackerName,
-    Map<String,String> expectedTaskStrings) throws IOException {
-    //Call assign task
-    List<Task> tasks = scheduler.assignTasks(
-      taskTrackerManager.getTaskTracker(
-        taskTrackerName));
-
-    if (tasks==null || tasks.isEmpty()) {
-      if (expectedTaskStrings.size() > 0) {
-        fail("Expected some tasks to be assigned, but got none.");  
-      } else {
-        return null;
-      }
-    }
-
-    if (expectedTaskStrings.size() > tasks.size()) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("Expected strings different from actual strings.");
-      sb.append(" Expected string count=").append(expectedTaskStrings.size());
-      sb.append(" Actual string count=").append(tasks.size());
-      sb.append(" Expected strings=");
-      for (String expectedTask : expectedTaskStrings.values()) {
-        sb.append(expectedTask).append(",");
-      }
-      sb.append("Actual strings=");
-      for (Task actualTask : tasks) {
-        sb.append(actualTask.toString()).append(",");
-      }
-      fail(sb.toString());
-    }
-    
-    for (Task task : tasks) {
-      LOG.info("tasks are : " + tasks.toString());
-      if (task.isMapTask()) {
-        //check if expected string is set for map or not.
-        if (expectedTaskStrings.get(MAP) != null) {
-          assertEquals(expectedTaskStrings.get(MAP), task.toString());
-        } else {
-          fail("No map task is expected, but got " + task.toString());
-        }
-      } else {
-        //check if expectedStrings is set for reduce or not.
-        if (expectedTaskStrings.get(REDUCE) != null) {
-          assertEquals(expectedTaskStrings.get(REDUCE), task.toString());
-        } else {
-          fail("No reduce task is expected, but got " + task.toString());
-        }
-      }
-    }
-    return tasks;
-  }
-
-  static void verifyCapacity(
-    FakeTaskTrackerManager taskTrackerManager,
-    String expectedCapacity,
-    String queue) throws IOException {
-    String schedInfo = taskTrackerManager.getQueueManager().
-      getSchedulerInfo(queue).toString();
-    assertTrue(
-      schedInfo.contains(
-        "Map tasks\nCapacity: "
-          + expectedCapacity + " slots"));
-  }
-
-  /*
-  * Fake job in progress object used for testing the schedulers scheduling
-  * decisions. The JobInProgress objects returns out FakeTaskInProgress
-  * objects when assignTasks is called. If speculative maps and reduces
-  * are configured then JobInProgress returns exactly one Speculative
-  * map and reduce task.
-  */
-  static class FakeJobInProgress extends JobInProgress {
-
-    protected FakeTaskTrackerManager taskTrackerManager;
-    private int mapTaskCtr;
-    private int redTaskCtr;
-    private Set<TaskInProgress> mapTips =
-      new HashSet<TaskInProgress>();
-    private Set<TaskInProgress> reduceTips =
-      new HashSet<TaskInProgress>();
-    private int speculativeMapTaskCounter = 0;
-    private int speculativeReduceTaskCounter = 0;
-
-    public FakeJobInProgress(
-      JobID jId, JobConf jobConf,
-      FakeTaskTrackerManager taskTrackerManager, String user, 
-      JobTracker jt) throws IOException {
-      super(jId, jobConf, jt);
-      this.taskTrackerManager = taskTrackerManager;
-      this.startTime = System.currentTimeMillis();
-      this.status = new JobStatus(
-        jId, 0f, 0f, JobStatus.PREP,
-        jobConf.getUser(),
-        jobConf.getJobName(), "", "");
-      this.status.setJobPriority(JobPriority.NORMAL);
-      this.status.setStartTime(startTime);
-      if (null == jobConf.getQueueName()) {
-        this.profile = new JobProfile(
-          user, jId,
-          null, null, null);
-      } else {
-        this.profile = new JobProfile(
-          user, jId,
-          null, null, null, jobConf.getQueueName());
-      }
-      mapTaskCtr = 0;
-      redTaskCtr = 0;
-      this.jobHistory = new FakeJobHistory();
-    }
-
-    @Override
-    public synchronized void initTasks() throws IOException {
-      getStatus().setRunState(JobStatus.RUNNING);
-    }
-
-    @Override
-    public synchronized Task obtainNewMapTask(
-      final TaskTrackerStatus tts, int clusterSize,
-      int ignored) throws IOException {
-      boolean areAllMapsRunning = (mapTaskCtr == numMapTasks);
-      if (areAllMapsRunning) {
-        if (!getJobConf().getMapSpeculativeExecution() ||
-          speculativeMapTasks > 0) {
-          return null;
-        }
-      }
-      TaskAttemptID attemptId = getTaskAttemptID(true, areAllMapsRunning);
-      JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
-      Task task = new MapTask(
-        "", attemptId, 0, split.getSplitIndex(), super.numSlotsPerMap) {
-        @Override
-        public String toString() {
-          return String.format("%s on %s", getTaskID(), tts.getTrackerName());
-        }
-      };
-      taskTrackerManager.startTask(tts.getTrackerName(), task);
-      runningMapTasks++;
-      // create a fake TIP and keep track of it
-      FakeTaskInProgress mapTip = new FakeTaskInProgress(
-        getJobID(),
-        getJobConf(), task, true, this, split);
-      mapTip.taskStatus.setRunState(TaskStatus.State.RUNNING);
-      if (areAllMapsRunning) {
-        speculativeMapTasks++;
-        //you have scheduled a speculative map. Now set all tips in the
-        //map tips not to have speculative task.
-        for (TaskInProgress t : mapTips) {
-          if (t instanceof FakeTaskInProgress) {
-            FakeTaskInProgress mt = (FakeTaskInProgress) t;
-            mt.hasSpeculativeMap = false;
-          }
-        }
-      } else {
-        //add only non-speculative tips.
-        mapTips.add(mapTip);
-        //add the tips to the JobInProgress TIPS
-        maps = mapTips.toArray(new TaskInProgress[mapTips.size()]);
-      }
-      return task;
-    }
-
-    @Override
-    public synchronized Task obtainNewReduceTask(
-      final TaskTrackerStatus tts,
-      int clusterSize, int ignored) throws IOException {
-      boolean areAllReducesRunning = (redTaskCtr == numReduceTasks);
-      if (areAllReducesRunning) {
-        if (!getJobConf().getReduceSpeculativeExecution() ||
-          speculativeReduceTasks > 0) {
-          return null;
-        }
-      }
-      TaskAttemptID attemptId = getTaskAttemptID(false, areAllReducesRunning);
-      Task task = new ReduceTask(
-        "", attemptId, 0, 10,
-        super.numSlotsPerReduce) {
-        @Override
-        public String toString() {
-          return String.format("%s on %s", getTaskID(), tts.getTrackerName());
-        }
-      };
-      taskTrackerManager.startTask(tts.getTrackerName(), task);
-      runningReduceTasks++;
-      // create a fake TIP and keep track of it
-      FakeTaskInProgress reduceTip = new FakeTaskInProgress(
-        getJobID(),
-        getJobConf(), task, false, this, null);
-      reduceTip.taskStatus.setRunState(TaskStatus.State.RUNNING);
-      if (areAllReducesRunning) {
-        speculativeReduceTasks++;
-        //you have scheduled a speculative map. Now set all tips in the
-        //map tips not to have speculative task.
-        for (TaskInProgress t : reduceTips) {
-          if (t instanceof FakeTaskInProgress) {
-            FakeTaskInProgress rt = (FakeTaskInProgress) t;
-            rt.hasSpeculativeReduce = false;
-          }
-        }
-      } else {
-        //add only non-speculative tips.
-        reduceTips.add(reduceTip);
-        //add the tips to the JobInProgress TIPS
-        reduces = reduceTips.toArray(new TaskInProgress[reduceTips.size()]);
-      }
-      return task;
-    }
-
-    public void mapTaskFinished() {
-      runningMapTasks--;
-      finishedMapTasks++;
-    }
-
-    public void reduceTaskFinished() {
-      runningReduceTasks--;
-      finishedReduceTasks++;
-    }
-
-    private TaskAttemptID getTaskAttemptID(
-      boolean isMap, boolean isSpeculative) {
-      JobID jobId = getJobID();
-      TaskType t = TaskType.REDUCE;
-      if (isMap) {
-        t = TaskType.MAP;
-      }
-      if (!isSpeculative) {
-        return new TaskAttemptID(
-          jobId.getJtIdentifier(), jobId.getId(), t,
-          (isMap) ? ++mapTaskCtr : ++redTaskCtr, 0);
-      } else {
-        return new TaskAttemptID(
-          jobId.getJtIdentifier(), jobId.getId(), t,
-          (isMap) ? mapTaskCtr : redTaskCtr, 1);
-      }
-    }
-
-    @Override
-    Set<TaskInProgress> getNonLocalRunningMaps() {
-      return (Set<TaskInProgress>) mapTips;
-    }
-
-    @Override
-    Set<TaskInProgress> getRunningReduces() {
-      return (Set<TaskInProgress>) reduceTips;
-    }
-
-  }
-
-  static class FakeFailingJobInProgress extends FakeJobInProgress {
-
-    public FakeFailingJobInProgress(
-      JobID id, JobConf jobConf,
-      FakeTaskTrackerManager taskTrackerManager, String user, 
-      JobTracker jt) throws IOException {
-      super(id, jobConf, taskTrackerManager, user, jt);
-    }
-
-    @Override
-    public synchronized void initTasks() throws IOException {
-      throw new IOException("Failed Initalization");
-    }
-
-    @Override
-    synchronized void fail() {
-      this.status.setRunState(JobStatus.FAILED);
-    }
-  }
-
-  static class FakeTaskInProgress extends TaskInProgress {
-    private boolean isMap;
-    private FakeJobInProgress fakeJob;
-    private TreeMap<TaskAttemptID, String> activeTasks;
-    private TaskStatus taskStatus;
-    boolean hasSpeculativeMap;
-    boolean hasSpeculativeReduce;
-
-    FakeTaskInProgress(
-      JobID jId, JobConf jobConf, Task t,
-      boolean isMap, FakeJobInProgress job, 
-      JobSplit.TaskSplitMetaInfo split) {
-      super(jId, "", split, null, jobConf, job, 0, 1);
-      this.isMap = isMap;
-      this.fakeJob = job;
-      activeTasks = new TreeMap<TaskAttemptID, String>();
-      activeTasks.put(t.getTaskID(), "tt");
-      // create a fake status for a task that is running for a bit
-      this.taskStatus = TaskStatus.createTaskStatus(isMap);
-      taskStatus.setProgress(0.5f);
-      taskStatus.setRunState(TaskStatus.State.RUNNING);
-      if (jobConf.getMapSpeculativeExecution()) {
-        //resetting of the hasSpeculativeMap is done
-        //when speculative map is scheduled by the job.
-        hasSpeculativeMap = true;
-      }
-      if (jobConf.getReduceSpeculativeExecution()) {
-        //resetting of the hasSpeculativeReduce is done
-        //when speculative reduce is scheduled by the job.
-        hasSpeculativeReduce = true;
-      }
-    }
-
-    @Override
-    TreeMap<TaskAttemptID, String> getActiveTasks() {
-      return activeTasks;
-    }
-
-    @Override
-    public TaskStatus getTaskStatus(TaskAttemptID taskid) {
-      // return a status for a task that has run a bit
-      return taskStatus;
-    }
-
-    @Override
-    boolean killTask(TaskAttemptID taskId, boolean shouldFail) {
-      if (isMap) {
-        fakeJob.mapTaskFinished();
-      } else {
-        fakeJob.reduceTaskFinished();
-      }
-      return true;
-    }
-
-    @Override
-      /*
-      *hasSpeculativeMap and hasSpeculativeReduce is reset by FakeJobInProgress
-      *after the speculative tip has been scheduled.
-      */
-    boolean canBeSpeculated(long currentTime) {
-      if (isMap && hasSpeculativeMap) {
-        return fakeJob.getJobConf().getMapSpeculativeExecution();
-      }
-      if (!isMap && hasSpeculativeReduce) {
-        return fakeJob.getJobConf().getReduceSpeculativeExecution();
-      }
-      return false;
-    }
-
-    @Override
-    public boolean isRunning() {
-      return !activeTasks.isEmpty();
-    }
-
-  }
-
-  static class FakeQueueManager extends QueueManager {
-    private Set<String> queueNames = null;
-    private static final AccessControlList allEnabledAcl =
-      new AccessControlList("*");
-
-    FakeQueueManager(Configuration conf) {
-      super(conf);
-    }
-
-    void setQueues(Set<String> queueNames) {
-      this.queueNames = queueNames;
-
-      // sync up queues with the parent class.
-      Queue[] queues = new Queue[queueNames.size()];
-      int i = 0;
-      for (String queueName : queueNames) {
-        HashMap<String, AccessControlList> aclsMap
-          = new HashMap<String, AccessControlList>();
-        for (QueueACL qAcl : QueueACL.values()) {
-          String key = toFullPropertyName(queueName, qAcl.getAclName());
-          aclsMap.put(key, allEnabledAcl);
-        }
-        queues[i++] = new Queue(queueName, aclsMap, QueueState.RUNNING);
-      }
-      super.setQueues(queues);
-    }
-
-    public synchronized Set<String> getLeafQueueNames() {
-      return queueNames;
-    }
-
-  }
-
-  static class FakeTaskTrackerManager implements TaskTrackerManager {
-    int maps = 0;
-    int reduces = 0;
-    int maxMapTasksPerTracker = 2;
-    int maxReduceTasksPerTracker = 1;
-    long ttExpiryInterval = 10 * 60 * 1000L; // default interval
-    List<JobInProgressListener> listeners =
-      new ArrayList<JobInProgressListener>();
-
-    QueueManager qm = null;
-
-    private Map<String, TaskTracker> trackers =
-      new HashMap<String, TaskTracker>();
-    private Map<String, TaskStatus> taskStatuses =
-      new HashMap<String, TaskStatus>();
-    private Map<JobID, JobInProgress> jobs =
-      new HashMap<JobID, JobInProgress>();
-    protected JobConf defaultJobConf;
-    private int jobCounter = 0;
-
-    public FakeTaskTrackerManager() {
-      this(2, 2, 1);
-    }
-
-    public FakeTaskTrackerManager(
-      int numTaskTrackers,
-      int maxMapTasksPerTracker, int maxReduceTasksPerTracker) {
-    Configuration cfn = new Configuration();
-    cfn.set("mapred.queue.names","default");
-    qm = new FakeQueueManager(cfn);
-      this.maxMapTasksPerTracker = maxMapTasksPerTracker;
-      this.maxReduceTasksPerTracker = maxReduceTasksPerTracker;
-      for (int i = 1; i < numTaskTrackers + 1; i++) {
-        String ttName = "tt" + i;
-        TaskTracker tt = new TaskTracker(ttName);
-        tt.setStatus(
-          new TaskTrackerStatus(
-            ttName, ttName + ".host", i,
-            new ArrayList<TaskStatus>(), 0,
-            maxMapTasksPerTracker,
-            maxReduceTasksPerTracker));
-        trackers.put(ttName, tt);
-      }
-
-      defaultJobConf = new JobConf();
-      defaultJobConf.set("mapred.queue.names","default");
-      //by default disable speculative execution.
-      defaultJobConf.setMapSpeculativeExecution(false);
-      defaultJobConf.setReduceSpeculativeExecution(false);
-    }
-
-    public void addTaskTracker(String ttName) {
-      TaskTracker tt = new TaskTracker(ttName);
-      tt.setStatus(
-        new TaskTrackerStatus(
-          ttName, ttName + ".host", 1,
-          new ArrayList<TaskStatus>(), 0,
-          maxMapTasksPerTracker,
-          maxReduceTasksPerTracker));
-      trackers.put(ttName, tt);
-    }
-
-    public ClusterStatus getClusterStatus() {
-      int numTrackers = trackers.size();
-      return new ClusterStatus(
-        numTrackers, 0,
-        ttExpiryInterval, maps, reduces,
-        numTrackers * maxMapTasksPerTracker,
-        numTrackers * maxReduceTasksPerTracker,
-        JobTrackerStatus.RUNNING);
-    }
-
-    public int getNumberOfUniqueHosts() {
-      return 0;
-    }
-
-    public int getNextHeartbeatInterval() {
-      return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
-    }
-
-    /**
-     * Kill Job, and all its tasks on the corresponding TaskTrackers.
-     */
-    @Override
-    public void killJob(JobID jobid) throws IOException {
-      killJob(jobid, true);
-    }
-
-    /**
-     * Kill the job, and kill the tasks on the corresponding TaskTrackers only
-     * if killTasks is true.
-     * 
-     * @param jobid
-     * @throws IOException
-     */
-    void killJob(JobID jobid, boolean killTasks)
-        throws IOException {
-      JobInProgress job = jobs.get(jobid);
-      // Finish all the tasks for this job
-      if (job instanceof FakeJobInProgress && killTasks) {
-        FakeJobInProgress fJob = (FakeJobInProgress) job;
-        for (String tipID : taskStatuses.keySet()) {
-          finishTask(tipID, fJob, TaskStatus.State.KILLED);
-        }
-      }
-      finalizeJob(job, JobStatus.KILLED);
-      job.kill();
-    }
-
-    public void initJob(JobInProgress jip) {
-      try {
-        JobStatus oldStatus = (JobStatus) jip.getStatus().clone();
-        jip.initTasks();
-        if (jip.isJobEmpty()) {
-          completeEmptyJob(jip);
-        } else if (!jip.isSetupCleanupRequired()) {
-          jip.completeSetup();
-        }
-        JobStatus newStatus = (JobStatus) jip.getStatus().clone();
-        JobStatusChangeEvent event = new JobStatusChangeEvent(
-          jip,
-          JobStatusChangeEvent.EventType.RUN_STATE_CHANGED, oldStatus,
-          newStatus);
-        for (JobInProgressListener listener : listeners) {
-          listener.jobUpdated(event);
-        }
-      } catch (Exception ioe) {
-        failJob(jip);
-      }
-    }
-
-    private synchronized void completeEmptyJob(JobInProgress jip) {
-      jip.completeEmptyJob();
-    }
-
-    public synchronized void failJob(JobInProgress jip) {
-      JobStatus oldStatus = (JobStatus) jip.getStatus().clone();
-      jip.fail();
-      JobStatus newStatus = (JobStatus) jip.getStatus().clone();
-      JobStatusChangeEvent event = new JobStatusChangeEvent(
-        jip,
-        JobStatusChangeEvent.EventType.RUN_STATE_CHANGED, oldStatus, newStatus);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobUpdated(event);
-      }
-    }
-
-    public void retireJob(JobID jobid) {
-      jobs.remove(jobid);
-    }
-
-    @Override
-    public JobInProgress getJob(JobID jobid) {
-      return jobs.get(jobid);
-    }
-
-    Collection<JobInProgress> getJobs() {
-      return jobs.values();
-    }
-
-    public Collection<TaskTrackerStatus> taskTrackers() {
-      List<TaskTrackerStatus> statuses = new ArrayList<TaskTrackerStatus>();
-      for (TaskTracker tt : trackers.values()) {
-        statuses.add(tt.getStatus());
-      }
-      return statuses;
-    }
-
-
-    public void addJobInProgressListener(JobInProgressListener listener) {
-      listeners.add(listener);
-    }
-
-    public void removeJobInProgressListener(JobInProgressListener listener) {
-      listeners.remove(listener);
-    }
-
-    public void submitJob(JobInProgress job) throws IOException {
-      jobs.put(job.getJobID(), job);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobAdded(job);
-      }
-    }
-
-    FakeJobInProgress submitJob(int state, JobConf jobConf)
-        throws IOException {
-      FakeJobInProgress job =
-          new FakeJobInProgress(new JobID("test", ++jobCounter),
-              (jobConf == null ? new JobConf(defaultJobConf) : jobConf), this,
-              jobConf.getUser(), UtilsForTests.getJobTracker());
-      job.getStatus().setRunState(state);
-      this.submitJob(job);
-      return job;
-    }
-
-    FakeJobInProgress submitJobAndInit(int state, JobConf jobConf)
-        throws IOException {
-      FakeJobInProgress j = submitJob(state, jobConf);
-      this.initJob(j);
-      return j;
-    }
-
-    FakeJobInProgress submitJob(int state, int maps, int reduces,
-        String queue, String user)
-        throws IOException {
-      JobConf jobConf = new JobConf(defaultJobConf);
-      jobConf.setNumMapTasks(maps);
-      jobConf.setNumReduceTasks(reduces);
-      if (queue != null) {
-        jobConf.setQueueName(queue);
-      }
-      jobConf.setUser(user);
-      return submitJob(state, jobConf);
-    }
-
-    // Submit a job and update the listeners
-    FakeJobInProgress submitJobAndInit(int state, int maps, int reduces,
-        String queue, String user)
-        throws IOException {
-      FakeJobInProgress j = submitJob(state, maps, reduces, queue, user);
-      this.initJob(j);
-      return j;
-    }
-
-    HashMap<String, ArrayList<FakeJobInProgress>> submitJobs(
-        int numberOfUsers, int numberOfJobsPerUser, String queue)
-        throws Exception {
-      HashMap<String, ArrayList<FakeJobInProgress>> userJobs =
-          new HashMap<String, ArrayList<FakeJobInProgress>>();
-      for (int i = 1; i <= numberOfUsers; i++) {
-        String user = String.valueOf("u" + i);
-        ArrayList<FakeJobInProgress> jips = new ArrayList<FakeJobInProgress>();
-        for (int j = 1; j <= numberOfJobsPerUser; j++) {
-          jips.add(this.submitJob(JobStatus.PREP, 1, 1, queue, user));
-        }
-        userJobs.put(user, jips);
-      }
-      return userJobs;
-    }
-
-    public TaskTracker getTaskTracker(String trackerID) {
-      return trackers.get(trackerID);
-    }
-
-    public void startTask(String taskTrackerName, final Task t) {
-      if (t.isMapTask()) {
-        maps++;
-      } else {
-        reduces++;
-      }
-      TaskStatus status = new TaskStatus() {
-        @Override
-        public TaskAttemptID getTaskID() {
-          return t.getTaskID();
-        }
-
-        @Override
-        public boolean getIsMap() {
-          return t.isMapTask();
-        }
-
-        @Override
-        public int getNumSlots() {
-          return t.getNumSlotsRequired();
-        }
-
-        @Override
-        public void addFetchFailedMap(TaskAttemptID mapTaskId) {
-          
-        }
-      };
-      taskStatuses.put(t.getTaskID().toString(), status);
-      status.setRunState(TaskStatus.State.RUNNING);
-      trackers.get(taskTrackerName).getStatus().getTaskReports().add(status);
-    }
-
-    public void finishTask(
-        String tipId,
-        FakeJobInProgress j) {
-      finishTask(tipId, j, TaskStatus.State.SUCCEEDED);
-    }
-
-    public void finishTask(String tipId, FakeJobInProgress j,
-        TaskStatus.State taskState) {
-      TaskStatus status = taskStatuses.get(tipId);
-      if (status.getIsMap()) {
-        maps--;
-        j.mapTaskFinished();
-      } else {
-        reduces--;
-        j.reduceTaskFinished();
-      }
-      status.setRunState(taskState);
-    }
-
-    void finalizeJob(FakeJobInProgress fjob) {
-      finalizeJob(fjob, JobStatus.SUCCEEDED);
-    }
-
-    void finalizeJob(JobInProgress fjob, int state) {
-      // take a snapshot of the status before changing it
-      JobStatus oldStatus = (JobStatus) fjob.getStatus().clone();
-      fjob.getStatus().setRunState(state);
-      JobStatus newStatus = (JobStatus) fjob.getStatus().clone();
-      JobStatusChangeEvent event =
-        new JobStatusChangeEvent(
-          fjob, JobStatusChangeEvent.EventType.RUN_STATE_CHANGED, oldStatus,
-          newStatus);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobUpdated(event);
-      }
-    }
-
-    public void setPriority(FakeJobInProgress fjob, JobPriority priority) {
-      // take a snapshot of the status before changing it
-      JobStatus oldStatus = (JobStatus) fjob.getStatus().clone();
-      fjob.setPriority(priority);
-      JobStatus newStatus = (JobStatus) fjob.getStatus().clone();
-      JobStatusChangeEvent event =
-        new JobStatusChangeEvent(
-          fjob, JobStatusChangeEvent.EventType.PRIORITY_CHANGED, oldStatus,
-          newStatus);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobUpdated(event);
-      }
-    }
-
-    public void setStartTime(FakeJobInProgress fjob, long start) {
-      // take a snapshot of the status before changing it
-      JobStatus oldStatus = (JobStatus) fjob.getStatus().clone();
-
-      fjob.startTime = start; // change the start time of the job
-      fjob.status.setStartTime(start); // change the start time of the jobstatus
-
-      JobStatus newStatus = (JobStatus) fjob.getStatus().clone();
-
-      JobStatusChangeEvent event =
-        new JobStatusChangeEvent(
-          fjob, JobStatusChangeEvent.EventType.START_TIME_CHANGED, oldStatus,
-          newStatus);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobUpdated(event);
-      }
-    }
-
-    void addQueues(String[] arr) {
-      Set<String> queues = new HashSet<String>();
-      for (String s : arr) {
-        queues.add(s);
-      }
-      ((FakeQueueManager)qm).setQueues(queues);
-    }
-
-    void setFakeQueues(List<CapacityTestUtils.FakeQueueInfo> queues) {
-      for (CapacityTestUtils.FakeQueueInfo q : queues) {
-        Properties p = new Properties();
-        p.setProperty(CapacitySchedulerConf.CAPACITY_PROPERTY,
-            String.valueOf(q.capacity));
-        p.setProperty(CapacitySchedulerConf.MAX_CAPACITY_PROPERTY,
-            String.valueOf(q.maxCapacity));
-        p.setProperty(CapacitySchedulerConf.SUPPORTS_PRIORITY_PROPERTY,
-            String.valueOf(q.supportsPrio));
-        p.setProperty(
-            CapacitySchedulerConf.MINIMUM_USER_LIMIT_PERCENT_PROPERTY,
-            String.valueOf(q.ulMin));
-        ((FakeQueueManager) qm).getQueue(q.queueName).setProperties(p);
-      }
-    }
-
-    void setQueueManager(QueueManager qManager) {
-      this.qm = qManager;
-    }
-
-    public QueueManager getQueueManager() {
-      return qm;
-    }
-
-    @Override
-    public boolean killTask(TaskAttemptID taskid, boolean shouldFail) {
-      return true;
-    }
-
-    public JobID getNextJobID() {
-      return new JobID("test", ++jobCounter);
-    }
-
-  }// represents a fake queue configuration info
-
-  static class FakeQueueInfo {
-    String queueName;
-    float capacity;
-    float maxCapacity = -1.0f;
-    boolean supportsPrio;
-    int ulMin;
-
-    public FakeQueueInfo(
-      String queueName, float capacity, boolean supportsPrio, int ulMin) {
-      this.queueName = queueName;
-      this.capacity = capacity;
-      this.supportsPrio = supportsPrio;
-      this.ulMin = ulMin;
-    }
-  }
-
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ClusterWithCapacityScheduler.java

@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Enumeration;
-import java.util.Properties;
-
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-
-/**
- * A test-cluster based on {@link MiniMRCluster} that is started with
- * CapacityTaskScheduler. It provides knobs to configure both the cluster as
- * well as the scheduler. Any test that intends to test capacity-scheduler
- * should extend this.
- * 
- */
-public class ClusterWithCapacityScheduler extends TestCase {
-
-  static final Log LOG = LogFactory.getLog(ClusterWithCapacityScheduler.class);
-  private MiniMRCluster mrCluster;
-
-  private JobConf jobConf;
-
-  static final String MY_SCHEDULER_CONF_PATH_PROPERTY = "my.resource.path";
-
-  protected void startCluster()
-      throws IOException {
-    startCluster(null, null);
-  }
-
-  /**
-   * Start the cluster with two TaskTrackers and two DataNodes and configure the
-   * cluster with clusterProperties and the scheduler with schedulerProperties.
-   * Uses default configuration whenever user provided properties are missing
-   * (null/empty)
-   * 
-   * @param clusterProperties
-   * @param schedulerProperties
-   * @throws IOException
-   */
-  protected void startCluster(Properties clusterProperties,
-      Properties schedulerProperties)
-      throws IOException {
-    startCluster(2, clusterProperties, schedulerProperties);
-  }
-
-  /**
-   * Start the cluster with numTaskTrackers TaskTrackers and numDataNodes
-   * DataNodes and configure the cluster with clusterProperties and the
-   * scheduler with schedulerProperties. Uses default configuration whenever
-   * user provided properties are missing (null/empty)
-   * 
-   * @param numTaskTrackers
-   * @param clusterProperties
-   * @param schedulerProperties
-   * @throws IOException
-   */
-  protected void startCluster(int numTaskTrackers,
-      Properties clusterProperties, Properties schedulerProperties)
-      throws IOException {
-    Thread.currentThread().setContextClassLoader(
-        new ClusterWithCapacityScheduler.MyClassLoader());
-    JobConf clusterConf = new JobConf();
-    if (clusterProperties != null) {
-      for (Enumeration<?> e = clusterProperties.propertyNames(); e
-          .hasMoreElements();) {
-        String key = (String) e.nextElement();
-        clusterConf.set(key, (String) clusterProperties.get(key));
-      }
-    }
-
-    if (schedulerProperties != null) {
-      setUpSchedulerConfigFile(schedulerProperties);
-    }
-
-    clusterConf.set(JTConfig.JT_TASK_SCHEDULER,
-        CapacityTaskScheduler.class.getName());
-    mrCluster =
-        new MiniMRCluster(numTaskTrackers, "file:///", 1, null, null,
-            clusterConf);
-
-    this.jobConf = mrCluster.createJobConf(clusterConf);
-  }
-
-  private void setUpSchedulerConfigFile(Properties schedulerConfProps)
-      throws IOException {
-    LocalFileSystem fs = FileSystem.getLocal(new Configuration());
-
-    String myResourcePath = System.getProperty("test.build.data");
-    Path schedulerConfigFilePath =
-        new Path(myResourcePath, CapacitySchedulerConf.SCHEDULER_CONF_FILE);
-    OutputStream out = fs.create(schedulerConfigFilePath);
-
-    Configuration config = new Configuration(false);
-    for (Enumeration<?> e = schedulerConfProps.propertyNames(); e
-        .hasMoreElements();) {
-      String key = (String) e.nextElement();
-      LOG.debug("Adding " + key + schedulerConfProps.getProperty(key));
-      config.set(key, schedulerConfProps.getProperty(key));
-    }
-
-    config.writeXml(out);
-    out.close();
-
-    LOG.info("setting resource path where capacity-scheduler's config file "
-        + "is placed to " + myResourcePath);
-    System.setProperty(MY_SCHEDULER_CONF_PATH_PROPERTY, myResourcePath);
-  }
-
-  private void cleanUpSchedulerConfigFile() throws IOException {
-    LocalFileSystem fs = FileSystem.getLocal(new Configuration());
-
-    String myResourcePath = System.getProperty("test.build.data");
-    Path schedulerConfigFilePath =
-        new Path(myResourcePath, CapacitySchedulerConf.SCHEDULER_CONF_FILE);
-    fs.delete(schedulerConfigFilePath, false);
-  }
-
-  protected JobConf getJobConf() {
-    return new JobConf(this.jobConf);
-  }
-
-  protected JobTracker getJobTracker() {
-    return this.mrCluster.getJobTrackerRunner().getJobTracker();
-  }
-
-  @Override
-  protected void tearDown()
-      throws Exception {
-    cleanUpSchedulerConfigFile();
-    
-    if (mrCluster != null) {
-      mrCluster.shutdown();
-    }
-  }
-
-  /**
-   * Wait till all the slots in the cluster are occupied with respect to the
-   * tasks of type specified isMap.
-   * 
-   * <p>
-   * 
-   * <b>Also, it is assumed that the tasks won't finish any time soon, like in
-   * the case of tasks of {@link ControlledMapReduceJob}</b>.
-   * 
-   * @param isMap
-   */
-  protected void waitTillAllSlotsAreOccupied(boolean isMap)
-      throws InterruptedException {
-    JobTracker jt = this.mrCluster.getJobTrackerRunner().getJobTracker();
-    ClusterStatus clusterStatus = jt.getClusterStatus();
-    int currentTasks =
-        (isMap ? clusterStatus.getMapTasks() : clusterStatus.getReduceTasks());
-    int maxTasks =
-        (isMap ? clusterStatus.getMaxMapTasks() : clusterStatus
-            .getMaxReduceTasks());
-    while (currentTasks != maxTasks) {
-      Thread.sleep(1000);
-      clusterStatus = jt.getClusterStatus();
-      currentTasks =
-          (isMap ? clusterStatus.getMapTasks() : clusterStatus
-              .getReduceTasks());
-      maxTasks =
-          (isMap ? clusterStatus.getMaxMapTasks() : clusterStatus
-              .getMaxReduceTasks());
-      LOG.info("Waiting till cluster reaches steady state. currentTasks : "
-          + currentTasks + " total cluster capacity : " + maxTasks);
-    }
-  }
-
-  static class MyClassLoader extends ClassLoader {
-    @Override
-    public URL getResource(String name) {
-      if (!name.equals(CapacitySchedulerConf.SCHEDULER_CONF_FILE)) {
-        return super.getResource(name);
-      }
-      return findResource(name);
-    }
-
-    @Override
-    protected URL findResource(String name) {
-      try {
-        String resourcePath =
-            System
-                .getProperty(ClusterWithCapacityScheduler.MY_SCHEDULER_CONF_PATH_PROPERTY);
-        // Check the resourcePath directory
-        File file = new File(resourcePath, name);
-        if (file.exists()) {
-          return new URL("file://" + file.getAbsolutePath());
-        }
-      } catch (MalformedURLException mue) {
-        LOG.warn("exception : " + mue);
-      }
-      return super.findResource(name);
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java

@@ -1,2736 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import junit.framework.TestCase;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobStatusChangeEvent.EventType;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-import static org.apache.hadoop.mapred.CapacityTestUtils.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-
-public class TestCapacityScheduler extends TestCase {
-
-  static final Log LOG =
-    LogFactory.getLog(org.apache.hadoop.mapred.TestCapacityScheduler.class);
-
-  String queueConfigPath =
-    System.getProperty("test.build.extraconf", "build/test/extraconf");
-  File queueConfigFile =
-    new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME);
-
-  private static int jobCounter;
-
-  private ControlledInitializationPoller controlledInitializationPoller;
-
-
-  protected JobConf conf;
-  protected CapacityTaskScheduler scheduler;
-  private FakeTaskTrackerManager taskTrackerManager;
-  private FakeClock clock;
-
-  @Override
-  protected void setUp() {
-    setUp(2, 2, 1);
-  }
-
-  private void setUp(
-    int numTaskTrackers, int numMapTasksPerTracker,
-    int numReduceTasksPerTracker) {
-    jobCounter = 0;
-    taskTrackerManager =
-      new FakeTaskTrackerManager(
-        numTaskTrackers, numMapTasksPerTracker,
-        numReduceTasksPerTracker);
-    clock = new FakeClock();
-    scheduler = new CapacityTaskScheduler(clock);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-
-    conf = new JobConf();
-    // Don't let the JobInitializationPoller come in our way.
-    conf.set("mapred.queue.names","default");
-    controlledInitializationPoller =
-        new ControlledInitializationPoller(scheduler.jobQueuesManager,
-            taskTrackerManager);
-    scheduler.setInitializationPoller(controlledInitializationPoller);
-    scheduler.setConf(conf);
-    //by default disable speculative execution.
-    conf.setMapSpeculativeExecution(false);
-    conf.setReduceSpeculativeExecution(false);
-  }
-
-  @Override
-  protected void tearDown() throws Exception {
-    if (scheduler != null) {
-      scheduler.terminate();
-    }
-  }
-
-  /**
-   * Test max capacity
-   * @throws IOException
-   */
-  public void testMaxCapacity() throws IOException {
-    this.setUp(4, 1, 1);
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 25.0f, false, 1));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-    scheduler.getRoot().getChildren().get(0).getQueueSchedulingContext()
-      .setMaxCapacityPercent(50.0f);
-
-    //submit the Job
-    FakeJobInProgress fjob1 = taskTrackerManager.submitJob(
-      JobStatus.PREP, 4, 4, "default", "user");
-
-    taskTrackerManager.initJob(fjob1);
-    HashMap<String, String> expectedStrings = new HashMap<String, String>();
-
-    expectedStrings.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    List<Task> task1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1", expectedStrings);
-
-
-    expectedStrings.put(MAP, "attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000002_0 on tt2");
-    List<Task> task2 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2", expectedStrings);
-
-    //we have already reached the limit
-    //this call would return null
-    List<Task> task3 = scheduler.assignTasks(tracker("tt3"));
-    assertNull(task3);
-
-    //Now complete the task 1 i.e map task.
-    for (Task task : task1) {
-        taskTrackerManager.finishTask(
-          task.getTaskID().toString(), fjob1);
-    }
-    
-    expectedStrings.put(MAP, "attempt_test_0001_m_000003_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000003_0 on tt1");
-    task2 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1", expectedStrings);
-  }
-
-  // test job run-state change
-  public void testJobRunStateChange() throws IOException {
-    // start the scheduler
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 1));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit the job
-    FakeJobInProgress fjob1 =
-      taskTrackerManager.submitJob(JobStatus.PREP, 1, 0, "default", "user");
-
-    FakeJobInProgress fjob2 =
-      taskTrackerManager.submitJob(JobStatus.PREP, 1, 0, "default", "user");
-
-    // test if changing the job priority/start-time works as expected in the 
-    // waiting queue
-    testJobOrderChange(fjob1, fjob2, true);
-
-    // Init the jobs
-    // simulate the case where the job with a lower priority becomes running 
-    // first (may be because of the setup tasks).
-
-    // init the lower ranked job first
-    taskTrackerManager.initJob(fjob2);
-
-    // init the higher ordered job later
-    taskTrackerManager.initJob(fjob1);
-
-    // check if the jobs are missing from the waiting queue
-    // The jobs are not removed from waiting queue until they are scheduled 
-    assertEquals(
-      "Waiting queue is garbled on job init", 2,
-      scheduler.jobQueuesManager.getJobQueue("default").getWaitingJobs()
-        .size());
-
-    // test if changing the job priority/start-time works as expected in the 
-    // running queue
-    testJobOrderChange(fjob1, fjob2, false);
-
-    // schedule a task
-    List<Task> tasks = scheduler.assignTasks(tracker("tt1"));
-
-    // complete the job
-    taskTrackerManager.finishTask(
-      tasks.get(0).getTaskID().toString(),
-      fjob1);
-
-    // mark the job as complete
-    taskTrackerManager.finalizeJob(fjob1);
-
-    Collection<JobInProgress> rqueue =
-      scheduler.jobQueuesManager.getJobQueue("default").getRunningJobs();
-
-    // check if the job is removed from the scheduler
-    assertFalse(
-      "Scheduler contains completed job",
-      rqueue.contains(fjob1));
-
-    // check if the running queue size is correct
-    assertEquals(
-      "Job finish garbles the queue",
-      1, rqueue.size());
-
-  }
-
-  // test if the queue reflects the changes
-  private void testJobOrderChange(
-    FakeJobInProgress fjob1,
-    FakeJobInProgress fjob2,
-    boolean waiting) {
-    String queueName = waiting ? "waiting" : "running";
-
-    // check if the jobs in the queue are the right order
-    JobInProgress[] jobs = getJobsInQueue(waiting);
-    assertTrue(
-      queueName + " queue doesnt contain job #1 in right order",
-      jobs[0].getJobID().equals(fjob1.getJobID()));
-    assertTrue(
-      queueName + " queue doesnt contain job #2 in right order",
-      jobs[1].getJobID().equals(fjob2.getJobID()));
-
-    // I. Check the start-time change
-    // Change job2 start-time and check if job2 bumps up in the queue 
-    taskTrackerManager.setStartTime(fjob2, fjob1.startTime - 1);
-
-    jobs = getJobsInQueue(waiting);
-    assertTrue(
-      "Start time change didnt not work as expected for job #2 in "
-        + queueName + " queue",
-      jobs[0].getJobID().equals(fjob2.getJobID()));
-    assertTrue(
-      "Start time change didnt not work as expected for job #1 in"
-        + queueName + " queue",
-      jobs[1].getJobID().equals(fjob1.getJobID()));
-
-    // check if the queue is fine
-    assertEquals(
-      "Start-time change garbled the " + queueName + " queue",
-      2, jobs.length);
-
-    // II. Change job priority change
-    // Bump up job1's priority and make sure job1 bumps up in the queue
-    taskTrackerManager.setPriority(fjob1, JobPriority.HIGH);
-
-    // Check if the priority changes are reflected
-    jobs = getJobsInQueue(waiting);
-    assertTrue(
-      "Priority change didnt not work as expected for job #1 in "
-        + queueName + " queue",
-      jobs[0].getJobID().equals(fjob1.getJobID()));
-    assertTrue(
-      "Priority change didnt not work as expected for job #2 in "
-        + queueName + " queue",
-      jobs[1].getJobID().equals(fjob2.getJobID()));
-
-    // check if the queue is fine
-    assertEquals(
-      "Priority change has garbled the " + queueName + " queue",
-      2, jobs.length);
-
-    // reset the queue state back to normal
-    taskTrackerManager.setStartTime(fjob1, fjob2.startTime - 1);
-    taskTrackerManager.setPriority(fjob1, JobPriority.NORMAL);
-  }
-
-  private JobInProgress[] getJobsInQueue(boolean waiting) {
-    Collection<JobInProgress> queue =
-      waiting
-        ? scheduler.jobQueuesManager.getJobQueue("default").getWaitingJobs()
-        : scheduler.jobQueuesManager.getJobQueue("default").getRunningJobs();
-    return queue.toArray(new JobInProgress[0]);
-  }
-
-  // tests if tasks can be assinged when there are multiple jobs from a same
-  // user
-  public void testJobFinished() throws Exception {
-    taskTrackerManager.addQueues(new String[]{"default"});
-
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit 2 jobs
-    FakeJobInProgress j1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 3, 0, "default", "u1");
-    FakeJobInProgress j2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 3, 0, "default", "u1");
-
-    // I. Check multiple assignments with running tasks within job
-    // ask for a task from first job
-    Task t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000001_0 on tt1");
-    //  ask for another task from the first job
-    t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000002_0 on tt1");
-
-    // complete tasks
-    taskTrackerManager.finishTask("attempt_test_0001_m_000001_0", j1);
-    taskTrackerManager.finishTask("attempt_test_0001_m_000002_0", j1);
-
-    // II. Check multiple assignments with running tasks across jobs
-    // ask for a task from first job
-    t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000003_0 on tt1");
-
-    //  ask for a task from the second job
-    t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000001_0 on tt1");
-
-    // complete tasks
-    taskTrackerManager.finishTask("attempt_test_0002_m_000001_0", j2);
-    taskTrackerManager.finishTask("attempt_test_0001_m_000003_0", j1);
-
-    // III. Check multiple assignments with completed tasks across jobs
-    // ask for a task from the second job
-    t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000002_0 on tt1");
-
-    // complete task
-    taskTrackerManager.finishTask("attempt_test_0002_m_000002_0", j2);
-
-    // IV. Check assignment with completed job
-    // finish first job
-    scheduler.jobQueuesManager.getJobQueue(j1).jobCompleted(j1);
-
-    // ask for another task from the second job
-    // if tasks can be assigned then the structures are properly updated 
-    t = checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000003_0 on tt1");
-
-    // complete task
-    taskTrackerManager.finishTask("attempt_test_0002_m_000003_0", j2);
-  }
-
-  /**
-   * tests the submission of jobs to container and job queues
-   * @throws Exception
-   */
-  public void testJobSubmission() throws Exception {
-    JobQueueInfo[] queues = TestQueueManagerRefresh.getSimpleQueueHierarchy();
-
-    queues[0].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(100));
-    queues[1].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-    queues[2].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-    setUp(1, 4, 4);
-    // use the queues from the config file.
-    taskTrackerManager.setQueueManager(new QueueManager());
-    scheduler.start();
-
-    // submit a job to the container queue
-    try {
-      taskTrackerManager.submitJobAndInit(JobStatus.PREP, 20, 0,
-          queues[0].getQueueName(), "user");
-      fail("Jobs are being able to be submitted to the container queue");
-    } catch (Exception e) {
-      assertTrue(scheduler.getJobs(queues[0].getQueueName()).isEmpty());
-    }
-
-    FakeJobInProgress job = taskTrackerManager.submitJobAndInit(JobStatus.PREP,
-        1, 0, queues[1].getQueueName(), "user");
-    assertEquals(1, scheduler.getJobs(queues[1].getQueueName()).size());
-    assertTrue(scheduler.getJobs(queues[1].getQueueName()).contains(job));
-
-    // check if the job is submitted
-    checkAssignment(taskTrackerManager, scheduler, "tt1", 
-    "attempt_test_0002_m_000001_0 on tt1");
-
-    // test for getJobs
-    HashMap<String, ArrayList<FakeJobInProgress>> subJobsList =
-      taskTrackerManager.submitJobs(1, 4, queues[2].getQueueName());
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    //Raise status change events for jobs submitted.
-    raiseStatusChangeEvents(mgr, queues[2].getQueueName());
-    Collection<JobInProgress> jobs =
-      scheduler.getJobs(queues[2].getQueueName());
-
-    assertTrue(
-      "Number of jobs returned by scheduler is wrong"
-      , jobs.size() == 4);
-
-    assertTrue(
-      "Submitted jobs and Returned jobs are not same",
-      subJobsList.get("u1").containsAll(jobs));
-  }
-
-  //Basic test to test capacity allocation across the queues which have no
-  //capacity configured.
-
-  public void testCapacityAllocationToQueues() throws Exception {
-    String[] qs = {"default", "qAZ1", "qAZ2", "qAZ3", "qAZ4"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 25.0f, true, 25));
-    queues.add(new FakeQueueInfo("qAZ1", -1.0f, true, 25));
-    queues.add(new FakeQueueInfo("qAZ2", -1.0f, true, 25));
-    queues.add(new FakeQueueInfo("qAZ3", -1.0f, true, 25));
-    queues.add(new FakeQueueInfo("qAZ4", -1.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-    JobQueuesManager jqm = scheduler.jobQueuesManager;
-    assertEquals(18.75f, jqm.getJobQueue("qAZ1").qsc.getCapacityPercent());
-    assertEquals(18.75f, jqm.getJobQueue("qAZ2").qsc.getCapacityPercent());
-    assertEquals(18.75f, jqm.getJobQueue("qAZ3").qsc.getCapacityPercent());
-    assertEquals(18.75f, jqm.getJobQueue("qAZ4").qsc.getCapacityPercent());
-  }
-
-  public void testCapacityAllocFailureWithLowerMaxCapacity() throws Exception {
-    String[] qs = {"default", "qAZ1"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 25.0f, true, 25));
-    FakeQueueInfo qi = new FakeQueueInfo("qAZ1", -1.0f, true, 25);
-    qi.maxCapacity = 40.0f;
-    queues.add(qi);
-    taskTrackerManager.setFakeQueues(queues);
-    try {
-      scheduler.start();
-      fail("scheduler start should fail ");
-    }catch(IOException ise) {
-      Throwable e = ise.getCause();
-      assertTrue(e instanceof IllegalStateException);
-      assertEquals(
-        e.getMessage(),
-        " Capacity share (" + 75.0f + ")for unconfigured queue " + "qAZ1" +
-          " is greater than its maximum-capacity percentage " + 40.0f);
-    }
-  }
-
-  // Tests how capacity is computed and assignment of tasks done
-  // on the basis of the capacity.
-  public void testCapacityBasedAllocation() throws Exception {
-    // set up some queues
-    String[] qs = {"default", "q2"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    // set the capacity % as 10%, so that capacity will be zero initially as 
-    // the cluster capacity increase slowly.
-    queues.add(new FakeQueueInfo("default", 10.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 90.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a job to the default queue
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 0, "default", "u1");
-
-    // submit a job to the second queue
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 0, "q2", "u1");
-
-    // job from q2 runs first because it has some non-zero capacity.
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000001_0 on tt1");
-    verifyCapacity(taskTrackerManager, "0", "default");
-    verifyCapacity(taskTrackerManager, "3", "q2");
-
-    // add another tt to increase tt slots
-    taskTrackerManager.addTaskTracker("tt3");
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0002_m_000002_0 on tt2");
-    verifyCapacity(taskTrackerManager, "0", "default");
-    verifyCapacity(taskTrackerManager, "5", "q2");
-
-    // add another tt to increase tt slots
-    taskTrackerManager.addTaskTracker("tt4");
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt3",
-      "attempt_test_0002_m_000003_0 on tt3");
-    verifyCapacity(taskTrackerManager, "0", "default");
-    verifyCapacity(taskTrackerManager, "7", "q2");
-
-    // add another tt to increase tt slots
-    taskTrackerManager.addTaskTracker("tt5");
-    // now job from default should run, as it is furthest away
-    // in terms of runningMaps / capacity.
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt4",
-      "attempt_test_0001_m_000001_0 on tt4");
-    verifyCapacity(taskTrackerManager, "1", "default");
-    verifyCapacity(taskTrackerManager, "9", "q2");
-  }
-
-  // test capacity transfer
-  public void testCapacityTransfer() throws Exception {
-    // set up some queues
-    String[] qs = {"default", "q2"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a job  
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u1");
-    // for queue 'q2', the capacity for maps is 2. Since we're the only user,
-    // we should get a task
-    Map<String,String> expectedStrings = new HashMap<String,String>();
-    expectedStrings.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    // I should get another map task.
-    //No redduces as there is 1 slot only for reduce on TT
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000002_0 on tt1");
-
-    // Now we're at full capacity for maps. If I ask for another map task,
-    // I should get a map task from the default queue's capacity.
-    //same with reduce
-    expectedStrings.put(MAP,"attempt_test_0001_m_000003_0 on tt2");
-    expectedStrings.put(REDUCE,"attempt_test_0001_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-    
-    // and another
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0001_m_000004_0 on tt2");
-  }
-
-  /**
-   * test the high memory blocking with max capacity.
-   * @throws IOException
-   */
-  public void testHighMemoryBlockingWithMaxCapacity()
-    throws IOException {
-    taskTrackerManager = new FakeTaskTrackerManager(2, 2, 2);
-
-    taskTrackerManager.addQueues(new String[]{"defaultXYZM"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("defaultXYZM", 25.0f, true, 50));
-
-
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    // enabled memory-based scheduling
-    // Normal job in the cluster would be 1GB maps/reduces
-    scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-    scheduler.getRoot().getChildren().get(0).getQueueSchedulingContext()
-      .setMaxCapacityPercent(50);
-
-    JobConf jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(2 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(1);
-    jConf.setQueueName("defaultXYZM");
-    jConf.setUser("u1");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(2 * 1024);
-    jConf.setNumMapTasks(1);
-    jConf.setNumReduceTasks(2);
-    jConf.setQueueName("defaultXYZM");
-    jConf.setUser("u1");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-  //high ram map from job 1 and normal reduce task from job 1
-    HashMap<String,String> expectedStrings = new HashMap<String,String>();
-    expectedStrings.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-
-    List<Task> tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt1", expectedStrings);
-
-    checkOccupiedSlots("defaultXYZM", TaskType.MAP, 1, 2, 200.0f,1,0);
-    checkOccupiedSlots("defaultXYZM", TaskType.REDUCE, 1, 1, 100.0f,0,2);
-    checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 1 * 1024L);
-
-    //we have reached the maximum limit for map, so no more map tasks.
-    //we have used 1 reduce already and 1 more reduce slot is left for the
-    //before we reach maxcapacity for reduces.
-    // But current 1 slot + 2 slots for high ram reduce would
-    //mean we are crossing the maxium capacity.hence nothing would be assigned
-    //in this call
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-
-    //complete the high ram job on tt1.
-    for (Task task : tasks) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(),
-        job1);
-    }
-
-    expectedStrings.put(MAP,"attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE,"attempt_test_0002_r_000001_0 on tt2");
-
-    tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt2", expectedStrings);
-
-    checkOccupiedSlots("defaultXYZM", TaskType.MAP, 1, 2, 200.0f,1,0);
-    checkOccupiedSlots("defaultXYZM", TaskType.REDUCE, 1, 2, 200.0f,0,2);
-    checkMemReservedForTasksOnTT("tt2", 2 * 1024L, 2 * 1024L);
-
-    //complete the high ram job on tt1.
-    for (Task task : tasks) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(),
-        job2);
-    }
-
-
-    expectedStrings.put(MAP,"attempt_test_0002_m_000001_0 on tt2");
-    expectedStrings.put(REDUCE,"attempt_test_0002_r_000002_0 on tt2");
-
-    tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt2", expectedStrings);
-  }
-
-  /**
-   * test if user limits automatically adjust to max map or reduce limit
-   */
-  public void testUserLimitsWithMaxCapacity() throws Exception {
-    setUp(2, 2, 2);
-    // set up some queues
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 50));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-    scheduler.getRoot().getChildren().get(0).getQueueSchedulingContext()
-      .setMaxCapacityPercent(75);
-
-    // submit a job
-    FakeJobInProgress fjob1 =
-      taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "default", "u1");
-    FakeJobInProgress fjob2 =
-      taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "default", "u2");
-
-    // for queue 'default', maxCapacity for map and reduce is 3.
-    // initial user limit for 50% assuming there are 2 users/queue is.
-    //  1 map and 1 reduce.
-    // after max capacity it is 1.5 each.
-
-    //first job would be given 1 job each.
-    HashMap<String,String> expectedStrings = new HashMap<String,String>();
-    expectedStrings.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-
-    List<Task> tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt1", expectedStrings);
-
-
-    //for user u1 we have reached the limit. that is 1 job.
-    //1 more map and reduce tasks.
-    expectedStrings.put(MAP,"attempt_test_0002_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE,"attempt_test_0002_r_000001_0 on tt1");
-
-    tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt1", expectedStrings);
-
-    expectedStrings.put(MAP,"attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE,"attempt_test_0001_r_000002_0 on tt2");
-
-    tasks = checkMultipleTaskAssignment(taskTrackerManager,scheduler,
-      "tt2", expectedStrings);
-
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-
-  // Utility method to construct a map of expected strings
-  // with exactly one map task and one reduce task.
-  private void populateExpectedStrings(Map<String, String> expectedTaskStrings,
-                        String mapTask, String reduceTask) {
-    expectedTaskStrings.clear();
-    expectedTaskStrings.put(CapacityTestUtils.MAP, mapTask);
-    expectedTaskStrings.put(CapacityTestUtils.REDUCE, reduceTask);
-  }
-
-
-  // test user limits
-  public void testUserLimits() throws Exception {
-    // set up some queues
-    String[] qs = {"default", "q2"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a job  
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u1");
-    // for queue 'q2', the capacity is 2 for maps and 1 for reduce. 
-    // Since we're the only user, we should get tasks
-    Map<String, String> expectedTaskStrings = new HashMap<String, String>();
-    populateExpectedStrings(expectedTaskStrings, 
-              "attempt_test_0001_m_000001_0 on tt1", 
-              "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-                                  "tt1", expectedTaskStrings);
-
-    // Submit another job, from a different user
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u2");
-    // Now if I ask for a task, it should come from the second job
-    checkAssignment(taskTrackerManager, scheduler, 
-        "tt1", "attempt_test_0002_m_000001_0 on tt1");
-
-    // Now we're at full capacity. If I ask for another task,
-    // I should get tasks from the default queue's capacity.
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0001_m_000002_0 on tt2", 
-        "attempt_test_0002_r_000001_0 on tt2");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-          "tt2", expectedTaskStrings);
-    // and another
-    checkAssignment(taskTrackerManager, scheduler, 
-            "tt2", "attempt_test_0002_m_000002_0 on tt2");
-  }
-
-  // test user limits when a 2nd job is submitted much after first job 
-  public void testUserLimits2() throws Exception {
-    // set up some queues
-    String[] qs = {"default", "q2"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a job  
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u1");
-    // for queue 'q2', the capacity for maps is 2 and reduce is 1. 
-    // Since we're the only user, we should get tasks
-    Map<String, String> expectedTaskStrings = new HashMap<String, String>();
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0001_m_000001_0 on tt1",
-        "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-        "tt1", expectedTaskStrings);
-
-    // since we're the only job, we get another map
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000002_0 on tt1");
-
-    // Submit another job, from a different user
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u2");
-    // Now if I ask for a task, it should come from the second job
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0002_m_000001_0 on tt2",
-        "attempt_test_0002_r_000001_0 on tt2");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-        "tt2", expectedTaskStrings);
-    // and another
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0002_m_000002_0 on tt2");
-  }
-
-  // test user limits when a 2nd job is submitted much after first job 
-  // and we need to wait for first job's task to complete
-  public void testUserLimits3() throws Exception {
-    // set up some queues
-    String[] qs = {"default", "q2"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a job  
-    FakeJobInProgress j1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u1");
-    // for queue 'q2', the capacity for maps is 2 and reduces is 1. 
-    // Since we're the only user, we should get a task
-    Map<String, String> expectedTaskStrings = new HashMap<String, String>();
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0001_m_000001_0 on tt1", 
-        "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-        "tt1", expectedTaskStrings);
-    // since we're the only job, we get another map
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000002_0 on tt1");
-    // we get more tasks from 'default queue'
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0001_m_000003_0 on tt2",
-        "attempt_test_0001_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler,
-        "tt2", expectedTaskStrings);
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0001_m_000004_0 on tt2");
-
-    // Submit another job, from a different user
-    FakeJobInProgress j2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, "q2", "u2");
-    // one of the task finishes of each type
-    taskTrackerManager.finishTask("attempt_test_0001_m_000001_0", j1);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000001_0", j1);
-    
-    // Now if I ask for a task, it should come from the second job
-    populateExpectedStrings(expectedTaskStrings, 
-        "attempt_test_0002_m_000001_0 on tt1",
-        "attempt_test_0002_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler,
-        "tt1", expectedTaskStrings);
-
-    // another task from job1 finishes, another new task to job2
-    taskTrackerManager.finishTask("attempt_test_0001_m_000002_0", j1);
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000002_0 on tt1");
-
-    // now we have equal number of tasks from each job. Whichever job's
-    // task finishes, that job gets a new task
-    taskTrackerManager.finishTask("attempt_test_0001_m_000003_0", j1);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000002_0", j1);
-    populateExpectedStrings(expectedTaskStrings,
-        "attempt_test_0001_m_000005_0 on tt2",
-        "attempt_test_0001_r_000003_0 on tt2");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler,
-        "tt2", expectedTaskStrings);
-    taskTrackerManager.finishTask("attempt_test_0002_m_000001_0", j2);
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_m_000003_0 on tt1");
-  }
-
-  // test user limits with many users, more slots
-  public void testUserLimits4() throws Exception {
-    // set up one queue, with 10 map slots and 5 reduce slots
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-    // add some more TTs 
-    taskTrackerManager.addTaskTracker("tt3");
-    taskTrackerManager.addTaskTracker("tt4");
-    taskTrackerManager.addTaskTracker("tt5");
-
-    // u1 submits job
-    FakeJobInProgress j1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, null, "u1");
-    // it gets the first 5 slots
-    Map<String, String> expectedTaskStrings = new HashMap<String, String>();
-    for (int i=0; i<5; i++) {
-      String ttName = "tt"+(i+1);
-      populateExpectedStrings(expectedTaskStrings,
-          "attempt_test_0001_m_00000"+(i+1)+"_0 on " + ttName, 
-          "attempt_test_0001_r_00000"+(i+1)+"_0 on " + ttName);
-      checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-          ttName, expectedTaskStrings);
-    }
-      
-    // u2 submits job with 4 slots
-    FakeJobInProgress j2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, 4, 4, null, "u2");
-    // u2 should get next 4 slots
-    for (int i=0; i<4; i++) {
-      String ttName = "tt"+(i+1);
-      checkAssignment(taskTrackerManager, scheduler, ttName,
-          "attempt_test_0002_m_00000"+(i+1)+"_0 on " + ttName);
-    }
-    // last slot should go to u1, since u2 has no more tasks
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt5",
-      "attempt_test_0001_m_000006_0 on tt5");
-    // u1 finishes tasks
-    taskTrackerManager.finishTask("attempt_test_0001_m_000006_0", j1);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000005_0", j1);
-    // u1 submits a few more jobs 
-    // All the jobs are inited when submitted
-    // because of addition of Eager Job Initializer all jobs in this
-    //case would e initialised.
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, null, "u1");
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, null, "u1");
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, null, "u1");
-    // u2 also submits a job
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 10, 10, null, "u2");
-    // now u3 submits a job
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 2, 2, null, "u3");
-    // next map slot should go to u3, even though u2 has an earlier job, since
-    // user limits have changed and u1/u2 are over limits
-    // reduce slot will go to job 2, as it is still under limit.
-    populateExpectedStrings(expectedTaskStrings,
-        "attempt_test_0007_m_000001_0 on tt5",
-        "attempt_test_0002_r_000001_0 on tt5");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, 
-        "tt5", expectedTaskStrings);
-    // some other task finishes and u3 gets it
-    taskTrackerManager.finishTask("attempt_test_0002_m_000004_0", j1);
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt4",
-      "attempt_test_0007_m_000002_0 on tt4");
-    // now, u2 finishes a task
-    taskTrackerManager.finishTask("attempt_test_0002_m_000002_0", j1);
-    // next slot will go to u1, since u3 has nothing to run and u1's job is 
-    // first in the queue
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0001_m_000007_0 on tt2");
-  }
-
-  /**
-   * Test to verify that high memory jobs hit user limits faster than any normal
-   * job.
-   *
-   * @throws IOException
-   */
-  public void testUserLimitsForHighMemoryJobs()
-    throws IOException {
-    taskTrackerManager = new FakeTaskTrackerManager(1, 10, 10);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 50));
-
-
-    // enabled memory-based scheduling
-    // Normal job in the cluster would be 1GB maps/reduces
-    scheduler.getConf().setLong(
-      JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(
-      MRConfig.MAPMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(
-      JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(
-      MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // Submit one normal job to the other queue.
-    JobConf jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(6);
-    jConf.setNumReduceTasks(6);
-    jConf.setUser("u1");
-    jConf.setQueueName("default");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    LOG.debug(
-      "Submit one high memory(2GB maps, 2GB reduces) job of "
-        + "6 map and 6 reduce tasks");
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(2 * 1024);
-    jConf.setMemoryForReduceTask(2 * 1024);
-    jConf.setNumMapTasks(6);
-    jConf.setNumReduceTasks(6);
-    jConf.setQueueName("default");
-    jConf.setUser("u2");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // Verify that normal job takes 5 task assignments to hit user limits
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    for (int i = 0; i < 5; i++) {
-      expectedStrings.clear();
-      expectedStrings.put(
-        CapacityTestUtils.MAP,
-        "attempt_test_0001_m_00000" + (i + 1) + "_0 on tt1");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE,
-        "attempt_test_0001_r_00000" + (i + 1) + "_0 on tt1");
-      checkMultipleTaskAssignment(
-        taskTrackerManager, scheduler, "tt1",
-        expectedStrings);
-    }
-    // u1 has 5 map slots and 5 reduce slots. u2 has none. So u1's user limits
-    // are hit. So u2 should get slots
-
-    for (int i = 0; i < 2; i++) {
-      expectedStrings.clear();
-      expectedStrings.put(
-        CapacityTestUtils.MAP,
-        "attempt_test_0002_m_00000" + (i + 1) + "_0 on tt1");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE,
-        "attempt_test_0002_r_00000" + (i + 1) + "_0 on tt1");
-      checkMultipleTaskAssignment(
-        taskTrackerManager, scheduler, "tt1",
-        expectedStrings);
-    }  // u1 has 5 map slots and 5 reduce slots. u2 has 4 map slots and 4 reduce
-    // slots. Because of high memory tasks, giving u2 another task would
-    // overflow limits. So, no more tasks should be given to anyone.
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-  }
-
-  /*
-   * Following is the testing strategy for testing scheduling information.
-   * - start capacity scheduler with two queues.
-   * - check the scheduling information with respect to the configuration
-   * which was used to configure the queues.
-   * - Submit 5 jobs to a queue.
-   * - Check the waiting jobs count, it should be 5.
-   * - Then run initializationPoller()
-   * - Check once again the waiting queue, it should be 5 jobs again.
-   * - Then raise status change events.
-   * - Assign tasks to a task tracker.
-   * - Check waiting job count, it should be 4 now and used map (%) = 100
-   * and used reduce (%) = 100
-   * - finish the job and then check the used percentage it should go
-   * back to zero
-   * - Then pick an initialized job but not scheduled job and fail it.
-   * - Run the poller
-   * - Check the waiting job count should now be 3.
-   * - Now fail a job which has not been initialized at all.
-   * - Run the poller, so that it can clean up the job queue.
-   * - Check the count, the waiting job count should be 2.
-   * - Now raise status change events to move the initialized jobs which
-   * should be two in count to running queue.
-   * - Then schedule a map and reduce of the job in running queue.
-   * - Run the poller because the poller is responsible for waiting
-   * jobs count. Check the count, it should be using 100% map, reduce and one
-   * waiting job
-   * - fail the running job.
-   * - Check the count, it should be now one waiting job and zero running
-   * tasks
-   */
-
-  public void testSchedulingInformation() throws Exception {
-    String[] qs = {"default", "q2"};
-    taskTrackerManager = new FakeTaskTrackerManager(2, 1, 1);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q2", 50.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    scheduler.assignTasks(tracker("tt1")); // heartbeat
-    scheduler.assignTasks(tracker("tt2")); // heartbeat
-    int totalMaps = taskTrackerManager.getClusterStatus().getMaxMapTasks();
-    int totalReduces =
-      taskTrackerManager.getClusterStatus().getMaxReduceTasks();
-    QueueManager queueManager = scheduler.taskTrackerManager.getQueueManager();
-    String schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    String schedulingInfo2 =
-      queueManager.getJobQueueInfo("q2").getSchedulingInfo();
-
-    String[] infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 18);
-    assertEquals(infoStrings[0], "Queue configuration");
-    assertEquals(infoStrings[1], "Capacity Percentage: 50.0%");
-    assertEquals(infoStrings[2], "User Limit: 25%");
-    assertEquals(infoStrings[3], "Priority Supported: YES");
-    assertEquals(infoStrings[4], "-------------");
-    assertEquals(infoStrings[5], "Map tasks");
-    assertEquals(
-      infoStrings[6], "Capacity: " + totalMaps * 50 / 100
-        + " slots");
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[9], "-------------");
-    assertEquals(infoStrings[10], "Reduce tasks");
-    assertEquals(
-      infoStrings[11], "Capacity: " + totalReduces * 50 / 100
-        + " slots");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[14], "-------------");
-    assertEquals(infoStrings[15], "Job info");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 0");
-    assertEquals(infoStrings[17], "Number of users who have submitted jobs: 0");
-
-    assertEquals(schedulingInfo, schedulingInfo2);
-
-    //Testing with actual job submission.
-    ArrayList<FakeJobInProgress> userJobs =
-      taskTrackerManager.submitJobs(1, 5, "default").get("u1");
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-
-    //waiting job should be equal to number of jobs submitted.
-    assertEquals(infoStrings.length, 18);
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 5");
-    assertEquals(infoStrings[17], "Number of users who have submitted jobs: 1");
-
-    //Initalize the jobs but don't raise events
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 18);
-    //should be previous value as nothing is scheduled because no events
-    //has been raised after initialization.
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 5");
-
-    //Raise status change event so that jobs can move to running queue.
-    raiseStatusChangeEvents(scheduler.jobQueuesManager);
-    raiseStatusChangeEvents(scheduler.jobQueuesManager, "q2");
-    //assign one job
-    Map<String, String> strs = new HashMap<String, String>();
-    strs.put(CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    strs.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    List<Task> t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-    //Initalize extra job.
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    //Get scheduling information, now the number of waiting job should have
-    //changed to 4 as one is scheduled and has become running.
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-
-    assertEquals(infoStrings.length, 22);
-    assertEquals(infoStrings[7], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 1");
-    assertEquals(infoStrings[9], "Active users:");
-    assertEquals(infoStrings[10], "User 'u1': 1 (100.0% of used capacity)");
-    assertEquals(infoStrings[14], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[15], "Running tasks: 1");
-    assertEquals(infoStrings[20], "Number of Waiting Jobs: 4");
-
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-
-    assertEquals(infoStrings.length, 22);
-    assertEquals(infoStrings[7], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 1");
-    assertEquals(infoStrings[9], "Active users:");
-    assertEquals(infoStrings[10], "User 'u1': 1 (100.0% of used capacity)");
-    assertEquals(infoStrings[14], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[15], "Running tasks: 1");
-    assertEquals(infoStrings[16], "Active users:");
-    assertEquals(infoStrings[17], "User 'u1': 1 (100.0% of used capacity)");
-    assertEquals(infoStrings[20], "Number of Waiting Jobs: 4");
-
-    //Complete the job and check the running tasks count
-    FakeJobInProgress u1j1 = userJobs.get(0);
-    for (Task task : t1) {
-      taskTrackerManager.finishTask(task.getTaskID().toString(), u1j1);
-    }
-    taskTrackerManager.finalizeJob(u1j1);
-
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-
-    assertEquals(infoStrings.length, 18);
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 4");
-
-    //Fail a job which is initialized but not scheduled and check the count.
-    FakeJobInProgress u1j2 = userJobs.get(1);
-    assertTrue(
-      "User1 job 2 not initalized ",
-      u1j2.getStatus().getRunState() == JobStatus.RUNNING);
-    taskTrackerManager.finalizeJob(u1j2, JobStatus.FAILED);
-    //Run initializer to clean up failed jobs
-    controlledInitializationPoller.selectJobsToInitialize();
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 18);
-    //should be previous value as nothing is scheduled because no events
-    //has been raised after initialization.
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 3");
-
-    //Fail a job which is not initialized but is in the waiting queue.
-    FakeJobInProgress u1j5 = userJobs.get(4);
-    assertFalse(
-      "User1 job 5 initalized ",
-      u1j5.getStatus().getRunState() == JobStatus.RUNNING);
-
-    taskTrackerManager.finalizeJob(u1j5, JobStatus.FAILED);
-    //run initializer to clean up failed job
-    controlledInitializationPoller.selectJobsToInitialize();
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 18);
-    //should be previous value as nothing is scheduled because no events
-    //has been raised after initialization.
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[12], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[13], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 2");
-
-    //Raise status change events as none of the intialized jobs would be
-    //in running queue as we just failed the second job which was initialized
-    //and completed the first one.
-    raiseStatusChangeEvents(scheduler.jobQueuesManager);
-    raiseStatusChangeEvents(scheduler.jobQueuesManager, "q2");
-
-    //Now schedule a map should be job3 of the user as job1 succeeded job2
-    //failed and now job3 is running
-    strs.clear();
-    strs.put(CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt1");
-    strs.put(CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt1");
-    t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-    FakeJobInProgress u1j3 = userJobs.get(2);
-    assertTrue(
-      "User Job 3 not running ",
-      u1j3.getStatus().getRunState() == JobStatus.RUNNING);
-
-    //now the running count of map should be one and waiting jobs should be
-    //one. run the poller as it is responsible for waiting count
-    controlledInitializationPoller.selectJobsToInitialize();
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 22);
-    assertEquals(infoStrings[7], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 1");
-    assertEquals(infoStrings[9], "Active users:");
-    assertEquals(infoStrings[10], "User 'u1': 1 (100.0% of used capacity)");
-    assertEquals(infoStrings[14], "Used capacity: 1 (100.0% of Capacity)");
-    assertEquals(infoStrings[15], "Running tasks: 1");
-    assertEquals(infoStrings[16], "Active users:");
-    assertEquals(infoStrings[17], "User 'u1': 1 (100.0% of used capacity)");
-    assertEquals(infoStrings[20], "Number of Waiting Jobs: 1");
-
-    //Fail the executing job
-    taskTrackerManager.finalizeJob(u1j3, JobStatus.FAILED);
-    // make sure we update our stats
-    scheduler.updateContextInfoForTests();
-    //Now running counts should become zero
-    schedulingInfo =
-      queueManager.getJobQueueInfo("default").getSchedulingInfo();
-    infoStrings = schedulingInfo.split("\n");
-    assertEquals(infoStrings.length, 18);
-    assertEquals(infoStrings[7], "Used capacity: 0 (0.0% of Capacity)");
-    assertEquals(infoStrings[8], "Running tasks: 0");
-    assertEquals(infoStrings[16], "Number of Waiting Jobs: 1");
-  }
-
-  /**
-   * Test to verify that highMemoryJobs are scheduled like all other jobs when
-   * memory-based scheduling is not enabled.
-   *
-   * @throws IOException
-   */
-  public void testDisabledMemoryBasedScheduling()
-    throws IOException {
-
-    LOG.debug("Starting the scheduler.");
-    taskTrackerManager = new FakeTaskTrackerManager(1, 1, 1);
-
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    // memory-based scheduling disabled by default.
-    scheduler.start();
-
-    LOG.debug(
-      "Submit one high memory job of 1 3GB map task "
-        + "and 1 1GB reduce task.");
-    JobConf jConf = new JobConf();
-    jConf.setMemoryForMapTask(3 * 1024L); // 3GB
-    jConf.setMemoryForReduceTask(1 * 1024L); // 1 GB
-    jConf.setNumMapTasks(1);
-    jConf.setNumReduceTasks(1);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    taskTrackerManager.submitJobAndInit(JobStatus.RUNNING, jConf);
-
-    // assert that all tasks are launched even though they transgress the
-    // scheduling limits.
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-  }
-
-  /**
-   * Test reverting HADOOP-4979. If there is a high-mem job, we should now look
-   * at reduce jobs (if map tasks are high-mem) or vice-versa.
-   *
-   * @throws IOException
-   */
-  public void testHighMemoryBlockingAcrossTaskTypes()
-    throws IOException {
-
-    // 2 map and 1 reduce slots
-    taskTrackerManager = new FakeTaskTrackerManager(1, 2, 2);
-
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-
-
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    // enabled memory-based scheduling
-    // Normal job in the cluster would be 1GB maps/reduces
-    scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // The situation : Two jobs in the queue. First job with only maps and no
-    // reduces and is a high memory job. Second job is a normal job with both
-    // maps and reduces.
-    // First job cannot run for want of memory for maps. In this case, second
-    // job's reduces should run.
-
-    LOG.debug(
-      "Submit one high memory(2GB maps, 0MB reduces) job of "
-        + "2 map tasks");
-    JobConf jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(2 * 1024);
-    jConf.setMemoryForReduceTask(0);
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(0);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    LOG.debug(
-      "Submit another regular memory(1GB vmem maps/reduces) job of "
-        + "2 map/red tasks");
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(2);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // first, a map from j1 and a reduce from other job j2
-    Map<String,String> strs = new HashMap<String,String>();
-    strs.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-    strs.put(REDUCE,"attempt_test_0002_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-    // Total 2 map slots should be accounted for.
-    checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f);
-    checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 50.0f);
-    checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 1 * 1024L);
-
-    //TT has 2 slots for reduces hence this call should get a reduce task
-    //from other job
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0002_r_000002_0 on tt1");
-    checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f);
-    checkOccupiedSlots("default", TaskType.REDUCE, 1, 2, 100.0f);
-    checkMemReservedForTasksOnTT("tt1", 2 * 1024L, 2 * 1024L);
-
-    //now as all the slots are occupied hence no more tasks would be
-    //assigned.
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-  }
-
-    /**
-   * Tests that scheduler schedules normal jobs once high RAM jobs
-   * have been reserved to the limit.
-   *
-   * The test causes the scheduler to schedule a normal job on two
-   * trackers, and one task of the high RAM job on a third. Then it
-   * asserts that one of the first two trackers gets a reservation
-   * for the remaining task of the high RAM job. After this, it
-   * asserts that a normal job submitted later is allowed to run
-   * on a free slot, as all tasks of the high RAM job are either
-   * scheduled or reserved.
-   *
-   * @throws IOException
-   */
-  public void testClusterBlockingForLackOfMemory()
-      throws IOException {
-
-      LOG.debug("Starting the scheduler.");
-      taskTrackerManager = new FakeTaskTrackerManager(3, 2, 2);
-
-      ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-      queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-      taskTrackerManager.addQueues(new String[]{"default"});
-      scheduler.setTaskTrackerManager(taskTrackerManager);
-      // enabled memory-based scheduling
-      // Normal jobs 1GB maps/reduces. 2GB limit on maps/reduces
-      scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-      scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
-      scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024);
-      scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-      taskTrackerManager.setFakeQueues(queues);
-      scheduler.start();
-
-      LOG.debug(
-        "Submit one normal memory(1GB maps/reduces) job of "
-          + "2 map, 2 reduce tasks.");
-      JobConf jConf = new JobConf(conf);
-      jConf.setMemoryForMapTask(1 * 1024);
-      jConf.setMemoryForReduceTask(1 * 1024);
-      jConf.setNumMapTasks(2);
-      jConf.setNumReduceTasks(2);
-      jConf.setQueueName("default");
-      jConf.setUser("u1");
-      FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, jConf);
-
-      // Fill a tt with this job's tasks.
-      Map<String, String> expectedStrings = new HashMap<String, String>();
-      expectedStrings.put(
-        CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-      checkMultipleTaskAssignment(
-        taskTrackerManager, scheduler, "tt1",
-        expectedStrings);
-      // Total 1 map slot should be accounted for.
-      checkOccupiedSlots("default", TaskType.MAP, 1, 1, 16.7f);
-      checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 16.7f);
-      assertEquals(JobQueue.getJobQueueSchedInfo(1, 1, 0, 1, 1, 0),
-                   job1.getSchedulingInfo().toString());
-      checkMemReservedForTasksOnTT("tt1", 1 * 1024L, 1 * 1024L);
-
-      expectedStrings.clear();
-      expectedStrings.put(
-        CapacityTestUtils.MAP, "attempt_test_0001_m_000002_0 on tt2");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE, "attempt_test_0001_r_000002_0 on tt2");
-
-      // fill another TT with the rest of the tasks of the job
-      checkMultipleTaskAssignment(
-        taskTrackerManager, scheduler, "tt2",
-        expectedStrings);
-
-      LOG.debug(
-        "Submit one high memory(2GB maps/reduces) job of "
-          + "2 map, 2 reduce tasks.");
-      jConf = new JobConf(conf);
-      jConf.setMemoryForMapTask(2 * 1024);
-      jConf.setMemoryForReduceTask(2 * 1024);
-      jConf.setNumMapTasks(2);
-      jConf.setNumReduceTasks(2);
-      jConf.setQueueName("default");
-      jConf.setUser("u1");
-      FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, jConf);
-
-      // Have another TT run one task of each type of the high RAM
-      // job. This will fill up the TT.
-      expectedStrings.clear();
-      expectedStrings.put(
-        CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt3");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt3");
-
-      checkMultipleTaskAssignment(taskTrackerManager, scheduler,
-          "tt3", expectedStrings);
-      checkOccupiedSlots("default", TaskType.MAP, 1, 4, 66.7f);
-      checkOccupiedSlots("default", TaskType.REDUCE, 1, 4, 66.7f);
-      assertEquals(JobQueue.getJobQueueSchedInfo(1, 2, 0, 1, 2, 0),
-                   job2.getSchedulingInfo().toString());
-      checkMemReservedForTasksOnTT("tt3", 2 * 1024L, 2 * 1024L);
-
-      LOG.debug(
-        "Submit one normal memory(1GB maps/reduces) job of "
-          + "1 map, 1 reduce tasks.");
-      jConf = new JobConf(conf);
-      jConf.setMemoryForMapTask(1 * 1024);
-      jConf.setMemoryForReduceTask(1 * 1024);
-      jConf.setNumMapTasks(1);
-      jConf.setNumReduceTasks(1);
-      jConf.setQueueName("default");
-      jConf.setUser("u1");
-      FakeJobInProgress job3 = taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, jConf);
-
-      // Send a TT with insufficient space for task assignment,
-      // This will cause a reservation for the high RAM job.
-      assertNull(scheduler.assignTasks(tracker("tt1")));
-
-      // reserved tasktrackers contribute to occupied slots for maps and reduces
-      checkOccupiedSlots("default", TaskType.MAP, 1, 6, 100.0f);
-      checkOccupiedSlots("default", TaskType.REDUCE, 1, 6, 100.0f);
-      checkMemReservedForTasksOnTT("tt1", 1 * 1024L, 1 * 1024L);
-      LOG.info(job2.getSchedulingInfo());
-      assertEquals(JobQueue.getJobQueueSchedInfo(1, 2, 2, 1, 2, 2),
-                   job2.getSchedulingInfo().toString());
-      assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 0, 0, 0, 0),
-                   job3.getSchedulingInfo().toString());
-
-      // Reservations are already done for job2. So job3 should go ahead.
-      expectedStrings.clear();
-      expectedStrings.put(
-        CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt2");
-      expectedStrings.put(
-        CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt2");
-
-      checkMultipleTaskAssignment(
-        taskTrackerManager, scheduler, "tt2",
-        expectedStrings);
-    }
-
-  /**
-   * Testcase to verify fix for a NPE (HADOOP-5641), when memory based
-   * scheduling is enabled and jobs are retired from memory when tasks
-   * are still active on some Tasktrackers.
-   *
-   * @throws IOException
-   */
-  public void testMemoryMatchingWithRetiredJobs() throws IOException {
-    // create a cluster with a single node.
-    LOG.debug("Starting cluster with 1 tasktracker, 2 map and 2 reduce slots");
-    taskTrackerManager = new FakeTaskTrackerManager(1, 2, 2);
-
-    // create scheduler
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-    taskTrackerManager.addQueues(new String[]{"default"});
-
-
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    // enabled memory-based scheduling
-    LOG.debug("Assume TT has 2GB for maps and 2GB for reduces");
-    scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024L);
-    scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 512);
-    scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 2 * 1024L);
-    scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 512);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    // submit a normal job
-    LOG.debug("Submitting a normal job with 2 maps and 2 reduces");
-    JobConf jConf = new JobConf();
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(2);
-    jConf.setMemoryForMapTask(512);
-    jConf.setMemoryForReduceTask(512);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // 1st cycle - 1 map and reduce gets assigned.
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    List<Task> t = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    // Total 1 map slot and 1 reduce slot should be accounted for.
-    checkOccupiedSlots("default", TaskType.MAP, 1, 1, 50.0f);
-    checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 50.0f);
-    checkMemReservedForTasksOnTT("tt1", 512L, 512L);
-
-    // kill this job !
-    taskTrackerManager.killJob(job1.getJobID(), false);
-    // No more map/reduce slots should be accounted for.
-    checkOccupiedSlots("default", TaskType.MAP, 0, 0, 0.0f);
-    checkOccupiedSlots(
-      "default", TaskType.REDUCE, 0, 0,
-      0.0f);
-
-    // retire the job
-    taskTrackerManager.retireJob(job1.getJobID());
-
-    // submit another job.
-    LOG.debug("Submitting another normal job with 2 maps and 2 reduces");
-    jConf = new JobConf();
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(2);
-    jConf.setMemoryForMapTask(512);
-    jConf.setMemoryForReduceTask(512);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // since with HADOOP-5964, we don't rely on a job conf to get
-    // the memory occupied, scheduling should be able to work correctly.
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt1");
-
-    List<Task> t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    checkOccupiedSlots("default", TaskType.MAP, 1, 1, 50);
-    checkOccupiedSlots("default", TaskType.REDUCE, 1, 1, 50);
-    checkMemReservedForTasksOnTT("tt1", 1024L, 1024L);
-
-    // now, no more can be assigned because all the slots are blocked.
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-
-    // finish the tasks on the tracker.
-    for (Task task : t) {
-      taskTrackerManager.finishTask(task.getTaskID().toString(), job1);
-    }
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000002_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000002_0 on tt1");
-
-    // now a new task can be assigned.
-    t = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    checkOccupiedSlots("default", TaskType.MAP, 1, 2, 100.0f);
-    checkOccupiedSlots("default", TaskType.REDUCE, 1, 2, 100.0f);
-    // memory used will change because of the finished task above.
-    checkMemReservedForTasksOnTT("tt1", 1024L, 1024L);
-  }
-
-  /*
-   * Test cases for Job Initialization poller.
-   */
-
-  /*
-  * This test verifies that the correct number of jobs for
-  * correct number of users is initialized.
-  * It also verifies that as jobs of users complete, new jobs
-  * from the correct users are initialized.
-  */
-
-  public void testJobInitialization() throws Exception {
-    // set up the scheduler
-    String[] qs = {"default"};
-    taskTrackerManager = new FakeTaskTrackerManager(2, 1, 1);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    JobInitializationPoller initPoller = scheduler.getInitializationPoller();
-
-    // submit 4 jobs each for 3 users.
-    HashMap<String, ArrayList<FakeJobInProgress>> userJobs =
-      taskTrackerManager.submitJobs(
-        3,
-        4, "default");
-
-    // get the jobs submitted.
-    ArrayList<FakeJobInProgress> u1Jobs = userJobs.get("u1");
-    ArrayList<FakeJobInProgress> u2Jobs = userJobs.get("u2");
-    ArrayList<FakeJobInProgress> u3Jobs = userJobs.get("u3");
-
-    // reference to the initializedJobs data structure
-    // changes are reflected in the set as they are made by the poller
-    Set<JobID> initializedJobs = initPoller.getInitializedJobList();
-
-    // we should have 12 (3 x 4) jobs in the job queue
-    assertEquals(mgr.getJobQueue("default").getWaitingJobs().size(), 12);
-
-    // run one poller iteration.
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // the poller should initialize 6 jobs
-    // 3 users and 2 jobs from each
-    assertEquals(initializedJobs.size(), 6);
-
-    assertTrue(
-      "Initialized jobs didnt contain the user1 job 1",
-      initializedJobs.contains(u1Jobs.get(0).getJobID()));
-    assertTrue(
-      "Initialized jobs didnt contain the user1 job 2",
-      initializedJobs.contains(u1Jobs.get(1).getJobID()));
-    assertTrue(
-      "Initialized jobs didnt contain the user2 job 1",
-      initializedJobs.contains(u2Jobs.get(0).getJobID()));
-    assertTrue(
-      "Initialized jobs didnt contain the user2 job 2",
-      initializedJobs.contains(u2Jobs.get(1).getJobID()));
-    assertTrue(
-      "Initialized jobs didnt contain the user3 job 1",
-      initializedJobs.contains(u3Jobs.get(0).getJobID()));
-    assertTrue(
-      "Initialized jobs didnt contain the user3 job 2",
-      initializedJobs.contains(u3Jobs.get(1).getJobID()));
-
-    // now submit one more job from another user.
-    FakeJobInProgress u4j1 =
-      taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "default", "u4");
-
-    // run the poller again.
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // since no jobs have started running, there should be no
-    // change to the initialized jobs.
-    assertEquals(initializedJobs.size(), 6);
-    assertFalse(
-      "Initialized jobs contains user 4 jobs",
-      initializedJobs.contains(u4j1.getJobID()));
-
-    // This event simulates raising the event on completion of setup task
-    // and moves the job to the running list for the scheduler to pick up.
-    raiseStatusChangeEvents(mgr);
-
-    // get some tasks assigned.
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-
-    List<Task> t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt2");
-
-    List<Task> t2 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-    for (Task task : t1) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(), u1Jobs.get(
-          0));
-    }
-
-    for (Task task : t2) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(), u1Jobs.get(
-          0));
-    }
-    // as some jobs have running tasks, the poller will now
-    // pick up new jobs to initialize.
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // count should still be the same
-    assertEquals(initializedJobs.size(), 6);
-
-    // new jobs that have got into the list
-    assertTrue(initializedJobs.contains(u1Jobs.get(2).getJobID()));
-    assertTrue(initializedJobs.contains(u1Jobs.get(3).getJobID()));
-    raiseStatusChangeEvents(mgr);
-
-    // the first two jobs are done, no longer in the initialized list.
-    assertFalse(
-      "Initialized jobs contains the user1 job 1",
-      initializedJobs.contains(u1Jobs.get(0).getJobID()));
-    assertFalse(
-      "Initialized jobs contains the user1 job 2",
-      initializedJobs.contains(u1Jobs.get(1).getJobID()));
-
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt1");
-
-    // finish one more job
-    t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    for (Task task : t1) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(), u1Jobs.get(
-          2));
-    }
-
-    // no new jobs should be picked up, because max user limit
-    // is still 3.
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    assertEquals(initializedJobs.size(), 5);
-
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0004_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0004_r_000001_0 on tt1");
-
-    // run 1 more jobs.. 
-    t1 = checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    for (Task task : t1) {
-      taskTrackerManager.finishTask(
-        task.getTaskID().toString(), u1Jobs.get(
-          3));
-    }
-
-    // Now initialised jobs should contain user 4's job, as
-    // user 1's jobs are all done and the number of users is
-    // below the limit
-    controlledInitializationPoller.selectJobsToInitialize();
-    assertEquals(initializedJobs.size(), 5);
-    assertTrue(initializedJobs.contains(u4j1.getJobID()));
-
-    controlledInitializationPoller.stopRunning();
-  }
-
-  /*
-   * testHighPriorityJobInitialization() shows behaviour when high priority job
-   * is submitted into a queue and how initialisation happens for the same.
-   */
-  public void testHighPriorityJobInitialization() throws Exception {
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    JobInitializationPoller initPoller = scheduler.getInitializationPoller();
-    Set<JobID> initializedJobsList = initPoller.getInitializedJobList();
-
-    // submit 3 jobs for 3 users
-    taskTrackerManager.submitJobs(3, 3, "default");
-    controlledInitializationPoller.selectJobsToInitialize();
-    assertEquals(initializedJobsList.size(), 6);
-
-    // submit 2 job for a different user. one of them will be made high priority
-    FakeJobInProgress u4j1 = taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "default", "u4");
-    FakeJobInProgress u4j2 = taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "default", "u4");
-
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // shouldn't change
-    assertEquals(initializedJobsList.size(), 6);
-
-    assertFalse(
-      "Contains U4J1 high priority job ",
-      initializedJobsList.contains(u4j1.getJobID()));
-    assertFalse(
-      "Contains U4J2 Normal priority job ",
-      initializedJobsList.contains(u4j2.getJobID()));
-
-    // change priority of one job
-    taskTrackerManager.setPriority(u4j1, JobPriority.VERY_HIGH);
-
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // the high priority job should get initialized, but not the
-    // low priority job from u4, as we have already exceeded the
-    // limit.
-    assertEquals(initializedJobsList.size(), 7);
-    assertTrue(
-      "Does not contain U4J1 high priority job ",
-      initializedJobsList.contains(u4j1.getJobID()));
-    assertFalse(
-      "Contains U4J2 Normal priority job ",
-      initializedJobsList.contains(u4j2.getJobID()));
-    controlledInitializationPoller.stopRunning();
-  }
-
-  public void testJobMovement() throws Exception {
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-
-    // check proper running job movement and completion
-    checkRunningJobMovementAndCompletion();
-
-    // check failed running job movement
-    checkFailedRunningJobMovement();
-
-    // Check job movement of failed initalized job
-    checkFailedInitializedJobMovement();
-
-    // Check failed waiting job movement
-    checkFailedWaitingJobMovement();
-  }
-
-  public void testStartWithoutDefaultQueueConfigured() throws Exception {
-    //configure a single queue which is not default queue
-    String[] qs = {"q1"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("q1", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    //Start the scheduler.
-    scheduler.start();
-    //Submit a job and wait till it completes
-    FakeJobInProgress job =
-      taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "q1", "u1");
-    controlledInitializationPoller.selectJobsToInitialize();
-    raiseStatusChangeEvents(scheduler.jobQueuesManager, "q1");
-    Map<String,String> strs = new HashMap<String,String>();
-    strs.put(CapacityTestUtils.MAP,"attempt_test_0001_m_000001_0 on tt1");
-    strs.put(CapacityTestUtils.REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-
-  }
-
-  public void testFailedJobInitalizations() throws Exception {
-    String[] qs = {"default"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-
-    //Submit a job whose initialization would fail always.
-    FakeJobInProgress job =
-      new FakeFailingJobInProgress(
-        new JobID("test", ++jobCounter),
-        new JobConf(), taskTrackerManager, "u1", UtilsForTests.getJobTracker());
-    job.getStatus().setRunState(JobStatus.PREP);
-    taskTrackerManager.submitJob(job);
-    //check if job is present in waiting list.
-    assertEquals(
-      "Waiting job list does not contain submitted job",
-      1, mgr.getJobQueue("default").getWaitingJobCount());
-    assertTrue(
-      "Waiting job does not contain submitted job",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-    //initialization should fail now.
-    controlledInitializationPoller.selectJobsToInitialize();
-    //Check if the job has been properly cleaned up.
-    assertEquals(
-      "Waiting job list contains submitted job",
-      0, mgr.getJobQueue("default").getWaitingJobCount());
-    assertFalse(
-      "Waiting job contains submitted job",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-    assertFalse(
-      "Waiting job contains submitted job",
-      mgr.getJobQueue("default").getRunningJobs().contains(job));
-  }
-
-  /**
-   * Test case deals with normal jobs which have speculative maps and reduce.
-   * Following is test executed
-   * <ol>
-   * <li>Submit one job with speculative maps and reduce.</li>
-   * <li>Submit another job with no speculative execution.</li>
-   * <li>Observe that all tasks from first job get scheduled, speculative
-   * and normal tasks</li>
-   * <li>Finish all the first jobs tasks second jobs tasks get scheduled.</li>
-   * </ol>
-   *
-   * @throws IOException
-   */
-  public void testSpeculativeTaskScheduling() throws IOException {
-    String[] qs = {"default"};
-    taskTrackerManager = new FakeTaskTrackerManager(2, 1, 1);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 100));
-
-
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    JobConf conf = new JobConf();
-    conf.setNumMapTasks(1);
-    conf.setNumReduceTasks(1);
-    conf.setMapSpeculativeExecution(true);
-    conf.setReduceSpeculativeExecution(true);
-    //Submit a job which would have one speculative map and one speculative
-    //reduce.
-    FakeJobInProgress fjob1 = taskTrackerManager.submitJob(
-      JobStatus.PREP, conf);
-
-    conf = new JobConf();
-    conf.setNumMapTasks(1);
-    conf.setNumReduceTasks(1);
-    //Submit a job which has no speculative map or reduce.
-    FakeJobInProgress fjob2 = taskTrackerManager.submitJob(
-      JobStatus.PREP, conf);
-
-    //Ask the poller to initalize all the submitted job and raise status
-    //change event.
-    controlledInitializationPoller.selectJobsToInitialize();
-    raiseStatusChangeEvents(mgr);
-    Map<String, String> strs = new HashMap<String, String>();
-    strs.put(CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    strs.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-    assertTrue(
-      "Pending maps of job1 greater than zero",
-      (fjob1.pendingMaps() == 0));
-
-    assertTrue(
-      "Pending reduces of job1 greater than zero",
-      (fjob1.pendingReduces() == 0));
-
-    Map<String, String> str = new HashMap<String, String>();
-    str.put(CapacityTestUtils.MAP, "attempt_test_0001_m_000001_1 on tt2");
-    str.put(CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_1 on tt2");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      str);
-
-    taskTrackerManager.finishTask("attempt_test_0001_m_000001_0", fjob1);
-    taskTrackerManager.finishTask("attempt_test_0001_m_000001_1", fjob1);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000001_0", fjob1);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000001_1", fjob1);
-    taskTrackerManager.finalizeJob(fjob1);
-
-    str.clear();
-    str.put(CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt1");
-    str.put(CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      str);
-    taskTrackerManager.finishTask("attempt_test_0002_m_000001_0", fjob2);
-    taskTrackerManager.finishTask("attempt_test_0002_r_000001_0", fjob2);
-    taskTrackerManager.finalizeJob(fjob2);
-  }
-
-  /**
-   * Test to verify that TTs are reserved for high memory jobs, but only till a
-   * TT is reserved for each of the pending task.
-   *
-   * @throws IOException
-   */
-  public void testTTReservingWithHighMemoryJobs()
-    throws IOException {
-    // 3 taskTrackers, 2 map and 0 reduce slots on each TT
-    taskTrackerManager = new FakeTaskTrackerManager(3, 2, 0);
-
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-
-
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    // enabled memory-based scheduling
-    // Normal job in the cluster would be 1GB maps/reduces
-    scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    LOG.debug(
-      "Submit a regular memory(1GB vmem maps/reduces) job of "
-        + "3 map/red tasks");
-    JobConf jConf = new JobConf(conf);
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(3);
-    jConf.setNumReduceTasks(3);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
-
-    // assign one map task of job1 on all the TTs
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt3",
-      "attempt_test_0001_m_000003_0 on tt3");
-    scheduler.updateContextInfoForTests();
-
-    LOG.info(job1.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(3, 3, 0, 0, 0, 0), 
-                 job1.getSchedulingInfo().toString());
-
-    LOG.debug(
-      "Submit one high memory(2GB maps, 0MB reduces) job of "
-        + "2 map tasks");
-    jConf.setMemoryForMapTask(2 * 1024);
-    jConf.setMemoryForReduceTask(0);
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(0);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
-
-    LOG.debug(
-      "Submit another regular memory(1GB vmem maps/reduces) job of "
-        + "2 map/red tasks");
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(2);
-    jConf.setNumReduceTasks(2);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job3 = taskTrackerManager.submitJobAndInit(JobStatus.PREP, jConf);
-
-    // Job2, a high memory job cannot be accommodated on a any TT. But with each
-    // trip to the scheduler, each of the TT should be reserved by job2.
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    scheduler.updateContextInfoForTests();
-    LOG.info(job2.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 2, 0, 0, 0), 
-                 job2.getSchedulingInfo().toString()); 
-
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    scheduler.updateContextInfoForTests();
-    LOG.info(job2.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 4, 0, 0, 0), 
-                 job2.getSchedulingInfo().toString());
-
-    // Job2 has only 2 pending tasks. So no more reservations. Job3 should get
-    // slots on tt3. tt1 and tt2 should not be assigned any slots with the
-    // reservation stats intact.
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    scheduler.updateContextInfoForTests();
-    LOG.info(job2.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 4, 0, 0, 0), 
-                 job2.getSchedulingInfo().toString()); 
-
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    scheduler.updateContextInfoForTests();
-    LOG.info(job2.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 4, 0, 0, 0), 
-                 job2.getSchedulingInfo().toString());
-
-    checkAssignment(
-      taskTrackerManager, scheduler, "tt3",
-      "attempt_test_0003_m_000001_0 on tt3");
-    scheduler.updateContextInfoForTests();
-    LOG.info(job2.getSchedulingInfo());
-    assertEquals(JobQueue.getJobQueueSchedInfo(0, 0, 4, 0, 0, 0), 
-                 job2.getSchedulingInfo().toString());
-
-    // No more tasks there in job3 also
-    assertNull(scheduler.assignTasks(tracker("tt3")));
-  }
-
-  /**
-   * Test to verify that queue ordering is based on the number of slots occupied
-   * and hence to verify that presence of high memory jobs is reflected properly
-   * while determining used capacities of queues and hence the queue ordering.
-   *
-   * @throws IOException
-   */
-  public void testQueueOrdering()
-    throws IOException {
-    taskTrackerManager = new FakeTaskTrackerManager(2, 6, 6);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    String[] qs = {"default", "q1"};
-    String[] reversedQs = {qs[1], qs[0]};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 100));
-    queues.add(new FakeQueueInfo("q1", 50.0f, true, 100));
-
-
-    // enabled memory-based scheduling
-    // Normal job in the cluster would be 1GB maps/reduces
-    scheduler.getConf().setLong(JTConfig.JT_MAX_MAPMEMORY_MB, 2 * 1024);
-    scheduler.getConf().setLong(MRConfig.MAPMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(JTConfig.JT_MAX_REDUCEMEMORY_MB, 1 * 1024);
-    scheduler.getConf().setLong(MRConfig.REDUCEMEMORY_MB, 1 * 1024);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    LOG.debug(
-      "Submit one high memory(2GB maps, 2GB reduces) job of "
-        + "6 map and 6 reduce tasks");
-    JobConf jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(2 * 1024);
-    jConf.setMemoryForReduceTask(2 * 1024);
-    jConf.setNumMapTasks(6);
-    jConf.setNumReduceTasks(6);
-    jConf.setQueueName("default");
-    jConf.setUser("u1");
-    FakeJobInProgress job1 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // Submit a normal job to the other queue.
-    jConf = new JobConf(conf);
-    jConf.setMemoryForMapTask(1 * 1024);
-    jConf.setMemoryForReduceTask(1 * 1024);
-    jConf.setNumMapTasks(6);
-    jConf.setNumReduceTasks(6);
-    jConf.setUser("u1");
-    jConf.setQueueName("q1");
-    FakeJobInProgress job2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, jConf);
-
-    // Map and reduce of high memory job should be assigned
-    HashMap<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    checkQueuesOrder(
-      qs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-
-    checkQueuesOrder(
-      qs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-
-    // 1st map and reduce of normal job should be assigned
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-
-    // 2nd map and reduce of normal job should be assigned
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000002_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000002_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-
-    // Now both the queues are equally served. But the comparator doesn't change
-    // the order if queues are equally served.
-    // Hence, 3rd map and reduce of normal job should be assigned
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000003_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000003_0 on tt2");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-
-    // 2nd map and reduce of high memory job should be assigned
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000002_0 on tt2");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-    checkQueuesOrder(
-      qs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-
-    checkQueuesOrder(
-      qs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-
-    // 4th map and reduce of normal job should be assigned.
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000004_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000004_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.MAP));
-
-    checkQueuesOrder(
-      reversedQs, scheduler
-        .getOrderedQueues(TaskType.REDUCE));
-  }
-
-  /**
-   * Tests whether 1 map and 1 reduce are assigned even if reduces span across
-   * multiple jobs or multiple queues.
-   *
-   * creates a cluster of 6 maps and 2 reduces.
-   * Submits 2 jobs:
-   * job1 , with 6 map and 1 reduces
-   * job2 with  2 map and 1 reduces
-   *
-   *
-   * check that first assignment assigns a map and a reduce.
-   * check that second assignment assigns a map and a reduce
-   * (both from other job and other queue)
-   *
-   * the last 2 calls just checks to make sure that we dont get further reduces
-   * 
-   * @throws Exception
-   */
-  public void testMultiTaskAssignmentInMultipleQueues() throws Exception {
-    setUp(1, 6, 2);
-    // set up some queues
-    String[] qs = {"default", "q1"};
-    taskTrackerManager.addQueues(qs);
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 50.0f, true, 25));
-    queues.add(new FakeQueueInfo("q1", 50.0f, true, 25));
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    //Submit the job with 6 maps and 2 reduces
-    taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 6, 1, "default", "u1");
-
-    FakeJobInProgress j2 = taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 2, 1, "q1", "u2");
-
-    Map<String, String> str = new HashMap<String, String>();
-    str.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
-    str.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", str);
-
-    // next assignment will be for job in second queue.
-    str.clear();
-    str.put(MAP, "attempt_test_0002_m_000001_0 on tt1");
-    str.put(REDUCE, "attempt_test_0002_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", str);
-
-    //now both the reduce slots are being used , hence we sholdnot get only 1
-    //map task in this assignTasks call.
-    str.clear();
-    str.put(MAP, "attempt_test_0002_m_000002_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", str);
-
-    str.clear();
-    str.put(MAP, "attempt_test_0001_m_000002_0 on tt1");
-    checkMultipleTaskAssignment(taskTrackerManager, scheduler, "tt1", str);
-  }
-
-
-  private void checkRunningJobMovementAndCompletion() throws IOException {
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    JobInitializationPoller p = scheduler.getInitializationPoller();
-    // submit a job
-    FakeJobInProgress job =
-      taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "default", "u1");
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    assertEquals(p.getInitializedJobList().size(), 1);
-
-    // make it running.
-    raiseStatusChangeEvents(mgr);
-
-    // it should be there in both the queues.
-    assertTrue(
-      "Job not present in Job Queue",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-    assertTrue(
-      "Job not present in Running Queue",
-      mgr.getJobQueue("default").getRunningJobs().contains(job));
-
-    // assign a task
-    Map<String,String> strs = new HashMap<String,String>();
-    strs.put(MAP,"attempt_test_0001_m_000001_0 on tt1");
-    strs.put(REDUCE,"attempt_test_0001_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      strs);
-
-    controlledInitializationPoller.selectJobsToInitialize();
-
-    // now this task should be removed from the initialized list.
-    assertTrue(p.getInitializedJobList().isEmpty());
-
-    // the job should also be removed from the job queue as tasks
-    // are scheduled
-    assertFalse(
-      "Job present in Job Queue",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-
-    // complete tasks and job
-    taskTrackerManager.finishTask("attempt_test_0001_m_000001_0", job);
-    taskTrackerManager.finishTask("attempt_test_0001_r_000001_0", job);
-    taskTrackerManager.finalizeJob(job);
-
-    // make sure it is removed from the run queue
-    assertFalse(
-      "Job present in running queue",
-      mgr.getJobQueue("default").getRunningJobs().contains(job));
-  }
-
-  private void checkFailedRunningJobMovement() throws IOException {
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-
-    //submit a job and initalized the same
-    FakeJobInProgress job =
-      taskTrackerManager.submitJobAndInit(JobStatus.RUNNING, 1, 1, "default", "u1");
-
-    //check if the job is present in running queue.
-    assertTrue(
-      "Running jobs list does not contain submitted job",
-      mgr.getJobQueue("default").getRunningJobs().contains(job));
-
-    taskTrackerManager.finalizeJob(job, JobStatus.KILLED);
-
-    //check if the job is properly removed from running queue.
-    assertFalse(
-      "Running jobs list does not contain submitted job",
-      mgr.getJobQueue("default").getRunningJobs().contains(job));
-
-  }
-
-  private void checkFailedInitializedJobMovement() throws IOException {
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    JobInitializationPoller p = scheduler.getInitializationPoller();
-
-    //submit a job
-    FakeJobInProgress job = taskTrackerManager.submitJob(JobStatus.PREP, 1, 1, "default", "u1");
-    //Initialize the job
-    p.selectJobsToInitialize();
-    //Don't raise the status change event.
-
-    //check in waiting and initialized jobs list.
-    assertTrue(
-      "Waiting jobs list does not contain the job",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-
-    assertTrue(
-      "Initialized job does not contain the job",
-      p.getInitializedJobList().contains(job.getJobID()));
-
-    //fail the initalized job
-    taskTrackerManager.finalizeJob(job, JobStatus.KILLED);
-
-    //Check if the job is present in waiting queue
-    assertFalse(
-      "Waiting jobs list contains failed job",
-      mgr.getJobQueue("default").getWaitingJobs().contains(job));
-
-    //run the poller to do the cleanup
-    p.selectJobsToInitialize();
-
-    //check for failed job in the initialized job list
-    assertFalse(
-      "Initialized jobs  contains failed job",
-      p.getInitializedJobList().contains(job.getJobID()));
-  }
-
-  private void checkFailedWaitingJobMovement() throws IOException {
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    // submit a job
-    FakeJobInProgress job = taskTrackerManager.submitJob(
-      JobStatus.PREP, 1, 1, "default",
-      "u1");
-
-    // check in waiting and initialized jobs list.
-    assertTrue(
-      "Waiting jobs list does not contain the job", mgr
-        .getJobQueue("default").getWaitingJobs().contains(job));
-    // fail the waiting job
-    taskTrackerManager.finalizeJob(job, JobStatus.KILLED);
-
-    // Check if the job is present in waiting queue
-    assertFalse(
-      "Waiting jobs list contains failed job", mgr
-        .getJobQueue("default").getWaitingJobs().contains(job));
-  }
-
-  private void raiseStatusChangeEvents(JobQueuesManager mgr) {
-    raiseStatusChangeEvents(mgr, "default");
-  }
-
-  private void raiseStatusChangeEvents(JobQueuesManager mgr, String queueName) {
-    Collection<JobInProgress> jips = mgr.getJobQueue(queueName)
-      .getWaitingJobs();
-    for (JobInProgress jip : jips) {
-      if (jip.getStatus().getRunState() == JobStatus.RUNNING) {
-        JobStatusChangeEvent evt = new JobStatusChangeEvent(
-          jip,
-          EventType.RUN_STATE_CHANGED, jip.getStatus());
-        mgr.jobUpdated(evt);
-      }
-    }
-  }
-
-  protected TaskTracker tracker(String taskTrackerName) {
-    return taskTrackerManager.getTaskTracker(taskTrackerName);
-  }
-
-  /**
-   * Get the amount of memory that is reserved for tasks on the taskTracker and
-   * verify that it matches what is expected.
-   *
-   * @param taskTracker
-   * @param expectedMemForMapsOnTT
-   * @param expectedMemForReducesOnTT
-   */
-  private void checkMemReservedForTasksOnTT(
-    String taskTracker,
-    Long expectedMemForMapsOnTT, Long expectedMemForReducesOnTT) {
-    Long observedMemForMapsOnTT =
-      scheduler.memoryMatcher.getMemReservedForTasks(
-        tracker(taskTracker).getStatus(),
-        TaskType.MAP);
-    Long observedMemForReducesOnTT =
-      scheduler.memoryMatcher.getMemReservedForTasks(
-        tracker(taskTracker).getStatus(),
-        TaskType.REDUCE);
-    if (expectedMemForMapsOnTT == null) {
-      assertEquals(observedMemForMapsOnTT,null);
-    } else {
-      assertEquals(observedMemForMapsOnTT,expectedMemForMapsOnTT);
-    }
-    if (expectedMemForReducesOnTT == null) {
-      assertEquals(observedMemForReducesOnTT,null);
-    } else {
-      assertEquals(observedMemForReducesOnTT,expectedMemForReducesOnTT);
-    }
-  }
-
-  /**
-   * Verify the number of slots of type 'type' from the queue 'queue'.
-   * incrMapIndex and incrReduceIndex are set , when expected output string is
-   * changed.these values can be set if the index of
-   * "Used capacity: %d (%.1f%% of Capacity)"
-   * is changed.
-   *
-   * @param queue
-   * @param type
-   * @param numActiveUsers               in the queue at present.
-   * @param expectedOccupiedSlots
-   * @param expectedOccupiedSlotsPercent
-   * @param incrMapIndex
-   * @param incrReduceIndex
-   */
-  private void checkOccupiedSlots(
-    String queue, TaskType type, int numActiveUsers, int expectedOccupiedSlots,
-    float expectedOccupiedSlotsPercent, int incrMapIndex, int incrReduceIndex) {
-    scheduler.updateContextInfoForTests();
-    QueueManager queueManager = scheduler.taskTrackerManager.getQueueManager();
-    String schedulingInfo = queueManager.getJobQueueInfo(queue)
-      .getSchedulingInfo();
-    String[] infoStrings = schedulingInfo.split("\n");
-    int index = -1;
-    if (type.equals(TaskType.MAP)) {
-      index = 7 + incrMapIndex;
-    } else if (type.equals(TaskType.REDUCE)) {
-      index =
-        (numActiveUsers == 0 ? 12 : 13 + numActiveUsers) + incrReduceIndex;
-    }
-    LOG.info(infoStrings[index]);
-    assertEquals(
-      String.format(
-        "Used capacity: %d (%.1f%% of Capacity)", expectedOccupiedSlots,
-        expectedOccupiedSlotsPercent), infoStrings[index]);
-  }
-
-  /**
-   * @param queue
-   * @param type
-   * @param numActiveUsers
-   * @param expectedOccupiedSlots
-   * @param expectedOccupiedSlotsPercent
-   */
-  private void checkOccupiedSlots(
-    String queue,
-    TaskType type, int numActiveUsers,
-    int expectedOccupiedSlots, float expectedOccupiedSlotsPercent
-  ) {
-    checkOccupiedSlots(
-      queue, type, numActiveUsers, expectedOccupiedSlots,
-      expectedOccupiedSlotsPercent, 0, 0);
-  }
-
-  private void checkQueuesOrder(
-    String[] expectedOrder, String[] observedOrder) {
-    assertTrue(
-      "Observed and expected queues are not of same length.",
-      expectedOrder.length == observedOrder.length);
-    int i = 0;
-    for (String expectedQ : expectedOrder) {
-      assertTrue(
-        "Observed and expected queues are not in the same order. "
-          + "Differ at index " + i + ". Got " + observedOrder[i]
-          + " instead of " + expectedQ, expectedQ.equals(observedOrder[i]));
-      i++;
-    }
-  }
-
-  public void testDeprecatedMemoryValues() throws IOException {
-    // 2 map and 1 reduce slots
-    taskTrackerManager.addQueues(new String[]{"default"});
-    ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
-    queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
-
-    JobConf conf = (JobConf) (scheduler.getConf());
-    conf.set(
-      JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, String.valueOf(
-        1024 * 1024 * 3));
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    taskTrackerManager.setFakeQueues(queues);
-    scheduler.start();
-
-    assertEquals(MemoryMatcher.getLimitMaxMemForMapSlot(), 3);
-    assertEquals(MemoryMatcher.getLimitMaxMemForReduceSlot(), 3);
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java

@@ -1,171 +0,0 @@
-/** Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.PrintWriter;
-import java.util.Properties;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.fs.Path;
-
-public class TestCapacitySchedulerConf extends TestCase {
-
-  private static String testDataDir = System.getProperty("test.build.data");
-  private static String testConfFile;
-  
-  //private Map<String, String> defaultProperties;
-  private CapacitySchedulerConf testConf;
-  private PrintWriter writer;
-  
-  static {
-    if (testDataDir == null) {
-      testDataDir = ".";
-    } else {
-      new File(testDataDir).mkdirs();
-    }
-    testConfFile = new File(testDataDir, "test-conf.xml").getAbsolutePath();
-  }
-  
-  public TestCapacitySchedulerConf() {
-  }
-
-  
-  public void setUp() throws IOException {
-    openFile();
-  }
-  
-  public void tearDown() throws IOException {
-    File confFile = new File(testConfFile);
-    if (confFile.exists()) {
-      confFile.delete();  
-    }
-  }
-
-
-  public void testInitializationPollerProperties() 
-    throws Exception {
-    /*
-     * Test case to check properties of poller when no configuration file
-     * is present.
-     */
-    testConf = new CapacitySchedulerConf();
-    long pollingInterval = testConf.getSleepInterval();
-    int maxWorker = testConf.getMaxWorkerThreads();
-    assertTrue("Invalid polling interval ",pollingInterval > 0);
-    assertTrue("Invalid working thread pool size" , maxWorker > 0);
-    
-    //test case for custom values configured for initialization 
-    //poller.
-    openFile();
-    startConfig();
-    writeProperty("mapred.capacity-scheduler.init-worker-threads", "1");
-    writeProperty("mapred.capacity-scheduler.init-poll-interval", "1");
-    endConfig();
-    
-    testConf = new CapacitySchedulerConf(new Path(testConfFile));
-    
-    pollingInterval = testConf.getSleepInterval();
-    
-    maxWorker = testConf.getMaxWorkerThreads();
-    
-    assertEquals("Invalid polling interval ",pollingInterval ,1);
-    assertEquals("Invalid working thread pool size" , maxWorker, 1);
-    
-    //Test case for invalid values configured for initialization
-    //poller
-    openFile();
-    startConfig();
-    writeProperty("mapred.capacity-scheduler.init-worker-threads", "0");
-    writeProperty("mapred.capacity-scheduler.init-poll-interval", "0");
-    endConfig();
-    
-    testConf = new CapacitySchedulerConf(new Path(testConfFile));
-    
-    try {
-      pollingInterval = testConf.getSleepInterval();
-      fail("Polling interval configured is illegal");
-    } catch (IllegalArgumentException e) {}
-    try {
-      maxWorker = testConf.getMaxWorkerThreads();
-      fail("Max worker thread configured is illegal");
-    } catch (IllegalArgumentException e) {}
-  }
-  
-
-  private void openFile() throws IOException {
-    
-    if (testDataDir != null) {
-      File f = new File(testDataDir);
-      f.mkdirs();
-    }
-    FileWriter fw = new FileWriter(testConfFile);
-    BufferedWriter bw = new BufferedWriter(fw);
-    writer = new PrintWriter(bw);
-  }
-  
-  private void startConfig() {
-    writer.println("<?xml version=\"1.0\"?>");
-    writer.println("<configuration>");
-  }
-
-
-  private void writeProperty(String name, String value) {
-    writer.println("<property>");
-    writer.println("<name> " + name + "</name>");
-    writer.println("<value>"+ value+"</value>");
-    writer.println("</property>");
-    
-  }
-  
-  private void endConfig() {
-    writer.println("</configuration>");
-    writer.close();
-  }
-
-  public void testConfigurationValuesConversion() throws IOException {
-    Properties prp = new Properties();
-
-    prp.setProperty("capacity","10");
-    prp.setProperty("maximum-capacity","20.5");
-    prp.setProperty("supports-priority","false");
-    prp.setProperty("minimum-user-limit-percent","23");
-
-    CapacitySchedulerConf conf = new CapacitySchedulerConf();
-    conf.setProperties("default",prp);
-
-    assertTrue(conf.getCapacity("default") == 10f);
-    assertTrue(conf.getMaxCapacity("default") == 20.5f);
-    assertTrue(conf.isPrioritySupported("default") == false);
-    assertTrue(conf.getMinimumUserLimitPercent("default")==23);
-
-    //check for inproper stuff
-    prp.setProperty("capacity","h");
-    prp.setProperty("maximum-capacity","20");
-
-    //This is because h is invalid value.
-    assertTrue(conf.getCapacity("default") == -1);
-    
-    assertFalse(conf.getMaxCapacity("default") != 20);
-  }
-  
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java

@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.Properties;
-
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
-import org.apache.hadoop.mapreduce.SleepJob;
-
-public class TestCapacitySchedulerWithJobTracker extends
-                                                 ClusterWithCapacityScheduler {
-
-  /**
-   * Test case which checks if the jobs which
-   * fail initialization are removed from the
-   * {@link CapacityTaskScheduler} waiting queue.
-   *
-   * @throws Exception
-   */
-  public void testFailingJobInitalization() throws Exception {
-    Properties schedulerProps = new Properties();
-    Properties clusterProps = new Properties();
-    clusterProps.put("mapred.queue.names","default");
-    clusterProps.put(TTConfig.TT_MAP_SLOTS, String.valueOf(1));
-    clusterProps.put(TTConfig.TT_REDUCE_SLOTS, String.valueOf(1));
-    clusterProps.put(JTConfig.JT_TASKS_PER_JOB, String.valueOf(1));
-    clusterProps.put(JTConfig.JT_PERSIST_JOBSTATUS, "false");
-    // cluster capacity 1 maps, 1 reduces
-    startCluster(1, clusterProps, schedulerProps);
-    CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
-      .getTaskScheduler();
-
-     AbstractQueue root = scheduler.getRoot();
-     root.getChildren().get(0).getQueueSchedulingContext().setCapacityPercent(100);
-
-    JobConf conf = getJobConf();
-    conf.setSpeculativeExecution(false);
-    conf.setNumTasksToExecutePerJvm(-1);
-    SleepJob sleepJob = new SleepJob();
-    sleepJob.setConf(conf);
-    Job job = sleepJob.createJob(3, 3, 1, 1, 1, 1);
-    job.waitForCompletion(false);
-    assertFalse(
-      "The submitted job successfully completed",
-      job.isSuccessful());
-
-    JobQueuesManager mgr = scheduler.jobQueuesManager;
-    assertEquals(
-      "Failed job present in Waiting queue", 0, mgr
-        .getJobQueue("default").getWaitingJobCount());
-  }
-
-  /**
-   * Test case which checks {@link JobTracker} and {@link CapacityTaskScheduler}
-   * <p/>
-   * Test case submits 2 jobs in two different capacity scheduler queues.
-   * And checks if the jobs successfully complete.
-   *
-   * @throws Exception
-   */
-
-  public void testJobTrackerIntegration() throws Exception {
-
-    Properties schedulerProps = new Properties();
-    String[] queues = new String[]{"Q1", "Q2"};
-    Job jobs[] = new Job[2];
-
-    Properties clusterProps = new Properties();
-    clusterProps.put(TTConfig.TT_MAP_SLOTS, String.valueOf(2));
-    clusterProps.put(TTConfig.TT_REDUCE_SLOTS, String.valueOf(2));
-    clusterProps.put("mapred.queue.names", queues[0] + "," + queues[1]);
-    clusterProps.put(JTConfig.JT_PERSIST_JOBSTATUS, "false");
-    startCluster(2, clusterProps, schedulerProps);
-    CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
-      .getTaskScheduler();
-
-
-
-    AbstractQueue root = scheduler.getRoot();
-
-    for(AbstractQueue q : root.getChildren()) {
-      q.getQueueSchedulingContext().setCapacityPercent(50);
-      q.getQueueSchedulingContext().setUlMin(100);
-    }
-
-
-    LOG.info("WE CREATED THE QUEUES TEST 2");
-   // scheduler.taskTrackerManager.getQueueManager().setQueues(qs);
-   // scheduler.start();
-
-    JobConf conf = getJobConf();
-    conf.setSpeculativeExecution(false);
-    conf.set(MRJobConfig.SETUP_CLEANUP_NEEDED, "false");
-    conf.setNumTasksToExecutePerJvm(-1);
-    conf.setQueueName(queues[0]);
-    SleepJob sleepJob1 = new SleepJob();
-    sleepJob1.setConf(conf);
-    jobs[0] = sleepJob1.createJob(1, 1, 1, 1, 1, 1);
-    jobs[0].submit();
-
-    JobConf conf2 = getJobConf();
-    conf2.setSpeculativeExecution(false);
-    conf2.setNumTasksToExecutePerJvm(-1);
-    conf2.setQueueName(queues[1]);
-    SleepJob sleepJob2 = new SleepJob();
-    sleepJob2.setConf(conf2);
-    jobs[1] = sleepJob2.createJob(3, 3, 5, 3, 5, 3);
-    jobs[0].waitForCompletion(false);
-    jobs[1].waitForCompletion(false);
-    assertTrue(
-      "Sleep job submitted to queue 1 is not successful", jobs[0]
-        .isSuccessful());
-    assertTrue(
-      "Sleep job submitted to queue 2 is not successful", jobs[1]
-        .isSuccessful());
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestContainerQueue.java

@@ -1,597 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import junit.framework.TestCase;
-
-import java.util.List;
-import java.util.HashMap;
-import java.util.Map;
-import java.io.IOException;
-import static org.apache.hadoop.mapred.CapacityTestUtils.*;
-
-public class TestContainerQueue extends TestCase {
-
-  CapacityTestUtils.FakeTaskTrackerManager taskTrackerManager = null;
-  CapacityTaskScheduler scheduler = null;
-  CapacityTestUtils.FakeClock clock = null;
-  CapacityTestUtils.ControlledInitializationPoller
-    controlledInitializationPoller;
-
-  @Override
-  protected void setUp() {
-    setUp(2, 2, 1);
-  }
-
-  private void setUp(
-    int numTaskTrackers, int numMapTasksPerTracker,
-    int numReduceTasksPerTracker) {
-    taskTrackerManager =
-      new CapacityTestUtils.FakeTaskTrackerManager(
-        numTaskTrackers, numMapTasksPerTracker,
-        numReduceTasksPerTracker);
-    clock = new CapacityTestUtils.FakeClock();
-    scheduler = new CapacityTaskScheduler(clock);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-
-    // Don't let the JobInitializationPoller come in our way.
-    controlledInitializationPoller =
-        new CapacityTestUtils.ControlledInitializationPoller(
-            scheduler.jobQueuesManager, taskTrackerManager);
-    scheduler.setInitializationPoller(controlledInitializationPoller);
-    scheduler.setConf(taskTrackerManager.defaultJobConf);
-  }
-
-  /**
-   * check the minCapacity distribution across children
-   * and grand children.
-   * <p/>
-   * Check for both capacity % and also actual no of slots allocated.
-   */
-  public void testMinCapacity() {
-
-    AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
-
-    //Simple check to make sure that capacity is properly distributed among
-    // its children.
-    //level 1 children
-    QueueSchedulingContext a1 = new QueueSchedulingContext(
-      "a", 25, -1, -1);
-    QueueSchedulingContext a2 = new QueueSchedulingContext(
-      "b", 25, -1, -1);
-
-    AbstractQueue q = new ContainerQueue(rt, a1);
-    AbstractQueue ql = new ContainerQueue(rt, a2);
-
-    //level 2 children
-    QueueSchedulingContext a = new QueueSchedulingContext(
-      "aa", 50, -1, -1);
-    QueueSchedulingContext b = new QueueSchedulingContext(
-      "ab", 50, -1, -1);
-    QueueSchedulingContext c = new QueueSchedulingContext(
-      "ac", 50, -1, -1);
-    QueueSchedulingContext d = new QueueSchedulingContext(
-      "ad", 50, -1, -1);
-
-    AbstractQueue q1 = new JobQueue(q, a);
-    AbstractQueue q2 = new JobQueue(q, b);
-    AbstractQueue q3 = new JobQueue(ql, c);
-    AbstractQueue q4 = new JobQueue(ql, d);
-
-    rt.update(1000, 1000);
-
-    //share at level 0.
-    // (1000 * 25) / 100
-    assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 250);
-    assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 250);
-
-    //share would be (1000 * 25 / 100 ) * (50 / 100)
-    assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
-    assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
-    assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
-    assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 125);
-
-    //
-    rt.update(1, 1);
-    assertEquals(q.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-    assertEquals(ql.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-
-    //share would be (1000 * 25 / 100 ) * (50 / 100)
-    assertEquals(q1.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-    assertEquals(q2.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-    assertEquals(q3.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-    assertEquals(q4.getQueueSchedulingContext().getMapTSC().getCapacity(), 0);
-
-  }
-
-  public void testMaxCapacity() throws IOException {
-    this.setUp(4, 1, 1);
-    taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
-
-    AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
-
-    QueueSchedulingContext a1 = new QueueSchedulingContext(
-      "R.a", 25, 50, -1);
-    QueueSchedulingContext a2 = new QueueSchedulingContext(
-      "R.b", 25, 30, -1);
-    QueueSchedulingContext a3 = new QueueSchedulingContext(
-      "R.c", 50, -1, -1);
-
-
-    //Test for max capacity
-    AbstractQueue q = new JobQueue(rt, a1);
-    AbstractQueue q1 = new JobQueue(rt, a2);
-    AbstractQueue q2 = new JobQueue(rt, a3);
-    scheduler.jobQueuesManager.addQueue((JobQueue) q);
-    scheduler.jobQueuesManager.addQueue((JobQueue) q1);
-    scheduler.jobQueuesManager.addQueue((JobQueue) q2);
-
-    scheduler.setRoot(rt);
-    rt.update(4, 4);
-
-    scheduler.updateContextInfoForTests();
-
-    // submit a job to the second queue
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 20, 20, "R.a", "u1");
-
-    //Queue R.a should not more than 2 slots
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP,
-      "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE,
-      "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP,
-      "attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE,
-      "attempt_test_0001_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-    assertNull(scheduler.assignTasks(
-      taskTrackerManager.getTaskTracker(
-        "tt3")));
-
-  }
-
-  public void testDistributedUnConfiguredCapacity() {
-
-    AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
-
-    //generate Queuecontext for the children
-    QueueSchedulingContext a1 = new QueueSchedulingContext(
-      "a", 50, -1, -1);
-    QueueSchedulingContext a2 = new QueueSchedulingContext(
-      "b", -1, -1, -1);
-
-    AbstractQueue rtChild1 = new ContainerQueue(rt, a1);
-    AbstractQueue rtChild2 = new ContainerQueue(rt, a2);
-
-    //Add further children to rtChild1.    
-    QueueSchedulingContext b = new QueueSchedulingContext(
-      "ab", 30, -1, -1);
-    QueueSchedulingContext c = new QueueSchedulingContext(
-      "ac", -1, -1, -1);
-    QueueSchedulingContext d = new QueueSchedulingContext(
-      "ad", 100, -1, -1);
-
-    AbstractQueue q0 = new JobQueue(rtChild1, b);
-    AbstractQueue q1 = new JobQueue(rtChild1, c);
-    AbstractQueue q2 = new JobQueue(rtChild2, d);
-
-    rt.distributeUnConfiguredCapacity();
-
-    //after distribution the rtChild2 capacity should be 50.
-    assertEquals(
-      rtChild2.getQueueSchedulingContext().getCapacityPercent(), 50.0f);
-
-    //Now check the capacity of q1. It should be 60%.
-    assertTrue(q1.getQueueSchedulingContext().getCapacityPercent() == 70);
-  }
-
-  private Map setUpHierarchy() {
-    return setUpHierarchy(60, 40, 80, 20);
-  }
-
-  /**
-   * @param a capacity for sch q
-   * @param b capacity for gta q
-   * @param c capacity for sch.prod q
-   * @param d capacity for sch.misc q
-   * @return
-   */
-  private Map<String, AbstractQueue> setUpHierarchy(
-                                    int a, int b, int c, int d) {
-    HashMap<String, AbstractQueue> map = new HashMap<String, AbstractQueue>();
-
-    AbstractQueue rt = QueueHierarchyBuilder.createRootAbstractQueue();
-    map.put(rt.getName(), rt);
-    scheduler.setRoot(rt);
-
-    // Create 2 levels of hierarchy.
-
-    //Firt level
-    QueueSchedulingContext sch =
-      new QueueSchedulingContext("rt.sch", a, -1, -1);
-    QueueSchedulingContext gta =
-      new QueueSchedulingContext("rt.gta", b, -1, -1);
-
-    AbstractQueue schq = new ContainerQueue(rt, sch);
-
-    //Cannot declare a ContainerQueue if no children.
-    AbstractQueue gtaq = new JobQueue(rt, gta);
-    map.put(schq.getName(), schq);
-    map.put(gtaq.getName(), gtaq);
-    scheduler.jobQueuesManager.addQueue((JobQueue) gtaq);
-
-    //Create further children.
-    QueueSchedulingContext prod =
-      new QueueSchedulingContext("rt.sch.prod", c, -1, -1);
-    QueueSchedulingContext misc =
-      new QueueSchedulingContext("rt.sch.misc", d, -1, -1);
-
-    AbstractQueue prodq = new JobQueue(schq, prod);
-    AbstractQueue miscq = new JobQueue(schq, misc);
-    map.put(prodq.getName(), prodq);
-    map.put(miscq.getName(), miscq);
-    scheduler.jobQueuesManager.addQueue(
-      (JobQueue) prodq);
-    scheduler.jobQueuesManager.addQueue((JobQueue) miscq);
-    return map;
-  }
-
-  /**
-   * Verifies that capacities are allocated properly in hierarchical queues.
-   * 
-   * The test case sets up a hierarchy of queues and submits jobs to 
-   * all the queues. It verifies that computation of capacities, sorting,
-   * etc are working correctly.
-   * @throws Exception
-   */
-  public void testCapacityAllocationInHierarchicalQueues() throws Exception {
-    this.setUp(9, 1, 1);
-    taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
-    // set up some queues
-    Map<String, AbstractQueue> map = setUpHierarchy();
-
-    scheduler.updateContextInfoForTests();
-
-    // verify initial capacity distribution
-    TaskSchedulingContext mapTsc
-      = map.get("rt.gta").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTsc.getCapacity(), 3);
-
-    mapTsc = map.get("rt.sch").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTsc.getCapacity(), 5);
-
-    mapTsc = map.get("rt.sch.prod").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTsc.getCapacity(), 4);
-
-    mapTsc = map.get("rt.sch.misc").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTsc.getCapacity(), 1);
-
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{0, 0, 0, 0});
-
-    //Only Allow job submission to leaf queue
-    taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 4, 4, "rt.sch.prod",
-      "u1");
-
-    // submit a job to the second queue
-    taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 4, 4, "rt.sch.misc",
-      "u1");
-
-    //submit a job in gta level queue
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 4, 4, "rt.gta", "u1");
-
-    int counter = 0;
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{0, 1, 1, 0});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0003_m_000001_0 on tt2");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0003_r_000001_0 on tt2");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{1, 1, 1, 0});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000001_0 on tt3");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000001_0 on tt3");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt3",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{1, 2, 1, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0003_m_000002_0 on tt4");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0003_r_000002_0 on tt4");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt4",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 2, 1, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000002_0 on tt5");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000002_0 on tt5");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt5",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 3, 2, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000003_0 on tt6");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000003_0 on tt6");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt6",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 4, 3, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0003_m_000003_0 on tt7");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0003_r_000003_0 on tt7");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt7",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{3, 4, 3, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0001_m_000004_0 on tt8");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0001_r_000004_0 on tt8");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt8",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{3, 5, 4, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(
-      CapacityTestUtils.MAP, "attempt_test_0002_m_000002_0 on tt9");
-    expectedStrings.put(
-      CapacityTestUtils.REDUCE, "attempt_test_0002_r_000002_0 on tt9");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt9",
-      expectedStrings);
-  }
-  
-  /**
-   * Test to make sure that capacity is divided at each level properly.
-   *
-   * The test case sets up a two level hierarchy of queues as follows:
-   *   - rt.sch
-   *     - rt.sch.prod
-   *     - rt.sch.misc
-   *   - rt.gta  
-   * Jobs are submitted to rt.sch.misc and rt.gta, and the test verifies 
-   * that as long as rt.sch is below rt.gta's capacity, it still gets 
-   * allocated slots even if rt.sch.misc is over its capacity. 
-   * @throws IOException
-   */
-  public void testHierarchicalCapacityAllocation() throws IOException {
-    this.setUp(8, 1, 1);
-    taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
-    // set up some queues
-    Map<String, AbstractQueue> map = setUpHierarchy(70, 30, 80, 20);
-
-    scheduler.updateContextInfoForTests();
-
-    // verify capacities as per the setup.
-    TaskSchedulingContext mapTSC
-      = map.get("rt.gta").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTSC.getCapacity(), 2);
-
-    mapTSC = map.get("rt.sch").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTSC.getCapacity(), 5);
-
-    mapTSC = map.get("rt.sch.prod").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTSC.getCapacity(), 4);
-
-    mapTSC = map.get("rt.sch.misc").getQueueSchedulingContext().getMapTSC();
-    assertEquals(mapTSC.getCapacity(), 1);
-
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{0, 0, 0, 0});
-
-    // submit a job to the second queue
-    taskTrackerManager.submitJobAndInit(
-      JobStatus.PREP, 20, 20, "rt.sch.misc",
-      "u1");
-
-    //submit a job in gta level queue
-    taskTrackerManager.submitJobAndInit(JobStatus.PREP, 20, 20, "rt.gta", "u1");
-    int counter = 0;
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{0, 1, 0, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0002_m_000001_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0002_r_000001_0 on tt2");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{1, 1, 0, 1});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000002_0 on tt3");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000002_0 on tt3");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt3",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{1, 2, 0, 2});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000003_0 on tt4");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000003_0 on tt4");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt4",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{1, 3, 0, 3});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0002_m_000002_0 on tt5");
-    expectedStrings.put(REDUCE, "attempt_test_0002_r_000002_0 on tt5");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt5",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 3, 0, 3});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000004_0 on tt6");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000004_0 on tt6");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt6",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 4, 0, 4});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000005_0 on tt7");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000005_0 on tt7");
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt7",
-      expectedStrings);
-    assertUsedCapacity(
-      map,
-      new String[]{"rt.gta", "rt.sch", "rt.sch.prod", "rt.sch.misc"},
-      new int[]{2, 5, 0, 5});
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000006_0 on tt8");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000006_0 on tt8");
-
-
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt8",
-      expectedStrings);
-  }
-
-  // verify that the number of slots used for each queue
-  // matches the expected value.
-  private void assertUsedCapacity(Map<String, AbstractQueue> queueMap,
-      String[] queueNames, int[] expectedUsedSlots) {
-  
-    scheduler.updateContextInfoForTests();
-    assertEquals(queueNames.length, expectedUsedSlots.length);
-    
-    for (int i=0; i<queueNames.length; i++) {
-      TaskSchedulingContext mapTSC = 
-        queueMap.get(queueNames[i]).getQueueSchedulingContext().getMapTSC();
-      assertEquals(mapTSC.getNumSlotsOccupied(), expectedUsedSlots[i]);
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java

@@ -1,479 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.CapacityTestUtils.ControlledInitializationPoller;
-import org.apache.hadoop.mapred.CapacityTestUtils.FakeJobInProgress;
-import org.apache.hadoop.mapred.CapacityTestUtils.FakeTaskTrackerManager;
-import static org.apache.hadoop.mapred.CapacityTestUtils.*;
-import org.junit.After;
-import org.junit.Test;
-
-/**
- * Test the Queue-Refresh feature.
- */
-public class TestRefreshOfQueues {
-
-  private static final Log LOG =
-      LogFactory.getLog(org.apache.hadoop.mapred.TestRefreshOfQueues.class);
-
-  String queueConfigPath =
-      System.getProperty("test.build.extraconf", "build/test/extraconf");
-  File queueConfigFile =
-      new File(queueConfigPath, QueueManager.QUEUE_CONF_FILE_NAME);
-
-  private CapacityTaskScheduler scheduler;
-  private FakeTaskTrackerManager taskTrackerManager;
-
-  /**
-   * Remove the queueConfigFile once the test is done.
-   */
-  @After
-  public void tearDown() {
-    if (queueConfigFile.exists()) {
-      queueConfigFile.delete();
-    }
-  }
-
-  /**
-   * Sets up the scheduler, TaskTrackerManager, QueueManager, initializer and
-   * starts the scheduler.
-   * 
-   * @throws IOException
-   */
-  private void setupAndStartSchedulerFramework(int numTTs, int numMapsPerTT,
-      int numReducesPerTT)
-      throws IOException {
-    scheduler = new CapacityTaskScheduler();
-    taskTrackerManager =
-        new FakeTaskTrackerManager(numTTs, numMapsPerTT, numReducesPerTT);
-    taskTrackerManager.setQueueManager(new QueueManager());
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    scheduler.setConf(new Configuration());
-    ControlledInitializationPoller controlledInitializationPoller =
-        new ControlledInitializationPoller(scheduler.jobQueuesManager,
-            taskTrackerManager);
-    scheduler.setInitializationPoller(controlledInitializationPoller);
-    taskTrackerManager.addJobInProgressListener(scheduler.jobQueuesManager);
-    scheduler.start();
-  }
-
-  /**
-   * Helper method that ensures TaskScheduler is locked before calling
-   * {@link QueueManager#refreshQueues(Configuration, 
-   *    org.apache.hadoop.mapred.TaskScheduler.QueueRefresher)}.
-   */
-  private static void refreshQueues(QueueManager qm, Configuration conf,
-      TaskScheduler ts) throws IOException {
-    synchronized (ts) {
-      qm.refreshQueues(conf, ts.getQueueRefresher());
-    }
-  }
-
-  /**
-   * @throws Throwable
-   */
-  @Test
-  public void testRefreshOfQueuesSanity()
-      throws Throwable {
-
-    JobQueueInfo[] queues = TestQueueManagerRefresh.getSimpleQueueHierarchy();
-
-    Properties[] props = new Properties[3];
-    for (int i = 0; i < props.length; i++) {
-      props[i] = queues[i].getProperties();
-      props[i].setProperty(CapacitySchedulerConf.CAPACITY_PROPERTY,
-          String.valueOf(i + 10));
-      props[i].setProperty(CapacitySchedulerConf.MAX_CAPACITY_PROPERTY,
-          String.valueOf(i + 15));
-      props[i].setProperty(CapacitySchedulerConf.SUPPORTS_PRIORITY_PROPERTY,
-          String.valueOf(false));
-      props[i].setProperty(
-          CapacitySchedulerConf.MAXIMUM_INITIALIZED_JOBS_PER_USER_PROPERTY,
-          String.valueOf(i + 11));
-      props[i].setProperty(
-          CapacitySchedulerConf.MINIMUM_USER_LIMIT_PERCENT_PROPERTY,
-          String.valueOf(i + 16));
-    }
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-
-    setupAndStartSchedulerFramework(0, 0, 0);
-
-    Map<String, AbstractQueue> allQueues = getAllQueues(scheduler);
-
-    // Verify the configuration.
-    for (int i = 0; i < queues.length; i++) {
-      String qName = queues[i].getQueueName();
-      LOG.info("Queue name : " + qName);
-      QueueSchedulingContext qsc =
-          allQueues.get(qName).getQueueSchedulingContext();
-      LOG.info("Context for queue " + qName + " is : " + qsc);
-      assertEquals(i + 10, qsc.getCapacityPercent(), 0);
-      assertEquals(i + 15, qsc.getMaxCapacityPercent(), 0);
-      assertEquals(Boolean.valueOf(false),
-          Boolean.valueOf(qsc.supportsPriorities()));
-      assertEquals(i + 16, qsc.getUlMin());
-    }
-
-    // change configuration
-    for (int i = 0; i < props.length; i++) {
-      props[i] = queues[i].getProperties();
-      props[i].setProperty(CapacitySchedulerConf.CAPACITY_PROPERTY,
-          String.valueOf(i + 20));
-      props[i].setProperty(CapacitySchedulerConf.MAX_CAPACITY_PROPERTY,
-          String.valueOf(i + 25));
-      props[i].setProperty(CapacitySchedulerConf.SUPPORTS_PRIORITY_PROPERTY,
-          String.valueOf(false));
-      props[i].setProperty(
-          CapacitySchedulerConf.MAXIMUM_INITIALIZED_JOBS_PER_USER_PROPERTY,
-          String.valueOf(i + 5));
-      props[i].setProperty(
-          CapacitySchedulerConf.MINIMUM_USER_LIMIT_PERCENT_PROPERTY,
-          String.valueOf(i + 10));
-    }
-
-    // Re-write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-
-    // Now do scheduler refresh.
-    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
-
-    allQueues = getAllQueues(scheduler);
-
-    for (int i = 0; i < queues.length; i++) {
-      String qName = queues[i].getQueueName();
-      LOG.info("Queue name : " + qName);
-      QueueSchedulingContext qsc =
-        allQueues.get(qName).getQueueSchedulingContext();
-      assertEquals(qName, qsc.getQueueName());
-      LOG.info("Context for queue " + qName + " is : " + qsc);
-      assertEquals(i + 20, qsc.getCapacityPercent(), 0);
-      assertEquals(i + 25, qsc.getMaxCapacityPercent(), 0);
-      assertEquals(Boolean.valueOf(false),
-          Boolean.valueOf(qsc.supportsPriorities()));
-    }
-  }
-
-  /**
-   * @throws Throwable
-   */
-   @Test
-  public void testSuccessfulCapacityRefresh()
-      throws Throwable {
-
-    JobQueueInfo[] queues = TestQueueManagerRefresh.getSimpleQueueHierarchy();
-
-    queues[0].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(100));
-    queues[1].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-    queues[2].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-      queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
-
-    setupAndStartSchedulerFramework(2, 2, 2);
-
-    FakeJobInProgress job1 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[1].getQueueName(), "user");
-    FakeJobInProgress job2 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[2].getQueueName(), "user");
-
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-//===========================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0002_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0002_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0002_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0002_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-//============================================
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-    taskTrackerManager.killJob(job1.getJobID());
-    taskTrackerManager.killJob(job2.getJobID());
-
-    // change configuration
-    queues[1].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(25));
-    queues[2].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(75));
-
-    // Re-write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-      queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
-
-    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
-
-    job1 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[1].getQueueName(), "user");
-    job2 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 4, 4,
-        queues[2].getQueueName(), "user");
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0003_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0003_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0004_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0004_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0004_m_000002_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0004_r_000002_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0004_m_000003_0 on tt2");
-    expectedStrings.put(REDUCE, "attempt_test_0004_r_000003_0 on tt2");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt2",
-      expectedStrings);
-
-  }
-
-  /**
-   * Test to verify that the refresh of the scheduler fails when modified
-   * configuration overflows 100%
-   * 
-   * @throws Throwable
-   */
-   @Test
-  public void testFailingCapacityRefresh()
-      throws Throwable {
-
-    JobQueueInfo[] queues = TestQueueManagerRefresh.getSimpleQueueHierarchy();
-
-    queues[0].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(100));
-    queues[1].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(70));
-    queues[2].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-
-    try {
-      setupAndStartSchedulerFramework(2, 2, 2);
-      fail("Scheduler should have failed to start!");
-    } catch (IOException ioe) {
-      assertTrue(ioe.getMessage().contains(
-          String.format(QueueHierarchyBuilder.TOTAL_CAPACITY_OVERFLOWN_MSG,
-              queues[1].getQueueName() + "," + queues[2].getQueueName(),
-              Float.valueOf(120.0f))));
-    }
-
-    // Rectify the properties and start the scheduler
-    queues[1].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-
-    setupAndStartSchedulerFramework(2, 2, 2);
-
-    // Now change configuration.
-    queues[1].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(35));
-    queues[2].getProperties().setProperty(
-        CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(95));
-
-    // Re-write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-        queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
-
-    try {
-      refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
-    } catch (IOException ioe) {
-      assertTrue(ioe.getMessage().contains(
-          String.format(QueueHierarchyBuilder.TOTAL_CAPACITY_OVERFLOWN_MSG,
-              queues[1].getQueueName() + "," + queues[2].getQueueName(),
-              Float.valueOf(130.0f))));
-    }
-  }
-
-  /**
-   * @throws Throwable
-   */
-   @Test
-  public void testRefreshUserLimits()
-      throws Throwable {
-
-    JobQueueInfo[] queues = TestQueueManagerRefresh.getSimpleQueueHierarchy();
-
-    queues[0].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(100));
-    queues[1].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-    queues[2].getProperties().setProperty(
-      CapacitySchedulerConf.CAPACITY_PROPERTY, String.valueOf(50));
-
-    queues[2].getProperties().setProperty(
-      CapacitySchedulerConf.MINIMUM_USER_LIMIT_PERCENT_PROPERTY,
-      String.valueOf(100));
-
-    // write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-      queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
-
-    setupAndStartSchedulerFramework(1, 2, 2);
-
-    FakeJobInProgress job1 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[2].getQueueName(), "user1");
-    FakeJobInProgress job2 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[2].getQueueName(), "user2");
-
-    Map<String, String> expectedStrings = new HashMap<String, String>();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-    
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0001_m_000002_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0001_r_000002_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    assertNull(scheduler.assignTasks(taskTrackerManager.getTaskTracker("tt1")));
-    taskTrackerManager.killJob(job1.getJobID());
-    taskTrackerManager.killJob(job2.getJobID());
-
-    // change configuration
-    queues[2].getProperties().setProperty(
-      CapacitySchedulerConf.MINIMUM_USER_LIMIT_PERCENT_PROPERTY,
-      String.valueOf(50));
-
-    // Re-write the configuration file
-    QueueManagerTestUtils.writeQueueConfigurationFile(
-      queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
-
-    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
-
-    job1 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[1].getQueueName(), "user1");
-    job2 =
-      taskTrackerManager.submitJobAndInit(
-        JobStatus.PREP, 2, 2,
-        queues[2].getQueueName(), "user2");
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0003_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0003_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-
-    expectedStrings.clear();
-    expectedStrings.put(MAP, "attempt_test_0004_m_000001_0 on tt1");
-    expectedStrings.put(REDUCE, "attempt_test_0004_r_000001_0 on tt1");
-    checkMultipleTaskAssignment(
-      taskTrackerManager, scheduler, "tt1",
-      expectedStrings);
-  }
-
-  /**
-   * Get a map of all {@link AbstractQueue}s.
-   * 
-   * @param sched
-   * @return
-   */
-  private static Map<String, AbstractQueue> getAllQueues(
-      CapacityTaskScheduler sched) {
-    AbstractQueue rootQueue = sched.getRoot();
-    HashMap<String, AbstractQueue> allQueues =
-        new HashMap<String, AbstractQueue>();
-    List<AbstractQueue> allQueuesList = new ArrayList<AbstractQueue>();
-    allQueuesList.addAll(rootQueue.getDescendentJobQueues());
-    allQueuesList.addAll(rootQueue.getDescendantContainerQueues());
-    for (AbstractQueue q : allQueuesList) {
-      LOG.info("Putting in allQueues list " + q.getName());
-      allQueues.put(q.getName(), q);
-    }
-    return allQueues;
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/README

@@ -1,167 +0,0 @@
-# Copyright 2008 The Apache Software Foundation Licensed under the
-# Apache License, Version 2.0 (the "License"); you may not use this
-# file except in compliance with the License.  You may obtain a copy
-# of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
-# required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.  See the License for the specific language governing
-# permissions and limitations under the License.
-
-This package implements dynamic priority scheduling for MapReduce jobs.
-
-Overview
---------
-The purpose of this scheduler is to allow users to increase and decrease
-their queue priorities continuosly to meet the requirements of their
-current workloads. The scheduler is aware of the current demand and makes
-it more expensive to boost the priority under peak usage times. Thus
-users who move their workload to low usage times are rewarded with
-discounts. Priorities can only be boosted within a limited quota.
-All users are given a quota or a budget which is deducted periodically
-in configurable accounting intervals. How much of the budget is 
-deducted is determined by a per-user spending rate, which may
-be modified at any time directly by the user. The cluster slots 
-share allocated to a particular user is computed as that users
-spending rate over the sum of all spending rates in the same accounting
-period.
-
-Configuration
--------------
-This scheduler comprises two components, an accounting or resource allocation part that 
-manages and bills for queue shares, and a scheduler that
-enforces the queue shares in the form of map and reduce slots of running jobs.
-
-Hadoop Configuration (e.g. hadoop-site.xml):
-mapreduce.jobtracker.taskscheduler      
-    This needs to be set to 
-    org.apache.hadoop.mapred.DynamicPriorityScheduler
-    to use the dynamic scheduler.
-Scheduler Configuration:
-mapred.dynamic-scheduler.scheduler   
-    The Java path of the MapReduce scheduler that should
-    enforce the allocated shares.
-    Has been tested with (which is the default):
-    org.apache.hadoop.mapred.PriorityScheduler
-mapred.priority-scheduler.acl-file
-    Full path of ACL with syntax:
-      <user> <role> <secret key>
-    separated by line feeds
-mapred.dynamic-scheduler.budget-file  
-    The full OS path of the file from which the
-    budgets are read and stored. The syntax of this file is:
-    <queue name> <budget> <spending rate>
-    separated by newlines where budget can be specified
-    as a Java float. The file should not be edited
-    directly, if the server is running, but through the 
-    servlet API to ensure proper synchronization.
-
-mapred.dynamic-scheduler.alloc-interval       
-    Allocation interval, when the scheduler rereads the
-    spending rates and recalculates the cluster shares.
-    Specified as seconds between allocations.
-    Default is 20 seconds.
-
-Servlet API
-----------
-The queue should be managed through the Servlet REST API
-if the jobtracker server is running.
-
-It is installed at
-[job tracker URL]/scheduler
-operations supported: 
-  price       i
-    get the current price of the cluster 
-    (aggregate spending rates of queues with running or pending jobs)
-  time        
-    get start time of server and current time in epoch units 
-  info=queue_to_query 
-    get info about queue (requires user or admin privilege>
-  infos 
-    get info about all queues (requires admin privilege)
-  addBudget=budget_to_add,queue=queue_to_change 
-    add budget to queue (requires admin privilege)
-  setSpending=spending_to_set,queue=queue_to_change 
-    set spending rate of queue (requires user or admin privilege)
-  addQueue=queue_to_add 
-    add new queue  (requires admin privilege)
-  removeQueue=queue_to_remove 
-    remove queue (requires admin privilege)
-
-Example:
-  http://myhost:50030/scheduler?setSpending=0.01&queue=myqueue
-  The Authorization header is used for signing
-
-The signature is created akin to the AWS Query Authentication scheme
-HMAC_SHA1("<query path>&user=<user>&timestamp=<timestamp>", key)
-For the servlet operations query path is everything that comes after /scheduler?
-in the url. For job submission the query path is just the empty string "".
-Job submissions also need to set the following job properties:
--Dmapred.job.timestamp=<ms epoch time> 
--Dmapred.job.signature=<signature as above> -Dmapreduce.job.queue.name=<queue>
-Note queue must match the user submitting the job.
-
-Example python query 
---------------------------------
-import base64
-import hmac
-import sha
-import httplib, urllib
-import sys
-import time
-from popen2 import popen3
-import os
-
-def hmac_sha1(data, key):
-    return urllib.quote(base64.encodestring(hmac.new(key, data, sha).digest()).strip())
-
-stdout, stdin, stderr = popen3("id -un")
-USER = stdout.read().strip()
-f = open(os.path.expanduser("~/.ssh/hadoop_key"))
-KEY = f.read().strip()
-f.close()
-f = open(os.path.expanduser("/etc/hadoop_server"))
-SERVER = f.read().strip()
-f.close()
-URL = "/scheduler"
-conn = httplib.HTTPConnection(SERVER)
-params = sys.argv[1]
-params = params + "&user=%s&timestamp=%d" % (USER,long(time.time()*1000))
-print params
-headers = {"Authorization": hmac_sha1(params, KEY)}
-print headers
-conn.request("GET",URL + "?" + params,None, headers)
-response = conn.getresponse()
-print response.status, response.reason
-data =  response.read()
-conn.close()
-print data
-
-Example python job submission parameter generation
---------------------------------------------------
-import base64
-import hmac
-import sha
-import httplib, urllib
-import sys
-import time
-import os
-from popen2 import popen3
-
-def hmac_sha1(data, key):
-    return urllib.quote(base64.encodestring(hmac.new(key, data, sha).digest()).strip())
-
-stdout, stdin, stderr = popen3("id -un")
-USER = stdout.read().strip()
-f = open(os.path.expanduser("~/.ssh/hadoop_key"))
-KEY = f.read().strip()
-f.close()
-if len(sys.argv) > 1:
-  params = sys.argv[1]
-else:
-  params = ""
-timestamp = long(time.time()*1000)
-params = params + "&user=%s&timestamp=%d" % (USER,timestamp)
-print "-Dmapred.job.timestamp=%d -Dmapred.job.signature=%s -Dmapreduce.job.queuename=%s" % (timestamp, hmac_sha1(params, KEY), USER)
-
-

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/build.xml

@@ -1,27 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- 
-Before you can run these subtargets directly, you need 
-to call at top-level: ant deploy-contrib compile-core-test
--->
-<project name="dynamic-priority" default="jar">
-  <import file="../build-contrib.xml"/>
-
-</project>

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/ivy.xml

@@ -1,66 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<ivy-module version="1.0">
-  <info organisation="org.apache.hadoop" module="${ant.project.name}">
-    <license name="Apache 2.0"/>
-    <ivyauthor name="Apache Hadoop Team" url="http://hadoop.apache.org"/>
-    <description> Apache Hadoop contrib </description>
-  </info>
-  <configurations defaultconfmapping="default">
-    <!--these match the Maven configurations-->
-    <conf name="default" extends="master,runtime"/>
-    <conf name="master" description="contains the artifact but no dependencies"/>
-    <conf name="runtime" description="runtime but not the artifact" />
-
-    <conf name="common" visibility="private" extends="runtime" description="common artifacts"/>
-    <conf name="test" visibility="private" extends="runtime"/>
-  </configurations>
-
-  <publications>
-    <artifact conf="master"/>
-  </publications>
-
-  <dependencies>
-   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
-               rev="${yarn.version}" conf="common->default"/>
-   <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
-               rev="${yarn.version}" conf="common->default"/>
-    <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master"/>
-    <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}" conf="common->master"/>
-    <dependency org="tomcat" name="jasper-runtime" rev="${jasper.version}" conf="common->master"/>
-    <dependency org="tomcat" name="jasper-compiler" rev="${jasper.version}" conf="common->master"/>
-    <dependency org="commons-el" name="commons-el" rev="${commons-el.version}" conf="common->master"/>
-    <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->master"/>
-    <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
-    <dependency org="net.java.dev.jets3t" name="jets3t" rev="${jets3t.version}" conf="common->master"/>
-    <dependency org="commons-net" name="commons-net" rev="${commons-net.version}" conf="common->master"/>  
-    <dependency org="org.mortbay.jetty" name="servlet-api-2.5" rev="${servlet-api-2.5.version}" conf="common->master"/>
-    <dependency org="junit" name="junit" rev="${junit.version}" conf="common->default"/>
-    <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}" conf="common->master"/>
-
-   <!-- Exclusions for transitive dependencies pulled in by log4j -->
-   <exclude org="com.sun.jdmk"/>
-   <exclude org="com.sun.jmx"/>
-   <exclude org="javax.jms"/> 
-   <exclude org="javax.mail"/> 
-
-  </dependencies>
-</ivy-module>

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/ivy/libraries.properties

@@ -1,17 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#This properties file lists the versions of the various artifacts used by streaming.
-#It drives ivy and the generation of a maven POM
-
-#Please list the dependencies name with version if they are different from the ones 
-#listed in the global libraries.properties file (in alphabetical order)

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/AllocationStore.java

@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.conf.Configuration;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Collection;
-
-/**
- * Abstract class for implementing a persistent store
- * of allocation information.
- */
-public abstract class AllocationStore {
-  Map<String,BudgetQueue> queueCache = new HashMap<String,BudgetQueue>();
-
-  /**
-   * Initializes configuration
-   * @param conf MapReduce configuration
-   */ 
-  public abstract void init(Configuration conf);
-
-  /**
-   * Loads allocations from persistent store
-   */ 
-  public abstract void load();
-
-  /**
-   * Saves allocations to persistent store
-   */ 
-  public abstract void save();
-
-  /**
-   * Gets current remaining budget associated with queue.
-   * @param queue name of queue
-   * @return budget in credits
-   */ 
-  public float getBudget(String queue) {
-    float budget = 0.0f;
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue != null) {
-      budget = budgetQueue.budget;
-    }
-    return budget;
-  }
-
-  /**
-   * Gets current spending rate associated with queue.
-   * @param queue name of queue
-   * @return spending rate in credits per allocation interval to be
-   * deducted from budget
-   */ 
-  public float getSpending(String queue) {
-    float spending = 0;
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue != null) {
-      spending = budgetQueue.spending;
-    }
-    return spending;
-  }
-
-  /**
-   * Adds budget to queue.
-   * @param queue name of queue
-   * @param budget in credits to be added to queue
-   */
-  public synchronized void addBudget(String queue, float budget) {
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue == null) {
-        return;
-    }
-    budgetQueue.addBudget(budget);
-  }
-
-
-  /**
-   * Adds new queue.
-   * @param queue name of queue
-   */
-  public synchronized void addQueue(String queue) {
-    queueCache.put(queue, new BudgetQueue(queue,0.0f,0.0f));
-  }
-
-  /**
-   * Gets queue info.
-   * @param queue name of queue
-   * @return xml representation of queue info as a string
-   */
-  public String getQueueInfo(String queue) {
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue == null) {
-        return "";
-    }
-    return "<budget>" + Float.toString(budgetQueue.budget) + "</budget>\n" +
-        "<spending>" + Float.toString(budgetQueue.spending) + "</spending>\n" +
-        "<used>" + Integer.toString(budgetQueue.used) + "</used>\n" +
-        "<pending>" + budgetQueue.pending + "</pending>\n";
-  }
-
-  /**
-   * Remove queue.
-   * @param queue name of queue
-   */
-  public synchronized void removeQueue(String queue) {
-    queueCache.remove(queue);
-  }
-
-  /**
-   * Sets spending rate for queue.
-   * @param queue name of queue
-   * @param spending spending rate in credits per allocation interval to be
-   * deducted from budget
-   */ 
-  public synchronized void setSpending(String queue, float spending) {
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue == null) {
-        return;
-    }
-    budgetQueue.spending = spending;
-  }
-
-  /**
-   * Sets queue usage for accounting
-   * @param queue name of queue
-   * @param used slots currently in use
-   * @param pending pending tasks
-   */ 
-  public synchronized void setUsage(String queue, int used, int pending) {
-    BudgetQueue budgetQueue = queueCache.get(queue);
-    if (budgetQueue == null) {
-        return;
-    }
-    budgetQueue.used = used;
-    budgetQueue.pending = pending;
-  }
-
-  /**
-  * Gets queue status (budget, spending, usage)
-  * @return collection of queue status objects
-  */
-  public Collection<BudgetQueue> getQueues() {
-    return queueCache.values();
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/BudgetQueue.java

@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * Class to hold accounting info about a queue
- * such as remaining budget, spending rate and
- * whether queue usage
- */
-public class BudgetQueue {
-  String name;
-  volatile float budget;
-  volatile float spending;
-  volatile int used;
-  volatile int pending;
-  /**
-   * @param name queue name
-   * @param budget queue budget in credits
-   * @param spending queue spending rate in credits per allocation interval
-   * to deduct from budget
-   */
-  public BudgetQueue(String name, float budget, float spending) {
-      this.name = name;
-      this.budget = budget;
-      this.spending = spending;
-      this.used = 0;
-      this.pending = 0;
-  }
-  /**
-   * Thread safe addition of budget
-   * @param newBudget budget to add
-   */
-  public synchronized void addBudget(float newBudget) {
-    budget += newBudget;
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityScheduler.java

@@ -1,275 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.http.HttpServer;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-
-/**
- * A {@link TaskScheduler} that 
- * provides the following features: 
- * The purpose of this scheduler is to allow users to increase and decrease
- * their queue priorities continuosly to meet the requirements of their
- * current workloads. The scheduler is aware of the current demand and makes
- * it more expensive to boost the priority under peak usage times. Thus
- * users who move their workload to low usage times are rewarded with
- * discounts. Priorities can only be boosted within a limited quota.
- * All users are given a quota or a budget which is deducted periodically
- * in configurable accounting intervals. How much of the budget is
- * deducted is determined by a per-user spending rate, which may
- * be modified at any time directly by the user. The cluster slots
- * share allocated to a particular user is computed as that users
- * spending rate over the sum of all spending rates in the same accounting
- * period.
- *
- * This scheduler has been designed as a meta-scheduler on top of 
- * existing MapReduce schedulers, which are responsible for enforcing
- * shares computed by the dynamic scheduler in the cluster. 
- */
-class DynamicPriorityScheduler extends TaskScheduler {
-  /**
-   * This class periodically checks spending rates for queues and
-   * updates queue capacity shares and budgets
-   */
-  static class Allocations extends TimerTask implements QueueAllocator {
-    Map<String,QueueAllocation> allocation = 
-        new HashMap<String,QueueAllocation>();
-    Configuration conf;
-    HashMap<String,String> queueInfo = new HashMap<String,String>();
-    float totalSpending;
-    Set<String> infoQueues;
-    QueueManager queueManager;
-    AllocationStore store;
-    Allocations(Configuration conf, QueueManager queueManager) {
-      this.conf = conf;
-      this.queueManager = queueManager;
-      this.infoQueues = queueManager.getLeafQueueNames();
-      
-      this.store = ReflectionUtils.newInstance(
-          conf.getClass(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_STORE,
-              FileAllocationStore.class, AllocationStore.class), conf);
-      this.store.init(this.conf);
-      this.store.load();
-    }
-    void addBudget(String queue, float budget) {
-      store.addBudget(queue, budget);
-    }
-    void addQueue(String queue) {
-      store.addQueue(queue);
-    }
-    synchronized float getPrice() {
-      return totalSpending;
-    }
-    void removeQueue(String queue) {
-      store.removeQueue(queue);
-      queueManager.setSchedulerInfo(queue, "");
-    }
-    void setSpending(String queue, float spending) {
-      store.setSpending(queue, spending);
-    }
-    String getInfo(String queue) {
-      return store.getQueueInfo(queue);
-    }
-    String getQueueInfos() {
-      String info = "<price>" + Float.toString(totalSpending) + "</price>\n";
-      for (BudgetQueue queue: store.getQueues()) {
-        info += "<queue name=\"" + queue.name + "\">" + 
-            queueInfo.get(queue.name) + "</queue>\n";
-      }
-      return info;
-    }
-    private synchronized void updateAllocation() {
-      String queueList = "";
-      totalSpending = 0.0f;
-      for (BudgetQueue queue: store.getQueues()) {
-        if (!infoQueues.contains(queue.name)) {
-          Queue[] newQueues = new Queue[infoQueues.size()+1];
-          int i = 0;
-          for (String infoQueue: infoQueues) {
-            newQueues[i] = queueManager.getQueue(infoQueue);
-            i++;
-          }
-          Queue newQueue = new Queue();
-          newQueue.setName(queue.name);
-          newQueues[i] = newQueue;
-          queueManager.setQueues(newQueues);
-
-          QueueInfo newQueueInfo = new QueueInfo(queue.name, null, this); 
-          queueManager.setSchedulerInfo(queue.name, newQueueInfo);
-
-          infoQueues = queueManager.getLeafQueueNames();
-        }
-        if (!queueList.equals("")) {
-          queueList += ",";
-        }
-        queueList += queue.name;
-        // What to include in the published price in spending per slot
-        if (queue.spending <= queue.budget && 
-            (queue.used != 0 || queue.pending != 0)) {
-          totalSpending += queue.spending;
-        } 
-      } 
-      conf.set(PrioritySchedulerOptions.MAPRED_QUEUE_NAMES, queueList);
-      setShares();
-    }
-    // Calculates shares in proportion to spending rates
-    // and sets the appropriate configuration parameter
-    // for schedulers to read
-    private synchronized void setShares() {
-      Map<String,QueueAllocation> shares = new HashMap<String,QueueAllocation>();
-      for (BudgetQueue queue: store.getQueues()) {
-        float spending = queue.spending;
-        if (queue.budget < (queue.spending * queue.used) || 
-            (queue.used == 0 && queue.pending == 0)) {
-          spending = 0.0f;
-        } else {
-          queue.addBudget(-(queue.spending*queue.used));
-        }
-        float queueShare = 0.0f;
-        if (totalSpending > 0.0f) {
-          queueShare = (spending/totalSpending);
-        }
-        queueInfo.put(queue.name, "<budget>" + Float.toString(queue.budget) + 
-            "</budget>\n<spending>" + Float.toString(spending) + "</spending>\n<share>" + 
-            Float.toString(queueShare) + "</share>\n<used>" + 
-            Integer.toString(queue.used) + "</used>\n<pending>" +
-            Integer.toString(queue.pending) + "</pending>\n"); 
-        shares.put(queue.name,new QueueAllocation(queue.name,queueShare));
-      }
-      setAllocation(shares);
-    }
-    private synchronized void setAllocation(Map<String,QueueAllocation> shares) {
-      allocation = shares;
-    }
-    /** {@inheritDoc} */
-    public synchronized Map<String,QueueAllocation> getAllocation() {
-      return allocation;
-    }
-    /** {@inheritDoc} */
-    public synchronized void setUsage(String queue, int used, int pending) {
-      store.setUsage(queue, used, pending);
-    }
-    // Used to expose the QueueInfo in the JobTracker web UI 
-    synchronized String getQueueInfo(String queue) {
-      return queueInfo.get(queue);
-    }
-    // run once in each allocation interval to 
-    // calculate new shares based on updated
-    // budgets and spending rates
-    @Override
-    public void run() {
-      store.load();
-      updateAllocation();
-      store.save();
-    }
-  }
-  /**
-   * this class merges the queue info from the underlying
-   * MapReduce scheduler and the dynamic scheduler
-   * to be displayed in the JobTracker web UI
-   */
-  private static class QueueInfo {
-    String queue;
-    Object info; 
-    Allocations allocations;
-    QueueInfo(String queue, Object info, Allocations allocations) {
-      this.queue = queue;
-      this.info = info;  
-      this.allocations = allocations;
-    }
-    public String toString() {
-      String buffer = "";
-      if (info != null) {
-        buffer += info.toString();
-      }
-      String queueInfo = allocations.getQueueInfo(queue);
-      buffer += queueInfo;
-      return buffer;
-    }
-  } 
-
-  // this is the actual scheduler that picks
-  // the jobs to run, e.g. PriorityScheduler
-  protected QueueTaskScheduler scheduler;
-  private Timer timer = new Timer(true);
-  protected Allocations allocations;
-  private static final Log LOG = LogFactory.getLog(DynamicPriorityScheduler.class);
-
-  // Used for testing in discrete time
-  void setTimer(Timer timer) {
-    this.timer = timer;
-  }
-
-  @Override
-  public void start() throws IOException {
-    Configuration conf = getConf();
-    QueueManager queueManager = taskTrackerManager.getQueueManager();
-    allocations = new Allocations(conf,queueManager);
-    scheduler = ReflectionUtils.newInstance(
-        conf.getClass(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_SCHEDULER,
-        PriorityScheduler.class, QueueTaskScheduler.class), conf);
-    scheduler.setAllocator(allocations);
-    scheduler.setConf(conf);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    scheduler.start();
-    long interval = conf.getLong(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_ALLOC_INTERVAL,20)*1000;
-     
-    timer.scheduleAtFixedRate(allocations, interval, interval);   
-    for (String queue: queueManager.getLeafQueueNames()) {
-      Object info = queueManager.getSchedulerInfo(queue);
-      QueueInfo queueInfo = new QueueInfo(queue, info, allocations); 
-      queueManager.setSchedulerInfo(queue, queueInfo);
-    }
-    if (taskTrackerManager instanceof JobTracker) {
-      JobTracker jobTracker = (JobTracker) taskTrackerManager;
-      HttpServer infoServer = jobTracker.infoServer;
-      infoServer.setAttribute("scheduler", this);
-      infoServer.addServlet("scheduler", "/scheduler",
-          DynamicPriorityServlet.class);
-    }
-  }
-
-  @Override
-  public void terminate() throws IOException {
-    scheduler.terminate();
-  }
-
-  @Override
-  public List<Task> assignTasks(TaskTracker taskTracker)
-      throws IOException {
-    return scheduler.assignTasks(taskTracker);  
-  }
-
-  @Override
-  public Collection<JobInProgress> getJobs(String queueName) {
-    return scheduler.getJobs(queueName);
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/DynamicPriorityServlet.java

@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.List;
-
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Servlet for controlling queue allocations, installed at
- * [job tracker URL]/scheduler when the {@link DynamicPriorityScheduler} 
- * is in use.
- * operations supported: <br>
- * price <br>
- * time <br>
- * info=queue_to_query (requires user or admin privilege><br>
- * infos (requires admin privilege) <br> 
- * addBudget=budget_to_add,queue=queue_to_change 
- * (requires admin privilege) <br>
- * setSpending=spending_to_set,queue=queue_to_change 
- * (requires user or admin privilege) <br>
- * addQueue=queue_to_add (requires admin privilege) <br>
- * removeQueue=queue_to_remove (requires admin privilege) <br>
- */
-public class DynamicPriorityServlet extends HttpServlet {
-
-  private DynamicPriorityScheduler scheduler;
-  private JobTracker jobTracker;
-  private PriorityAuthorization auth;
-  @Override
-  public void init() throws ServletException {
-    super.init();
-    ServletContext servletContext = getServletContext();
-    scheduler = 
-        (DynamicPriorityScheduler) servletContext.getAttribute("scheduler");
-    jobTracker = (JobTracker) scheduler.taskTrackerManager;
-    auth = new PriorityAuthorization();
-    auth.init(scheduler.conf);
-  }
-
-  @Override
-  protected void doPost(HttpServletRequest req, HttpServletResponse resp)
-      throws ServletException, IOException {
-    doGet(req, resp); // Same handler for both GET and POST
-  }
-
-  private void checkAdmin(int role, String query) throws IOException {
-    if (role != PriorityAuthorization.ADMIN) {
-      throw new IOException("ACCESS DENIED: " + query);
-    }
-  }
-  private void checkUser(int role, HttpServletRequest request, 
-      String queue, String query) throws IOException {
-    if (role == PriorityAuthorization.ADMIN) {
-      return;
-    } 
-    if (role == PriorityAuthorization.USER &&
-        request.getParameter("user").equals(queue)) {
-      return;
-    }
-    throw new IOException("ACCESS DENIED: " + query);
-  }
-
-  @Override
-  public void doGet(HttpServletRequest request, 
-      HttpServletResponse response) throws ServletException, IOException {
-    String query = request.getQueryString();
-    int role = auth.authorize(query,
-        request.getHeader("Authorization"),
-        request.getParameter("user"),
-        request.getParameter("timestamp"));
-
-    String queue = request.getParameter("queue");
-    String info = "";
-    // admin
-    if (request.getParameter("addQueue") != null) {
-      checkAdmin(role, query);
-      queue = request.getParameter("addQueue");
-      scheduler.allocations.addQueue(queue);
-      info = scheduler.allocations.getInfo(queue); 
-    }
-    // admin
-    if (request.getParameter("removeQueue") != null) {
-      checkAdmin(role, query);
-      queue = request.getParameter("removeQueue");
-      scheduler.allocations.removeQueue(queue);
-      info = scheduler.allocations.getInfo(queue); 
-    }
-    // admin
-    if (request.getParameter("addBudget") != null) {
-      checkAdmin(role, query);
-      float budget = Float.parseFloat(request.getParameter("addBudget"));
-      scheduler.allocations.addBudget(queue, budget);
-      info = scheduler.allocations.getInfo(queue); 
-    }
-    // user
-    if (request.getParameter("setSpending") != null) {
-      checkUser(role, request, queue, query);
-      float spending = Float.parseFloat(request.getParameter("setSpending"));
-      scheduler.allocations.setSpending(queue, spending);
-      info = scheduler.allocations.getInfo(queue); 
-    }
-    // user
-    if (request.getParameter("info") != null) {
-      queue = request.getParameter("info");
-      checkUser(role, request, queue, query);
-      info = scheduler.allocations.getQueueInfo(queue);
-    }
-    // admin
-    if (request.getParameter("infos") != null) {
-      checkAdmin(role, query);
-      info = scheduler.allocations.getQueueInfos();
-    }
-
-    // all
-    if (request.getParameter("price") != null) {
-      info = Float.toString(scheduler.allocations.getPrice());
-      info = "<price>" + info + "</price>\n";
-    }
-    // all
-    if (request.getParameter("time") != null) {
-      info = "<start>" + Long.toString(PriorityAuthorization.START_TIME) + 
-          "</start>\n";
-      info += "<time>" + Long.toString(System.currentTimeMillis()) + 
-          "</time>\n";
-    }
-    if (info == null) {
-      info = "";
-    }
-    response.setContentType("text/xml");
-    PrintWriter out = new PrintWriter(response.getOutputStream());
-    String hostname = StringUtils.simpleHostname(
-        jobTracker.getJobTrackerMachine());
-    out.print("<QueueInfo>");
-    out.printf("<host>%s</host>\n", hostname);
-    out.printf("%s", info);
-    out.print("</QueueInfo>\n");
-    out.close();
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/FileAllocationStore.java

@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.FileReader;
-import java.io.BufferedReader;
-import java.io.PrintWriter;
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.Closeable;
-
-/**
- * Implements persistent storage for queue budget and spending
- * information in a file.
- */
-public class FileAllocationStore extends AllocationStore {
-  private static final Log LOG = LogFactory.getLog(FileAllocationStore.class);
-  private String fileName = "";
-  private boolean loaded = false;
-
-  /** {@inheritDoc} */
-  public void init(Configuration conf) {
-    fileName = conf.get(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_BUDGET_FILE,
-        "/etc/hadoop.budget");
-  }
-
-  /** {@inheritDoc} */
-  public void save() {
-    PrintWriter out = null; 
-    try {
-      out = new PrintWriter(new BufferedWriter(new FileWriter(fileName)));
-      for (BudgetQueue queue: getQueues()) {
-        out.printf("%s %.20f %.20f\n", queue.name, queue.budget, 
-            queue.spending);
-      }
-    } catch (Exception e) {
-      LOG.error("Error writing to file: " + fileName, e);
-    } finally {
-      close(out);
-    }
-  }
-
-  private void close(Closeable closeable) {
-    if (closeable != null) {
-      try {
-        closeable.close();
-      } catch (Exception ce) {
-        LOG.error("Error closing file: " + fileName, ce);
-      }
-    }
-  }
-
-  /** {@inheritDoc} */
-  public void load() {
-    if (loaded) {
-      return;
-    }
-    BufferedReader in = null;
-    try {
-      in = new BufferedReader(new FileReader(fileName));
-      String line = in.readLine();
-      while (line != null) {
-        String[] nameValue = line.split(" ");
-        if (nameValue.length != 3) {
-          continue;
-        }
-        queueCache.put(nameValue[0], new BudgetQueue(nameValue[0],
-            Float.parseFloat(nameValue[1]), Float.parseFloat(nameValue[2])));
-        line = in.readLine();
-      } 
-      loaded = true;
-    } catch (Exception e) {
-      LOG.error("Error reading file: " + fileName, e);
-    } finally {
-       close(in);
-    }
-  }
-}
-

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityAuthorization.java

@@ -1,205 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.security.SignatureException;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.File;
-import java.net.URLDecoder;
-
-
-import java.util.HashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.commons.codec.binary.Base64;
-
-
-/**
- * This class implements symmetric key HMAC/SHA1 signature
- * based authorization of users and admins.
- */
-public class PriorityAuthorization {
-  public static final int USER = 0;
-  public static final int ADMIN = 1;
-  public static final int NO_ACCESS = 2;
-  private HashMap<String,UserACL> acl = new HashMap<String,UserACL>();
-  private long lastSuccessfulReload = 0;
-  public static final long START_TIME = System.currentTimeMillis();
-  private String aclFile;
-  private static final Log LOG = LogFactory.getLog(PriorityAuthorization.class);
-  private static final boolean debug = LOG.isDebugEnabled();
-  private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
-
-  /**
-   * Initializes authorization configuration
-   * @param conf MapReduce configuration handle 
-   */
-  public void init(Configuration conf) {
-    aclFile = conf.get("mapred.priority-scheduler.acl-file","/etc/hadoop.acl");
-  }
-
-  /**
-   * Adapted from AWS Query Authentication cookbook:
-   * Computes RFC 2104-compliant HMAC signature.
-   *
-   * @param data
-   *     The data to be signed.
-   * @param key
-   *     The signing key.
-   * @return
-   *     The base64-encoded RFC 2104-compliant HMAC signature.
-   * @throws
-   *     java.security.SignatureException when signature generation fails
-   */
-  public static String hmac(String data, String key)
-    throws java.security.SignatureException {
-    String result;
-    try {
-      // get an hmac_sha1 key from the raw key bytes
-      SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), 
-          HMAC_SHA1_ALGORITHM);
-           
-      // get an hmac_sha1 Mac instance and initialize with the signing key
-      Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
-      mac.init(signingKey);
-            
-      // compute the hmac on input data bytes
-      byte[] rawHmac = mac.doFinal(data.getBytes());
-            
-      // base64-encode the hmac
-      result = new String(Base64.encodeBase64(rawHmac));
-    } 
-    catch (Exception e) {
-      throw new SignatureException("Failed to generate HMAC : " + e, e);
-    }
-    return result;
-  }
-
-  class UserACL {
-    String user;
-    String role;
-    String key;
-    // for replay detection
-    long lastTimestamp = START_TIME;
-    UserACL(String user, String role, String key) {
-      this.user = user;
-      this.role = role;
-      this.key = key;
-    }
-  }
-
-  private void reloadACL() {
-    BufferedReader in = null;
-    try {
-      in = new BufferedReader(new FileReader(aclFile));
-      String line = in.readLine();
-      while (line != null) {
-        String[] nameValue = line.split(" ");
-        if (nameValue.length != 3) {
-          continue;
-        }
-        acl.put(nameValue[0], new UserACL(nameValue[0], nameValue[1], nameValue[2]));
-        if (debug) {
-          LOG.debug("Loading " + line);
-        }
-        line = in.readLine();
-      }
-    } catch (Exception e) {
-      LOG.error(e);
-    }
-    try {
-      in.close();
-    } catch (Exception e) {
-      LOG.error(e);
-    }
-  }
-
-  private void loadACL() {
-    long time = System.currentTimeMillis();
-    try {
-      File file = new File(aclFile);
-      long lastModified = file.lastModified();
-      if (lastModified > lastSuccessfulReload) {
-        reloadACL();
-        lastSuccessfulReload = time;
-      }
-    } catch (Exception e) {
-      LOG.error("Failed to reload acl file", e);
-    }
-  }
-
-  private boolean isReplay(String timestamp, String signature, UserACL userACL) {
-    long signatureTime = Long.parseLong(timestamp);
-    if (debug) {
-      LOG.debug("signaturetime: " + Long.toString(signatureTime));
-      LOG.debug("lasttime: " + Long.toString(userACL.lastTimestamp));
-    }
-    if (signatureTime <= userACL.lastTimestamp) {
-        return true;
-    }
-    userACL.lastTimestamp = signatureTime;   
-    return false;
-  }
-
-  /**
-   * Returns authorized role for user.
-   * Checks whether signature obtained by user was made by key stored in local acl.
-   * Also checks for replay attacks.
-   * @param data data that was signed by user
-   * @param signature user-provided signature
-   * @param user-provided nonce/timestamp of signature 
-   * @return the authorized role of the user:
-   *   ADMIN, USER or NO_ACCESS
-   */
-  public int authorize(String data, String signature, String user, String timestamp) {
-    try {
-      signature = URLDecoder.decode(signature, "UTF-8");
-    } catch (Exception e) {
-      LOG.error("Authorization exception:",e);
-      return NO_ACCESS;
-    }
-    if (debug) {
-      LOG.debug(data + " sig: " + signature + " user: " + user + " time: " + timestamp);
-    }
-    try {
-      loadACL();
-      UserACL userACL = acl.get(user);  
-      if (userACL == null) {
-        return NO_ACCESS;
-      }
-      String signatureTest = hmac(data, userACL.key);
-      if (debug) {
-        LOG.debug("SignatureTest " + signatureTest);
-        LOG.debug("Signature " + signature);
-      }
-      if (signatureTest.equals(signature) && !isReplay(timestamp, signature, userACL)) {
-        return (userACL.role.equals("admin")) ? ADMIN : USER; 
-      }
-    } catch (Exception e) {
-      LOG.error("Athorization exception:", e);
-    }
-    return NO_ACCESS;
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PriorityScheduler.java

@@ -1,537 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.PrintWriter;
-import java.util.Collection;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.HashMap;
-import java.util.TreeMap;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-
-/**
- * A {@link TaskScheduler} that 
- * provides the following features: 
- * (1) allows continuous enforcement of user controlled dynamic queue shares,
- * (2) preempts tasks exceeding their queue shares instantaneously when new 
- * jobs arrive,
- * (3) is work conserving,
- * (4) tracks queue usage to only charge when jobs are pending or running,
- * (5) authorizes queue submissions based on symmetric private key HMAC/SHA1 
- * signatures.
- */
-class PriorityScheduler extends QueueTaskScheduler {
-
-  private class InitThread extends Thread {
-    JobInProgress job;
-
-    InitThread(JobInProgress job) {
-      this.job = job;
-    }
-
-    @Override
-    public void run() {
-      taskTrackerManager.initJob(job);
-    }
-  }
-
-  private class JobListener extends JobInProgressListener {
-    @Override
-    public void jobAdded(JobInProgress job) {
-      new InitThread(job).start();
-      synchronized (PriorityScheduler.this) {
-        String queue = authorize(job);
-        if (queue.equals("")) {
-            job.kill(); 
-            return;
-        }
-        jobQueue.add(job);
-        QueueJobs jobs = queueJobs.get(queue);
-        if (jobs == null) {
-          jobs = new QueueJobs(queue);
-          queueJobs.put(queue,jobs);
-        }
-        jobs.jobs.add(job);
-        if (debug) {
-          LOG.debug("Add job " + job.getProfile().getJobID());
-        }
-      }
-    }
-    @Override
-    public void jobRemoved(JobInProgress job) {
-      synchronized (PriorityScheduler.this) {
-        jobQueue.remove(job);
-        String queue = getQueue(job);
-        queueJobs.get(queue).jobs.remove(job);
-      }
-    }
-    @Override
-    public void jobUpdated(JobChangeEvent event) {
-    }
-  }
-
-
-  static final Comparator<TaskInProgress> TASK_COMPARATOR
-    = new Comparator<TaskInProgress>() {
-    public int compare(TaskInProgress o1, TaskInProgress o2) {
-      int res = 0;
-      if (o1.getProgress() < o2.getProgress()) {
-        res = -1;
-      } else {
-        res = (o1.getProgress() == o2.getProgress() ? 0 : 1);
-      }
-      if (res == 0) {
-        if (o1.getExecStartTime() > o2.getExecStartTime()) {
-          res = -1;
-        } else {
-          res = (o1.getExecStartTime() == o2.getExecStartTime() ? 0 : 1);
-        }
-      }
-      return res;
-    }
-  };
-
-  static final Comparator<KillQueue> QUEUE_COMPARATOR
-    = new Comparator<KillQueue>() {
-    public int compare(KillQueue o1, KillQueue o2) {
-      if (o1.startTime < o2.startTime) {
-        return 1;
-      }
-      if (o1.startTime > o2.startTime) {
-        return -1;
-      }
-      return 0;
-    }
-  };
-
-  class QueueJobs {
-    String name;
-    LinkedList<JobInProgress> jobs = new LinkedList<JobInProgress>();
-    QueueJobs(String name) {
-      this.name = name;
-    }
-  }
-
-  class QueueQuota {
-    int quota;
-    int map_used;
-    int reduce_used;
-    int map_pending;
-    int reduce_pending;
-    int mappers;
-    int reducers;
-    String name;
-    QueueQuota(String name) {
-      this.name = name;
-    }
-  }
-
-  private QueueAllocator allocator;
-
-  private static final Log LOG = 
-    LogFactory.getLog(PriorityScheduler.class);
-
-  static final boolean MAP = true;
-  static final boolean REDUCE = false;
-  private static final boolean FILL = true;
-  private static final boolean NO_FILL = false;
-
-  private JobListener jobListener = new JobListener();
-  private static final boolean debug = LOG.isDebugEnabled();
-  private boolean sortTasks = true;
-  private long lastKill = 0;
-  private long killInterval = 0;
-  private PriorityAuthorization auth = new PriorityAuthorization();
-
-  private LinkedList<JobInProgress> jobQueue = 
-    new LinkedList<JobInProgress>();
-  private HashMap<String,QueueJobs> queueJobs = 
-    new HashMap<String,QueueJobs>();
-
-  @Override
-  public void start() throws IOException {
-    taskTrackerManager.addJobInProgressListener(jobListener);
-    sortTasks = conf.getBoolean("mapred.priority-scheduler.sort-tasks", true);
-    killInterval = conf.getLong("mapred.priority-scheduler.kill-interval", 0);
-    auth.init(conf);
-  }
-
-  @Override
-  public void terminate() throws IOException {
-  }
-
-  @Override
-  public void setAllocator(QueueAllocator allocator) {
-    this.allocator = allocator;
-  }
-
-  private boolean assignMapRedTask(JobInProgress job, 
-      TaskTrackerStatus taskTracker, int numTrackers, List<Task> assignedTasks,
-      Map<String,QueueQuota> queueQuota, boolean fill, boolean map) 
-      throws IOException {
-    String queue = getQueue(job);
-    QueueQuota quota = queueQuota.get(queue);
-    if (quota == null) {
-        LOG.error("Queue " + queue + " not configured properly");
-        return false;
-    }
-    if (quota.quota < 1 && !fill) {
-      return false;
-    }
-    Task t = null;
-    if (map) {
-      t = job.obtainNewLocalMapTask(taskTracker, numTrackers,
-                                 taskTrackerManager.getNumberOfUniqueHosts());
-      if (t != null) {
-        if (debug) {
-          LOG.debug("assigned local task for job " + job.getProfile().getJobID() + 
-              " " + taskType(map) );
-        }
-        assignedTasks.add(t);
-        if (map) {
-          quota.map_used++;
-        } else {
-          quota.reduce_used++;
-        }
-        quota.quota--;
-        return true;
-      }
-      t = job.obtainNewNonLocalMapTask(taskTracker, numTrackers,
-          taskTrackerManager.getNumberOfUniqueHosts());
-    } else {
-      t = job.obtainNewReduceTask(taskTracker, numTrackers,
-          taskTrackerManager.getNumberOfUniqueHosts());
-    }
-    if (t != null) {
-      if (debug) {
-        LOG.debug("assigned remote task for job " + job.getProfile().getJobID() + 
-            " " + taskType(map));
-      }
-      assignedTasks.add(t);
-      if (map) {
-        quota.map_used++;
-      } else {
-        quota.reduce_used++;
-      }
-      quota.quota--;
-      return true;
-    }
-    return false;
-  }
-
-  Map<String,QueueQuota> getQueueQuota(int maxMapTasks, int maxReduceTasks, 
-      boolean map) {
-    if (debug) {
-      LOG.debug("max map tasks " + Integer.toString(maxMapTasks) + " "  + 
-          taskType(map));
-      LOG.debug("max reduce tasks " + Integer.toString(maxReduceTasks) + " "  + 
-          taskType(map));
-    }
-    int maxTasks = (map) ? maxMapTasks : maxReduceTasks;
-    Map<String,QueueAllocation> shares = allocator.getAllocation();
-    Map<String,QueueQuota> quotaMap = new HashMap<String,QueueQuota>();
-    for (QueueAllocation share: shares.values()) {
-      QueueQuota quota = new QueueQuota(share.getName());
-      quota.mappers = Math.round(share.getShare() * maxMapTasks);
-      quota.reducers = Math.round(share.getShare() * maxReduceTasks);
-      quota.quota = (map) ? quota.mappers : quota.reducers;
-
-      if (debug) {
-        LOG.debug("queue " + quota.name + " initial quota " + 
-            Integer.toString(quota.quota) + " "  + taskType(map));
-      }
-      quota.map_used = 0;
-      quota.reduce_used = 0;
-      quota.map_pending = 0;
-      quota.reduce_pending = 0;
-      Collection<JobInProgress> jobs = getJobs(quota.name);
-      for (JobInProgress job : jobs) {
-        quota.map_pending += job.pendingMaps();
-        quota.reduce_pending += job.pendingReduces();
-        int running = (map) ? job.runningMapTasks : job.runningReduceTasks;
-        quota.quota -= running;
-        quota.map_used += job.runningMapTasks ;
-        quota.reduce_used += job.runningReduceTasks;
-      }
-      if (debug) {
-        LOG.debug("queue " + quota.name + " quota " + 
-            Integer.toString(quota.quota) + " "  + taskType(map));
-      }
-      quotaMap.put(quota.name,quota);
-    } 
-   return quotaMap;
-  }
-
-  private void scheduleJobs(int availableSlots, boolean map, boolean fill, 
-      TaskTrackerStatus taskTracker, int numTrackers, List<Task> assignedTasks, 
-      Map<String,QueueQuota> queueQuota) throws IOException {
-    for (int i = 0; i < availableSlots; i++) {
-      for (JobInProgress job : jobQueue) {
-        if ((job.getStatus().getRunState() != JobStatus.RUNNING) ||
-            (!map && job.numReduceTasks == 0)) {
-          continue;
-        }
-        if (assignMapRedTask(job, taskTracker, numTrackers, assignedTasks, 
-            queueQuota, fill, map)) {
-          break;
-        }
-      }
-    }
-  } 
-
-  private int countTasksToKill(Map<String,QueueQuota> queueQuota, boolean map) {
-    int killTasks = 0;
-    for (QueueQuota quota : queueQuota.values()) {
-      killTasks += Math.min((map) ? quota.map_pending : quota.reduce_pending,
-          Math.max(quota.quota,0));
-    } 
-    return killTasks;
-  }
-   
-  protected void markIdle(Map<String, QueueQuota> queueQuota) {
-    for (QueueQuota quota: queueQuota.values()) {
-      allocator.setUsage(quota.name, Math.min(quota.map_used, quota.mappers) + 
-          Math.min(quota.reduce_used, quota.reducers), 
-          (quota.map_pending + quota.reduce_pending));
-    }  
-  }
-
-  private synchronized void assignMapRedTasks(List<Task> assignedTasks, 
-      TaskTrackerStatus taskTracker, int numTrackers, boolean map)
-      throws IOException {
-    int taskOffset = assignedTasks.size();
-    int maxTasks = (map) ? taskTracker.getMaxMapSlots() : 
-        taskTracker.getMaxReduceSlots();
-    int countTasks =  (map) ? taskTracker.countMapTasks() : 
-        taskTracker.countReduceTasks();
-    int availableSlots = maxTasks - countTasks;
-    int map_capacity = 0;
-    int reduce_capacity = 0;
-    ClusterStatus status = taskTrackerManager.getClusterStatus();
-    if (status != null) {
-      map_capacity = status.getMaxMapTasks();
-      reduce_capacity = status.getMaxReduceTasks();
-    }
-    Map<String,QueueQuota> queueQuota = getQueueQuota(map_capacity, 
-        reduce_capacity,map);
-    if (debug) {
-      LOG.debug("available slots " + Integer.toString(availableSlots) + " " + 
-          taskType(map));
-      LOG.debug("queue size " + Integer.toString(jobQueue.size()));
-      LOG.debug("map capacity " + Integer.toString(map_capacity) + " " + 
-           taskType(map));
-      LOG.debug("reduce capacity " + Integer.toString(reduce_capacity) + " " + 
-           taskType(map));
-    }
-    scheduleJobs(availableSlots, map, NO_FILL, taskTracker, numTrackers, 
-        assignedTasks, queueQuota);
-    availableSlots -= assignedTasks.size() + taskOffset;
-    scheduleJobs(availableSlots, map, FILL, taskTracker, numTrackers, 
-        assignedTasks, queueQuota);
-    if (map) {
-      markIdle(queueQuota);
-    }
-
-    long currentTime = System.currentTimeMillis()/1000;
-    if ((killInterval > 0) && (currentTime - lastKill > killInterval)) {
-      lastKill = currentTime;
-    } else {
-      return;
-    }
- 
-    int killTasks = countTasksToKill(queueQuota, map);
-    if (debug) {
-      LOG.debug("trying to kill " + Integer.toString(killTasks) + " tasks " + 
-          taskType(map));
-    }
-    killMapRedTasks(killTasks, queueQuota, map);
-  }
-
-  class KillQueue {
-    String name;
-    long startTime;
-    QueueQuota quota;
-  }
-
-  private Collection<KillQueue> getKillQueues(Map<String, 
-      QueueQuota> queueQuota) {
-    TreeMap killQueues = new TreeMap(QUEUE_COMPARATOR);
-    for (QueueJobs queueJob : queueJobs.values()) {
-      QueueQuota quota = queueQuota.get(queueJob.name); 
-      if (quota.quota >= 0) {
-        continue;
-      }
-      for (JobInProgress job : queueJob.jobs) {
-        if (job.getStatus().getRunState() == JobStatus.RUNNING) {
-          KillQueue killQueue = new KillQueue();
-          killQueue.name = queueJob.name;
-          killQueue.startTime = job.getStartTime();
-          killQueue.quota = quota;
-          killQueues.put(killQueue, killQueue);
-        }
-      }
-    }
-    return killQueues.values();
-  }
-  private void killMapRedTasks(int killTasks, Map<String,QueueQuota> queueQuota, 
-      boolean map) {
-    int killed = 0;
-    // sort queues  exceeding quota in reverse order of time since starting 
-    // a running job
-    Collection<KillQueue> killQueues = getKillQueues(queueQuota);
-    for (KillQueue killQueue : killQueues) {
-      if (killed == killTasks) {
-        return;
-      }
-      QueueQuota quota = killQueue.quota;
-      // don't kill more than needed and not more than quota exceeded
-      int toKill = Math.min(killTasks-killed,-quota.quota);
-      killQueueTasks(quota.name, toKill, map);
-      killed += toKill;
-    }  
-  }
-
-  private String taskType(boolean map) {
-    return (map) ? "MAP" : "REDUCE";
-  }
-  private void killQueueTasks(String queue, int killTasks, boolean map) {
-    if (killTasks == 0) {
-      return;
-    }
-    if (debug) {
-      LOG.debug("trying to kill " + Integer.toString(killTasks) + 
-          " tasks from queue " + queue + " " + taskType(map));
-    }
-    int killed = 0;
-    Collection<JobInProgress> jobs = getJobs(queue); 
-    if (debug) {
-      LOG.debug("total jobs to kill from " + Integer.toString(jobs.size()) + 
-          " " + taskType(map));
-    }
-    for (JobInProgress job : jobs) {
-      TaskInProgress tasks[] = (map) ? job.maps.clone() : 
-                                       job.reduces.clone();
-      if (sortTasks) {
-        Arrays.sort(tasks, TASK_COMPARATOR);
-      }
-      if (debug) {
-        LOG.debug("total tasks to kill from " + 
-            Integer.toString(tasks.length) + " " + taskType(map));
-      }
-      for (int i=0; i < tasks.length; i++) {
-        if (debug) {
-          LOG.debug("total active tasks to kill from " + 
-              Integer.toString(tasks[i].getActiveTasks().keySet().size()) + 
-              " " + taskType(map));
-        }
-        for (TaskAttemptID id: tasks[i].getActiveTasks().keySet()) {
-          if (tasks[i].isCommitPending(id)) {
-            continue;
-          }
-          tasks[i].killTask(id, false);
-          if (debug) {
-            LOG.debug("killed task " + id + " progress " + 
-                Double.toString(tasks[i].getProgress()) +
-                " start time " + Long.toString(tasks[i].getExecStartTime()) + 
-                " " +  taskType(map));
-          }
-          killed += 1;
-          if (killed == killTasks) {
-            return;
-          }
-        }
-      }
-    }
-  }
-
-  @Override
-  public List<Task> assignTasks(TaskTracker taskTracker)
-    throws IOException {
-    long millis = 0;
-    if (debug) {
-      millis = System.currentTimeMillis();
-    } 
-    ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
-    int numTrackers = clusterStatus.getTaskTrackers();
- 
-    List<Task> assignedTasks = new ArrayList<Task>();
-
-    assignMapRedTasks(assignedTasks, taskTracker.getStatus(), numTrackers, MAP);
-    assignMapRedTasks(assignedTasks, taskTracker.getStatus(), numTrackers, REDUCE);
-    if (debug) {
-      long elapsed = System.currentTimeMillis() - millis;
-      LOG.debug("assigned total tasks: " + 
-          Integer.toString(assignedTasks.size()) + " in " + 
-          Long.toString(elapsed) + " ms");
-    }
-    return assignedTasks;
-  }
-
-  @Override
-  public Collection<JobInProgress> getJobs(String queueName) {
-    QueueJobs jobs = queueJobs.get(queueName);
-    if (jobs == null) {
-      return new ArrayList<JobInProgress>();
-    }
-    return jobs.jobs;
-  }
-
-  private String getQueue(JobInProgress job) {
-    JobConf conf = job.getJobConf();
-    return conf.getQueueName();
-  }
-  private String getUser(JobInProgress job) {
-    JobConf conf = job.getJobConf();
-    return conf.getUser();
-  }
-  private String authorize(JobInProgress job) {
-    JobConf conf = job.getJobConf();
-    String user = conf.getUser();
-    String queue = conf.getQueueName();
-    if (!user.equals(queue)) {
-      return "";
-    }
-    String timestamp = conf.get("mapred.job.timestamp");
-    String signature = conf.get("mapred.job.signature");
-    int role = auth.authorize("&user=" + user + "&timestamp=" + timestamp, 
-        signature, user, timestamp);
-    if (role != PriorityAuthorization.NO_ACCESS) {
-      return queue;
-    }
-    return "";
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/PrioritySchedulerOptions.java

@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.hadoop.mapred;
-
-/**
- * Configuration Options used in the priority schedulers
- * -all in one place for ease of referencing in code.
- */
-public class PrioritySchedulerOptions {
-  /** {@value} */
-  public static final String DYNAMIC_SCHEDULER_BUDGET_FILE = "mapred.dynamic-scheduler.budget-file";
-  /** {@value} */
-  public static final String DYNAMIC_SCHEDULER_STORE = "mapred.dynamic-scheduler.store";
-  /** {@value} */
-  public static final String MAPRED_QUEUE_NAMES = "mapred.queue.names";
-  /** {@value} */
-  public static final String DYNAMIC_SCHEDULER_SCHEDULER = "mapred.dynamic-scheduler.scheduler";
-  /** {@value} */
-  public static final String DYNAMIC_SCHEDULER_ALLOC_INTERVAL = "mapred.dynamic-scheduler.alloc-interval";
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocation.java

@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * Class to hold queue share info to be
- * communicated between scheduler and 
- * queue share manager
- */
-public class QueueAllocation {
-  private String name;
-  private float share;
-  /**
-   * @param name queue name
-   * @param share queue share of total capacity (0..1)
-   */
-  public QueueAllocation(String name, float share) {
-    this.name = name;
-    this.share = share;
-  }
-  /**
-   * Gets queue share
-   * @return queue share of total capacity (0..1)
-   */
-  public float getShare() {
-    return this.share;
-  }
-  /**
-   * Gets queue name
-   * @return queue name
-   */
-  public String getName() {
-    return this.name;
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueAllocator.java

@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.util.Map;
-/**
- * This interface is intended for allowing schedulers to 
- * communicate with the queue share management implementation.
- * Schedulers can periodically poll this interface to
- * obtain the latest queue allocations.
- */
-public interface QueueAllocator {
-  /**
-   * Used by schedulers to obtain queue allocations periodically
-   * @return hashtable of queue names and their allocations (shares)
-   */
-  Map<String,QueueAllocation> getAllocation();
-  /**
-   * Used by schedulers to push queue usage info for
-   * accounting purposes.
-   * @param queue the queue name
-   * @param used of slots currently used
-   * @param pending number of tasks pending
-   */
-  void setUsage(String queue, int used, int pending);
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapred/QueueTaskScheduler.java

@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/** 
- * This class allows the scheduler to retrieve periodic
- * queue allocation info from the queue share manager.
- */ 
-abstract public class QueueTaskScheduler extends TaskScheduler {
-  /**
-   * Sets the queue share manager of a scheduler
-   * @param allocator the queue share manager of this scheduler
-   */
-  public abstract void setAllocator(QueueAllocator allocator);
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/BaseSchedulerTest.java

@@ -1,146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.hadoop.mapred;
-
-import junit.framework.TestCase;
-
-import java.util.Collection;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.Arrays;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.io.IOException;
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Base class for various scheduler tests
- */
-public class BaseSchedulerTest extends TestCase {
-  final static String[] QUEUES = new String[] {"queue1","queue2"};
-  protected FakeDynamicTimer timer = new FakeDynamicTimer();
-  protected FakeTaskTrackerManager taskTracker = new FakeTaskTrackerManager();
-  protected String budgetFile;
-  protected Configuration conf;
-   /**
-   * Create the test budget file
-   * @throws Exception
-   */
-  @Override
-  protected void setUp() throws Exception {
-    super.setUp();
-     String pathname = System.getProperty("test.build.data",
-                                          "build/contrib/dynamic-scheduler/test/data");
-     String testDir = new File(pathname).getAbsolutePath();
-    budgetFile = new File(testDir, "test-budget").getAbsolutePath();
-    new File(testDir).mkdirs();
-    new File(budgetFile).createNewFile();
-         conf = new Configuration();
-    conf.set(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_ALLOC_INTERVAL, "2");
-    conf.set(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_BUDGET_FILE, budgetFile);
-  }
-
-  /**
-    * deletes the test budget file
-    * @throws Exception
-    */
-   @Override
-   protected void tearDown() throws Exception {
-     new File(budgetFile).delete();
-   }
-
-  static class FakeTaskTrackerManager implements TaskTrackerManager {
-    FakeQueueManager qm = new FakeQueueManager();
-    public FakeTaskTrackerManager() {
-    }
-    public void addTaskTracker(String ttName) {
-    }
-    public ClusterStatus getClusterStatus() {
-      return null;
-    }
-    public int getNumberOfUniqueHosts() {
-      return 0;
-    }
-    public int getNextHeartbeatInterval() {
-      return 0;
-    }
-    public Collection<TaskTrackerStatus> taskTrackers() {
-      return null;
-    }
-    public void addJobInProgressListener(JobInProgressListener listener) {
-    }
-    public void removeJobInProgressListener(JobInProgressListener listener) {
-    }
-    public void submitJob(JobInProgress job) {
-    }
-    public TaskTrackerStatus getTaskTracker(String trackerID) {
-      return null;
-    }
-    public void killJob(JobID jobid) throws IOException {
-    }
-    public JobInProgress getJob(JobID jobid) {
-      return null;
-    }
-    public void initJob(JobInProgress job) {
-    }
-    public void failJob(JobInProgress job) {
-    }
-    public void startTask(String taskTrackerName, final Task t) {
-    }
-    public boolean killTask(TaskAttemptID attemptId, boolean shouldFail) {
-      return true;
-    }
-    void addQueues(String[] arr) {
-      Set<String> queues = new HashSet<String>();
-      queues.addAll(Arrays.asList(arr));
-      qm.setQueues(queues);
-    }
-    public QueueManager getQueueManager() {
-      return qm;
-    }
-  }
-
-
-
-
-  static class FakeDynamicTimer extends Timer {
-    private TimerTask task;
-    public void scheduleAtFixedRate(TimerTask task, long delay, long period) {
-      this.task = task;
-    }
-    public void runTask() {
-      task.run();
-    }
-  }
-
-  static class FakeQueueManager extends QueueManager {
-    private Set<String> queues = null;
-    FakeQueueManager() {
-      super(new Configuration());
-    }
-    void setQueues(Set<String> queues) {
-      this.queues = queues;
-    }
-    public synchronized Set<String> getLeafQueueNames() {
-      return queues;
-    }
-  }
-
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/FakeDynamicScheduler.java

@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Collection;
-
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-
-/**
- * Mock queue scheduler for testing only
- */
-public class FakeDynamicScheduler extends QueueTaskScheduler {
-  public void start() throws IOException {
-  }
-  public void terminate() throws IOException {
-  }
-  public List<Task> assignTasks(TaskTracker taskTracker)
-    throws IOException {
-    return null;
-  }
-  public Collection<JobInProgress> getJobs(String queueName) {
-    return null;
-  }
-  public void setAllocator(QueueAllocator allocator) {
-  }
-}
-

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestDynamicScheduler.java

@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-
-/**
- Test the dynamic scheduler.
- Use the System Property test.build.data to drive the test run
- */
-public class TestDynamicScheduler extends BaseSchedulerTest {
-
-  private DynamicPriorityScheduler scheduler;
-
-
-  /**
-   * Create the test queues
-   * @throws Exception
-   */
-  @Override
-  protected void setUp() throws Exception {
-    super.setUp();
-    conf.set(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_SCHEDULER,
-             "org.apache.hadoop.mapred.FakeDynamicScheduler");
-    scheduler = new DynamicPriorityScheduler();
-    scheduler.setTimer(timer); 
-    scheduler.setConf(conf);
-    scheduler.setTaskTrackerManager(taskTracker);
-    taskTracker.addQueues(QUEUES);
-    scheduler.start();
-  }
-
-  /**
-   * Remove the queues
-   * @throws Exception
-   */
-  @Override
-  protected void tearDown() throws Exception {
-    super.tearDown();
-    removeQueues(QUEUES);
-  }
-  
-
-  private void setSpending(String queue, float spending) throws IOException {
-    scheduler.allocations.setSpending(queue, spending);
-  }
-
-  private void setBudgets(String[] queue, float[] budget) throws IOException {
-    for (int i = 0; i < queue.length; i++) {
-      scheduler.allocations.addBudget(queue[i], budget[i]);
-    }
-  }
-
-  private void addQueues(String[] queue) throws IOException {
-    for (String aQueue : queue) {
-      scheduler.allocations.addQueue(aQueue);
-    }
-  }
-  private void removeQueues(String[] queue) throws IOException {
-    for (String aQueue : queue) {
-      scheduler.allocations.removeQueue(aQueue);
-    }
-  }
-
-
-  public void testAllocation() throws IOException {
-    addQueues(QUEUES);
-    setSpending("queue1", 1.0f);
-    setSpending("queue2", 2.0f);
-    setBudgets(QUEUES, new float[] {100.0f, 100.0f});
-    scheduler.allocations.setUsage("queue1",2,0);
-    scheduler.allocations.setUsage("queue2",3,0);
-    timer.runTask();
-    assertNotNull(scheduler.allocations);
-    assertNotNull(scheduler.allocations.allocation);
-    assertNotNull(scheduler.allocations.allocation.get("queue1"));
-    assertNotNull(scheduler.allocations.allocation.get("queue2"));
-    Collection<BudgetQueue> budgetQueues =
-      scheduler.allocations.store.getQueues();
-    assertNotNull(budgetQueues);
-    assertEquals(2, budgetQueues.size());
-    BudgetQueue queue1Budget = null;
-    BudgetQueue queue2Budget = null;
-    for (BudgetQueue queue: budgetQueues) {
-      if (queue.name.equals("queue1")) {
-        queue1Budget = queue;
-      } else {
-        queue2Budget = queue;
-      }
-    }
-    assertNotNull(queue1Budget);
-    assertNotNull(queue2Budget);
-
-    assertEquals(98.0f, queue1Budget.budget, 0.1f);
-    assertEquals(94.0f, queue2Budget.budget, 0.1f);
-    assertEquals(1.0f, queue1Budget.spending, 0.1f);
-    assertEquals(2.0f, queue2Budget.spending, 0.1f);
-
-    Map<String,QueueAllocation> shares = scheduler.allocations.getAllocation();
-    assertNotNull(shares);
-    assertEquals(2, shares.size());
-    assertNotNull(shares.get("queue1")); 
-    assertNotNull(shares.get("queue2")); 
-    assertEquals(1.0f/3.0f, shares.get("queue1").getShare(), 0.1f);
-    assertEquals(2.0f/3.0f, shares.get("queue2").getShare(), 0.1f);
-  }
-
-  public void testBudgetUpdate() throws IOException {
-    addQueues(QUEUES);
-    setSpending("queue1", 1.0f);
-    setSpending("queue2", 2.0f);
-    setBudgets(QUEUES, new float[] {100.0f, 200.0f});
-    timer.runTask();
-    Collection<BudgetQueue> budgetQueues =
-      scheduler.allocations.store.getQueues();
-    BudgetQueue queue1Budget = null;
-    BudgetQueue queue2Budget = null;
-    for (BudgetQueue queue: budgetQueues) {
-      if (queue.name.equals("queue1")) {
-        queue1Budget = queue;
-      } else {
-        queue2Budget = queue;
-      }
-    }
-    assertNotNull(queue1Budget);
-    assertNotNull(queue2Budget);
-    assertEquals(100.0f, queue1Budget.budget, 0.1f);
-    assertEquals(200.0f, queue2Budget.budget, 0.1f);
-    setBudgets(QUEUES, new float[] {200.0f, 300.0f});
-    timer.runTask();
-    budgetQueues =  scheduler.allocations.store.getQueues();
-    for (BudgetQueue queue: budgetQueues) {
-      if (queue.name.equals("queue1")) {
-        queue1Budget = queue;
-      } else {
-        queue2Budget = queue;
-      }
-    }
-    assertEquals(300.0f, queue1Budget.budget, 0.1f);
-    assertEquals(500.0f, queue2Budget.budget, 0.1f);
-    removeQueues(QUEUES);
-  }
-
-  public void testSpendingUpdate() throws IOException {
-    addQueues(QUEUES);
-    setSpending("queue1", 1.0f);
-    setSpending("queue2", 2.0f);
-    setBudgets(QUEUES, new float[] {100.0f, 100.0f});
-    scheduler.allocations.setUsage("queue1", 1, 0);
-    scheduler.allocations.setUsage("queue2", 1, 0);
-    timer.runTask();
-    Map<String,QueueAllocation> shares =
-      scheduler.allocations.getAllocation();
-    assertEquals(1.0f/3.0f, shares.get("queue1").getShare(), 0.1f);
-    assertEquals(2.0f/3.0f, shares.get("queue2").getShare(), 0.1f);
-    setSpending("queue1", 5.0f);
-    setSpending("queue2", 1.0f);
-    timer.runTask();
-    shares = scheduler.allocations.getAllocation();
-    assertEquals(5.0f/6.0f, shares.get("queue1").getShare(), 0.1f);
-    assertEquals(1.0f/6.0f, shares.get("queue2").getShare(), 0.1f);
-  }
-}

hadoop-mapreduce-project/src/contrib/dynamic-scheduler/src/test/org/apache/hadoop/mapred/TestPriorityScheduler.java

@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-public class TestPriorityScheduler extends BaseSchedulerTest {
-
-  private DynamicPriorityScheduler scheduler;
-
-  @Override
-  protected void setUp() throws Exception {
-    super.setUp();
-    conf.set(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_SCHEDULER,
-      "org.apache.hadoop.mapred.PriorityScheduler");
-    scheduler = new DynamicPriorityScheduler();
-    scheduler.setTimer(timer); 
-    scheduler.setConf(conf);
-    scheduler.setTaskTrackerManager(taskTracker);
-    taskTracker.addQueues(QUEUES);
-    scheduler.start();
-  }
-
-
-  /**
-   * Remove the queues
-   * @throws Exception
-   */
-  @Override
-  protected void tearDown() throws Exception {
-    super.tearDown();
-    removeQueues(QUEUES);
-  }
-
-  private void setSpending(String queue, float spending) throws IOException {
-    scheduler.allocations.setSpending(queue, spending);
-  }
-
-  private void setBudgets(String[] queue, float[] budget) throws IOException {
-    for (int i = 0; i < queue.length; i++) {
-      scheduler.allocations.addBudget(queue[i], budget[i]);
-    }
-  }
-
-  private void addQueues(String[] queue) throws IOException {
-    for (String aQueue : queue) {
-      scheduler.allocations.addQueue(aQueue);
-    }
-  }
-  private void removeQueues(String[] queue) throws IOException {
-    for (String aQueue : queue) {
-      scheduler.allocations.removeQueue(aQueue);
-    }
-  }
-
-  public void testQueueAllocation() throws IOException {
-    addQueues(QUEUES);
-    setSpending("queue1", 1.0f);
-    setSpending("queue2", 2.0f);
-    setBudgets(QUEUES, new float[] {100.0f, 100.0f});
-    scheduler.allocations.setUsage("queue1", 2,0);
-    scheduler.allocations.setUsage("queue2", 3,0);
-    timer.runTask();
-    Map<String,PriorityScheduler.QueueQuota> queueQuota = 
-      ((PriorityScheduler)scheduler.scheduler).
-      getQueueQuota(100, 10, PriorityScheduler.MAP); 
-    assertEquals(2, queueQuota.size());
-    for (PriorityScheduler.QueueQuota quota: queueQuota.values()) {
-      if (quota.name.equals("queue1")) {
-        assertEquals(Math.round(100 * 1.0f/3.0f), quota.quota, 0.1f);
-      } else {
-        assertEquals(Math.round(100 * 2.0f/3.0f), quota.quota, 0.1f);
-      }
-      assertTrue(quota.mappers == quota.quota);
-    }     
-    queueQuota = ((PriorityScheduler)scheduler.scheduler).getQueueQuota(100, 10,
-        PriorityScheduler.REDUCE); 
-    assertEquals(2, queueQuota.size());
-    for (PriorityScheduler.QueueQuota quota: queueQuota.values()) {
-      if (quota.name.equals("queue1")) {
-        assertEquals( Math.round(10 * 1.0f/3.0f), quota.quota, 0.1f);
-      } else {
-        assertEquals(Math.round(10 * 2.0f/3.0f), quota.quota, 0.1f);
-      }
-      assertTrue(quota.reducers == quota.quota);
-    }     
-  }
-
-  public void testUsage() throws IOException {
-    addQueues(QUEUES);
-    setSpending("queue1", 1.0f);
-    setSpending("queue2", 2.0f);
-    setBudgets(QUEUES, new float[] {1000.0f, 1000.0f});
-    scheduler.allocations.setUsage("queue1", 0, 1);
-    scheduler.allocations.setUsage("queue2", 0, 1);
-    timer.runTask();
-    Map<String,PriorityScheduler.QueueQuota> queueQuota = 
-      ((PriorityScheduler)scheduler.scheduler).getQueueQuota(100, 10,
-          PriorityScheduler.MAP); 
-    PriorityScheduler.QueueQuota quota1 = queueQuota.get("queue1");
-    PriorityScheduler.QueueQuota quota2 = queueQuota.get("queue2");
-    quota1.map_used = 10;
-    quota2.map_used = 90; 
-    ((PriorityScheduler)scheduler.scheduler).markIdle(queueQuota);
-    timer.runTask();
-
-    Collection<BudgetQueue> budgetQueues =
-      scheduler.allocations.store.getQueues();
-    assertNotNull(budgetQueues);
-    assertEquals(2, budgetQueues.size());
-    BudgetQueue queue1Budget = null;
-    BudgetQueue queue2Budget = null;
-    for (BudgetQueue queue: budgetQueues) {
-      if (queue.name.equals("queue1")) {
-        queue1Budget = queue;
-      } else {
-        queue2Budget = queue;
-      }
-    }
-    assertNotNull(queue1Budget);
-    assertNotNull(queue2Budget);
-    assertEquals("Budget incorrect", 990.0f, queue1Budget.budget, 0.1f);
-    assertEquals("Budget incorrect", 866.0f, queue2Budget.budget, 0.1f);
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/README

@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-This package implements a fair scheduler for MapReduce jobs with additional
-support for guaranteed shares and job limits.
-
-The functionality of this scheduler is described in the Forrest 
-documentation at http://hadoop.apache.org/core/ or alternatively, in the 
-hadoop release it can be found at $HADOOP_PREFIX/docs. In order to build the 
-documentation on your own from source please use the following command in 
-the downloaded source folder:
-
-ant docs -Dforrest.home=path to forrest -Djava5.home= path to jdk5. 
-
-The documentation so built would be under $HADOOP_PREFIX/build/docs

hadoop-mapreduce-project/src/contrib/fairscheduler/build.xml

@@ -1,28 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- 
-Before you can run these subtargets directly, you need 
-to call at top-level: ant deploy-contrib compile-core-test
--->
-<project name="fairscheduler" default="jar">
-
-  <import file="../build-contrib.xml"/>
-
-</project>

hadoop-mapreduce-project/src/contrib/fairscheduler/designdoc/fair_scheduler_design_doc.tex

@@ -1,253 +0,0 @@
-% Licensed to the Apache Software Foundation (ASF) under one
-% or more contributor license agreements.  See the NOTICE file
-% distributed with this work for additional information
-% regarding copyright ownership.  The ASF licenses this file
-% to you under the Apache License, Version 2.0 (the
-% "License"); you may not use this file except in compliance
-% with the License.  You may obtain a copy of the License at
-% 
-%     http://www.apache.org/licenses/LICENSE-2.0
-% 
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\documentclass[11pt]{article}
-\usepackage{geometry}
-\geometry{letterpaper}
-
-\begin{document}
-
-\title{Hadoop Fair Scheduler Design Document}
-\author{}
-\maketitle
-\tableofcontents
-
-\section{Introduction}
-
-The Hadoop Fair Scheduler started as a simple means to share MapReduce clusters. Over time, it has grown in functionality to support hierarchical scheduling, preemption, and multiple ways of organizing and weighing jobs. This document explains the goals and features of the Fair Scheduler and its internal design.
-
-\section{Fair Scheduler Goals}
-
-The Fair Scheduler was designed with four main goals:
-\begin{enumerate}
-  \item Run small jobs quickly even if they are sharing a cluster with large jobs. Unlike Hadoop's built-in FIFO scheduler, fair scheduling lets small jobs make progress even if a large job is running, without starving the large job.
-  \item Provide guaranteed service levels to ``production" jobs, to let them run alongside experimental jobs in a shared cluster.
-  \item Be simple to administer and configure. The scheduler should do something reasonable ``out of the box," and users should only need to configure it as they discover that they want to use more advanced features.
-  \item Support reconfiguration at runtime, without requiring a cluster restart.
-\end{enumerate}
-
-\section{Scheduler Features}
-
-This section provides a quick overview of the features of the Fair Scheduler. A detailed usage guide is available in the Hadoop documentation in {\tt build/docs/fair\_scheduler.html}.
-
-\subsection{Pools}
-
-The Fair Scheduler groups jobs into ``pools" and performs fair sharing between these pools. Each pool can use either FIFO or fair sharing to schedule jobs internal to the pool. The pool that a job is placed in is determined by a JobConf property, the ``pool name property". By default, this is {\tt mapreduce.job.user.name}, so that there is one pool per user. However, different properties can be used, e.g.~{\tt group.name} to have one pool per Unix group.
-
-A common trick is to set the pool name property to an unused property name such as {\tt pool.name} and make this default to {\tt mapreduce.job.user.name}, so that there is one pool per user but it is also possible to place jobs into ``special" pools by setting their {\tt pool.name} directly. The {\tt mapred-site.xml} snippet below shows how to do this:
-
-\begin{verbatim}
-<property>
-  <name>mapred.fairscheduler.poolnameproperty</name>
-  <value>pool.name</value>
-</property>
-
-<property>
-  <name>pool.name</name>
-  <value>${mapreduce.job.user.name}</value>
-</property>
-\end{verbatim}
-
-\subsection{Minimum Shares}
-
-Normally, active pools (those that contain jobs) will get equal shares of the map and reduce task slots in the cluster. However, it is also possible to set a \emph{minimum share} of map and reduce slots on a given pool, which is a number of slots that it will always get when it is active, even if its fair share would be below this number. This is useful for guaranteeing that production jobs get a certain desired level of service when sharing a cluster with non-production jobs. Minimum shares have three effects:
-\begin{enumerate}
-  \item The pool's fair share will always be at least as large as its minimum share. Slots are taken from the share of other pools to achieve this. The only exception is if the minimum shares of the active pools add up to more than the total number of slots in the cluster; in this case, each pool's share will be scaled down proportionally.
-  \item Pools whose running task count is below their minimum share get assigned slots first when slots are available.
-  \item It is possible to set a \emph{preemption timeout} on the pool after which, if it has not received enough task slots to meet its minimum share, it is allowed to kill tasks in other jobs to meet its share. Minimum shares with preemption timeouts thus act like SLAs.
-\end{enumerate}
-
-Note that when a pool is inactive (contains no jobs), its minimum share is not ``reserved" for it -- the slots are split up among the other pools.
-
-\subsection{Preemption}
-
-As explained above, the scheduler may kill tasks from a job in one pool in order to meet the minimum share of another pool. We call this preemption, although this usage of the word is somewhat strange given the normal definition of preemption as pausing; really it is the \emph{job} that gets preempted, while the task gets killed. The feature explained above is called \emph{min share preemption}. In addition, the scheduler supports \emph{fair share preemption}, to kill tasks when a pool's fair share is not being met. Fair share preemption is much more conservative than min share preemption, because pools without min shares are expected to be non-production jobs where some amount of unfairness is tolerable. In particular, fair share preemption activates if a pool has been below \emph{half} of its fair share for a configurable fair share preemption timeout, which is recommended to be set fairly high (e.g. 10 minutes).
-
-In both types of preemption, the scheduler kills the most recently launched tasks from over-scheduled pools, to minimize the amount of computation wasted by preemption.
-
-\subsection{Running Job Limits}
-
-The fair scheduler can limit the number of concurrently running jobs from each user and from each pool. This is useful for limiting the amount of intermediate data generated on the cluster. The jobs that will run are chosen in order of submit time and priority. Jobs submitted beyond the limit wait for one of the running jobs to finish.
-
-\subsection{Job Priorities}
-
-Within a pool, job priorities can be used to control the scheduling of jobs, whether the pool's internal scheduling mode is FIFO or fair sharing:
-\begin{itemize}
-  \item In FIFO pools, jobs are ordered first by priority and then by submit time, as in Hadoop's default scheduler.
-  \item In fair sharing pools, job priorities are used as weights to control how much share a job gets. The normal priority corresponds to a weight of 1.0, and each level gives 2x more weight. For example, a high-priority job gets a weight of 2.0, and will therefore get 2x the share of a normal-priority job. 
-\end{itemize}
-
-\subsection{Pool Weights}
-
-Pools can be given weights to achieve unequal sharing of the cluster. For example, a pool with weight 2.0 gets 2x the share of a pool with weight 1.0.
-
-\subsection{Delay Scheduling}
-
-The Fair Scheduler contains an algorithm called delay scheduling to improve data locality. Jobs that cannot launch a data-local map task wait for some period of time before they are allowed to launch non-data-local tasks, ensuring that they will run locally if some node in the cluster has the relevant data. Delay scheduling is described in detail in Section \ref{sec:delay-scheduling}.
-
-\subsection{Administration}
-
-The Fair Scheduler includes a web UI displaying the active pools and jobs and their fair shares, moving jobs between pools, and changing job priorities.
-In addition, the Fair Scheduler's allocation file (specifying min shares and preemption timeouts for the pools) is automatically reloaded if it is modified on disk, to allow runtime reconfiguration.
-
-\section{Implementation}
-
-\subsection{Hadoop Scheduling Background}
-
-Hadoop jobs consist of a number of map and reduce \emph{tasks}. These task run in \emph{slots} on the nodes on the cluster. Each node is configured with a number of map slots and reduce slots based on its computational resources (typically one slot per core). The role of the scheduler is to assign tasks to any slots that are free.
-
-All schedulers in Hadoop, including the Fair Scheduler, inherit from the {\tt TaskScheduler} abstract class. This class provides access to a {\tt TaskTrackerManager} -- an interface to the JobTracker -- as well as a {\tt Configuration} instance. It also ask the scheduler to implement three abstract methods: the lifecycle methods {\tt start} and {\tt terminate}, and a method called {\tt assignTasks} to launch tasks on a given TaskTracker.
-Task assignment in Hadoop is reactive. TaskTrackers periodically send heartbeats to the JobTracker with their {\tt TaskTrackerStatus}, which contains a list of running tasks, the number of slots on the node, and other information. The JobTracker then calls {\tt assignTasks} on the scheduler to obtain tasks to launch. These are returned with the heartbeat response.
-
-Apart from reacting to heartbeats through {\tt assignTasks}, schedulers can also be notified when jobs have been submitted to the cluster, killed, or removed by adding listeners to the {\tt TaskTrackerManager}. The Fair Scheduler sets up these listeners in its {\tt start} method. An important role of the listeners is to initialize jobs that are submitted -- until a job is initialized, it cannot launch tasks. The Fair Scheduler currently initializes all jobs right away, but it may also be desirable to hold off initializing jobs if too many are submitted to limit memory usage on the JobTracker.
-
-Selection of tasks \emph{within} a job is mostly done by the {\tt JobInProgress} class, and not by individual schedulers. {\tt JobInProgress} exposes two methods, {\tt obtainNewMapTask} and {\tt obtainNewReduceTask}, to launch a task of either type. Both methods may either return a {\tt Task} object or {\tt null} if the job does not wish to launch a task. Whether a job wishes to launch a task may change back and forth during its lifetime. Even after all tasks in the job have been started, the job may wish to run another task for speculative execution. In addition, if the node containing a map task failed, the job will wish to re-run it to rebuild its output for use in the reduce tasks. Schedulers may therefore need to poll multiple jobs until they find one with a task to run.
-
-Finally, for map tasks, an important scheduling criterion is data locality: running the task on a node or rack that contains its input data. Normally, {\tt JobInProgress.obtainNewMapTask} returns the ``closest" map task to a given node. However, to give schedulers slightly more control over data locality, there is also a version of {\tt obtainNewMapTask} that allow the scheduler to cap the level of non-locality allowed for the task (e.g.~request a task only on the same node, or {\tt null} if none is available). The Fair Scheduler uses this method with an algorithm called delay scheduling (Section \ref{sec:delay-scheduling}) to optimize data locality.
-
-\subsection{Fair Scheduler Basics}
-
-At a high level, the Fair Scheduler uses hierarchical scheduling to assign tasks. First it selects a pool to assign a task to according to the fair sharing algorithm in Section \ref{sec:fair-sharing-alg}. Then it asks the pool obtain a task. The pool chooses among its jobs according to its internal scheduling order (FIFO or fair sharing).
-
-In fact, because jobs might not have tasks to launch ({\tt obtainNew(Map|Reduce)Task} can return null), the scheduler actually establishes an ordering on jobs and asks them for tasks in turn. Within a pool, jobs are sorted either by priority and start time (for FIFO) or by distance below fair share. If the first job in the ordering does not have a task to launch, the pool will ask the second, third, etc jobs. Pools themselves are sorted by distance below min share and fair share, so if the first pool does not have any jobs that can launch tasks, the second pool is asked, etc. This makes it straightforward to implement features like delay scheduling (Section \ref{sec:delay-scheduling}) that may cause jobs to ``pass" on a slot.
-
-Apart from the assign tasks code path, the Fair Scheduler also has a periodic update thread that calls {\tt update} every few seconds. This thread is responsible for recomputing fair shares to display them on the UI (Section \ref{sec:fair-share-computation}), checking whether jobs need to be preempted (Section \ref{sec:preemption}), and checking whether the allocations file has changed to reload pool allocations (through {\tt PoolManager}).
-
-\subsection{The {\tt Schedulable} Class}
-
-To allow the same fair sharing algorithm to be used both between pools and within a pool, the Fair Scheduler uses an abstract class called {\tt Schedulable} to represent both pools and jobs. Its subclasses for these roles are {\tt PoolSchedulable} and {\tt JobSchedulable}. A {\tt Schedulable} is responsible for three roles:
-\begin{enumerate}
-  \item It can be asked to obtain a task through {\tt assignTask}. This may return {\tt null} if the {\tt Schedulable} has no tasks to launch.
-  \item It can be queried for information about the pool/job to use in scheduling, such as:
-  \begin{itemize}
-    \item Number of running tasks.
-    \item Demand (number of tasks the {\tt Schedulable} \emph{wants} to run; this is equal to number of running tasks + number of unlaunched tasks).
-    \item Min share assigned through config file.
-    \item Weight (for fair sharing).
-    \item Priority and start time (for FIFO scheduling).
-  \end{itemize}
-  \item It can be assigned a fair share through {\tt setFairShare}.
-\end{enumerate}
-
-There are separate {\tt Schedulable}s for map and reduce tasks, to make it possible to use the same algorithm on both types of tasks.
-
-\subsection{Fair Sharing Algorithm}
-\label{sec:fair-sharing-alg}
-
-A simple way to achieve fair sharing is the following: whenever a slot is available, assign it to the pool that has the fewest running tasks. This will ensure that all pool get an equal number of slots, unless a pool's demand is less than its fair share, in which case the extra slots are divided evenly among the other pools. Two features of the Fair Scheduler complicate this algorithm a little:
-\begin{itemize}
-  \item Pool weights mean that some pools should get more slots than others. For example, a pool with weight 2 should get 2x more slots than a pool with weight 1. This is accomplished by changing the scheduling rule to ``assign the slot to the pool whose value of $runningTasks/weight$ is smallest."
-  \item Minimum shares mean that pools below their min share should get slots first. When we sort pools to choose which ones to schedule next, we place pools below their min share ahead of pools above their min share. We order the pools below their min share by how far they are below it as a percentage of the share.
-\end{itemize}
-
-This fair sharing algorithm is implemented in {\tt FairShareComparator} in the {\tt SchedulingAlgorithms} class. The comparator orders jobs by distance below min share and then by $runningTasks/weight$.
-
-\subsection{Preemption}
-\label{sec:preemption}
-
-To determine when to preempt tasks, the Fair Schedulers maintains two values for each {\tt PoolSchedulable}: the last time when the pool was at its min share, and the last time when the pool was at half its fair share. These conditions are checked periodically by the update thread in {\tt FairScheduler.updatePreemptionVariables}, using the methods {\tt isStarvedForMinShare} and {\tt isStarvedForFairShare}. These methods also take into account the demand of the pool, so that a pool is not counted as starving if its demand is below its min/fair share but is otherwise met.
-
-When preempting tasks, the scheduler kills the most recently launched tasks from over-scheduled pools. This minimizes the amount of computation wasted by preemption and ensures that all jobs can eventually finish (it is as if the preempted jobs just never got their last few slots). The tasks are chosen and preempted in {\tt preemptTasks}.
-
-Note that for min share preemption, it is clear when a pool is below its min share because the min share is given as a number of slots, but for fair share preemption, we must be able to compute a pool's fair share to determine when it is being starved. This computation is trickier than dividing the number of slots by the number of pools due to weights, min shares and demands. Section \ref{sec:fair-share-computation} explains how fair shares are computed.
-
-\subsection{Fair Share Computation}
-\label{sec:fair-share-computation}
-
-The scheduling algorithm in Section \ref{sec:fair-sharing-alg} achieves fair shares without actually needing to compute pools' numerical shares beforehand. However, for preemption and for displaying shares in the Web UI, we want to know what a pool's fair share is even if the pool is not currently at its share. That is, we want to know how many slots the pool \emph{would} get if we started with all slots being empty and ran the algorithm in Section \ref{sec:fair-sharing-alg} until we filled them.
-One way to compute these shares would be to simulate starting out with empty slots and calling {\tt assignTasks} repeatedly until they filled, but this is expensive, because each scheduling decision takes $O(numJobs)$ time and we need to make $O(numSlots)$ decisions.
-
-To compute fair shares efficiently, the Fair Scheduler includes an algorithm based on binary search in {\tt SchedulingAlgorithms.computeFairShares}. This algorithm is based on the following observation. If all slots had been assigned according to weighted fair sharing respecting pools' demands and min shares, then there would exist a ratio $r$ such that:
-\begin{enumerate}
-  \item Pools whose demand $d_i$ is less than $r w_i$ (where $w_i$ is the weight of the pool) are assigned $d_i$ slots.
-  \item Pools whose min share $m_i$ is more than $r w_i$ are assigned $\min(m_i, d_i)$ slots.
-  \item All other pools are assigned $r w_i$ slots.
-  \item The pools' shares sum up to the total number of slots $t$.
-\end{enumerate}
-
-The Fair Scheduler uses binary search to compute the correct $r$. We define a function $f(r)$ as the number of slots that would be used for a given $r$ if conditions 1-3 above were met, and then find a value of $r$ that makes $f(r)=t$. More precisely, $f(r)$ is defined as:
-$$f(r) = \sum_i{\min(d_i, \max(r w_i, m_i)).}$$
-
-Note that $f(r)$ is increasing in $r$ because every term of the sum is increasing, so the equation $f(r) = t$ can be solved by binary search. We choose 0 as a lower bound of our binary search because with $r=0$, only min shares are assigned. (An earlier check in {\tt computeFairShares} checks whether the min shares add up to more than the total number of slots, and if so, computes fair shares by scaling down the min shares proportionally and returns.) To compute an upper bound for the binary search, we try $r=1,2,4,8,\dots$ until we find a value large enough that either more than $t$ slots are used or all pools' demands are met (in case the demands added up to less than $t$).
-
-The steps of the algorithm are explained in detail in {\tt SchedulingAlgorithms.java}.
-
-This algorithm runs in time $O(NP)$, where $N$ is the number of jobs/pools and $P$ is the desired number of bits of precision in the computed values (number of iterations of binary search), which we've set to 25. It thus scales linearly in the number of jobs and pools.
-
-\subsection{Running Job Limits}
-
-Running job limits are implemented by marking jobs as not runnable if there are too many jobs submitted by the same user or pool. This is done in {\tt FairScheduler.updateRunnability}. A job that is not runnable declares its demand as 0 and always returns {\tt null} from {\tt assignTasks}.
-
-\subsection{Delay Scheduling}
-\label{sec:delay-scheduling}
-
-In Hadoop, running map tasks on the nodes or racks that contain their input data is critical for performance, because it avoids shipping the data over the network. However, always assigning slots to the first job in order of pool shares and in-pool ordering (the ``head-of-line job") can sometimes lead to poor locality:
-\begin{itemize}
-  \item If the head-of-line job is small, the chance of it having data on the node that a heartbeat was received from is small. Therefore, locality would be poor in a small-job workload if we always assigned slots to the head-of-line job.
-  \item When fair sharing is used, there is a strong bias for a job to be reassigned into a slot that it just finished a task in, because when it finishes the task, the job falls below its fair share. This can mean that jobs have a difficult time running in slots that other jobs have taken and thus achieve poor locality.
-\end{itemize}
-
-To deal with both of these situations, the Fair Scheduler can sacrifice fairness temporarily to improve locality through an algorithm called delay scheduling. If the head-of-line job cannot launch a local task on the TaskTracker that sent a heartbeat, then it is skipped, and other running jobs are looked at in order of pool shares and in-pool scheduling rules to find a job with a local task. However, if the head-of-line job has been skipped for a sufficiently long time, it is allowed to launch rack-local tasks. Then, if it is skipped for a longer time, it is also allowed to launch off-rack tasks. These skip times are called locality delays. Delays of a few seconds are sufficient to drastically increase locality.
-
-The Fair Scheduler allows locality delays to be set through {\tt mapred-site.xml} or to be turned off by setting them to zero. However, by default, it computes the delay automatically based on the heartbeat interval of the cluster. The delay is set to 1.5x the heartbeat interval.
-
-When a job that has been allowed to launch non-local tasks ends up launching a local task again, its ``locality level" resets and it must wait again before launching non-local tasks. This is done so that a job that gets ``unlucky" early in its lifetime does not continue to launch non-local tasks throughout its life.
-
-Delay scheduling is implemented by keeping track of two variables on each job: the locality level of the last map it launched (0 for node-local, 1 for rack-local and 2 for off-rack) and the time it has spent being skipped for a task. These are kept in a {\tt JobInfo} structure associated with each job in {\tt FairScheduler.java}. Whenever a job is asked for tasks, it checks the locality level it is allowed to launch them at through {\tt FairScheduler.getAllowedLocalityLevel}. If it does not launch a task, it is marked as ``visited" on that heartbeat by appending itself to a {\tt visited} job list that is passed around between calls to {\tt assignTasks} on the same heartbeat. Jobs that are visited on a heartbeat but do not launch any tasks during it are considered as skipped for the time interval between this heartbeat and the next. Code at the beginning of {\tt FairScheduler.assignTasks} increments the wait time of each skipped job by the time elapsed since the last heartbeat. Once a job has been skipped for more than the locality delay, {\tt getAllowedLocalityLevel} starts returning higher locality so that it is allowed to launch less-local tasks. Whenever the job launches a task, its wait time is reset, but we remember the locality level of the launched task so that the job is allowed to launch more tasks at this level without further waiting.
-
-\subsection{Locking Order}
-
-Fair Scheduler data structures can be touched by several threads. Most commonly, the JobTracker invokes {\tt assignTasks}. This happens inside a block of code where the JobTracker has locked itself already. Therefore, to prevent deadlocks, we always ensure that \emph{if both the FairScheduler and the JobTracker must be locked, the JobTracker is locked first}. Other threads that can lock the FairScheduler include the update thread and the web UI.
-
-\subsection{Unit Tests}
-
-The Fair Scheduler contains extensive unit tests using mock {\tt TaskTrackerManager}, {\tt JobInProgress}, {\tt TaskInProgress}, and {\tt Schedulable} objects. Scheduler tests are in {\tt TestFairScheduler.java}. The {\tt computeFairShares} algorithm is tested separately in {\tt TestComputeFairShares.java}. All tests use accelerated time via a fake {\tt Clock} class.
-
-\pagebreak
-\section{Code Guide}
-
-The following table lists some key source files in the Fair Scheduler:
-
-\begin{center}
-\begin{tabular}{|l|p{0.7\columnwidth}|}
-  \hline
-  {\bf File} & {\bf Contents} 
-  \\ \hline
-  {\tt FairScheduler.java} & Scheduler entry point. Also contains update thread, and logic for preemption, delay scheduling, and running job limits.
-  \\ \hline
-  {\tt Schedulable.java} & Definition of the {\tt Schedulable} class. Extended by {\tt JobSchedulable} and {\tt PoolSchedulable}.
-  \\ \hline
-  {\tt SchedulingAlgorithms.java} & Contains FIFO and fair sharing comparators, as well as the {\tt computeFairShares} algorithm in Section \ref{sec:fair-share-computation}.
-  \\ \hline
-  {\tt PoolManager.java} & Reads pool properties from the allocation file and maintains a collection of {\tt Pool} objects. Pools are created on demand.
-  \\ \hline
-  {\tt Pool.java} & Represents a pool and stores its map and reduce {\tt Schedulables}.
-  \\ \hline
-  {\tt FairSchedulerServlet.java} & Implements the scheduler's web UI.
-  \\ \hline
-  {\tt FairSchedulerEventLog.java} & An easy-to-parse event log for debugging. Must be enabled through {\tt mapred.fairscheduler.eventlog.enabled}.
-  If enabled, logs are placed in {\tt \$HADOOP\_LOG\_DIR/fairscheduler}.
-  \\ \hline
-  {\tt TaskSelector.java} & A pluggable class responsible for picking tasks within a job. Currently, {\tt DefaultTaskSelector} delegates to {\tt JobInProgress}, but this would be a useful place to experiment with new algorithms for speculative execution and locality.
-  \\ \hline
-  {\tt LoadManager.java} & A pluggable class responsible for determining when to launch more tasks on a TaskTracker. Currently, {\tt CapBasedLoadManager} uses slot counts, but this would be a useful place to experiment with scheduling based on machine load.
-  \\ \hline
-  {\tt WeightAdjuster.java} & A pluggable class responsible for setting job weights. An example, {\tt NewJobWeightBooster}, is provided, which increases weight temporarily for new jobs.
-  \\ \hline
-\end{tabular}
-\end{center}
-
-\end{document}

hadoop-mapreduce-project/src/contrib/fairscheduler/ivy.xml

@@ -1,120 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<ivy-module version="1.0" xmlns:m="http://ant.apache.org/ivy/maven">
-  <info organisation="org.apache.hadoop" module="${ant.project.name}">
-    <license name="Apache 2.0"/>
-    <ivyauthor name="Apache Hadoop Team" url="http://hadoop.apache.org"/>
-    <description>
-        Apache Hadoop contrib
-    </description>
-  </info>
-  <configurations defaultconfmapping="default">
-    <!--these match the Maven configurations-->
-    <conf name="default" extends="master,runtime"/>
-    <conf name="master" description="contains the artifact but no dependencies"/>
-    <conf name="runtime" description="runtime but not the artifact" />
-
-    <conf name="common" visibility="private" 
-      description="artifacts needed to compile/test the application"/>
-    <conf name="test" visibility="private" extends="runtime"/>
-  </configurations>
-
-  <publications>
-    <!--get the artifact from our module name-->
-    <artifact conf="master"/>
-  </publications>
-  <dependencies>
-   <dependency org="org.apache.hadoop" name="hadoop-annotations" rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" 
-                rev="${hadoop-common.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" 
-                rev="${hadoop-common.version}" conf="test->default">
-      <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs" 
-                rev="${hadoop-hdfs.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs" 
-                rev="${hadoop-hdfs.version}" conf="test->default">
-      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
-               rev="${yarn.version}" conf="common->default"/>
-   <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
-               rev="${yarn.version}" conf="common->default"/>
-    <dependency org="commons-logging"
-      name="commons-logging"
-      rev="${commons-logging.version}"
-      conf="common->default"/>
-    <dependency org="log4j"
-      name="log4j"
-      rev="${log4j.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="servlet-api-2.5"
-      rev="${servlet-api-2.5.version}"
-      conf="common->default"/> 
-   <dependency org="junit"
-      name="junit"
-      rev="${junit.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.avro"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
-    <dependency org="org.codehaus.jackson"
-      name="jackson-mapper-asl"
-      rev="${jackson.version}"
-      conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer-ant"
-      rev="${paranamer.version}"
-      conf="common->default"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty-util"
-      rev="${jetty-util.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-api-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-
-   <!-- Exclusions for transitive dependencies pulled in by log4j -->
-   <exclude org="com.sun.jdmk"/>
-   <exclude org="com.sun.jmx"/>
-   <exclude org="javax.jms"/> 
-   <exclude org="javax.mail"/> 
-
-  </dependencies>
-</ivy-module>

hadoop-mapreduce-project/src/contrib/fairscheduler/ivy/libraries.properties

@@ -1,17 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#This properties file lists the versions of the various artifacts used by streaming.
-#It drives ivy and the generation of a maven POM
-
-#Please list the dependencies name with version if they are different from the ones 
-#listed in the global libraries.properties file (in alphabetical order)

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/AllocationConfigurationException.java

@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-/**
- * Thrown when the allocation file for {@link PoolManager} is malformed.  
- */
-public class AllocationConfigurationException extends Exception {
-  private static final long serialVersionUID = 4046517047810854249L;
-  
-  public AllocationConfigurationException(String message) {
-    super(message);
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/CapBasedLoadManager.java

@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A {@link LoadManager} for use by the {@link FairScheduler} that allocates
- * tasks evenly across nodes up to their per-node maximum, using the default
- * load management algorithm in Hadoop.
- */
-public class CapBasedLoadManager extends LoadManager {
-  
-  float maxDiff = 0.0f;
-  
-  public void setConf(Configuration conf) {
-    super.setConf(conf);
-    maxDiff = conf.getFloat("mapred.fairscheduler.load.max.diff", 0.0f);
-  }
-  
-  /**
-   * Determine how many tasks of a given type we want to run on a TaskTracker. 
-   * This cap is chosen based on how many tasks of that type are outstanding in
-   * total, so that when the cluster is used below capacity, tasks are spread
-   * out uniformly across the nodes rather than being clumped up on whichever
-   * machines sent out heartbeats earliest.
-   */
-  int getCap(int totalRunnableTasks, int localMaxTasks, int totalSlots) {
-    double load = maxDiff + ((double)totalRunnableTasks) / totalSlots;
-    return (int) Math.ceil(localMaxTasks * Math.min(1.0, load));
-  }
-
-  @Override
-  public boolean canAssignMap(TaskTrackerStatus tracker,
-      int totalRunnableMaps, int totalMapSlots) {
-    return tracker.countMapTasks() < getCap(totalRunnableMaps,
-        tracker.getMaxMapSlots(), totalMapSlots);
-  }
-
-  @Override
-  public boolean canAssignReduce(TaskTrackerStatus tracker,
-      int totalRunnableReduces, int totalReduceSlots) {
-    return tracker.countReduceTasks() < getCap(totalRunnableReduces,
-        tracker.getMaxReduceSlots(), totalReduceSlots);
-  }
-
-  @Override
-  public boolean canLaunchTask(TaskTrackerStatus tracker,
-      JobInProgress job,  TaskType type) {
-    return true;
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/DefaultTaskSelector.java

@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-/**
- * A {@link TaskSelector} implementation that wraps around the default
- * {@link JobInProgress#obtainNewMapTask(TaskTrackerStatus, int)} and
- * {@link JobInProgress#obtainNewReduceTask(TaskTrackerStatus, int)} methods
- * in {@link JobInProgress}, using the default Hadoop locality and speculative
- * threshold algorithms.
- */
-public class DefaultTaskSelector extends TaskSelector {
-
-  @Override
-  public int neededSpeculativeMaps(JobInProgress job) {
-    int count = 0;
-    long time = System.currentTimeMillis();
-    for (TaskInProgress tip: job.maps) {
-      if (tip.isRunning() && tip.canBeSpeculated(time)) {
-        count++;
-      }
-    }
-    return count;
-  }
-
-  @Override
-  public int neededSpeculativeReduces(JobInProgress job) {
-    int count = 0;
-    long time = System.currentTimeMillis();
-    double avgProgress = job.getStatus().reduceProgress();
-    for (TaskInProgress tip: job.reduces) {
-      if (tip.isRunning() && tip.canBeSpeculated(time)) {
-        count++;
-      }
-    }
-    return count;
-  }
-
-  @Override
-  public Task obtainNewMapTask(TaskTrackerStatus taskTracker, JobInProgress job,
-      int localityLevel) throws IOException {
-    ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
-    int numTaskTrackers = clusterStatus.getTaskTrackers();
-    return job.obtainNewMapTask(taskTracker, numTaskTrackers,
-        taskTrackerManager.getNumberOfUniqueHosts(), localityLevel);
-  }
-
-  @Override
-  public Task obtainNewReduceTask(TaskTrackerStatus taskTracker, JobInProgress job)
-      throws IOException {
-    ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
-    int numTaskTrackers = clusterStatus.getTaskTrackers();
-    return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
-        taskTrackerManager.getNumberOfUniqueHosts());
-  }
-
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java

@@ -1,1101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.apache.hadoop.metrics.MetricsUtil;
-import org.apache.hadoop.metrics.Updater;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * A {@link TaskScheduler} that implements fair sharing.
- */
-public class FairScheduler extends TaskScheduler {
-  public static final Log LOG = LogFactory.getLog(
-      "org.apache.hadoop.mapred.FairScheduler");
-
-  // How often fair shares are re-calculated
-  protected long updateInterval = 500;
-
-  // How often to dump scheduler state to the event log
-  protected long dumpInterval = 10000;
-  
-  // How often tasks are preempted (must be longer than a couple
-  // of heartbeats to give task-kill commands a chance to act).
-  protected long preemptionInterval = 15000;
-  
-  // Used to iterate through map and reduce task types
-  private static final TaskType[] MAP_AND_REDUCE = 
-    new TaskType[] {TaskType.MAP, TaskType.REDUCE};
-  
-  // Maximum locality delay when auto-computing locality delays
-  private static final long MAX_AUTOCOMPUTED_LOCALITY_DELAY = 15000;
-  
-  protected PoolManager poolMgr;
-  protected LoadManager loadMgr;
-  protected TaskSelector taskSelector;
-  protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
-  protected Map<JobInProgress, JobInfo> infos = // per-job scheduling variables
-    new HashMap<JobInProgress, JobInfo>();
-  protected long lastUpdateTime;           // Time when we last updated infos
-  protected long lastPreemptionUpdateTime; // Time when we last updated preemption vars
-  protected boolean initialized;  // Are we initialized?
-  protected volatile boolean running; // Are we running?
-  protected boolean assignMultiple; // Simultaneously assign map and reduce?
-  protected int mapAssignCap = -1;    // Max maps to launch per heartbeat
-  protected int reduceAssignCap = -1; // Max reduces to launch per heartbeat
-  protected long nodeLocalityDelay;   // Time to wait for node locality
-  protected long rackLocalityDelay;   // Time to wait for rack locality
-  protected boolean autoComputeLocalityDelay = false; // Compute locality delay
-                                                      // from heartbeat interval
-  protected boolean sizeBasedWeight; // Give larger weights to larger jobs
-  protected boolean waitForMapsBeforeLaunchingReduces = true;
-  protected boolean preemptionEnabled;
-  protected boolean onlyLogPreemption; // Only log when tasks should be killed
-  private Clock clock;
-  private JobListener jobListener;
-  private JobInitializer jobInitializer;
-  private boolean mockMode; // Used for unit tests; disables background updates
-                            // and scheduler event log
-  private FairSchedulerEventLog eventLog;
-  protected long lastDumpTime;       // Time when we last dumped state to log
-  protected long lastHeartbeatTime;  // Time we last ran assignTasks 
-  private long lastPreemptCheckTime; // Time we last ran preemptTasksIfNecessary
-  
-  /**
-   * A class for holding per-job scheduler variables. These always contain the
-   * values of the variables at the last update(), and are used along with a
-   * time delta to update the map and reduce deficits before a new update().
-   */
-  static class JobInfo {
-    boolean runnable = false;   // Can the job run given user/pool limits?
-    // Does this job need to be initialized?
-    volatile boolean needsInitializing = true;
-    public JobSchedulable mapSchedulable;
-    public JobSchedulable reduceSchedulable;
-    // Variables used for delay scheduling
-    LocalityLevel lastMapLocalityLevel; // Locality level of last map launched
-    long timeWaitedForLocalMap; // Time waiting for local map since last map
-    boolean skippedAtLastHeartbeat;  // Was job skipped at previous assignTasks?
-                                     // (used to update timeWaitedForLocalMap)
-    public JobInfo(JobSchedulable mapSched, JobSchedulable reduceSched) {
-      this.mapSchedulable = mapSched;
-      this.reduceSchedulable = reduceSched;
-      this.lastMapLocalityLevel = LocalityLevel.NODE;
-    }
-  }
-  
-  public FairScheduler() {
-    this(new Clock(), false);
-  }
-  
-  /**
-   * Constructor used for tests, which can change the clock and disable updates.
-   */
-  protected FairScheduler(Clock clock, boolean mockMode) {
-    this.clock = clock;
-    this.mockMode = mockMode;
-    this.jobListener = new JobListener();
-  }
-
-  @Override
-  public void start() {
-    try {
-      Configuration conf = getConf();
-      // Create scheduling log and initialize it if it is enabled
-      eventLog = new FairSchedulerEventLog();
-      boolean logEnabled = conf.getBoolean(
-          "mapred.fairscheduler.eventlog.enabled", false);
-      if (!mockMode && logEnabled) {
-        String hostname = "localhost";
-        if (taskTrackerManager instanceof JobTracker) {
-          hostname = ((JobTracker) taskTrackerManager).getJobTrackerMachine();
-        }
-        eventLog.init(conf, hostname);
-      }
-      // Initialize other pieces of the scheduler
-      jobInitializer = new JobInitializer(conf, taskTrackerManager);
-      taskTrackerManager.addJobInProgressListener(jobListener);
-      poolMgr = new PoolManager(this);
-      poolMgr.initialize();
-      loadMgr = (LoadManager) ReflectionUtils.newInstance(
-          conf.getClass("mapred.fairscheduler.loadmanager", 
-              CapBasedLoadManager.class, LoadManager.class), conf);
-      loadMgr.setTaskTrackerManager(taskTrackerManager);
-      loadMgr.setEventLog(eventLog);
-      loadMgr.start();
-      taskSelector = (TaskSelector) ReflectionUtils.newInstance(
-          conf.getClass("mapred.fairscheduler.taskselector", 
-              DefaultTaskSelector.class, TaskSelector.class), conf);
-      taskSelector.setTaskTrackerManager(taskTrackerManager);
-      taskSelector.start();
-      Class<?> weightAdjClass = conf.getClass(
-          "mapred.fairscheduler.weightadjuster", null);
-      if (weightAdjClass != null) {
-        weightAdjuster = (WeightAdjuster) ReflectionUtils.newInstance(
-            weightAdjClass, conf);
-      }
-      updateInterval = conf.getLong(
-          "mapred.fairscheduler.update.interval", 500);
-      dumpInterval = conf.getLong(
-          "mapred.fairscheduler.dump.interval", 10000);
-      preemptionInterval = conf.getLong(
-          "mapred.fairscheduler.preemption.interval", 15000);
-      assignMultiple = conf.getBoolean(
-          "mapred.fairscheduler.assignmultiple", true);
-      mapAssignCap = conf.getInt(
-          "mapred.fairscheduler.assignmultiple.maps", -1);
-      reduceAssignCap = conf.getInt(
-          "mapred.fairscheduler.assignmultiple.reduces", -1);
-      sizeBasedWeight = conf.getBoolean(
-          "mapred.fairscheduler.sizebasedweight", false);
-      preemptionEnabled = conf.getBoolean(
-          "mapred.fairscheduler.preemption", false);
-      onlyLogPreemption = conf.getBoolean(
-          "mapred.fairscheduler.preemption.only.log", false);
-      long defaultDelay = conf.getLong(
-          "mapred.fairscheduler.locality.delay", -1);
-      nodeLocalityDelay = conf.getLong(
-          "mapred.fairscheduler.locality.delay.node", defaultDelay);
-      rackLocalityDelay = conf.getLong(
-          "mapred.fairscheduler.locality.delay.rack", defaultDelay);
-      if (defaultDelay == -1 && 
-          (nodeLocalityDelay == -1 || rackLocalityDelay == -1)) {
-        autoComputeLocalityDelay = true; // Compute from heartbeat interval
-      }
-      initialized = true;
-      running = true;
-      lastUpdateTime = clock.getTime();
-      // Start a thread to update deficits every UPDATE_INTERVAL
-      if (!mockMode) {
-        new UpdateThread().start();
-      }
-      // Register servlet with JobTracker's Jetty server
-      if (taskTrackerManager instanceof JobTracker) {
-        JobTracker jobTracker = (JobTracker) taskTrackerManager;
-        HttpServer infoServer = jobTracker.infoServer;
-        infoServer.setAttribute("scheduler", this);
-        infoServer.addServlet("scheduler", "/scheduler",
-            FairSchedulerServlet.class);
-      }
-      
-      initMetrics();
-      
-      eventLog.log("INITIALIZED");
-    } catch (Exception e) {
-      // Can't load one of the managers - crash the JobTracker now while it is
-      // starting up so that the user notices.
-      throw new RuntimeException("Failed to start FairScheduler", e);
-    }
-    LOG.info("Successfully configured FairScheduler");
-  }
-
-  private MetricsUpdater metricsUpdater; // responsible for pushing hadoop metrics
-
-  /**
-   * Returns the LoadManager object used by the Fair Share scheduler
-   */
-  LoadManager getLoadManager() {
-    return loadMgr;
-  }
-
-  /**
-   * Register metrics for the fair scheduler, and start a thread
-   * to update them periodically.
-   */
-  private void initMetrics() {
-    MetricsContext context = MetricsUtil.getContext("fairscheduler");
-    metricsUpdater = new MetricsUpdater();
-    context.registerUpdater(metricsUpdater);
-  }
-
-  @Override
-  public void terminate() throws IOException {
-    if (eventLog != null)
-      eventLog.log("SHUTDOWN");
-    running = false;
-    jobInitializer.terminate();
-    if (jobListener != null)
-      taskTrackerManager.removeJobInProgressListener(jobListener);
-    if (eventLog != null)
-      eventLog.shutdown();
-    if (metricsUpdater != null) {
-      MetricsContext context = MetricsUtil.getContext("fairscheduler");
-      context.unregisterUpdater(metricsUpdater);
-      metricsUpdater = null;
-    }
-  }
- 
-
-  private class JobInitializer {
-    private final int DEFAULT_NUM_THREADS = 1;
-    private ExecutorService threadPool;
-    private TaskTrackerManager ttm;
-    public JobInitializer(Configuration conf, TaskTrackerManager ttm) {
-      int numThreads = conf.getInt("mapred.jobinit.threads",
-          DEFAULT_NUM_THREADS);
-      threadPool = Executors.newFixedThreadPool(numThreads);
-      this.ttm = ttm;
-    }
-    public void initJob(JobInfo jobInfo, JobInProgress job) {
-      if (!mockMode) {
-        threadPool.execute(new InitJob(jobInfo, job));
-      } else {
-        new InitJob(jobInfo, job).run();
-      }
-    }
-    class InitJob implements Runnable {
-      private JobInfo jobInfo;
-      private JobInProgress job;
-      public InitJob(JobInfo jobInfo, JobInProgress job) {
-        this.jobInfo = jobInfo;
-        this.job = job;
-      }
-      public void run() {
-        ttm.initJob(job);
-      }
-    }
-    void terminate() {
-      LOG.info("Shutting down thread pool");
-      threadPool.shutdownNow();
-      try {
-        threadPool.awaitTermination(1, TimeUnit.MINUTES);
-      } catch (InterruptedException e) {
-        // Ignore, we are in shutdown anyway.
-      }
-    }
-  }
-
-/**
-   * Used to listen for jobs added/removed by our {@link TaskTrackerManager}.
-   */
-  private class JobListener extends JobInProgressListener {
-    @Override
-    public void jobAdded(JobInProgress job) {
-      synchronized (FairScheduler.this) {
-        eventLog.log("JOB_ADDED", job.getJobID());
-        JobInfo info = new JobInfo(new JobSchedulable(FairScheduler.this, job, TaskType.MAP),
-            new JobSchedulable(FairScheduler.this, job, TaskType.REDUCE));
-        infos.put(job, info);
-        poolMgr.addJob(job); // Also adds job into the right PoolScheduable
-        update();
-      }
-    }
-    
-    @Override
-    public void jobRemoved(JobInProgress job) {
-      synchronized (FairScheduler.this) {
-        eventLog.log("JOB_REMOVED", job.getJobID());
-        jobNoLongerRunning(job);
-      }
-    }
-  
-    @Override
-    public void jobUpdated(JobChangeEvent event) {
-      eventLog.log("JOB_UPDATED", event.getJobInProgress().getJobID());
-    }
-  }
-
-  /**
-   * A thread which calls {@link FairScheduler#update()} ever
-   * <code>UPDATE_INTERVAL</code> milliseconds.
-   */
-  private class UpdateThread extends Thread {
-    private UpdateThread() {
-      super("FairScheduler update thread");
-    }
-
-    public void run() {
-      while (running) {
-        try {
-          Thread.sleep(updateInterval);
-          update();
-          dumpIfNecessary();
-          preemptTasksIfNecessary();
-        } catch (Exception e) {
-          LOG.error("Exception in fair scheduler UpdateThread", e);
-        }
-      }
-    }
-  }
-
-  /**
-   * Responsible for updating metrics when the metrics context requests it.
-   */
-  private class MetricsUpdater implements Updater {
-    @Override
-    public void doUpdates(MetricsContext context) {
-      updateMetrics();
-    }    
-  }
-  
-  synchronized void updateMetrics() {
-    poolMgr.updateMetrics();
-  }
-  
-  @Override
-  public synchronized List<Task> assignTasks(TaskTracker tracker)
-      throws IOException {
-    if (!initialized) // Don't try to assign tasks if we haven't yet started up
-      return null;
-    String trackerName = tracker.getTrackerName();
-    eventLog.log("HEARTBEAT", trackerName);
-    long currentTime = clock.getTime();
-    
-    // Compute total runnable maps and reduces, and currently running ones
-    int runnableMaps = 0;
-    int runningMaps = 0;
-    int runnableReduces = 0;
-    int runningReduces = 0;
-    for (Pool pool: poolMgr.getPools()) {
-      runnableMaps += pool.getMapSchedulable().getDemand();
-      runningMaps += pool.getMapSchedulable().getRunningTasks();
-      runnableReduces += pool.getReduceSchedulable().getDemand();
-      runningReduces += pool.getReduceSchedulable().getRunningTasks();
-    }
-
-    ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
-    // Compute total map/reduce slots
-    // In the future we can precompute this if the Scheduler becomes a 
-    // listener of tracker join/leave events.
-    int totalMapSlots = getTotalSlots(TaskType.MAP, clusterStatus);
-    int totalReduceSlots = getTotalSlots(TaskType.REDUCE, clusterStatus);
-    
-    eventLog.log("RUNNABLE_TASKS", 
-        runnableMaps, runningMaps, runnableReduces, runningReduces);
-
-    // Update time waited for local maps for jobs skipped on last heartbeat
-    updateLocalityWaitTimes(currentTime);
-    
-    TaskTrackerStatus tts = tracker.getStatus();
-
-    int mapsAssigned = 0; // loop counter for map in the below while loop
-    int reducesAssigned = 0; // loop counter for reduce in the below while
-    int mapCapacity = maxTasksToAssign(TaskType.MAP, tts);
-    int reduceCapacity = maxTasksToAssign(TaskType.REDUCE, tts);
-    boolean mapRejected = false; // flag used for ending the loop
-    boolean reduceRejected = false; // flag used for ending the loop
-
-    // Keep track of which jobs were visited for map tasks and which had tasks
-    // launched, so that we can later mark skipped jobs for delay scheduling
-    Set<JobInProgress> visitedForMap = new HashSet<JobInProgress>();
-    Set<JobInProgress> visitedForReduce = new HashSet<JobInProgress>();
-    Set<JobInProgress> launchedMap = new HashSet<JobInProgress>();
-
-    ArrayList<Task> tasks = new ArrayList<Task>();
-    // Scan jobs to assign tasks until neither maps nor reduces can be assigned
-    while (true) {
-      // Computing the ending conditions for the loop
-      // Reject a task type if one of the following condition happens
-      // 1. number of assigned task reaches per heatbeat limit
-      // 2. number of running tasks reaches runnable tasks
-      // 3. task is rejected by the LoadManager.canAssign
-      if (!mapRejected) {
-        if (mapsAssigned == mapCapacity ||
-            runningMaps == runnableMaps ||
-            !loadMgr.canAssignMap(tts, runnableMaps, totalMapSlots)) {
-          eventLog.log("INFO", "Can't assign another MAP to " + trackerName);
-          mapRejected = true;
-        }
-      }
-      if (!reduceRejected) {
-        if (reducesAssigned == reduceCapacity ||
-            runningReduces == runnableReduces ||
-            !loadMgr.canAssignReduce(tts, runnableReduces, totalReduceSlots)) {
-          eventLog.log("INFO", "Can't assign another REDUCE to " + trackerName);
-          reduceRejected = true;
-        }
-      }
-      // Exit while (true) loop if
-      // 1. neither maps nor reduces can be assigned
-      // 2. assignMultiple is off and we already assigned one task
-      if (mapRejected && reduceRejected ||
-          !assignMultiple && tasks.size() > 0) {
-        break; // This is the only exit of the while (true) loop
-      }
-
-      // Determine which task type to assign this time
-      // First try choosing a task type which is not rejected
-      TaskType taskType;
-      if (mapRejected) {
-        taskType = TaskType.REDUCE;
-      } else if (reduceRejected) {
-        taskType = TaskType.MAP;
-      } else {
-        // If both types are available, choose the task type with fewer running
-        // tasks on the task tracker to prevent that task type from starving
-        if (tts.countMapTasks() <= tts.countReduceTasks()) {
-          taskType = TaskType.MAP;
-        } else {
-          taskType = TaskType.REDUCE;
-        }
-      }
-
-      // Get the map or reduce schedulables and sort them by fair sharing
-      List<PoolSchedulable> scheds = getPoolSchedulables(taskType);
-      Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator());
-      boolean foundTask = false;
-      for (Schedulable sched: scheds) { // This loop will assign only one task
-        eventLog.log("INFO", "Checking for " + taskType +
-            " task in " + sched.getName());
-        Task task = taskType == TaskType.MAP ? 
-                    sched.assignTask(tts, currentTime, visitedForMap) : 
-                    sched.assignTask(tts, currentTime, visitedForReduce);
-        if (task != null) {
-          foundTask = true;
-          JobInProgress job = taskTrackerManager.getJob(task.getJobID());
-          eventLog.log("ASSIGN", trackerName, taskType,
-              job.getJobID(), task.getTaskID());
-          // Update running task counts, and the job's locality level
-          if (taskType == TaskType.MAP) {
-            launchedMap.add(job);
-            mapsAssigned++;
-            runningMaps++;
-            updateLastMapLocalityLevel(job, task, tts);
-          } else {
-            reducesAssigned++;
-            runningReduces++;
-          }
-          // Add task to the list of assignments
-          tasks.add(task);
-          break; // This break makes this loop assign only one task
-        } // end if(task != null)
-      } // end for(Schedulable sched: scheds)
-
-      // Reject the task type if we cannot find a task
-      if (!foundTask) {
-        if (taskType == TaskType.MAP) {
-          mapRejected = true;
-        } else {
-          reduceRejected = true;
-        }
-      }
-    } // end while (true)
-
-    // Mark any jobs that were visited for map tasks but did not launch a task
-    // as skipped on this heartbeat
-    for (JobInProgress job: visitedForMap) {
-      if (!launchedMap.contains(job)) {
-        infos.get(job).skippedAtLastHeartbeat = true;
-      }
-    }
-    
-    // If no tasks were found, return null
-    return tasks.isEmpty() ? null : tasks;
-  }
-
-  /**
-   * Get maximum number of tasks to assign on a TaskTracker on a heartbeat.
-   * The scheduler may launch fewer than this many tasks if the LoadManager
-   * says not to launch more, but it will never launch more than this number.
-   */
-  private int maxTasksToAssign(TaskType type, TaskTrackerStatus tts) {
-    if (!assignMultiple)
-      return 1;
-    int cap = (type == TaskType.MAP) ? mapAssignCap : reduceAssignCap;
-    if (cap == -1) // Infinite cap; use the TaskTracker's slot count
-      return (type == TaskType.MAP) ?
-          tts.getAvailableMapSlots(): tts.getAvailableReduceSlots();
-    else
-      return cap;
-  }
-
-  /**
-   * Update locality wait times for jobs that were skipped at last heartbeat.
-   */
-  private void updateLocalityWaitTimes(long currentTime) {
-    long timeSinceLastHeartbeat = 
-      (lastHeartbeatTime == 0 ? 0 : currentTime - lastHeartbeatTime);
-    lastHeartbeatTime = currentTime;
-    for (JobInfo info: infos.values()) {
-      if (info.skippedAtLastHeartbeat) {
-        info.timeWaitedForLocalMap += timeSinceLastHeartbeat;
-        info.skippedAtLastHeartbeat = false;
-      }
-    }
-  }
-
-  /**
-   * Update a job's locality level and locality wait variables given that that 
-   * it has just launched a map task on a given task tracker.
-   */
-  private void updateLastMapLocalityLevel(JobInProgress job,
-      Task mapTaskLaunched, TaskTrackerStatus tracker) {
-    JobInfo info = infos.get(job);
-    LocalityLevel localityLevel = LocalityLevel.fromTask(
-        job, mapTaskLaunched, tracker);
-    info.lastMapLocalityLevel = localityLevel;
-    info.timeWaitedForLocalMap = 0;
-    eventLog.log("ASSIGNED_LOC_LEVEL", job.getJobID(), localityLevel);
-  }
-
-  /**
-   * Get the maximum locality level at which a given job is allowed to
-   * launch tasks, based on how long it has been waiting for local tasks.
-   * This is used to implement the "delay scheduling" feature of the Fair
-   * Scheduler for optimizing data locality.
-   * If the job has no locality information (e.g. it does not use HDFS), this 
-   * method returns LocalityLevel.ANY, allowing tasks at any level.
-   * Otherwise, the job can only launch tasks at its current locality level
-   * or lower, unless it has waited at least nodeLocalityDelay or
-   * rackLocalityDelay milliseconds depends on the current level. If it
-   * has waited (nodeLocalityDelay + rackLocalityDelay) milliseconds,
-   * it can go to any level.
-   */
-  protected LocalityLevel getAllowedLocalityLevel(JobInProgress job,
-      long currentTime) {
-    JobInfo info = infos.get(job);
-    if (info == null) { // Job not in infos (shouldn't happen)
-      LOG.error("getAllowedLocalityLevel called on job " + job
-          + ", which does not have a JobInfo in infos");
-      return LocalityLevel.ANY;
-    }
-    if (job.nonLocalMaps.size() > 0) { // Job doesn't have locality information
-      return LocalityLevel.ANY;
-    }
-    // Don't wait for locality if the job's pool is starving for maps
-    Pool pool = poolMgr.getPool(job);
-    PoolSchedulable sched = pool.getMapSchedulable();
-    long minShareTimeout = poolMgr.getMinSharePreemptionTimeout(pool.getName());
-    long fairShareTimeout = poolMgr.getFairSharePreemptionTimeout();
-    if (currentTime - sched.getLastTimeAtMinShare() > minShareTimeout ||
-        currentTime - sched.getLastTimeAtHalfFairShare() > fairShareTimeout) {
-      eventLog.log("INFO", "No delay scheduling for "
-          + job.getJobID() + " because it is being starved");
-      return LocalityLevel.ANY;
-    }
-    // In the common case, compute locality level based on time waited
-    switch(info.lastMapLocalityLevel) {
-    case NODE: // Last task launched was node-local
-      if (info.timeWaitedForLocalMap >=
-          nodeLocalityDelay + rackLocalityDelay)
-        return LocalityLevel.ANY;
-      else if (info.timeWaitedForLocalMap >= nodeLocalityDelay)
-        return LocalityLevel.RACK;
-      else
-        return LocalityLevel.NODE;
-    case RACK: // Last task launched was rack-local
-      if (info.timeWaitedForLocalMap >= rackLocalityDelay)
-        return LocalityLevel.ANY;
-      else
-        return LocalityLevel.RACK;
-    default: // Last task was non-local; can launch anywhere
-      return LocalityLevel.ANY;
-    }
-  }
-  
-  /**
-   * Recompute the internal variables used by the scheduler - per-job weights,
-   * fair shares, deficits, minimum slot allocations, and numbers of running
-   * and needed tasks of each type. 
-   */
-  protected void update() {
-    // Making more granular locking so that clusterStatus can be fetched 
-    // from Jobtracker without locking the scheduler.
-    ClusterStatus clusterStatus = taskTrackerManager.getClusterStatus();
-    
-    // Recompute locality delay from JobTracker heartbeat interval if enabled.
-    // This will also lock the JT, so do it outside of a fair scheduler lock.
-    if (autoComputeLocalityDelay) {
-      JobTracker jobTracker = (JobTracker) taskTrackerManager;
-      nodeLocalityDelay = Math.min(MAX_AUTOCOMPUTED_LOCALITY_DELAY,
-          (long) (1.5 * jobTracker.getNextHeartbeatInterval()));
-      rackLocalityDelay = nodeLocalityDelay;
-    }
-    
-    // Got clusterStatus hence acquiring scheduler lock now.
-    synchronized (this) {
-      // Reload allocations file if it hasn't been loaded in a while
-      poolMgr.reloadAllocsIfNecessary();
-      
-      // Remove any jobs that have stopped running
-      List<JobInProgress> toRemove = new ArrayList<JobInProgress>();
-      for (JobInProgress job: infos.keySet()) { 
-        int runState = job.getStatus().getRunState();
-        if (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED
-          || runState == JobStatus.KILLED) {
-            toRemove.add(job);
-        }
-      }
-      for (JobInProgress job: toRemove) {
-        jobNoLongerRunning(job);
-      }
-      
-      updateRunnability(); // Set job runnability based on user/pool limits 
-      
-      // Update demands of jobs and pools
-      for (Pool pool: poolMgr.getPools()) {
-        pool.getMapSchedulable().updateDemand();
-        pool.getReduceSchedulable().updateDemand();
-      }
-      
-      // Compute fair shares based on updated demands
-      List<PoolSchedulable> mapScheds = getPoolSchedulables(TaskType.MAP);
-      List<PoolSchedulable> reduceScheds = getPoolSchedulables(TaskType.REDUCE);
-      SchedulingAlgorithms.computeFairShares(
-          mapScheds, clusterStatus.getMaxMapTasks());
-      SchedulingAlgorithms.computeFairShares(
-          reduceScheds, clusterStatus.getMaxReduceTasks());
-      
-      // Use the computed shares to assign shares within each pool
-      for (Pool pool: poolMgr.getPools()) {
-        pool.getMapSchedulable().redistributeShare();
-        pool.getReduceSchedulable().redistributeShare();
-      }
-      
-      if (preemptionEnabled)
-        updatePreemptionVariables();
-    }
-  }
-
-  private void jobNoLongerRunning(JobInProgress job) {
-    assert Thread.holdsLock(this);
-    JobInfo info = infos.remove(job);
-    if (info != null) {
-      info.mapSchedulable.cleanupMetrics();
-      info.reduceSchedulable.cleanupMetrics();
-    }
-    poolMgr.removeJob(job);
-  }
-  
-  public List<PoolSchedulable> getPoolSchedulables(TaskType type) {
-    List<PoolSchedulable> scheds = new ArrayList<PoolSchedulable>();
-    for (Pool pool: poolMgr.getPools()) {
-      scheds.add(pool.getSchedulable(type));
-    }
-    return scheds;
-  }
-  
-  private void updateRunnability() {
-    // Start by marking everything as not runnable
-    for (JobInfo info: infos.values()) {
-      info.runnable = false;
-    }
-    // Create a list of sorted jobs in order of start time and priority
-    List<JobInProgress> jobs = new ArrayList<JobInProgress>(infos.keySet());
-    Collections.sort(jobs, new FifoJobComparator());
-    // Mark jobs as runnable in order of start time and priority, until
-    // user or pool limits have been reached.
-    Map<String, Integer> userJobs = new HashMap<String, Integer>();
-    Map<String, Integer> poolJobs = new HashMap<String, Integer>();
-    for (JobInProgress job: jobs) {
-      String user = job.getJobConf().getUser();
-      String pool = poolMgr.getPoolName(job);
-      int userCount = userJobs.containsKey(user) ? userJobs.get(user) : 0;
-      int poolCount = poolJobs.containsKey(pool) ? poolJobs.get(pool) : 0;
-      if (userCount < poolMgr.getUserMaxJobs(user) &&
-          poolCount < poolMgr.getPoolMaxJobs(pool)) {
-        if (job.getStatus().getRunState() == JobStatus.RUNNING ||
-            job.getStatus().getRunState() == JobStatus.PREP) {
-          userJobs.put(user, userCount + 1);
-          poolJobs.put(pool, poolCount + 1);
-          JobInfo jobInfo = infos.get(job);
-          if (job.getStatus().getRunState() == JobStatus.RUNNING) {
-            jobInfo.runnable = true;
-          } else {
-            // The job is in the PREP state. Give it to the job initializer
-            // for initialization if we have not already done it.
-            if (jobInfo.needsInitializing) {
-              jobInfo.needsInitializing = false;
-              jobInitializer.initJob(jobInfo, job);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  public double getJobWeight(JobInProgress job, TaskType taskType) {
-    if (!isRunnable(job)) {
-      // Job won't launch tasks, but don't return 0 to avoid division errors
-      return 1.0;
-    } else {
-      double weight = 1.0;
-      if (sizeBasedWeight) {
-        // Set weight based on runnable tasks
-        JobInfo info = infos.get(job);
-        int runnableTasks = (taskType == TaskType.MAP) ?
-            info.mapSchedulable.getDemand() : 
-            info.reduceSchedulable.getDemand();
-        weight = Math.log1p(runnableTasks) / Math.log(2);
-      }
-      weight *= getPriorityFactor(job.getPriority());
-      if (weightAdjuster != null) {
-        // Run weight through the user-supplied weightAdjuster
-        weight = weightAdjuster.adjustWeight(job, taskType, weight);
-      }
-      return weight;
-    }
-  }
-
-  private double getPriorityFactor(JobPriority priority) {
-    switch (priority) {
-    case VERY_HIGH: return 4.0;
-    case HIGH:      return 2.0;
-    case NORMAL:    return 1.0;
-    case LOW:       return 0.5;
-    default:        return 0.25; // priority = VERY_LOW
-    }
-  }
-  
-  public PoolManager getPoolManager() {
-    return poolMgr;
-  }
-
-  private int getTotalSlots(TaskType type, ClusterStatus clusterStatus) {
-    return (type == TaskType.MAP ?
-      clusterStatus.getMaxMapTasks() : clusterStatus.getMaxReduceTasks());
-  }
-
-  /**
-   * Update the preemption fields for all PoolScheduables, i.e. the times since
-   * each pool last was at its guaranteed share and at > 1/2 of its fair share
-   * for each type of task.
-   */
-  private void updatePreemptionVariables() {
-    long now = clock.getTime();
-    lastPreemptionUpdateTime = now;
-    for (TaskType type: MAP_AND_REDUCE) {
-      for (PoolSchedulable sched: getPoolSchedulables(type)) {
-        if (!isStarvedForMinShare(sched)) {
-          sched.setLastTimeAtMinShare(now);
-        }
-        if (!isStarvedForFairShare(sched)) {
-          sched.setLastTimeAtHalfFairShare(now);
-        }
-        eventLog.log("PREEMPT_VARS", sched.getName(), type,
-            now - sched.getLastTimeAtMinShare(),
-            now - sched.getLastTimeAtHalfFairShare());
-      }
-    }
-  }
-
-  /**
-   * Is a pool below its min share for the given task type?
-   */
-  boolean isStarvedForMinShare(PoolSchedulable sched) {
-    int desiredShare = Math.min(sched.getMinShare(), sched.getDemand());
-    return (sched.getRunningTasks() < desiredShare);
-  }
-  
-  /**
-   * Is a pool being starved for fair share for the given task type?
-   * This is defined as being below half its fair share.
-   */
-  boolean isStarvedForFairShare(PoolSchedulable sched) {
-    int desiredFairShare = (int) Math.floor(Math.min(
-        sched.getFairShare() / 2, sched.getDemand()));
-    return (sched.getRunningTasks() < desiredFairShare);
-  }
-
-  /**
-   * Check for pools that need tasks preempted, either because they have been
-   * below their guaranteed share for minSharePreemptionTimeout or they
-   * have been below half their fair share for the fairSharePreemptionTimeout.
-   * If such pools exist, compute how many tasks of each type need to be
-   * preempted and then select the right ones using preemptTasks.
-   * 
-   * This method computes and logs the number of tasks we want to preempt even
-   * if preemption is disabled, for debugging purposes.
-   */
-  protected void preemptTasksIfNecessary() {
-    if (!preemptionEnabled)
-      return;
-    
-    long curTime = clock.getTime();
-    if (curTime - lastPreemptCheckTime < preemptionInterval)
-      return;
-    lastPreemptCheckTime = curTime;
-    
-    // Acquire locks on both the JobTracker (task tracker manager) and this
-    // because we might need to call some JobTracker methods (killTask).
-    synchronized (taskTrackerManager) {
-      synchronized (this) {
-        for (TaskType type: MAP_AND_REDUCE) {
-          List<PoolSchedulable> scheds = getPoolSchedulables(type);
-          int tasksToPreempt = 0;
-          for (PoolSchedulable sched: scheds) {
-            tasksToPreempt += tasksToPreempt(sched, curTime);
-          }
-          if (tasksToPreempt > 0) {
-            eventLog.log("SHOULD_PREEMPT", type, tasksToPreempt);
-            if (!onlyLogPreemption) {
-              preemptTasks(scheds, tasksToPreempt);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Preempt a given number of tasks from a list of PoolSchedulables. 
-   * The policy for this is to pick tasks from pools that are over their fair 
-   * share, but make sure that no pool is placed below its fair share in the 
-   * process. Furthermore, we want to minimize the amount of computation
-   * wasted by preemption, so out of the tasks in over-scheduled pools, we
-   * prefer to preempt tasks that started most recently.
-   */
-  private void preemptTasks(List<PoolSchedulable> scheds, int tasksToPreempt) {
-    if (scheds.isEmpty() || tasksToPreempt == 0)
-      return;
-    
-    TaskType taskType = scheds.get(0).getTaskType();
-    
-    // Collect running tasks of our type from over-scheduled pools
-    List<TaskStatus> runningTasks = new ArrayList<TaskStatus>();
-    for (PoolSchedulable sched: scheds) {
-      if (sched.getRunningTasks() > sched.getFairShare())
-      for (JobSchedulable js: sched.getJobSchedulables()) {
-        runningTasks.addAll(getRunningTasks(js.getJob(), taskType));
-      }
-    }
-    
-    // Sort tasks into reverse order of start time
-    Collections.sort(runningTasks, new Comparator<TaskStatus>() {
-      public int compare(TaskStatus t1, TaskStatus t2) {
-        if (t1.getStartTime() < t2.getStartTime())
-          return 1;
-        else if (t1.getStartTime() == t2.getStartTime())
-          return 0;
-        else
-          return -1;
-      }
-    });
-    
-    // Maintain a count of tasks left in each pool; this is a bit
-    // faster than calling runningTasks() on the pool repeatedly
-    // because the latter must scan through jobs in the pool
-    HashMap<Pool, Integer> tasksLeft = new HashMap<Pool, Integer>(); 
-    for (Pool p: poolMgr.getPools()) {
-      tasksLeft.put(p, p.getSchedulable(taskType).getRunningTasks());
-    }
-    
-    // Scan down the sorted list of task statuses until we've killed enough
-    // tasks, making sure we don't kill too many from any pool
-    for (TaskStatus status: runningTasks) {
-      JobID jobID = status.getTaskID().getJobID();
-      JobInProgress job = taskTrackerManager.getJob(jobID);
-      Pool pool = poolMgr.getPool(job);
-      PoolSchedulable sched = pool.getSchedulable(taskType);
-      int tasksLeftForPool = tasksLeft.get(pool);
-      if (tasksLeftForPool > sched.getFairShare()) {
-        eventLog.log("PREEMPT", status.getTaskID(),
-            status.getTaskTracker());
-        try {
-          taskTrackerManager.killTask(status.getTaskID(), false);
-          tasksToPreempt--;
-          if (tasksToPreempt == 0)
-            break;
-          
-          // reduce tasks left for pool
-          tasksLeft.put(pool, --tasksLeftForPool);
-        } catch (IOException e) {
-          LOG.error("Failed to kill task " + status.getTaskID(), e);
-        }
-      }
-    }
-  }
-
-  /**
-   * Count how many tasks of a given type the pool needs to preempt, if any.
-   * If the pool has been below its min share for at least its preemption
-   * timeout, it should preempt the difference between its current share and
-   * this min share. If it has been below half its fair share for at least the
-   * fairSharePreemptionTimeout, it should preempt enough tasks to get up to
-   * its full fair share. If both conditions hold, we preempt the max of the
-   * two amounts (this shouldn't happen unless someone sets the timeouts to
-   * be identical for some reason).
-   */
-  protected int tasksToPreempt(PoolSchedulable sched, long curTime) {
-    String pool = sched.getName();
-    long minShareTimeout = poolMgr.getMinSharePreemptionTimeout(pool);
-    long fairShareTimeout = poolMgr.getFairSharePreemptionTimeout();
-    int tasksDueToMinShare = 0;
-    int tasksDueToFairShare = 0;
-    if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
-      int target = Math.min(sched.getMinShare(), sched.getDemand());
-      tasksDueToMinShare = Math.max(0, target - sched.getRunningTasks());
-    }
-    if (curTime - sched.getLastTimeAtHalfFairShare() > fairShareTimeout) {
-      int target = (int) Math.min(sched.getFairShare(), sched.getDemand());
-      tasksDueToFairShare = Math.max(0, target - sched.getRunningTasks());
-    }
-    int tasksToPreempt = Math.max(tasksDueToMinShare, tasksDueToFairShare);
-    if (tasksToPreempt > 0) {
-      String message = "Should preempt " + tasksToPreempt + " " 
-          + sched.getTaskType() + " tasks for pool " + sched.getName() 
-          + ": tasksDueToMinShare = " + tasksDueToMinShare
-          + ", tasksDueToFairShare = " + tasksDueToFairShare;
-      eventLog.log("INFO", message);
-      LOG.info(message);
-    }
-    return tasksToPreempt;
-  }
-
-  private List<TaskStatus> getRunningTasks(JobInProgress job, TaskType type) {
-    // Create a list of all running TaskInProgress'es in the job
-    Set<TaskInProgress> tips = new HashSet<TaskInProgress>();
-    if (type == TaskType.MAP) {
-      // Jobs may have both "non-local maps" which have a split with no locality
-      // info (e.g. the input file is not in HDFS), and maps with locality info,
-      // which are stored in the runningMapCache map from location to task list
-      tips.addAll(job.nonLocalRunningMaps);
-      for (Set<TaskInProgress> set: job.runningMapCache.values()) {
-        tips.addAll(set);
-      }
-    }
-    else {
-      tips.addAll(job.runningReduces);
-    }
-    // Get the active TaskStatus'es for each TaskInProgress (there may be
-    // more than one if the task has multiple copies active due to speculation)
-    List<TaskStatus> statuses = new ArrayList<TaskStatus>();
-    for (TaskInProgress tip: tips) {
-      for (TaskAttemptID id: tip.getActiveTasks().keySet()) {
-        TaskStatus stat = tip.getTaskStatus(id);
-        // status is null when the task has been scheduled but not yet running
-        if (stat != null) {
-          statuses.add(stat);
-        }
-      }
-    }
-    return statuses;
-  }
-
-  protected boolean isRunnable(JobInProgress job) {
-    JobInfo info = infos.get(job);
-    if (info == null) return false;
-    return info.runnable;
-  }
-
-  @Override
-  public synchronized Collection<JobInProgress> getJobs(String queueName) {
-    Pool myJobPool = poolMgr.getPool(queueName);
-    return myJobPool.getJobs();
-  }
-
-  protected void dumpIfNecessary() {
-    long now = clock.getTime();
-    long timeDelta = now - lastDumpTime;
-    if (timeDelta > dumpInterval && eventLog.isEnabled()) {
-      dump();
-      lastDumpTime = now;
-    }
-  }
-
-  /**
-   * Dump scheduler state to the fairscheduler log.
-   */
-  private synchronized void dump() {
-    synchronized (eventLog) {
-      eventLog.log("BEGIN_DUMP");
-      // List jobs in order of submit time
-      ArrayList<JobInProgress> jobs = 
-        new ArrayList<JobInProgress>(infos.keySet());
-      Collections.sort(jobs, new Comparator<JobInProgress>() {
-        public int compare(JobInProgress j1, JobInProgress j2) {
-          return (int) Math.signum(j1.getStartTime() - j2.getStartTime());
-        }
-      });
-      // Dump info for each job
-      for (JobInProgress job: jobs) {
-        JobProfile profile = job.getProfile();
-        JobInfo info = infos.get(job);
-        Schedulable ms = info.mapSchedulable;
-        Schedulable rs = info.reduceSchedulable;
-        eventLog.log("JOB",
-            profile.getJobID(), profile.name, profile.user,
-            job.getPriority(), poolMgr.getPoolName(job),
-            job.numMapTasks, ms.getRunningTasks(),
-            ms.getDemand(), ms.getFairShare(), ms.getWeight(),
-            job.numReduceTasks, rs.getRunningTasks(),
-            rs.getDemand(), rs.getFairShare(), rs.getWeight());
-      }
-      // List pools in alphabetical order
-      List<Pool> pools = new ArrayList<Pool>(poolMgr.getPools());
-      Collections.sort(pools, new Comparator<Pool>() {
-        public int compare(Pool p1, Pool p2) {
-          if (p1.isDefaultPool())
-            return 1;
-          else if (p2.isDefaultPool())
-            return -1;
-          else return p1.getName().compareTo(p2.getName());
-        }});
-      for (Pool pool: pools) {
-        int runningMaps = 0;
-        int runningReduces = 0;
-        for (JobInProgress job: pool.getJobs()) {
-          JobInfo info = infos.get(job);
-          if (info != null) {
-            // TODO: Fix
-            //runningMaps += info.runningMaps;
-            //runningReduces += info.runningReduces;
-          }
-        }
-        String name = pool.getName();
-        eventLog.log("POOL",
-            name, poolMgr.getPoolWeight(name), pool.getJobs().size(),
-            poolMgr.getAllocation(name, TaskType.MAP), runningMaps,
-            poolMgr.getAllocation(name, TaskType.REDUCE), runningReduces);
-      }
-      // Dump info for each pool
-      eventLog.log("END_DUMP");
-    }
-  }
-
-  public Clock getClock() {
-    return clock;
-  }
-  
-  public FairSchedulerEventLog getEventLog() {
-    return eventLog;
-  }
-
-  public JobInfo getJobInfo(JobInProgress job) {
-    return infos.get(job);
-  }
-  
-  boolean isPreemptionEnabled() {
-    return preemptionEnabled;
-  }
-  long getLastPreemptionUpdateTime() {
-    return lastPreemptionUpdateTime;
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerEventLog.java

@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.DailyRollingFileAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * Event log used by the fair scheduler for machine-readable debug info.
- * This class uses a log4j rolling file appender to write the log, but uses
- * a custom tab-separated event format of the form:
- * <pre>
- * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
- * </pre>
- * Various event types are used by the fair scheduler. The purpose of logging
- * in this format is to enable tools to parse the history log easily and read
- * internal scheduler variables, rather than trying to make the log human
- * readable. The fair scheduler also logs human readable messages in the
- * JobTracker's main log.
- * 
- * Constructing this class creates a disabled log. It must be initialized
- * using {@link FairSchedulerEventLog#init(Configuration, String)} to begin
- * writing to the file.
- */
-class FairSchedulerEventLog {
-  private static final Log LOG = LogFactory.getLog(
-    "org.apache.hadoop.mapred.FairSchedulerEventLog");
-  
-  /** Set to true if logging is disabled due to an error. */
-  private boolean logDisabled = true;
-  
-  /**
-   * Log directory, set by mapred.fairscheduler.eventlog.location in conf file;
-   * defaults to {hadoop.log.dir}/fairscheduler.
-   */
-  private String logDir;
-  
-  /** 
-   * Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.{host}.log.
-   * Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD).
-   */ 
-  private String logFile;
-  
-  /** Log4j appender used to write to the log file */
-  private DailyRollingFileAppender appender;
-
-  boolean init(Configuration conf, String jobtrackerHostname) {
-    try {
-      logDir = conf.get("mapred.fairscheduler.eventlog.location",
-          new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()
-          + File.separator + "fairscheduler");
-      Path logDirPath = new Path(logDir);
-      FileSystem fs = logDirPath.getFileSystem(conf);
-      if (!fs.exists(logDirPath)) {
-        if (!fs.mkdirs(logDirPath)) {
-          throw new IOException(
-              "Mkdirs failed to create " + logDirPath.toString());
-        }
-      }
-      String username = System.getProperty("user.name");
-      logFile = String.format("%s%shadoop-%s-fairscheduler-%s.log",
-          logDir, File.separator, username, jobtrackerHostname);
-      logDisabled = false;
-      PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
-      appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
-      appender.activateOptions();
-      LOG.info("Initialized fair scheduler event log, logging to " + logFile);
-    } catch (IOException e) {
-      LOG.error(
-          "Failed to initialize fair scheduler event log. Disabling it.", e);
-      logDisabled = true;
-    }
-    return !(logDisabled);
-  }
-  
-  /**
-   * Log an event, writing a line in the log file of the form
-   * <pre>
-   * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
-   * </pre>
-   */
-  synchronized void log(String eventType, Object... params) {
-    try {
-      if (logDisabled)
-        return;
-      StringBuffer buffer = new StringBuffer();
-      buffer.append(eventType);
-      for (Object param: params) {
-        buffer.append("\t");
-        buffer.append(param);
-      }
-      String message = buffer.toString();
-      Logger logger = Logger.getLogger(getClass());
-      appender.append(new LoggingEvent("", logger, Level.INFO, message, null));
-    } catch (Exception e) {
-      LOG.error("Failed to append to fair scheduler event log", e);
-      logDisabled = true;
-    }
-  }
-  
-  /**
-   * Flush and close the log.
-   */
-  void shutdown() {
-    try {
-      if (appender != null)
-        appender.close();
-    } catch (Exception e) {}
-    logDisabled = true;
-  }
-
-  boolean isEnabled() {
-    return !logDisabled;
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java

@@ -1,342 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.hadoop.mapred.FairScheduler.JobInfo;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Servlet for displaying fair scheduler information, installed at
- * [job tracker URL]/scheduler when the {@link FairScheduler} is in use.
- * 
- * The main features are viewing each job's task count and fair share,
- * and admin controls to change job priorities and pools from the UI.
- * 
- * There is also an "advanced" view for debugging that can be turned on by
- * going to [job tracker URL]/scheduler?advanced.
- */
-public class FairSchedulerServlet extends HttpServlet {
-  private static final long serialVersionUID = 9104070533067306659L;
-  private static final DateFormat DATE_FORMAT = 
-    new SimpleDateFormat("MMM dd, HH:mm");
-  
-  private FairScheduler scheduler;
-  private JobTracker jobTracker;
-  private static long lastId = 0; // Used to generate unique element IDs
-
-  @Override
-  public void init() throws ServletException {
-    super.init();
-    ServletContext servletContext = this.getServletContext();
-    this.scheduler = (FairScheduler) servletContext.getAttribute("scheduler");
-    this.jobTracker = (JobTracker) scheduler.taskTrackerManager;
-  }
-  
-  @Override
-  protected void doPost(HttpServletRequest req, HttpServletResponse resp)
-      throws ServletException, IOException {
-    doGet(req, resp); // Same handler for both GET and POST
-  }
-  
-  @Override
-  public void doGet(HttpServletRequest request, HttpServletResponse response)
-    throws ServletException, IOException {
-    // If the request has a set* param, handle that and redirect to the regular
-    // view page so that the user won't resubmit the data if they hit refresh.
-    boolean advancedView = request.getParameter("advanced") != null;
-    if (JSPUtil.privateActionsAllowed(jobTracker.conf)
-        && request.getParameter("setPool") != null) {
-      Collection<JobInProgress> runningJobs = getInitedJobs();
-      PoolManager poolMgr = null;
-      synchronized (scheduler) {
-        poolMgr = scheduler.getPoolManager();
-      }
-      String pool = request.getParameter("setPool");
-      String jobId = request.getParameter("jobid");
-      for (JobInProgress job: runningJobs) {
-        if (job.getProfile().getJobID().toString().equals(jobId)) {
-          synchronized(scheduler){
-            poolMgr.setPool(job, pool);
-          }
-          scheduler.update();
-          break;
-        }
-      }      
-      response.sendRedirect("/scheduler" + (advancedView ? "?advanced" : ""));
-      return;
-    }
-    if (JSPUtil.privateActionsAllowed(jobTracker.conf)
-        && request.getParameter("setPriority") != null) {
-      Collection<JobInProgress> runningJobs = getInitedJobs();
-      JobPriority priority = JobPriority.valueOf(request.getParameter(
-          "setPriority"));
-      String jobId = request.getParameter("jobid");
-      for (JobInProgress job: runningJobs) {
-        if (job.getProfile().getJobID().toString().equals(jobId)) {
-          job.setPriority(priority);
-          scheduler.update();
-          break;
-        }
-      }      
-      response.sendRedirect("/scheduler" + (advancedView ? "?advanced" : ""));
-      return;
-    }
-    // Print out the normal response
-    response.setContentType("text/html");
-
-    // Because the client may read arbitrarily slow, and we hold locks while
-    // the servlet outputs, we want to write to our own buffer which we know
-    // won't block.
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintWriter out = new PrintWriter(baos);
-    String hostname = StringUtils.simpleHostname(
-        jobTracker.getJobTrackerMachine());
-    out.print("<html><head>");
-    out.printf("<title>%s Fair Scheduler Administration</title>\n", hostname);
-    out.print("<link rel=\"stylesheet\" type=\"text/css\" " + 
-        "href=\"/static/hadoop.css\">\n");
-    out.print("</head><body>\n");
-    out.printf("<h1><a href=\"/jobtracker.jsp\">%s</a> " + 
-        "Fair Scheduler Administration</h1>\n", hostname);
-    showPools(out, advancedView);
-    showJobs(out, advancedView);
-    out.print("</body></html>\n");
-    out.close();
-
-    // Flush our buffer to the real servlet output
-    OutputStream servletOut = response.getOutputStream();
-    baos.writeTo(servletOut);
-    servletOut.close();
-  }
-
-  /**
-   * Print a view of pools to the given output writer.
-   */
-  private void showPools(PrintWriter out, boolean advancedView) {
-    synchronized(scheduler) {
-      boolean warnInverted = false;
-      PoolManager poolManager = scheduler.getPoolManager();
-      out.print("<h2>Pools</h2>\n");
-      out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
-      out.print("<tr><th rowspan=2>Pool</th>" +
-          "<th rowspan=2>Running Jobs</th>" + 
-          "<th colspan=4>Map Tasks</th>" + 
-          "<th colspan=4>Reduce Tasks</th>" +
-          "<th rowspan=2>Scheduling Mode</th></tr>\n<tr>" + 
-          "<th>Min Share</th><th>Max Share</th><th>Running</th><th>Fair Share</th>" + 
-          "<th>Min Share</th><th>Max Share</th><th>Running</th><th>Fair Share</th></tr>\n");
-      List<Pool> pools = new ArrayList<Pool>(poolManager.getPools());
-      Collections.sort(pools, new Comparator<Pool>() {
-        public int compare(Pool p1, Pool p2) {
-          if (p1.isDefaultPool())
-            return 1;
-          else if (p2.isDefaultPool())
-            return -1;
-          else return p1.getName().compareTo(p2.getName());
-        }});
-      for (Pool pool: pools) {
-        String name = pool.getName();
-        int runningMaps = pool.getMapSchedulable().getRunningTasks();
-        int runningReduces = pool.getReduceSchedulable().getRunningTasks();
-        int maxMaps = poolManager.getMaxSlots(name, TaskType.MAP);
-        int maxReduces = poolManager.getMaxSlots(name, TaskType.REDUCE);
-        boolean invertedMaps = poolManager.invertedMinMax(TaskType.MAP, name);
-        boolean invertedReduces = poolManager.invertedMinMax(TaskType.REDUCE, name);
-        warnInverted = warnInverted || invertedMaps || invertedReduces;
-        out.print("<tr>");
-        out.printf("<td>%s</td>", name);
-        out.printf("<td>%d</td>", pool.getJobs().size());
-        // Map Tasks
-        out.printf("<td>%d</td>", poolManager.getAllocation(name,
-            TaskType.MAP));
-        out.print("<td>");
-        if(maxMaps == Integer.MAX_VALUE) {
-          out.print("-");
-        } else {
-          out.print(maxMaps);
-        }
-        if(invertedMaps) {
-          out.print("*");
-        }
-        out.print("</td>");
-        out.printf("<td>%d</td>", runningMaps);
-        out.printf("<td>%.1f</td>", pool.getMapSchedulable().getFairShare());
-        // Reduce Tasks
-        out.printf("<td>%d</td>", poolManager.getAllocation(name,
-            TaskType.REDUCE));
-        out.print("<td>");
-        if(maxReduces == Integer.MAX_VALUE) {
-          out.print("-");
-        } else {
-          out.print(maxReduces);
-        }
-        if(invertedReduces) {
-          out.print("*");
-        }
-        out.print("</td>");
-        out.printf("<td>%d</td>", runningReduces);
-        out.printf("<td>%.1f</td>", pool.getReduceSchedulable().getFairShare());
-        out.printf("<td>%s</td>", pool.getSchedulingMode());
-        out.print("</tr>\n");
-      }
-      out.print("</table>\n");
-      if(warnInverted) {
-        out.print("<p>* One or more pools have max share set lower than min share. Max share will be used and minimum will be treated as if set equal to max.</p>");
-      }
-    }
-  }
-
-  /**
-   * Print a view of running jobs to the given output writer.
-   */
-  private void showJobs(PrintWriter out, boolean advancedView) {
-    out.print("<h2>Running Jobs</h2>\n");
-    out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
-    int colsPerTaskType = advancedView ? 4 : 3;
-    out.printf("<tr><th rowspan=2>Submitted</th>" + 
-        "<th rowspan=2>JobID</th>" +
-        "<th rowspan=2>User</th>" +
-        "<th rowspan=2>Name</th>" +
-        "<th rowspan=2>Pool</th>" +
-        "<th rowspan=2>Priority</th>" +
-        "<th colspan=%d>Map Tasks</th>" +
-        "<th colspan=%d>Reduce Tasks</th>",
-        colsPerTaskType, colsPerTaskType);
-    out.print("</tr><tr>\n");
-    out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>" +
-        (advancedView ? "<th>Weight</th>" : ""));
-    out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>" +
-        (advancedView ? "<th>Weight</th>" : ""));
-    out.print("</tr>\n");
-    synchronized (jobTracker) {
-      Collection<JobInProgress> runningJobs = getInitedJobs();
-      synchronized (scheduler) {
-        for (JobInProgress job: runningJobs) {
-          JobProfile profile = job.getProfile();
-          JobInfo info = scheduler.infos.get(job);
-          if (info == null) { // Job finished, but let's show 0's for info
-            info = new JobInfo(null, null);
-          }
-          out.print("<tr>\n");
-          out.printf("<td>%s</td>\n", DATE_FORMAT.format(
-              new Date(job.getStartTime())));
-          out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
-              profile.getJobID(), profile.getJobID());
-          out.printf("<td>%s</td>\n", profile.getUser());
-          out.printf("<td>%s</td>\n", profile.getJobName());
-          if (JSPUtil.privateActionsAllowed(jobTracker.conf)) {
-            out.printf("<td>%s</td>\n", generateSelect(scheduler
-                .getPoolManager().getPoolNames(), scheduler.getPoolManager()
-                .getPoolName(job), "/scheduler?setPool=<CHOICE>&jobid="
-                + profile.getJobID() + (advancedView ? "&advanced" : "")));
-            out.printf("<td>%s</td>\n", generateSelect(Arrays
-                .asList(new String[] { "VERY_LOW", "LOW", "NORMAL", "HIGH",
-                    "VERY_HIGH" }), job.getPriority().toString(),
-                "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID()
-                    + (advancedView ? "&advanced" : "")));
-          } else {
-            out.printf("<td>%s</td>\n", scheduler.getPoolManager().getPoolName(job));
-            out.printf("<td>%s</td>\n", job.getPriority().toString());
-          }
-          Pool pool = scheduler.getPoolManager().getPool(job);
-          String mapShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
-              String.format("%.1f", info.mapSchedulable.getFairShare()) : "NA";
-          out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
-              job.finishedMaps(), job.desiredMaps(), 
-              info.mapSchedulable.getRunningTasks(),
-              mapShare);
-          if (advancedView) {
-            out.printf("<td>%.1f</td>\n", info.mapSchedulable.getWeight());
-          }
-          String reduceShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
-              String.format("%.1f", info.reduceSchedulable.getFairShare()) : "NA";
-          out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
-              job.finishedReduces(), job.desiredReduces(), 
-              info.reduceSchedulable.getRunningTasks(),
-              reduceShare);
-          if (advancedView) {
-            out.printf("<td>%.1f</td>\n", info.reduceSchedulable.getWeight());
-          }
-          out.print("</tr>\n");
-        }
-      }
-    }
-    out.print("</table>\n");
-  }
-
-  /**
-   * Generate a HTML select control with a given list of choices and a given
-   * option selected. When the selection is changed, take the user to the
-   * <code>submitUrl</code>. The <code>submitUrl</code> can be made to include
-   * the option selected -- the first occurrence of the substring
-   * <code>&lt;CHOICE&gt;</code> will be replaced by the option chosen.
-   */
-  private String generateSelect(Iterable<String> choices, 
-      String selectedChoice, String submitUrl) {
-    StringBuilder html = new StringBuilder();
-    String id = "select" + lastId++;
-    html.append("<select id=\"" + id + "\" name=\"" + id + "\" " + 
-        "onchange=\"window.location = '" + submitUrl + 
-        "'.replace('<CHOICE>', document.getElementById('" + id +
-        "').value);\">\n");
-    for (String choice: choices) {
-      html.append(String.format("<option value=\"%s\"%s>%s</option>\n",
-          choice, (choice.equals(selectedChoice) ? " selected" : ""), choice));
-    }
-    html.append("</select>\n");
-    return html.toString();
-  }
-
-  /**
-   * Obtained all initialized jobs
-   */
-  private Collection<JobInProgress> getInitedJobs() {
-    Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
-    for (Iterator<JobInProgress> it = runningJobs.iterator(); it.hasNext();) {
-      JobInProgress job = it.next();
-      if (!job.inited()) {
-        it.remove();
-      }
-    }
-    return runningJobs;
-  }
-
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FifoJobComparator.java

@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.Comparator;
-
-/**
- * Order {@link JobInProgress} objects by priority and then by submit time, as
- * in the default scheduler in Hadoop.
- */
-public class FifoJobComparator implements Comparator<JobInProgress> {
-  public int compare(JobInProgress j1, JobInProgress j2) {
-    int res = j1.getPriority().compareTo(j2.getPriority());
-    if (res == 0) {
-      if (j1.getStartTime() < j2.getStartTime()) {
-        res = -1;
-      } else {
-        res = (j1.getStartTime() == j2.getStartTime() ? 0 : 1);
-      }
-    }
-    if (res == 0) {
-      // If there is a tie, break it by job ID to get a deterministic order
-      res = j1.getJobID().compareTo(j2.getJobID());
-    }
-    return res;
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java

@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.mapred.FairScheduler.JobInfo;
-import org.apache.hadoop.mapreduce.TaskType;
-
-public class JobSchedulable extends Schedulable {
-  private FairScheduler scheduler;
-  private JobInProgress job;
-  private TaskType taskType;
-  private int demand = 0;
-
-  public JobSchedulable(FairScheduler scheduler, JobInProgress job, 
-      TaskType taskType) {
-    this.scheduler = scheduler;
-    this.job = job;
-    this.taskType = taskType;
-    
-    initMetrics();
-  }
-  
-  @Override
-  public TaskType getTaskType() {
-    return taskType;
-  }
-  
-  @Override
-  public String getName() {
-    return job.getJobID().toString();
-  }
-
-  public JobInProgress getJob() {
-    return job;
-  }
-  
-  @Override
-  public void updateDemand() {
-    demand = 0;
-    if (isRunnable()) {
-      // For reduces, make sure enough maps are done that reduces can launch
-      if (taskType == TaskType.REDUCE && !job.scheduleReduces())
-        return;
-      // Add up demand from each TaskInProgress; each TIP can either
-      // - have no attempts running, in which case it demands 1 slot
-      // - have N attempts running, in which case it demands N slots, and may
-      //   potentially demand one more slot if it needs to be speculated
-      TaskInProgress[] tips = (taskType == TaskType.MAP ? 
-          job.getTasks(TaskType.MAP) : job.getTasks(TaskType.REDUCE));
-      boolean speculationEnabled = (taskType == TaskType.MAP ?
-          job.hasSpeculativeMaps() : job.hasSpeculativeReduces());
-      long time = scheduler.getClock().getTime();
-      for (TaskInProgress tip: tips) {
-        if (!tip.isComplete()) {
-          if (tip.isRunning()) {
-            // Count active tasks and any speculative task we want to launch
-            demand += tip.getActiveTasks().size();
-            if (speculationEnabled && tip.canBeSpeculated(time))
-              demand += 1;
-          } else {
-            // Need to launch 1 task
-            demand += 1;
-          }
-        }
-      }
-    }
-  }
-
-  private boolean isRunnable() {
-    JobInfo info = scheduler.getJobInfo(job);
-    int runState = job.getStatus().getRunState();
-    return (info != null && info.runnable && runState == JobStatus.RUNNING);
-  }
-
-  @Override
-  public int getDemand() {
-    return demand;
-  }
-  
-  @Override
-  public void redistributeShare() {}
-
-  @Override
-  public JobPriority getPriority() {
-    return job.getPriority();
-  }
-
-  @Override
-  public int getRunningTasks() {
-    if (!job.inited()) {
-      return 0;
-    }
-    return taskType == TaskType.MAP ? job.runningMaps() : job.runningReduces();
-  }
-
-  @Override
-  public long getStartTime() {
-    return job.startTime;
-  }
-  
-  @Override
-  public double getWeight() {
-    return scheduler.getJobWeight(job, taskType);
-  }
-  
-  @Override
-  public int getMinShare() {
-    return 0;
-  }
-
-  @Override
-  public Task assignTask(TaskTrackerStatus tts, long currentTime,
-      Collection<JobInProgress> visited) throws IOException {
-    if (isRunnable()) {
-      visited.add(job);
-      TaskTrackerManager ttm = scheduler.taskTrackerManager;
-      ClusterStatus clusterStatus = ttm.getClusterStatus();
-      int numTaskTrackers = clusterStatus.getTaskTrackers();
-
-      // check with the load manager whether it is safe to 
-      // launch this task on this taskTracker.
-      LoadManager loadMgr = scheduler.getLoadManager();
-      if (!loadMgr.canLaunchTask(tts, job, taskType)) {
-        return null;
-      }
-      if (taskType == TaskType.MAP) {
-        LocalityLevel localityLevel = scheduler.getAllowedLocalityLevel(
-            job, currentTime);
-        scheduler.getEventLog().log(
-            "ALLOWED_LOC_LEVEL", job.getJobID(), localityLevel);
-        // obtainNewMapTask needs to be passed 1 + the desired locality level
-        return job.obtainNewMapTask(tts, numTaskTrackers,
-            ttm.getNumberOfUniqueHosts(), localityLevel.toCacheLevelCap());
-      } else {
-        return job.obtainNewReduceTask(tts, numTaskTrackers,
-            ttm.getNumberOfUniqueHosts());
-      }
-    } else {
-      return null;
-    }
-  }
-
-  
-  @Override
-  protected String getMetricsContextName() {
-    return "jobs";
-  }
-  
-  @Override
-  void updateMetrics() {
-    assert metrics != null;
-    
-    super.setMetricValues(metrics);
-    metrics.update();
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/LoadManager.java

@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.TaskType;
-
-/**
- * A pluggable object that manages the load on each {@link TaskTracker}, telling
- * the {@link TaskScheduler} when it can launch new tasks. 
- */
-public abstract class LoadManager implements Configurable {
-  protected Configuration conf;
-  protected TaskTrackerManager taskTrackerManager;
-  protected FairSchedulerEventLog schedulingLog;
-  
-  public Configuration getConf() {
-    return conf;
-  }
-
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-  }
-
-  public synchronized void setTaskTrackerManager(
-      TaskTrackerManager taskTrackerManager) {
-    this.taskTrackerManager = taskTrackerManager;
-  }
-
-  public void setEventLog(FairSchedulerEventLog schedulingLog) {
-    this.schedulingLog = schedulingLog;
-  }
-  
-  /**
-   * Lifecycle method to allow the LoadManager to start any work in separate
-   * threads.
-   */
-  public void start() throws IOException {
-    // do nothing
-  }
-  
-  /**
-   * Lifecycle method to allow the LoadManager to stop any work it is doing.
-   */
-  public void terminate() throws IOException {
-    // do nothing
-  }
-  
-  /**
-   * Can a given {@link TaskTracker} run another map task?
-   * This method may check whether the specified tracker has
-   * enough resources to run another map task.
-   * @param tracker The machine we wish to run a new map on
-   * @param totalRunnableMaps Set of running jobs in the cluster
-   * @param totalMapSlots The total number of map slots in the cluster
-   * @return true if another map can be launched on <code>tracker</code>
-   */
-  public abstract boolean canAssignMap(TaskTrackerStatus tracker,
-      int totalRunnableMaps, int totalMapSlots);
-
-  /**
-   * Can a given {@link TaskTracker} run another reduce task?
-   * This method may check whether the specified tracker has
-   * enough resources to run another reduce task.
-   * @param tracker The machine we wish to run a new map on
-   * @param totalRunnableReduces Set of running jobs in the cluster
-   * @param totalReduceSlots The total number of reduce slots in the cluster
-   * @return true if another reduce can be launched on <code>tracker</code>
-   */
-  public abstract boolean canAssignReduce(TaskTrackerStatus tracker,
-      int totalRunnableReduces, int totalReduceSlots);
-
-  /**
-   * Can a given {@link TaskTracker} run another new task from a given job? 
-   * This method is provided for use by LoadManagers that take into 
-   * account jobs' individual resource needs when placing tasks.
-   * @param tracker The machine we wish to run a new map on
-   * @param job The job from which we want to run a task on this machine
-   * @param type The type of task that we want to run on
-   * @return true if this task can be launched on <code>tracker</code>
-   */
-  public abstract boolean canLaunchTask(TaskTrackerStatus tracker,
-      JobInProgress job,  TaskType type);
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/LocalityLevel.java

@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-/**
- * Represents the level of data-locality at which a job in the fair scheduler
- * is allowed to launch tasks. By default, jobs are not allowed to launch
- * non-data-local tasks until they have waited a small number of seconds to
- * find a slot on a node that they have data on. If a job has waited this
- * long, it is allowed to launch rack-local tasks as well (on nodes that may
- * not have the task's input data, but share a rack with a node that does).
- * Finally, after a further wait, jobs are allowed to launch tasks anywhere
- * in the cluster.
- * 
- * This enum defines three levels - NODE, RACK and ANY (for allowing tasks
- * to be launched on any node). A map task's level can be obtained from
- * its job through {@link #fromTask(JobInProgress, Task, TaskTrackerStatus)}. In
- * addition, for any locality level, it is possible to get a "level cap" to pass
- * to {@link JobInProgress#obtainNewMapTask(TaskTrackerStatus, int, int, int)}
- * to ensure that only tasks at this level or lower are launched, through
- * the {@link #toCacheLevelCap()} method.
- */
-public enum LocalityLevel {
-  NODE, RACK, ANY;
-  
-  public static LocalityLevel fromTask(JobInProgress job, Task mapTask,
-      TaskTrackerStatus tracker) {
-    TaskID tipID = mapTask.getTaskID().getTaskID();
-    TaskInProgress tip = job.getTaskInProgress(tipID);
-    switch (job.getLocalityLevel(tip, tracker)) {
-    case 0: return LocalityLevel.NODE;
-    case 1: return LocalityLevel.RACK;
-    default: return LocalityLevel.ANY;
-    }
-  }
-  
-  /**
-   * Obtain a JobInProgress cache level cap to pass to
-   * {@link JobInProgress#obtainNewMapTask(TaskTrackerStatus, int, int, int)}
-   * to ensure that only tasks of this locality level and lower are launched.
-   */
-  public int toCacheLevelCap() {
-    switch(this) {
-    case NODE: return 1;
-    case RACK: return 2;
-    default: return Integer.MAX_VALUE;
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/NewJobWeightBooster.java

@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.mapreduce.TaskType;
-
-/**
- * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs
- * for a certain amount of time -- by default, a 3x weight boost for 60 seconds.
- * This can be used to make shorter jobs finish faster, emulating Shortest Job
- * First scheduling while not starving long jobs. 
- */
-public class NewJobWeightBooster extends Configured implements WeightAdjuster {
-  private static final float DEFAULT_FACTOR = 3;
-  private static final long DEFAULT_DURATION = 5 * 60 * 1000;
-
-  private float factor;
-  private long duration;
-
-  public void setConf(Configuration conf) {
-    if (conf != null) {
-      factor = conf.getFloat("mapred.newjobweightbooster.factor",
-          DEFAULT_FACTOR);
-      duration = conf.getLong("mapred.newjobweightbooster.duration",
-          DEFAULT_DURATION);
-    }
-    super.setConf(conf);
-  }
-  
-  public double adjustWeight(JobInProgress job, TaskType taskType,
-      double curWeight) {
-    long start = job.getStartTime();
-    long now = System.currentTimeMillis();
-    if (now - start < duration) {
-      return curWeight * factor;
-    } else {
-      return curWeight;
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/Pool.java

@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.metrics.MetricsContext;
-
-/**
- * A schedulable pool of jobs.
- */
-public class Pool {
-  /** Name of the default pool, where jobs with no pool parameter go. */
-  public static final String DEFAULT_POOL_NAME = "default";
-  
-  /** Pool name. */
-  private String name;
-  
-  /** Jobs in this specific pool; does not include children pools' jobs. */
-  private Collection<JobInProgress> jobs = new ArrayList<JobInProgress>();
-  
-  /** Scheduling mode for jobs inside the pool (fair or FIFO) */
-  private SchedulingMode schedulingMode;
-
-  private PoolSchedulable mapSchedulable;
-  private PoolSchedulable reduceSchedulable;
-
-  public Pool(FairScheduler scheduler, String name) {
-    this.name = name;
-    mapSchedulable = new PoolSchedulable(scheduler, this, TaskType.MAP);
-    reduceSchedulable = new PoolSchedulable(scheduler, this, TaskType.REDUCE);
-  }
-  
-  public Collection<JobInProgress> getJobs() {
-    return jobs;
-  }
-  
-  public void addJob(JobInProgress job) {
-    jobs.add(job);
-    mapSchedulable.addJob(job);
-    reduceSchedulable.addJob(job);
-  }
-  
-  public void removeJob(JobInProgress job) {
-    jobs.remove(job);
-    mapSchedulable.removeJob(job);
-    reduceSchedulable.removeJob(job);
-  }
-  
-  public String getName() {
-    return name;
-  }
-
-  public SchedulingMode getSchedulingMode() {
-    return schedulingMode;
-  }
-  
-  public void setSchedulingMode(SchedulingMode schedulingMode) {
-    this.schedulingMode = schedulingMode;
-  }
-
-  public boolean isDefaultPool() {
-    return Pool.DEFAULT_POOL_NAME.equals(name);
-  }
-  
-  public PoolSchedulable getMapSchedulable() {
-    return mapSchedulable;
-  }
-  
-  public PoolSchedulable getReduceSchedulable() {
-    return reduceSchedulable;
-  }
-  
-  public PoolSchedulable getSchedulable(TaskType type) {
-    return type == TaskType.MAP ? mapSchedulable : reduceSchedulable;
-  }
-
-  public void updateMetrics() {
-    mapSchedulable.updateMetrics();
-    reduceSchedulable.updateMetrics();
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java

@@ -1,546 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.w3c.dom.Text;
-import org.xml.sax.SAXException;
-
-/**
- * Maintains a list of pools as well as scheduling parameters for each pool,
- * such as guaranteed share allocations, from the fair scheduler config file.
- */
-public class PoolManager {
-  public static final Log LOG = LogFactory.getLog(
-    "org.apache.hadoop.mapred.PoolManager");
-
-  /** Time to wait between checks of the allocation file */
-  public static final long ALLOC_RELOAD_INTERVAL = 10 * 1000;
-  
-  /**
-   * Time to wait after the allocation has been modified before reloading it
-   * (this is done to prevent loading a file that hasn't been fully written).
-   */
-  public static final long ALLOC_RELOAD_WAIT = 5 * 1000; 
-
-  public static final String EXPLICIT_POOL_PROPERTY = "mapred.fairscheduler.pool";
-
-  private final FairScheduler scheduler;
-  
-  // Map and reduce minimum allocations for each pool
-  private Map<String, Integer> mapAllocs = new HashMap<String, Integer>();
-  private Map<String, Integer> reduceAllocs = new HashMap<String, Integer>();
-
-  // If set, cap number of map and reduce tasks in a pool
-  private Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>();
-  private Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>();
-
-  // Sharing weights for each pool
-  private Map<String, Double> poolWeights = new HashMap<String, Double>();
-  
-  // Max concurrent running jobs for each pool and for each user; in addition,
-  // for users that have no max specified, we use the userMaxJobsDefault.
-  private Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
-  private Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
-  private int userMaxJobsDefault = Integer.MAX_VALUE;
-  private int poolMaxJobsDefault = Integer.MAX_VALUE;
-
-  // Min share preemption timeout for each pool in seconds. If a job in the pool
-  // waits this long without receiving its guaranteed share, it is allowed to
-  // preempt other jobs' tasks.
-  private Map<String, Long> minSharePreemptionTimeouts =
-    new HashMap<String, Long>();
-  
-  // Default min share preemption timeout for pools where it is not set
-  // explicitly.
-  private long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
-  
-  // Preemption timeout for jobs below fair share in seconds. If a job remains
-  // below half its fair share for this long, it is allowed to preempt tasks.
-  private long fairSharePreemptionTimeout = Long.MAX_VALUE;
-  
-  SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR;
-  
-  private Object allocFile; // Path to XML file containing allocations. This
-                            // is either a URL to specify a classpath resource
-                            // (if the fair-scheduler.xml on the classpath is
-                            // used) or a String to specify an absolute path (if
-                            // mapred.fairscheduler.allocation.file is used).
-  private String poolNameProperty; // Jobconf property to use for determining a
-                                   // job's pool name (default: mapreduce.job.user.name)
-  
-  private Map<String, Pool> pools = new HashMap<String, Pool>();
-  
-  private long lastReloadAttempt; // Last time we tried to reload the pools file
-  private long lastSuccessfulReload; // Last time we successfully reloaded pools
-  private boolean lastReloadAttemptFailed = false;
-
-  public PoolManager(FairScheduler scheduler) {
-    this.scheduler = scheduler;
-  }
-  
-  public void initialize() throws IOException, SAXException,
-      AllocationConfigurationException, ParserConfigurationException {
-    Configuration conf = scheduler.getConf();
-    this.poolNameProperty = conf.get(
-        "mapred.fairscheduler.poolnameproperty", JobContext.USER_NAME);
-    this.allocFile = conf.get("mapred.fairscheduler.allocation.file");
-    if (allocFile == null) {
-      // No allocation file specified in jobconf. Use the default allocation
-      // file, fair-scheduler.xml, looking for it on the classpath.
-      allocFile = new Configuration().getResource("fair-scheduler.xml");
-      if (allocFile == null) {
-        LOG.error("The fair scheduler allocation file fair-scheduler.xml was "
-            + "not found on the classpath, and no other config file is given "
-            + "through mapred.fairscheduler.allocation.file.");
-      }
-    }
-    reloadAllocs();
-    lastSuccessfulReload = System.currentTimeMillis();
-    lastReloadAttempt = System.currentTimeMillis();
-    // Create the default pool so that it shows up in the web UI
-    getPool(Pool.DEFAULT_POOL_NAME);
-  }
-  
-  /**
-   * Get a pool by name, creating it if necessary
-   */
-  public synchronized Pool getPool(String name) {
-    Pool pool = pools.get(name);
-    if (pool == null) {
-      pool = new Pool(scheduler, name);
-      pool.setSchedulingMode(defaultSchedulingMode);
-      pools.put(name, pool);
-    }
-    return pool;
-  }
-  
-  /**
-   * Get the pool that a given job is in.
-   */
-  public Pool getPool(JobInProgress job) {
-    return getPool(getPoolName(job));
-  }
-
-  /**
-   * Reload allocations file if it hasn't been loaded in a while
-   */
-  public void reloadAllocsIfNecessary() {
-    long time = System.currentTimeMillis();
-    if (time > lastReloadAttempt + ALLOC_RELOAD_INTERVAL) {
-      lastReloadAttempt = time;
-      if (null == allocFile) {
-        return;
-      }
-      try {
-        // Get last modified time of alloc file depending whether it's a String
-        // (for a path name) or an URL (for a classloader resource)
-        long lastModified;
-        if (allocFile instanceof String) {
-          File file = new File((String) allocFile);
-          lastModified = file.lastModified();
-        } else { // allocFile is an URL
-          URLConnection conn = ((URL) allocFile).openConnection();
-          lastModified = conn.getLastModified();
-        }
-        if (lastModified > lastSuccessfulReload &&
-            time > lastModified + ALLOC_RELOAD_WAIT) {
-          reloadAllocs();
-          lastSuccessfulReload = time;
-          lastReloadAttemptFailed = false;
-        }
-      } catch (Exception e) {
-        // Throwing the error further out here won't help - the RPC thread
-        // will catch it and report it in a loop. Instead, just log it and
-        // hope somebody will notice from the log.
-        // We log the error only on the first failure so we don't fill up the
-        // JobTracker's log with these messages.
-        if (!lastReloadAttemptFailed) {
-          LOG.error("Failed to reload fair scheduler config file - " +
-              "will use existing allocations.", e);
-        }
-        lastReloadAttemptFailed = true;
-      }
-    }
-  }
-  
-  /**
-   * Updates the allocation list from the allocation config file. This file is
-   * expected to be in the following whitespace-separated format:
-   * 
-   * <code>
-   * poolName1 mapAlloc reduceAlloc
-   * poolName2 mapAlloc reduceAlloc
-   * ...
-   * </code>
-   * 
-   * Blank lines and lines starting with # are ignored.
-   *  
-   * @throws IOException if the config file cannot be read.
-   * @throws AllocationConfigurationException if allocations are invalid.
-   * @throws ParserConfigurationException if XML parser is misconfigured.
-   * @throws SAXException if config file is malformed.
-   */
-  public void reloadAllocs() throws IOException, ParserConfigurationException, 
-      SAXException, AllocationConfigurationException {
-    if (allocFile == null) return;
-    // Create some temporary hashmaps to hold the new allocs, and we only save
-    // them in our fields if we have parsed the entire allocs file successfully.
-    Map<String, Integer> mapAllocs = new HashMap<String, Integer>();
-    Map<String, Integer> reduceAllocs = new HashMap<String, Integer>();
-    Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
-    Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
-    Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>();
-    Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>();
-    Map<String, Double> poolWeights = new HashMap<String, Double>();
-    Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>();
-    Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>();
-    int userMaxJobsDefault = Integer.MAX_VALUE;
-    int poolMaxJobsDefault = Integer.MAX_VALUE;
-    long fairSharePreemptionTimeout = Long.MAX_VALUE;
-    long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
-    SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR;
-    
-    // Remember all pool names so we can display them on web UI, etc.
-    List<String> poolNamesInAllocFile = new ArrayList<String>();
-    
-    // Read and parse the allocations file.
-    DocumentBuilderFactory docBuilderFactory =
-      DocumentBuilderFactory.newInstance();
-    docBuilderFactory.setIgnoringComments(true);
-    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
-    Document doc;
-    if (allocFile instanceof String) {
-      doc = builder.parse(new File((String) allocFile));
-    } else {
-      doc = builder.parse(allocFile.toString());
-    }
-    Element root = doc.getDocumentElement();
-    if (!"allocations".equals(root.getTagName()))
-      throw new AllocationConfigurationException("Bad fair scheduler config " + 
-          "file: top-level element not <allocations>");
-    NodeList elements = root.getChildNodes();
-    for (int i = 0; i < elements.getLength(); i++) {
-      Node node = elements.item(i);
-      if (!(node instanceof Element))
-        continue;
-      Element element = (Element)node;
-      if ("pool".equals(element.getTagName())) {
-        String poolName = element.getAttribute("name");
-        poolNamesInAllocFile.add(poolName);
-        NodeList fields = element.getChildNodes();
-        for (int j = 0; j < fields.getLength(); j++) {
-          Node fieldNode = fields.item(j);
-          if (!(fieldNode instanceof Element))
-            continue;
-          Element field = (Element) fieldNode;
-          if ("minMaps".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            mapAllocs.put(poolName, val);
-          } else if ("minReduces".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            reduceAllocs.put(poolName, val);
-          } else if ("maxMaps".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            poolMaxMaps.put(poolName, val);
-          } else if ("maxReduces".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            poolMaxReduces.put(poolName, val);
-          } else if ("maxRunningJobs".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            poolMaxJobs.put(poolName, val);
-          } else if ("weight".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            double val = Double.parseDouble(text);
-            poolWeights.put(poolName, val);
-          } else if ("minSharePreemptionTimeout".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            long val = Long.parseLong(text) * 1000L;
-            minSharePreemptionTimeouts.put(poolName, val);
-          } else if ("schedulingMode".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            poolModes.put(poolName, parseSchedulingMode(text));
-          }
-        }
-        if (poolMaxMaps.containsKey(poolName) && mapAllocs.containsKey(poolName)
-            && poolMaxMaps.get(poolName) < mapAllocs.get(poolName)) {
-          LOG.warn(String.format("Pool %s has max maps %d less than min maps %d",
-              poolName, poolMaxMaps.get(poolName), mapAllocs.get(poolName)));        
-        }
-        if(poolMaxReduces.containsKey(poolName) && reduceAllocs.containsKey(poolName)
-            && poolMaxReduces.get(poolName) < reduceAllocs.get(poolName)) {
-          LOG.warn(String.format("Pool %s has max reduces %d less than min reduces %d",
-              poolName, poolMaxReduces.get(poolName), reduceAllocs.get(poolName)));        
-        }
-      } else if ("user".equals(element.getTagName())) {
-        String userName = element.getAttribute("name");
-        NodeList fields = element.getChildNodes();
-        for (int j = 0; j < fields.getLength(); j++) {
-          Node fieldNode = fields.item(j);
-          if (!(fieldNode instanceof Element))
-            continue;
-          Element field = (Element) fieldNode;
-          if ("maxRunningJobs".equals(field.getTagName())) {
-            String text = ((Text)field.getFirstChild()).getData().trim();
-            int val = Integer.parseInt(text);
-            userMaxJobs.put(userName, val);
-          }
-        }
-      } else if ("userMaxJobsDefault".equals(element.getTagName())) {
-        String text = ((Text)element.getFirstChild()).getData().trim();
-        int val = Integer.parseInt(text);
-        userMaxJobsDefault = val;
-      } else if ("poolMaxJobsDefault".equals(element.getTagName())) {
-        String text = ((Text)element.getFirstChild()).getData().trim();
-        int val = Integer.parseInt(text);
-        poolMaxJobsDefault = val;
-      } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) {
-        String text = ((Text)element.getFirstChild()).getData().trim();
-        long val = Long.parseLong(text) * 1000L;
-        fairSharePreemptionTimeout = val;
-      } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) {
-        String text = ((Text)element.getFirstChild()).getData().trim();
-        long val = Long.parseLong(text) * 1000L;
-        defaultMinSharePreemptionTimeout = val;
-      } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) {
-        String text = ((Text)element.getFirstChild()).getData().trim();
-        defaultSchedulingMode = parseSchedulingMode(text);
-      } else {
-        LOG.warn("Bad element in allocations file: " + element.getTagName());
-      }
-    }
-    
-    // Commit the reload; also create any pool defined in the alloc file
-    // if it does not already exist, so it can be displayed on the web UI.
-    synchronized(this) {
-      this.mapAllocs = mapAllocs;
-      this.reduceAllocs = reduceAllocs;
-      this.poolMaxMaps = poolMaxMaps;
-      this.poolMaxReduces = poolMaxReduces;
-      this.poolMaxJobs = poolMaxJobs;
-      this.userMaxJobs = userMaxJobs;
-      this.poolWeights = poolWeights;
-      this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
-      this.userMaxJobsDefault = userMaxJobsDefault;
-      this.poolMaxJobsDefault = poolMaxJobsDefault;
-      this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
-      this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout;
-      this.defaultSchedulingMode = defaultSchedulingMode;
-      for (String name: poolNamesInAllocFile) {
-        Pool pool = getPool(name);
-        if (poolModes.containsKey(name)) {
-          pool.setSchedulingMode(poolModes.get(name));
-        } else {
-          pool.setSchedulingMode(defaultSchedulingMode);
-        }
-      }
-    }
-  }
-
-  /**
-   * Does the pool have incompatible max and min allocation set.
-   * 
-   * @param type
-   *          {@link TaskType#MAP} or {@link TaskType#REDUCE}
-   * @param pool
-   *          the pool name
-   * @return true if the max is less than the min
-   */
-  boolean invertedMinMax(TaskType type, String pool) {
-    Map<String, Integer> max = TaskType.MAP == type ? poolMaxMaps : poolMaxReduces;
-    Map<String, Integer> min = TaskType.MAP == type ? mapAllocs : reduceAllocs;
-    if (max.containsKey(pool) && min.containsKey(pool)
-        && max.get(pool) < min.get(pool)) {
-      return true;
-    }
-    return false;
-  }
-
-  private SchedulingMode parseSchedulingMode(String text)
-      throws AllocationConfigurationException {
-    text = text.toLowerCase();
-    if (text.equals("fair")) {
-      return SchedulingMode.FAIR;
-    } else if (text.equals("fifo")) {
-      return SchedulingMode.FIFO;
-    } else {
-      throw new AllocationConfigurationException(
-          "Unknown scheduling mode : " + text + "; expected 'fifo' or 'fair'");
-    }
-  }
-
-  /**
-   * Get the allocation for a particular pool
-   */
-  public int getAllocation(String pool, TaskType taskType) {
-    Map<String, Integer> allocationMap = (taskType == TaskType.MAP ?
-        mapAllocs : reduceAllocs);
-    Integer alloc = allocationMap.get(pool);
-    return (alloc == null ? 0 : alloc);
-  }
-
-  /**
-   * Get the maximum map or reduce slots for the given pool.
-   * @return the cap set on this pool, or Integer.MAX_VALUE if not set.
-   */
-  int getMaxSlots(String poolName, TaskType taskType) {
-    Map<String, Integer> maxMap = (taskType == TaskType.MAP ? poolMaxMaps : poolMaxReduces);
-    if (maxMap.containsKey(poolName)) {
-      return maxMap.get(poolName);
-    } else {
-      return Integer.MAX_VALUE;
-    }
-  }
- 
-  /**
-   * Add a job in the appropriate pool
-   */
-  public synchronized void addJob(JobInProgress job) {
-    getPool(getPoolName(job)).addJob(job);
-  }
-  
-  /**
-   * Remove a job
-   */
-  public synchronized void removeJob(JobInProgress job) {
-    getPool(getPoolName(job)).removeJob(job);
-  }
-  
-  /**
-   * Change the pool of a particular job
-   */
-  public synchronized void setPool(JobInProgress job, String pool) {
-    removeJob(job);
-    job.getJobConf().set(EXPLICIT_POOL_PROPERTY, pool);
-    addJob(job);
-  }
-
-  /**
-   * Get a collection of all pools
-   */
-  public synchronized Collection<Pool> getPools() {
-    return pools.values();
-  }
-  
-  /**
-   * Get the pool name for a JobInProgress from its configuration.  This uses
-   * the value of mapred.fairscheduler.pool if specified, otherwise the value 
-   * of the property named in mapred.fairscheduler.poolnameproperty if that is
-   * specified.  Otherwise if neither is specified it uses the "user.name" property 
-   * in the jobconf by default.
-   */
-  public String getPoolName(JobInProgress job) {
-    Configuration conf = job.getJobConf();
-    return conf.get(EXPLICIT_POOL_PROPERTY,
-      conf.get(poolNameProperty, Pool.DEFAULT_POOL_NAME)).trim();
-  }
-
-  /**
-   * Get all pool names that have been seen either in the allocation file or in
-   * a MapReduce job.
-   */
-  public synchronized Collection<String> getPoolNames() {
-    List<String> list = new ArrayList<String>();
-    for (Pool pool: getPools()) {
-      list.add(pool.getName());
-    }
-    Collections.sort(list);
-    return list;
-  }
-
-  public int getUserMaxJobs(String user) {
-    if (userMaxJobs.containsKey(user)) {
-      return userMaxJobs.get(user);
-    } else {
-      return userMaxJobsDefault;
-    }
-  }
-
-  public int getPoolMaxJobs(String pool) {
-    if (poolMaxJobs.containsKey(pool)) {
-      return poolMaxJobs.get(pool);
-    } else {
-      return poolMaxJobsDefault;
-    }
-  }
-
-  public double getPoolWeight(String pool) {
-    if (poolWeights.containsKey(pool)) {
-      return poolWeights.get(pool);
-    } else {
-      return 1.0;
-    }
-  }
-
-  /**
-   * Get a pool's min share preemption timeout, in milliseconds. This is the
-   * time after which jobs in the pool may kill other pools' tasks if they
-   * are below their min share.
-   */
-  public long getMinSharePreemptionTimeout(String pool) {
-    if (minSharePreemptionTimeouts.containsKey(pool)) {
-      return minSharePreemptionTimeouts.get(pool);
-    } else {
-      return defaultMinSharePreemptionTimeout;
-    }
-  }
-  
-  /**
-   * Get the fair share preemption, in milliseconds. This is the time
-   * after which any job may kill other jobs' tasks if it is below half
-   * its fair share.
-   */
-  public long getFairSharePreemptionTimeout() {
-    return fairSharePreemptionTimeout;
-  }
-
-  synchronized void updateMetrics() {
-    for (Pool pool : pools.values()) {
-      pool.updateMetrics();
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolSchedulable.java

@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.FairScheduler.JobInfo;
-import org.apache.hadoop.mapreduce.TaskType;
-
-public class PoolSchedulable extends Schedulable {
-  public static final Log LOG = LogFactory.getLog(
-      PoolSchedulable.class.getName());
-  
-  private FairScheduler scheduler;
-  private Pool pool;
-  private TaskType taskType;
-  private PoolManager poolMgr;
-  private List<JobSchedulable> jobScheds = new LinkedList<JobSchedulable>();
-  private int demand = 0;
-  
-  // Variables used for preemption
-  long lastTimeAtMinShare;
-  long lastTimeAtHalfFairShare;
-
-  public PoolSchedulable(FairScheduler scheduler, Pool pool, TaskType type) {
-    this.scheduler = scheduler;
-    this.pool = pool;
-    this.taskType = type;
-    this.poolMgr = scheduler.getPoolManager();
-    long currentTime = scheduler.getClock().getTime();
-    this.lastTimeAtMinShare = currentTime;
-    this.lastTimeAtHalfFairShare = currentTime;
-    
-    initMetrics();
-  }
-
-  public void addJob(JobInProgress job) {
-    JobInfo info = scheduler.getJobInfo(job);
-    jobScheds.add(taskType == TaskType.MAP ?
-        info.mapSchedulable : info.reduceSchedulable);
-  }
-  
-  public void removeJob(JobInProgress job) {
-    for (Iterator<JobSchedulable> it = jobScheds.iterator(); it.hasNext();) {
-      JobSchedulable jobSched = it.next();
-      if (jobSched.getJob() == job) {
-        it.remove();
-        break;
-      }
-    }
-  }
-
-  /**
-   * Update demand by asking jobs in the pool to update
-   */
-  @Override
-  public void updateDemand() {
-    demand = 0;
-    for (JobSchedulable sched: jobScheds) {
-      sched.updateDemand();
-      demand += sched.getDemand();
-    }
-    // if demand exceeds the cap for this pool, limit to the max
-    int maxTasks = poolMgr.getMaxSlots(pool.getName(), taskType);
-    if(demand > maxTasks) {
-      demand = maxTasks;
-    }
-  }
-  
-  /**
-   * Distribute the pool's fair share among its jobs
-   */
-  @Override
-  public void redistributeShare() {
-    if (pool.getSchedulingMode() == SchedulingMode.FAIR) {
-      SchedulingAlgorithms.computeFairShares(jobScheds, getFairShare());
-    } else {
-      for (JobSchedulable sched: jobScheds) {
-        sched.setFairShare(0);
-      }
-    } 
-  }
-
-  @Override
-  public int getDemand() {
-    return demand;
-  }
-
-  @Override
-  public int getMinShare() {
-    return poolMgr.getAllocation(pool.getName(), taskType);
-  }
-
-  @Override
-  public double getWeight() {
-    return poolMgr.getPoolWeight(pool.getName());
-  }
-
-  @Override
-  public JobPriority getPriority() {
-    return JobPriority.NORMAL;
-  }
-
-  @Override
-  public int getRunningTasks() {
-    int ans = 0;
-    for (JobSchedulable sched: jobScheds) {
-      ans += sched.getRunningTasks();
-    }
-    return ans;
-  }
-
-  @Override
-  public long getStartTime() {
-    return 0;
-  }
-
-  @Override
-  public Task assignTask(TaskTrackerStatus tts, long currentTime,
-      Collection<JobInProgress> visited) throws IOException {
-    int runningTasks = getRunningTasks();
-    if (runningTasks >= poolMgr.getMaxSlots(pool.getName(), taskType)) {
-      return null;
-    }
-    SchedulingMode mode = pool.getSchedulingMode();
-    Comparator<Schedulable> comparator;
-    if (mode == SchedulingMode.FIFO) {
-      comparator = new SchedulingAlgorithms.FifoComparator();
-    } else if (mode == SchedulingMode.FAIR) {
-      comparator = new SchedulingAlgorithms.FairShareComparator();
-    } else {
-      throw new RuntimeException("Unsupported pool scheduling mode " + mode);
-    }
-    Collections.sort(jobScheds, comparator);
-    for (JobSchedulable sched: jobScheds) {
-      Task task = sched.assignTask(tts, currentTime, visited);
-      if (task != null)
-        return task;
-    }
-    return null;
-  }
-  
-  @Override
-  public String getName() {
-    return pool.getName();
-  }
-
-  Pool getPool() {
-    return pool;
-  }
-
-  @Override
-  public TaskType getTaskType() {
-    return taskType;
-  }
-  
-  public Collection<JobSchedulable> getJobSchedulables() {
-    return jobScheds;
-  }
-  
-  public long getLastTimeAtMinShare() {
-    return lastTimeAtMinShare;
-  }
-  
-  public void setLastTimeAtMinShare(long lastTimeAtMinShare) {
-    this.lastTimeAtMinShare = lastTimeAtMinShare;
-  }
-  
-  public long getLastTimeAtHalfFairShare() {
-    return lastTimeAtHalfFairShare;
-  }
-  
-  public void setLastTimeAtHalfFairShare(long lastTimeAtHalfFairShare) {
-    this.lastTimeAtHalfFairShare = lastTimeAtHalfFairShare;
-  }
-
-  protected String getMetricsContextName() {
-    return "pools";
-  }
-  
-  @Override
-  public void updateMetrics() {
-    super.setMetricValues(metrics);
-    
-    if (scheduler.isPreemptionEnabled()) {
-      // These won't be set if preemption is off
-      long lastCheck = scheduler.getLastPreemptionUpdateTime();
-      metrics.setMetric("millisSinceAtMinShare", lastCheck - lastTimeAtMinShare);
-      metrics.setMetric("millisSinceAtHalfFairShare", lastCheck - lastTimeAtHalfFairShare);
-    }
-    metrics.update();
-
-    for (JobSchedulable job : jobScheds) {
-      job.updateMetrics();
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/Schedulable.java

@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.apache.hadoop.metrics.MetricsRecord;
-import org.apache.hadoop.metrics.MetricsUtil;
-
-/**
- * A Schedulable represents an entity that can launch tasks, such as a job
- * or a pool. It provides a common interface so that algorithms such as fair
- * sharing can be applied both within a pool and across pools. There are 
- * currently two types of Schedulables: JobSchedulables, which represent a
- * single job, and PoolSchedulables, which allocate among jobs in their pool.
- * 
- * Separate sets of Schedulables are used for maps and reduces. Each pool has
- * both a mapSchedulable and a reduceSchedulable, and so does each job.
- * 
- * A Schedulable is responsible for three roles:
- * 1) It can launch tasks through assignTask().
- * 2) It provides information about the job/pool to the scheduler, including:
- *    - Demand (maximum number of tasks required)
- *    - Number of currently running tasks
- *    - Minimum share (for pools)
- *    - Job/pool weight (for fair sharing)
- *    - Start time and priority (for FIFO)
- * 3) It can be assigned a fair share, for use with fair scheduling.
- * 
- * Schedulable also contains two methods for performing scheduling computations:
- * - updateDemand() is called periodically to compute the demand of the various
- *   jobs and pools, which may be expensive (e.g. jobs must iterate through all
- *   their tasks to count failed tasks, tasks that can be speculated, etc).
- * - redistributeShare() is called after demands are updated and a Schedulable's
- *   fair share has been set by its parent to let it distribute its share among
- *   the other Schedulables within it (e.g. for pools that want to perform fair
- *   sharing among their jobs).
- */
-abstract class Schedulable {
-  /** Fair share assigned to this Schedulable */
-  private double fairShare = 0;
-  protected MetricsRecord metrics;
-  
-  /**
-   * Name of job/pool, used for debugging as well as for breaking ties in
-   * scheduling order deterministically. 
-   */
-  public abstract String getName();
-  
-  /**
-   * @return the type of tasks that this pool schedules
-   */
-  public abstract TaskType getTaskType();
-  
-  /**
-   * Maximum number of tasks required by this Schedulable. This is defined as
-   * number of currently running tasks + number of unlaunched tasks (tasks that
-   * are either not yet launched or need to be speculated).
-   */
-  public abstract int getDemand();
-  
-  /** Number of tasks the schedulable is currently running. */
-  public abstract int getRunningTasks();
-  
-  /** Minimum share slots assigned to the schedulable. */
-  public abstract int getMinShare();
-  
-  /** Job/pool weight in fair sharing. */
-  public abstract double getWeight();
-  
-  /** Job priority for jobs in FIFO pools; meaningless for PoolSchedulables. */
-  public abstract JobPriority getPriority();
-  
-  /** Start time for jobs in FIFO pools; meaningless for PoolSchedulables. */
-  public abstract long getStartTime();
-  
-  /** Refresh the Schedulable's demand and those of its children if any. */
-  public abstract void updateDemand();
-  
-  /** 
-   * Distribute the fair share assigned to this Schedulable among its 
-   * children (used in pools where the internal scheduler is fair sharing). 
-   */
-  public abstract void redistributeShare();
-  
-  /**
-   * Obtain a task for a given TaskTracker, or null if the Schedulable has
-   * no tasks to launch at this moment or does not wish to launch a task on
-   * this TaskTracker (e.g. is waiting for a TaskTracker with local data). 
-   * In addition, if a job is skipped during this search because it is waiting
-   * for a TaskTracker with local data, this method is expected to add it to
-   * the <tt>visited</tt> collection passed in, so that the scheduler can
-   * properly mark it as skipped during this heartbeat. Please see
-   * {@link FairScheduler#getAllowedLocalityLevel(JobInProgress, long)}
-   * for details of delay scheduling (waiting for trackers with local data).
-   * 
-   * @param tts      TaskTracker that the task will be launched on
-   * @param currentTime Cached time (to prevent excessive calls to gettimeofday)
-   * @param visited  A Collection to which this method must add all jobs that
-   *                 were considered during the search for a job to assign.
-   * @return Task to launch, or null if Schedulable cannot currently launch one.
-   * @throws IOException Possible if obtainNew(Map|Reduce)Task throws exception.
-   */
-  public abstract Task assignTask(TaskTrackerStatus tts, long currentTime,
-      Collection<JobInProgress> visited) throws IOException;
-
-  /** Assign a fair share to this Schedulable. */
-  public void setFairShare(double fairShare) {
-    this.fairShare = fairShare;
-  }
-  
-  /** Get the fair share assigned to this Schedulable. */
-  public double getFairShare() {
-    return fairShare;
-  }
-  
-  /** Return the name of the metrics context for this schedulable */
-  protected abstract String getMetricsContextName();
-  
-  /**
-   * Set up metrics context
-   */
-  protected void initMetrics() {
-    MetricsContext metricsContext = MetricsUtil.getContext("fairscheduler");
-    this.metrics = MetricsUtil.createRecord(metricsContext,
-        getMetricsContextName());
-    metrics.setTag("name", getName());
-    metrics.setTag("taskType", getTaskType().toString());
-  }
-
-  void cleanupMetrics() {
-    metrics.remove();
-    metrics = null;
-  }
-
-  protected void setMetricValues(MetricsRecord metrics) {
-    metrics.setMetric("fairShare", (float)getFairShare());
-    metrics.setMetric("minShare", getMinShare());
-    metrics.setMetric("demand", getDemand());
-    metrics.setMetric("weight", (float)getWeight());
-    metrics.setMetric("runningTasks", getRunningTasks());
-  }
-  
-  abstract void updateMetrics();
-  
-  /** Convenient toString implementation for debugging. */
-  @Override
-  public String toString() {
-    return String.format("[%s, demand=%d, running=%d, share=%.1f, w=%.1f]",
-        getName(), getDemand(), getRunningTasks(), fairShare, getWeight());
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/SchedulingAlgorithms.java

@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.Collection;
-import java.util.Comparator;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-/**
- * Utility class containing scheduling algorithms used in the fair scheduler.
- */
-class SchedulingAlgorithms {
-  public static final Log LOG = LogFactory.getLog(
-      SchedulingAlgorithms.class.getName());
-  
-  /**
-   * Compare Schedulables in order of priority and then submission time, as in
-   * the default FIFO scheduler in Hadoop.
-   */
-  public static class FifoComparator implements Comparator<Schedulable> {
-    @Override
-    public int compare(Schedulable s1, Schedulable s2) {
-      int res = s1.getPriority().compareTo(s2.getPriority());
-      if (res == 0) {
-        res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
-      }
-      if (res == 0) {
-        // In the rare case where jobs were submitted at the exact same time,
-        // compare them by name (which will be the JobID) to get a deterministic
-        // ordering, so we don't alternately launch tasks from different jobs.
-        res = s1.getName().compareTo(s2.getName());
-      }
-      return res;
-    }
-  }
-
-  /**
-   * Compare Schedulables via weighted fair sharing. In addition, Schedulables
-   * below their min share get priority over those whose min share is met. 
-   * 
-   * Schedulables below their min share are compared by how far below it they
-   * are as a ratio. For example, if job A has 8 out of a min share of 10 tasks
-   * and job B has 50 out of a min share of 100, then job B is scheduled next, 
-   * because B is at 50% of its min share and A is at 80% of its min share.
-   * 
-   * Schedulables above their min share are compared by (runningTasks / weight).
-   * If all weights are equal, slots are given to the job with the fewest tasks;
-   * otherwise, jobs with more weight get proportionally more slots.
-   */
-  public static class FairShareComparator implements Comparator<Schedulable> {
-    @Override
-    public int compare(Schedulable s1, Schedulable s2) {
-      double minShareRatio1, minShareRatio2;
-      double tasksToWeightRatio1, tasksToWeightRatio2;
-      int minShare1 = Math.min(s1.getMinShare(), s1.getDemand());
-      int minShare2 = Math.min(s2.getMinShare(), s2.getDemand());
-      boolean s1Needy = s1.getRunningTasks() < minShare1;
-      boolean s2Needy = s2.getRunningTasks() < minShare2;
-      minShareRatio1 = s1.getRunningTasks() / Math.max(minShare1, 1.0);
-      minShareRatio2 = s2.getRunningTasks() / Math.max(minShare2, 1.0);
-      tasksToWeightRatio1 = s1.getRunningTasks() / s1.getWeight();
-      tasksToWeightRatio2 = s2.getRunningTasks() / s2.getWeight();
-      int res = 0;
-      if (s1Needy && !s2Needy)
-        res = -1;
-      else if (s2Needy && !s1Needy)
-        res = 1;
-      else if (s1Needy && s2Needy)
-        res = (int) Math.signum(minShareRatio1 - minShareRatio2);
-      else // Neither schedulable is needy
-        res = (int) Math.signum(tasksToWeightRatio1 - tasksToWeightRatio2);
-      if (res == 0) {
-        // Jobs are tied in fairness ratio. Break the tie by submit time and job 
-        // name to get a deterministic ordering, which is useful for unit tests.
-        res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
-        if (res == 0)
-          res = s1.getName().compareTo(s2.getName());
-      }
-      return res;
-    }
-  }
-
-  /** 
-   * Number of iterations for the binary search in computeFairShares. This is 
-   * equivalent to the number of bits of precision in the output. 25 iterations 
-   * gives precision better than 0.1 slots in clusters with one million slots.
-   */
-  private static final int COMPUTE_FAIR_SHARES_ITERATIONS = 25;
-  
-  /**
-   * Given a set of Schedulables and a number of slots, compute their weighted
-   * fair shares. The min shares and demands of the Schedulables are assumed to
-   * be set beforehand. We compute the fairest possible allocation of shares 
-   * to the Schedulables that respects their min shares and demands.
-   * 
-   * To understand what this method does, we must first define what weighted
-   * fair sharing means in the presence of minimum shares and demands. If there
-   * were no minimum shares and every Schedulable had an infinite demand (i.e.
-   * could launch infinitely many tasks), then weighted fair sharing would be
-   * achieved if the ratio of slotsAssigned / weight was equal for each
-   * Schedulable and all slots were assigned. Minimum shares and demands add
-   * two further twists:
-   * - Some Schedulables may not have enough tasks to fill all their share.
-   * - Some Schedulables may have a min share higher than their assigned share.
-   * 
-   * To deal with these possibilities, we define an assignment of slots as
-   * being fair if there exists a ratio R such that:
-   * - Schedulables S where S.demand < R * S.weight are assigned share S.demand
-   * - Schedulables S where S.minShare > R * S.weight are given share S.minShare
-   * - All other Schedulables S are assigned share R * S.weight
-   * - The sum of all the shares is totalSlots.
-   * 
-   * We call R the weight-to-slots ratio because it converts a Schedulable's
-   * weight to the number of slots it is assigned.
-   * 
-   * We compute a fair allocation by finding a suitable weight-to-slot ratio R.
-   * To do this, we use binary search. Given a ratio R, we compute the number
-   * of slots that would be used in total with this ratio (the sum of the shares
-   * computed using the conditions above). If this number of slots is less than
-   * totalSlots, then R is too small and more slots could be assigned. If the
-   * number of slots is more than totalSlots, then R is too large. 
-   * 
-   * We begin the binary search with a lower bound on R of 0 (which means that
-   * all Schedulables are only given their minShare) and an upper bound computed
-   * to be large enough that too many slots are given (by doubling R until we
-   * either use more than totalSlots slots or we fulfill all jobs' demands).
-   * The helper method slotsUsedWithWeightToSlotRatio computes the total number
-   * of slots used with a given value of R.
-   * 
-   * The running time of this algorithm is linear in the number of Schedulables,
-   * because slotsUsedWithWeightToSlotRatio is linear-time and the number of
-   * iterations of binary search is a constant (dependent on desired precision).
-   */
-  public static void computeFairShares(
-      Collection<? extends Schedulable> schedulables, double totalSlots) {
-    // Find an upper bound on R that we can use in our binary search. We start 
-    // at R = 1 and double it until we have either used totalSlots slots or we
-    // have met all Schedulables' demands (if total demand < totalSlots).
-    double totalDemand = 0;
-    for (Schedulable sched: schedulables) {
-      totalDemand += sched.getDemand();
-    }
-    double cap = Math.min(totalDemand, totalSlots);
-    double rMax = 1.0;
-    while (slotsUsedWithWeightToSlotRatio(rMax, schedulables) < cap) {
-      rMax *= 2.0;
-    }
-    // Perform the binary search for up to COMPUTE_FAIR_SHARES_ITERATIONS steps
-    double left = 0;
-    double right = rMax;
-    for (int i = 0; i < COMPUTE_FAIR_SHARES_ITERATIONS; i++) {
-      double mid = (left + right) / 2.0;
-      if (slotsUsedWithWeightToSlotRatio(mid, schedulables) < cap) {
-        left = mid;
-      } else {
-        right = mid;
-      }
-    }
-    // Set the fair shares based on the value of R we've converged to
-    for (Schedulable sched: schedulables) {
-      sched.setFairShare(computeShare(sched, right));
-    }
-  }
-  
-  /**
-   * Compute the number of slots that would be used given a weight-to-slot
-   * ratio w2sRatio, for use in the computeFairShares algorithm as described
-   * in #{@link SchedulingAlgorithms#computeFairShares(Collection, double)}.
-   */
-  private static double slotsUsedWithWeightToSlotRatio(double w2sRatio,
-      Collection<? extends Schedulable> schedulables) {
-    double slotsTaken = 0;
-    for (Schedulable sched: schedulables) {
-      double share = computeShare(sched, w2sRatio);
-      slotsTaken += share;
-    }
-    return slotsTaken;
-  }
-
-  /**
-   * Compute the number of slots assigned to a Schedulable given a particular
-   * weight-to-slot ratio w2sRatio, for use in computeFairShares as described
-   * in #{@link SchedulingAlgorithms#computeFairShares(Collection, double)}.
-   */
-  private static double computeShare(Schedulable sched, double w2sRatio) {
-    double share = sched.getWeight() * w2sRatio;
-    share = Math.max(share, sched.getMinShare());
-    share = Math.min(share, sched.getDemand());
-    return share;
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/SchedulingMode.java

@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-/**
- * Internal scheduling modes for pools.
- */
-public enum SchedulingMode {
-  FAIR, FIFO
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/TaskSelector.java

@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-
-/**
- * A pluggable object for selecting tasks to run from a {@link JobInProgress} on
- * a given {@link TaskTracker}, for use by the {@link TaskScheduler}. The
- * <code>TaskSelector</code> is in charge of managing both locality and
- * speculative execution. For the latter purpose, it must also provide counts of
- * how many tasks each speculative job needs to launch, so that the scheduler
- * can take this into account in its calculations.
- */
-public abstract class TaskSelector implements Configurable {
-  protected Configuration conf;
-  protected TaskTrackerManager taskTrackerManager;
-  
-  public Configuration getConf() {
-    return conf;
-  }
-
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-  }
-
-  public synchronized void setTaskTrackerManager(
-      TaskTrackerManager taskTrackerManager) {
-    this.taskTrackerManager = taskTrackerManager;
-  }
-  
-  /**
-   * Lifecycle method to allow the TaskSelector to start any work in separate
-   * threads.
-   */
-  public void start() throws IOException {
-    // do nothing
-  }
-  
-  /**
-   * Lifecycle method to allow the TaskSelector to stop any work it is doing.
-   */
-  public void terminate() throws IOException {
-    // do nothing
-  }
-  
-  /**
-   * How many speculative map tasks does the given job want to launch?
-   * @param job The job to count speculative maps for
-   * @return Number of speculative maps that can be launched for job
-   */
-  public abstract int neededSpeculativeMaps(JobInProgress job);
-
-  /**
-   * How many speculative reduce tasks does the given job want to launch?
-   * @param job The job to count speculative reduces for
-   * @return Number of speculative reduces that can be launched for job
-   */
-  public abstract int neededSpeculativeReduces(JobInProgress job);
-  
-  /**
-   * Choose a map task to run from the given job on the given TaskTracker.
-   * @param taskTracker {@link TaskTrackerStatus} of machine to run on
-   * @param job Job to select a task for
-   * @return A {@link Task} to run on the machine, or <code>null</code> if
-   *         no map should be launched from this job on the task tracker.
-   * @throws IOException 
-   */
-  public abstract Task obtainNewMapTask(TaskTrackerStatus taskTracker,
-      JobInProgress job, int localityLevel) throws IOException;
-
-  /**
-   * Choose a reduce task to run from the given job on the given TaskTracker.
-   * @param taskTracker {@link TaskTrackerStatus} of machine to run on
-   * @param job Job to select a task for
-   * @return A {@link Task} to run on the machine, or <code>null</code> if
-   *         no reduce should be launched from this job on the task tracker.
-   * @throws IOException 
-   */
-  public abstract Task obtainNewReduceTask(TaskTrackerStatus taskTracker,
-      JobInProgress job) throws IOException;
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/WeightAdjuster.java

@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.mapreduce.TaskType;
-
-/**
- * A pluggable object for altering the weights of jobs in the fair scheduler,
- * which is used for example by {@link NewJobWeightBooster} to give higher
- * weight to new jobs so that short jobs finish faster.
- * 
- * May implement {@link Configurable} to access configuration parameters.
- */
-public interface WeightAdjuster {
-  public double adjustWeight(JobInProgress job, TaskType taskType,
-      double curWeight);
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/FakeSchedulable.java

@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.hadoop.mapreduce.TaskType;
-
-/**
- * Dummy implementation of Schedulable for unit testing.
- */
-public class FakeSchedulable extends Schedulable {
-  private int demand;
-  private int runningTasks;
-  private int minShare;
-  private double weight;
-  private JobPriority priority;
-  private long startTime;
-  
-  public FakeSchedulable() {
-    this(0, 0, 1, 0, 0, JobPriority.NORMAL, 0);
-  }
-  
-  public FakeSchedulable(int demand) {
-    this(demand, 0, 1, 0, 0, JobPriority.NORMAL, 0);
-  }
-  
-  public FakeSchedulable(int demand, int minShare) {
-    this(demand, minShare, 1, 0, 0, JobPriority.NORMAL, 0);
-  }
-  
-  public FakeSchedulable(int demand, int minShare, double weight) {
-    this(demand, minShare, weight, 0, 0, JobPriority.NORMAL, 0);
-  }
-  
-  public FakeSchedulable(int demand, int minShare, double weight, int fairShare,
-      int runningTasks, JobPriority priority, long startTime) {
-    this.demand = demand;
-    this.minShare = minShare;
-    this.weight = weight;
-    setFairShare(fairShare);
-    this.runningTasks = runningTasks;
-    this.priority = priority;
-    this.startTime = startTime;
-  }
-  
-  @Override
-  public Task assignTask(TaskTrackerStatus tts, long currentTime,
-      Collection<JobInProgress> visited) throws IOException {
-    return null;
-  }
-
-  @Override
-  public int getDemand() {
-    return demand;
-  }
-
-  @Override
-  public String getName() {
-    return "FakeSchedulable" + this.hashCode();
-  }
-
-  @Override
-  public JobPriority getPriority() {
-    return priority;
-  }
-
-  @Override
-  public int getRunningTasks() {
-    return runningTasks;
-  }
-
-  @Override
-  public long getStartTime() {
-    return startTime;
-  }
-  
-  @Override
-  public double getWeight() {
-    return weight;
-  }
-  
-  @Override
-  public int getMinShare() {
-    return minShare;
-  }
-
-  @Override
-  public void redistributeShare() {}
-
-  @Override
-  public void updateDemand() {}
-
-  @Override
-  public TaskType getTaskType() {
-    return TaskType.MAP;
-  }
-
-  @Override
-  protected String getMetricsContextName() {
-    return "fake";
-  }
-
-  @Override
-  void updateMetrics() {
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestCapBasedLoadManager.java

@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.TaskStatus.State;
-
-import junit.framework.TestCase;
-
-/**
- * Exercise the canAssignMap and canAssignReduce methods in 
- * CapBasedLoadManager.
- */
-public class TestCapBasedLoadManager extends TestCase {
-  
-  /**
-   * Returns a running MapTaskStatus.
-   */
-  private TaskStatus getRunningMapTaskStatus() {
-    TaskStatus ts = new MapTaskStatus();
-    ts.setRunState(State.RUNNING);
-    return ts;
-  }
-
-  /**
-   * Returns a running ReduceTaskStatus.
-   */
-  private TaskStatus getRunningReduceTaskStatus() {
-    TaskStatus ts = new ReduceTaskStatus();
-    ts.setRunState(State.RUNNING);
-    return ts;
-  }
-  
-  /**
-   * Returns a TaskTrackerStatus with the specified statistics. 
-   * @param mapCap        The capacity of map tasks 
-   * @param reduceCap     The capacity of reduce tasks
-   * @param runningMap    The number of running map tasks
-   * @param runningReduce The number of running reduce tasks
-   */
-  private TaskTrackerStatus getTaskTrackerStatus(int mapCap, int reduceCap, 
-      int runningMap, int runningReduce) {
-    List<TaskStatus> ts = new ArrayList<TaskStatus>();
-    for (int i = 0; i < runningMap; i++) {
-      ts.add(getRunningMapTaskStatus());
-    }
-    for (int i = 0; i < runningReduce; i++) {
-      ts.add(getRunningReduceTaskStatus());
-    }
-    TaskTrackerStatus tracker = new TaskTrackerStatus("tracker", 
-        "tracker_host", 1234, ts, 0, mapCap, reduceCap);
-    return tracker;
-  }
-
-  /**
-   * A single test of canAssignMap.
-   */
-  private void oneTestCanAssignMap(float maxDiff, int mapCap, int runningMap,
-      int totalMapSlots, int totalRunnableMap, boolean expected) {
-    
-    CapBasedLoadManager manager = new CapBasedLoadManager();
-    Configuration conf = new Configuration();
-    conf.setFloat("mapred.fairscheduler.load.max.diff", maxDiff);
-    manager.setConf(conf);
-    
-    TaskTrackerStatus ts = getTaskTrackerStatus(mapCap, 1, runningMap, 1);
-    
-    assertEquals( "When maxDiff=" + maxDiff + ", with totalRunnableMap=" 
-        + totalRunnableMap + " and totalMapSlots=" + totalMapSlots
-        + ", a tracker with runningMap=" + runningMap + " and mapCap="
-        + mapCap + " should " + (expected ? "" : "not ")
-        + "be able to take more Maps.",
-        expected,
-        manager.canAssignMap(ts, totalRunnableMap, totalMapSlots)
-        );
-  }
-  
-  
-  /** 
-   * Test canAssignMap method.
-   */
-  public void testCanAssignMap() {
-    oneTestCanAssignMap(0.0f, 5, 0, 50, 1, true);
-    oneTestCanAssignMap(0.0f, 5, 1, 50, 10, false);
-    oneTestCanAssignMap(0.2f, 5, 1, 50, 10, true);
-    oneTestCanAssignMap(0.0f, 5, 1, 50, 11, true);
-    oneTestCanAssignMap(0.0f, 5, 2, 50, 11, false);
-    oneTestCanAssignMap(0.3f, 5, 2, 50, 6, true);
-    oneTestCanAssignMap(1.0f, 5, 5, 50, 50, false);
-  }
-  
-  
-  /**
-   * A single test of canAssignReduce.
-   */
-  private void oneTestCanAssignReduce(float maxDiff, int ReduceCap,
-      int runningReduce, int totalReduceSlots, int totalRunnableReduce,
-      boolean expected) {
-    
-    CapBasedLoadManager manager = new CapBasedLoadManager();
-    Configuration conf = new Configuration();
-    conf.setFloat("mapred.fairscheduler.load.max.diff", maxDiff);
-    manager.setConf(conf);
-    
-    TaskTrackerStatus ts = getTaskTrackerStatus(1, ReduceCap, 1,
-        runningReduce);
-    
-    assertEquals( "When maxDiff=" + maxDiff + ", with totalRunnableReduce=" 
-        + totalRunnableReduce + " and totalReduceSlots=" + totalReduceSlots
-        + ", a tracker with runningReduce=" + runningReduce
-        + " and ReduceCap=" + ReduceCap + " should "
-        + (expected ? "" : "not ") + "be able to take more Reduces.",
-        expected,
-        manager.canAssignReduce(ts, totalRunnableReduce, totalReduceSlots)
-        );
-  }
-    
-  /** 
-   * Test canAssignReduce method.
-   */
-  public void testCanAssignReduce() {
-    oneTestCanAssignReduce(0.0f, 5, 0, 50, 1, true);
-    oneTestCanAssignReduce(0.0f, 5, 1, 50, 10, false);
-    oneTestCanAssignReduce(0.2f, 5, 1, 50, 10, true);
-    oneTestCanAssignReduce(0.0f, 5, 1, 50, 11, true);
-    oneTestCanAssignReduce(0.0f, 5, 2, 50, 11, false);
-    oneTestCanAssignReduce(0.3f, 5, 2, 50, 6, true);
-    oneTestCanAssignReduce(1.0f, 5, 5, 50, 50, false);
-  }
-  
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestComputeFairShares.java

@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-/**
- * Exercise the computeFairShares method in SchedulingAlgorithms.
- */
-public class TestComputeFairShares extends TestCase {
-  private List<Schedulable> scheds;
-  
-  @Override
-  protected void setUp() throws Exception {
-    scheds = new ArrayList<Schedulable>();
-  }
-  
-  /** 
-   * Basic test - pools with different demands that are all higher than their
-   * fair share (of 10 slots) should each get their fair share.
-   */
-  public void testEqualSharing() {
-    scheds.add(new FakeSchedulable(100));
-    scheds.add(new FakeSchedulable(50));
-    scheds.add(new FakeSchedulable(30));
-    scheds.add(new FakeSchedulable(20));
-    SchedulingAlgorithms.computeFairShares(scheds, 40);
-    verifyShares(10, 10, 10, 10);
-  }
-  
-  /**
-   * In this test, pool 4 has a smaller demand than the 40 / 4 = 10 slots that
-   * it would be assigned with equal sharing. It should only get the 3 slots
-   * it demands. The other pools must then split the remaining 37 slots, but
-   * pool 3, with 11 slots demanded, is now below its share of 37/3 ~= 12.3,
-   * so it only gets 11 slots. Pools 1 and 2 split the rest and get 13 each. 
-   */
-  public void testLowDemands() {
-    scheds.add(new FakeSchedulable(100));
-    scheds.add(new FakeSchedulable(50));
-    scheds.add(new FakeSchedulable(11));
-    scheds.add(new FakeSchedulable(3));
-    SchedulingAlgorithms.computeFairShares(scheds, 40);
-    verifyShares(13, 13, 11, 3);
-  }
-  
-  /**
-   * In this test, some pools have minimum shares set. Pool 1 has a min share
-   * of 20 so it gets 20 slots. Pool 2 also has a min share of 20, but its
-   * demand is only 10 so it can only get 10 slots. The remaining pools have
-   * 10 slots to split between them. Pool 4 gets 3 slots because its demand is
-   * only 3, and pool 3 gets the remaining 7 slots. Pool 4 also had a min share
-   * of 2 slots but this should not affect the outcome.
-   */
-  public void testMinShares() {
-    scheds.add(new FakeSchedulable(100, 20));
-    scheds.add(new FakeSchedulable(10, 20));
-    scheds.add(new FakeSchedulable(10, 0));
-    scheds.add(new FakeSchedulable(3, 2));
-    SchedulingAlgorithms.computeFairShares(scheds, 40);
-    verifyShares(20, 10, 7, 3);
-  }
-  
-  /**
-   * Basic test for weighted shares with no minimum shares and no low demands.
-   * Each pool should get slots in proportion to its weight.
-   */
-  public void testWeightedSharing() {
-    scheds.add(new FakeSchedulable(100, 0, 2.0));
-    scheds.add(new FakeSchedulable(50,  0, 1.0));
-    scheds.add(new FakeSchedulable(30,  0, 1.0));
-    scheds.add(new FakeSchedulable(20,  0, 0.5));
-    SchedulingAlgorithms.computeFairShares(scheds, 45);
-    verifyShares(20, 10, 10, 5);
-  }
-
-  /**
-   * Weighted sharing test where pools 1 and 2 are now given lower demands than
-   * above. Pool 1 stops at 10 slots, leaving 35. If the remaining pools split
-   * this into a 1:1:0.5 ratio, they would get 14:14:7 slots respectively, but
-   * pool 2's demand is only 11, so it only gets 11. The remaining 2 pools split
-   * the 24 slots left into a 1:0.5 ratio, getting 16 and 8 slots respectively.
-   */
-  public void testWeightedSharingWithLowDemands() {
-    scheds.add(new FakeSchedulable(10, 0, 2.0));
-    scheds.add(new FakeSchedulable(11, 0, 1.0));
-    scheds.add(new FakeSchedulable(30, 0, 1.0));
-    scheds.add(new FakeSchedulable(20, 0, 0.5));
-    SchedulingAlgorithms.computeFairShares(scheds, 45);
-    verifyShares(10, 11, 16, 8);
-  }
-
-  /**
-   * Weighted fair sharing test with min shares. As in the min share test above,
-   * pool 1 has a min share greater than its demand so it only gets its demand.
-   * Pool 3 has a min share of 15 even though its weight is very small, so it
-   * gets 15 slots. The remaining pools share the remaining 20 slots equally,
-   * getting 10 each. Pool 3's min share of 5 slots doesn't affect this.
-   */
-  public void testWeightedSharingWithMinShares() {
-    scheds.add(new FakeSchedulable(10, 20, 2.0));
-    scheds.add(new FakeSchedulable(11, 0, 1.0));
-    scheds.add(new FakeSchedulable(30, 5, 1.0));
-    scheds.add(new FakeSchedulable(20, 15, 0.5));
-    SchedulingAlgorithms.computeFairShares(scheds, 45);
-    verifyShares(10, 10, 10, 15);
-  }
-
-  /**
-   * Test that shares are computed accurately even when there are many more
-   * frameworks than available slots.
-   */
-  public void testSmallShares() {
-    scheds.add(new FakeSchedulable(10));
-    scheds.add(new FakeSchedulable(5));
-    scheds.add(new FakeSchedulable(3));
-    scheds.add(new FakeSchedulable(2));
-    SchedulingAlgorithms.computeFairShares(scheds, 1);
-    verifyShares(0.25, 0.25, 0.25, 0.25);
-  }
-
-  /**
-   * Test that shares are computed accurately even when the number of slots is
-   * very large.
-   */  
-  public void testLargeShares() {
-    int million = 1000 * 1000;
-    scheds.add(new FakeSchedulable(100 * million));
-    scheds.add(new FakeSchedulable(50 * million));
-    scheds.add(new FakeSchedulable(30 * million));
-    scheds.add(new FakeSchedulable(20 * million));
-    SchedulingAlgorithms.computeFairShares(scheds, 40 * million);
-    verifyShares(10 * million, 10 * million, 10 * million, 10 * million);
-  }
-
-  /**
-   * Test that having a pool with 0 demand doesn't confuse the algorithm.
-   */
-  public void testZeroDemand() {
-    scheds.add(new FakeSchedulable(100));
-    scheds.add(new FakeSchedulable(50));
-    scheds.add(new FakeSchedulable(30));
-    scheds.add(new FakeSchedulable(0));
-    SchedulingAlgorithms.computeFairShares(scheds, 30);
-    verifyShares(10, 10, 10, 0);
-  }
-  
-  /**
-   * Test that being called on an empty list doesn't confuse the algorithm.
-   */
-  public void testEmptyList() {
-    SchedulingAlgorithms.computeFairShares(scheds, 40);
-    verifyShares();
-  }
-  
-  /**
-   * Check that a given list of shares have been assigned to this.scheds.
-   */
-  private void verifyShares(double... shares) {
-    assertEquals(scheds.size(), shares.length);
-    for (int i = 0; i < shares.length; i++) {
-      assertEquals(shares[i], scheds.get(i).getFairShare(), 0.01);
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java

@@ -1,3022 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.FairScheduler.JobInfo;
-import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobHistory;
-import org.apache.hadoop.mapred.JobInProgress.KillInterruptedException;
-import org.apache.hadoop.mapred.UtilsForTests.FakeClock;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
-import org.apache.hadoop.mapreduce.split.JobSplit;
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.MetricsContext;
-import org.apache.hadoop.metrics.MetricsUtil;
-import org.apache.hadoop.metrics.spi.NoEmitMetricsContext;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-import org.apache.hadoop.net.Node;
-import org.mortbay.log.Log;
-
-public class TestFairScheduler extends TestCase {
-  final static String TEST_DIR = new File(System.getProperty("test.build.data",
-      "build/contrib/streaming/test/data")).getAbsolutePath();
-  final static String ALLOC_FILE = new File(TEST_DIR, 
-      "test-pools").getAbsolutePath();
-  
-  private static final String POOL_PROPERTY = "pool";
-  private static final String EXPLICIT_POOL_PROPERTY = "mapred.fairscheduler.pool";
-  
-  private static int jobCounter;
-  
-  class FakeJobInProgress extends JobInProgress {
-    
-    private FakeTaskTrackerManager taskTrackerManager;
-    private int mapCounter = 0;
-    private int reduceCounter = 0;
-    private final String[][] mapInputLocations; // Array of hosts for each map
-    private boolean initialized;
-    
-    public FakeJobInProgress(JobConf jobConf,
-        FakeTaskTrackerManager taskTrackerManager, 
-        String[][] mapInputLocations, JobTracker jt) throws IOException {
-      super(new JobID("test", ++jobCounter), jobConf, jt);
-      this.taskTrackerManager = taskTrackerManager;
-      this.mapInputLocations = mapInputLocations;
-      this.startTime = System.currentTimeMillis();
-      this.status = new JobStatus();
-      this.status.setRunState(JobStatus.PREP);
-      this.nonLocalMaps = new LinkedList<TaskInProgress>();
-      this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
-      this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
-      this.nonRunningReduces = new LinkedList<TaskInProgress>();   
-      this.runningReduces = new LinkedHashSet<TaskInProgress>();
-      this.jobHistory = new FakeJobHistory();
-      this.initialized = false;
-    }
-    
-    @Override
-    public synchronized void initTasks() throws IOException {
-      // initTasks is needed to create non-empty cleanup and setup TIP
-      // arrays, otherwise calls such as job.getTaskInProgress will fail
-      JobID jobId = getJobID();
-      JobConf conf = getJobConf();
-      String jobFile = "";
-      // create two cleanup tips, one map and one reduce.
-      cleanup = new TaskInProgress[2];
-      // cleanup map tip.
-      cleanup[0] = new TaskInProgress(jobId, jobFile, null, 
-              jobtracker, conf, this, numMapTasks, 1);
-      cleanup[0].setJobCleanupTask();
-      // cleanup reduce tip.
-      cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
-                         numReduceTasks, jobtracker, conf, this, 1);
-      cleanup[1].setJobCleanupTask();
-      // create two setup tips, one map and one reduce.
-      setup = new TaskInProgress[2];
-      // setup map tip.
-      setup[0] = new TaskInProgress(jobId, jobFile, null, 
-              jobtracker, conf, this, numMapTasks + 1, 1);
-      setup[0].setJobSetupTask();
-      // setup reduce tip.
-      setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
-                         numReduceTasks + 1, jobtracker, conf, this, 1);
-      setup[1].setJobSetupTask();
-      // create maps
-      numMapTasks = conf.getNumMapTasks();
-      maps = new TaskInProgress[numMapTasks];
-      // empty format
-      JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
-      for (int i = 0; i < numMapTasks; i++) {
-        String[] inputLocations = null;
-        if (mapInputLocations != null)
-          inputLocations = mapInputLocations[i];
-        maps[i] = new FakeTaskInProgress(getJobID(), i,
-            getJobConf(), this, inputLocations, split);
-        if (mapInputLocations == null) // Job has no locality info
-          nonLocalMaps.add(maps[i]);
-      }
-      // create reduces
-      numReduceTasks = conf.getNumReduceTasks();
-      reduces = new TaskInProgress[numReduceTasks];
-      for (int i = 0; i < numReduceTasks; i++) {
-        reduces[i] = new FakeTaskInProgress(getJobID(), i,
-            getJobConf(), this);
-      }
-      
-      initialized = true;
-    }
-    
-    @Override
-    public boolean inited() {
-      return initialized;
-    }
-
-    @Override
-    public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize,
-        int numUniqueHosts, int localityLevel) throws IOException {
-      for (int map = 0; map < maps.length; map++) {
-        FakeTaskInProgress tip = (FakeTaskInProgress) maps[map];
-        if (!tip.isRunning() && !tip.isComplete() &&
-            getLocalityLevel(tip, tts) < localityLevel) {
-          TaskAttemptID attemptId = getTaskAttemptID(tip);
-          JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
-          Task task = new MapTask("", attemptId, 0, split.getSplitIndex(), 1) {
-            @Override
-            public String toString() {
-              return String.format("%s on %s", getTaskID(), tts.getTrackerName());
-            }
-          };
-          runningMapTasks++;
-          tip.createTaskAttempt(task, tts.getTrackerName());
-          nonLocalRunningMaps.add(tip);
-          taskTrackerManager.startTask(tts.getTrackerName(), task, tip);
-          return task;
-        }
-      }
-      return null;
-    }
-    
-    @Override
-    public Task obtainNewReduceTask(final TaskTrackerStatus tts,
-        int clusterSize, int ignored) throws IOException {
-      for (int reduce = 0; reduce < reduces.length; reduce++) {
-        FakeTaskInProgress tip = 
-          (FakeTaskInProgress) reduces[reduce];
-        if (!tip.isRunning() && !tip.isComplete()) {
-          TaskAttemptID attemptId = getTaskAttemptID(tip);
-          Task task = new ReduceTask("", attemptId, 0, maps.length, 1) {
-            @Override
-            public String toString() {
-              return String.format("%s on %s", getTaskID(), tts.getTrackerName());
-            }
-          };
-          runningReduceTasks++;
-          tip.createTaskAttempt(task, tts.getTrackerName());
-          runningReduces.add(tip);
-          taskTrackerManager.startTask(tts.getTrackerName(), task, tip);
-          return task;
-        }
-      }
-      return null;
-    }
-    
-    public void mapTaskFinished(TaskInProgress tip) {
-      runningMapTasks--;
-      finishedMapTasks++;
-      nonLocalRunningMaps.remove(tip);
-    }
-    
-    public void reduceTaskFinished(TaskInProgress tip) {
-      runningReduceTasks--;
-      finishedReduceTasks++;
-      runningReduces.remove(tip);
-    }
-    
-    private TaskAttemptID getTaskAttemptID(TaskInProgress tip) {
-      JobID jobId = getJobID();
-      TaskType type = tip.isMapTask() ? TaskType.MAP : TaskType.REDUCE;
-      return new TaskAttemptID(jobId.getJtIdentifier(),
-          jobId.getId(), type, tip.getIdWithinJob(), tip.nextTaskId++);
-    }
-    
-    @Override
-    int getLocalityLevel(TaskInProgress tip, TaskTrackerStatus tts) {
-      FakeTaskInProgress ftip = (FakeTaskInProgress) tip;
-      if (ftip.inputLocations != null) {
-        // Check whether we're on the same host as an input split
-        for (String location: ftip.inputLocations) {
-          if (location.equals(tts.host)) {
-            return 0;
-          }
-        }
-        // Check whether we're on the same rack as an input split
-        for (String location: ftip.inputLocations) {
-          if (getRack(location).equals(getRack(tts.host))) {
-            return 1;
-          }
-        }
-        // Not on same rack or host
-        return 2;
-      } else {
-        // Job has no locality info  
-        return -1;
-      }
-    }
-  }
-  
-  class FakeTaskInProgress extends TaskInProgress {
-    private boolean isMap;
-    private FakeJobInProgress fakeJob;
-    private TreeMap<TaskAttemptID, String> activeTasks;
-    private TaskStatus taskStatus;
-    private boolean isComplete = false;
-    private String[] inputLocations;
-    
-    // Constructor for map
-    FakeTaskInProgress(JobID jId, int id, JobConf jobConf,
-        FakeJobInProgress job, String[] inputLocations, 
-        JobSplit.TaskSplitMetaInfo split) {
-      super(jId, "", split, null, jobConf, job, id, 1);
-      this.isMap = true;
-      this.fakeJob = job;
-      this.inputLocations = inputLocations;
-      activeTasks = new TreeMap<TaskAttemptID, String>();
-      taskStatus = TaskStatus.createTaskStatus(isMap);
-      taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
-    }
-
-    // Constructor for reduce
-    FakeTaskInProgress(JobID jId, int id, JobConf jobConf,
-        FakeJobInProgress job) {
-      super(jId, "", jobConf.getNumMapTasks(), id, null, jobConf, job, 1);
-      this.isMap = false;
-      this.fakeJob = job;
-      activeTasks = new TreeMap<TaskAttemptID, String>();
-      taskStatus = TaskStatus.createTaskStatus(isMap);
-      taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
-    }
-    
-    private void createTaskAttempt(Task task, String taskTracker) {
-      activeTasks.put(task.getTaskID(), taskTracker);
-      taskStatus = TaskStatus.createTaskStatus(isMap, task.getTaskID(),
-          0.5f, 1, TaskStatus.State.RUNNING, "", "", "", 
-          TaskStatus.Phase.STARTING, new Counters());
-      taskStatus.setStartTime(clock.getTime());
-    }
-    
-    @Override
-    TreeMap<TaskAttemptID, String> getActiveTasks() {
-      return activeTasks;
-    }
-    
-    public synchronized boolean isComplete() {
-      return isComplete;
-    }
-    
-    public boolean isRunning() {
-      return activeTasks.size() > 0;
-    }
-    
-    @Override
-    public TaskStatus getTaskStatus(TaskAttemptID taskid) {
-      return taskStatus;
-    }
-    
-    void killAttempt() {
-      if (isMap) {
-        fakeJob.mapTaskFinished(this);
-      }
-      else {
-        fakeJob.reduceTaskFinished(this);
-      }
-      activeTasks.clear();
-      taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
-    }
-    
-    void finishAttempt() {
-      isComplete = true;
-      if (isMap) {
-        fakeJob.mapTaskFinished(this);
-      }
-      else {
-        fakeJob.reduceTaskFinished(this);
-      }
-      activeTasks.clear();
-      taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
-    }
-  }
-  
-  static class FakeQueueManager extends QueueManager {
-    private Set<String> queues = null;
-    FakeQueueManager() {
-      super(new Configuration());
-    }
-    void setQueues(Set<String> queues) {
-      this.queues = queues;
-    }
-    public synchronized Set<String> getLeafQueueNames() {
-      return queues;
-    }
-  }
-  
-  static class FakeTaskTrackerManager implements TaskTrackerManager {
-    int maps = 0;
-    int reduces = 0;
-    int maxMapTasksPerTracker = 2;
-    int maxReduceTasksPerTracker = 2;
-    long ttExpiryInterval = 10 * 60 * 1000L; // default interval
-    List<JobInProgressListener> listeners =
-      new ArrayList<JobInProgressListener>();
-    Map<JobID, JobInProgress> jobs = new HashMap<JobID, JobInProgress>();
-    
-    private Map<String, TaskTracker> trackers =
-      new HashMap<String, TaskTracker>();
-    private Map<String, TaskStatus> statuses = 
-      new HashMap<String, TaskStatus>();
-    private Map<String, FakeTaskInProgress> tips = 
-      new HashMap<String, FakeTaskInProgress>();
-    private Map<String, TaskTrackerStatus> trackerForTip =
-      new HashMap<String, TaskTrackerStatus>();
-    
-    public FakeTaskTrackerManager(int numRacks, int numTrackersPerRack) {
-      int nextTrackerId = 1;
-      for (int rack = 1; rack <= numRacks; rack++) {
-        for (int node = 1; node <= numTrackersPerRack; node++) {
-          int id = nextTrackerId++;
-          String host = "rack" + rack + ".node" + node;
-          System.out.println("Creating TaskTracker tt" + id + " on " + host);
-          TaskTracker tt = new TaskTracker("tt" + id);
-          tt.setStatus(new TaskTrackerStatus("tt" + id, host, 0,
-              new ArrayList<TaskStatus>(), 0,
-              maxMapTasksPerTracker, maxReduceTasksPerTracker));
-          trackers.put("tt" + id, tt);
-        }
-      }
-    }
-    
-    @Override
-    public ClusterStatus getClusterStatus() {
-      int numTrackers = trackers.size();
-
-      return new ClusterStatus(numTrackers, 0,
-          ttExpiryInterval, maps, reduces,
-          numTrackers * maxMapTasksPerTracker,
-          numTrackers * maxReduceTasksPerTracker,
-          JobTrackerStatus.RUNNING);
-    }
-
-    @Override
-    public QueueManager getQueueManager() {
-      return null;
-    }
-    
-    @Override
-    public int getNumberOfUniqueHosts() {
-      return trackers.size();
-    }
-
-    @Override
-    public Collection<TaskTrackerStatus> taskTrackers() {
-      List<TaskTrackerStatus> statuses = new ArrayList<TaskTrackerStatus>();
-      for (TaskTracker tt : trackers.values()) {
-        statuses.add(tt.getStatus());
-      }
-      return statuses;
-    }
-
-
-    @Override
-    public void addJobInProgressListener(JobInProgressListener listener) {
-      listeners.add(listener);
-    }
-
-    @Override
-    public void removeJobInProgressListener(JobInProgressListener listener) {
-      listeners.remove(listener);
-    }
-    
-    @Override
-    public int getNextHeartbeatInterval() {
-      return JTConfig.JT_HEARTBEAT_INTERVAL_MIN_DEFAULT;
-    }
-
-    @Override
-    public void killJob(JobID jobid) {
-      return;
-    }
-
-    @Override
-    public JobInProgress getJob(JobID jobid) {
-      return jobs.get(jobid);
-    }
-
-    public void initJob (JobInProgress job) {
-      try {
-        job.initTasks();
-      } catch (KillInterruptedException e) {
-      } catch (IOException e) {
-      }
-    }
-    
-    public void failJob (JobInProgress job) {
-      // do nothing
-    }
-    
-    // Test methods
-    
-    public void submitJob(JobInProgress job) throws IOException {
-      jobs.put(job.getJobID(), job);
-      for (JobInProgressListener listener : listeners) {
-        listener.jobAdded(job);
-      }
-    }
-    
-    public TaskTracker getTaskTracker(String trackerID) {
-      return trackers.get(trackerID);
-    }
-    
-    public void startTask(String trackerName, Task t, FakeTaskInProgress tip) {
-      final boolean isMap = t.isMapTask();
-      if (isMap) {
-        maps++;
-      } else {
-        reduces++;
-      }
-      String attemptId = t.getTaskID().toString();
-      TaskStatus status = tip.getTaskStatus(t.getTaskID());
-      TaskTrackerStatus trackerStatus = trackers.get(trackerName).getStatus();
-      tips.put(attemptId, tip);
-      statuses.put(attemptId, status);
-      trackerForTip.put(attemptId, trackerStatus);
-      status.setRunState(TaskStatus.State.RUNNING);
-      trackerStatus.getTaskReports().add(status);
-    }
-    
-    public void finishTask(String taskTrackerName, String attemptId) {
-      FakeTaskInProgress tip = tips.get(attemptId);
-      if (tip.isMapTask()) {
-        maps--;
-      } else {
-        reduces--;
-      }
-      tip.finishAttempt();
-      TaskStatus status = statuses.get(attemptId);
-      trackers.get(taskTrackerName).getStatus().getTaskReports().remove(status);
-    }
-
-    @Override
-    public boolean killTask(TaskAttemptID attemptId, boolean shouldFail) {
-      String attemptIdStr = attemptId.toString();
-      FakeTaskInProgress tip = tips.get(attemptIdStr);
-      if (tip.isMapTask()) {
-        maps--;
-      } else {
-        reduces--;
-      }
-      tip.killAttempt();
-      TaskStatus status = statuses.get(attemptIdStr);
-      trackerForTip.get(attemptIdStr).getTaskReports().remove(status);
-      return true;
-    }
-  }
-  
-  protected JobConf conf;
-  protected FairScheduler scheduler;
-  private FakeTaskTrackerManager taskTrackerManager;
-  private FakeClock clock;
-
-  @Override
-  protected void setUp() throws Exception {
-    jobCounter = 0;
-    new File(TEST_DIR).mkdirs(); // Make sure data directory exists
-    // Create an empty pools file (so we can add/remove pools later)
-    FileWriter fileWriter = new FileWriter(ALLOC_FILE);
-    fileWriter.write("<?xml version=\"1.0\"?>\n");
-    fileWriter.write("<allocations />\n");
-    fileWriter.close();
-    setUpCluster(1, 2, false);
-  }
-
-  public String getRack(String hostname) {
-    // Host names are of the form rackN.nodeM, so split at the dot.
-    return hostname.split("\\.")[0];
-  }
-
-  private void setUpCluster(int numRacks, int numNodesPerRack,
-      boolean assignMultiple) throws IOException {
-    
-    resetMetrics();
-    
-    conf = new JobConf();
-    conf.set("mapred.fairscheduler.allocation.file", ALLOC_FILE);
-    conf.set("mapred.fairscheduler.poolnameproperty", POOL_PROPERTY);
-    conf.setBoolean("mapred.fairscheduler.assignmultiple", assignMultiple);
-    // Manually set locality delay because we aren't using a JobTracker so
-    // we can't auto-compute it from the heartbeat interval.
-    conf.setLong("mapred.fairscheduler.locality.delay.node", 5000);
-    conf.setLong("mapred.fairscheduler.locality.delay.rack", 10000);
-    taskTrackerManager = new FakeTaskTrackerManager(numRacks, numNodesPerRack);
-    clock = new FakeClock();
-    scheduler = new FairScheduler(clock, true);
-    scheduler.waitForMapsBeforeLaunchingReduces = false;
-    scheduler.setConf(conf);
-    scheduler.setTaskTrackerManager(taskTrackerManager);
-    scheduler.start();
-  }
-  
-  /**
-   * Set up a metrics context that doesn't emit anywhere but stores the data
-   * so we can verify it. Also clears it of any data so that different test
-   * cases don't pollute each other.
-   */
-  private void resetMetrics() throws IOException {
-    ContextFactory factory = ContextFactory.getFactory();
-    factory.setAttribute("fairscheduler.class",
-        NoEmitMetricsContext.class.getName());
-    
-    MetricsUtil.getContext("fairscheduler").createRecord("jobs").remove();
-    MetricsUtil.getContext("fairscheduler").createRecord("pools").remove();
-  }
-
-  @Override
-  protected void tearDown() throws Exception {
-    if (scheduler != null) {
-      scheduler.terminate();
-    }
-  }
-  
-  private JobInProgress submitJobNotInitialized(int state, int maps, int reduces)
-	    throws IOException {
-    return submitJob(state, maps, reduces, null, null, false);
-  }
-
-  private JobInProgress submitJob(int state, int maps, int reduces)
-      throws IOException {
-    return submitJob(state, maps, reduces, null, null, true);
-  }
-  
-  private JobInProgress submitJob(int state, int maps, int reduces, String pool)
-      throws IOException {
-    return submitJob(state, maps, reduces, pool, null, true);
-  }
-  
-  private JobInProgress submitJob(int state, int maps, int reduces, String pool,
-      String[][] mapInputLocations, boolean initializeJob) throws IOException {
-    JobConf jobConf = new JobConf(conf);
-    jobConf.setNumMapTasks(maps);
-    jobConf.setNumReduceTasks(reduces);
-    if (pool != null)
-      jobConf.set(POOL_PROPERTY, pool);
-    JobInProgress job = new FakeJobInProgress(jobConf, taskTrackerManager,
-        mapInputLocations, UtilsForTests.getJobTracker());
-    if (initializeJob) {
-      taskTrackerManager.initJob(job);
-    }
-    job.getStatus().setRunState(state);
-    taskTrackerManager.submitJob(job);
-    job.startTime = clock.time;
-    return job;
-  }
-  
-  protected void submitJobs(int number, int state, int maps, int reduces)
-    throws IOException {
-    for (int i = 0; i < number; i++) {
-      submitJob(state, maps, reduces);
-    }
-  }
-
-  public void testAllocationFileParsing() throws Exception {
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>"); 
-    // Give pool A a minimum of 1 map, 2 reduces
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>1</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("</pool>");
-    // Give pool B a minimum of 2 maps, 1 reduce
-    out.println("<pool name=\"poolB\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>1</minReduces>");
-    out.println("</pool>");
-    // Give pool C min maps but no min reduces
-    out.println("<pool name=\"poolC\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("</pool>");
-    // Give pool D a limit of 3 running jobs
-    out.println("<pool name=\"poolD\">");
-    out.println("<maxRunningJobs>3</maxRunningJobs>");
-    out.println("</pool>");
-    // Give pool E a preemption timeout of one minute
-    out.println("<pool name=\"poolE\">");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    // Set default limit of jobs per pool to 15
-    out.println("<poolMaxJobsDefault>15</poolMaxJobsDefault>");
-    // Set default limit of jobs per user to 5
-    out.println("<userMaxJobsDefault>5</userMaxJobsDefault>");
-    // Give user1 a limit of 10 jobs
-    out.println("<user name=\"user1\">");
-    out.println("<maxRunningJobs>10</maxRunningJobs>");
-    out.println("</user>");
-    // Set default min share preemption timeout to 2 minutes
-    out.println("<defaultMinSharePreemptionTimeout>120" 
-        + "</defaultMinSharePreemptionTimeout>"); 
-    // Set fair share preemption timeout to 5 minutes
-    out.println("<fairSharePreemptionTimeout>300</fairSharePreemptionTimeout>"); 
-    out.println("</allocations>"); 
-    out.close();
-    
-    PoolManager poolManager = scheduler.getPoolManager();
-    poolManager.reloadAllocs();
-    
-    assertEquals(6, poolManager.getPools().size()); // 5 in file + default pool
-    assertEquals(0, poolManager.getAllocation(Pool.DEFAULT_POOL_NAME,
-        TaskType.MAP));
-    assertEquals(0, poolManager.getAllocation(Pool.DEFAULT_POOL_NAME,
-        TaskType.REDUCE));
-    assertEquals(1, poolManager.getAllocation("poolA", TaskType.MAP));
-    assertEquals(2, poolManager.getAllocation("poolA", TaskType.REDUCE));
-    assertEquals(2, poolManager.getAllocation("poolB", TaskType.MAP));
-    assertEquals(1, poolManager.getAllocation("poolB", TaskType.REDUCE));
-    assertEquals(2, poolManager.getAllocation("poolC", TaskType.MAP));
-    assertEquals(0, poolManager.getAllocation("poolC", TaskType.REDUCE));
-    assertEquals(0, poolManager.getAllocation("poolD", TaskType.MAP));
-    assertEquals(0, poolManager.getAllocation("poolD", TaskType.REDUCE));
-    assertEquals(0, poolManager.getAllocation("poolE", TaskType.MAP));
-    assertEquals(0, poolManager.getAllocation("poolE", TaskType.REDUCE));
-    assertEquals(15, poolManager.getPoolMaxJobs(Pool.DEFAULT_POOL_NAME));
-    assertEquals(15, poolManager.getPoolMaxJobs("poolA"));
-    assertEquals(15, poolManager.getPoolMaxJobs("poolB"));
-    assertEquals(15, poolManager.getPoolMaxJobs("poolC"));
-    assertEquals(3, poolManager.getPoolMaxJobs("poolD"));
-    assertEquals(15, poolManager.getPoolMaxJobs("poolE"));
-    assertEquals(10, poolManager.getUserMaxJobs("user1"));
-    assertEquals(5, poolManager.getUserMaxJobs("user2"));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout(
-        Pool.DEFAULT_POOL_NAME));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("poolA"));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("poolB"));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("poolC"));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("poolD"));
-    assertEquals(120000, poolManager.getMinSharePreemptionTimeout("poolA"));
-    assertEquals(60000, poolManager.getMinSharePreemptionTimeout("poolE"));
-    assertEquals(300000, poolManager.getFairSharePreemptionTimeout());
-  }
-  
-  public void testTaskNotAssignedWhenNoJobsArePresent() throws IOException {
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-  }
-
-  public void testNonRunningJobsAreIgnored() throws IOException {
-    submitJobs(1, JobStatus.SUCCEEDED, 10, 10);
-    submitJobs(1, JobStatus.FAILED, 10, 10);
-    submitJobs(1, JobStatus.KILLED, 10, 10);
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    advanceTime(100); // Check that we still don't assign jobs after an update
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-  }
-
-  /**
-   * This test contains two jobs with fewer required tasks than there are slots.
-   * We check that all tasks are assigned, but job 1 gets them first because it
-   * was submitted earlier.
-   */
-  public void testSmallJobs() throws IOException {
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 2, 1);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,    info1.mapSchedulable.getDemand());
-    assertEquals(1,    info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
-    verifyMetrics();
-
-    // Advance time before submitting another job j2, to make j1 run before j2
-    // deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 2);
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,    info1.mapSchedulable.getDemand());
-    assertEquals(1,    info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(1,    info2.mapSchedulable.getDemand());
-    assertEquals(2,    info2.reduceSchedulable.getDemand());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    verifyMetrics();
-    
-    // Assign tasks and check that jobs alternate in filling slots
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // Check that the scheduler has started counting the tasks as running
-    // as soon as it launched them.
-    assertEquals(2,  info1.mapSchedulable.getRunningTasks());
-    assertEquals(1,  info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,  info1.mapSchedulable.getDemand());
-    assertEquals(1,  info1.reduceSchedulable.getDemand());
-    assertEquals(1,  info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info2.reduceSchedulable.getRunningTasks());
-    assertEquals(1, info2.mapSchedulable.getDemand());
-    assertEquals(2, info2.reduceSchedulable.getDemand());
-    verifyMetrics();
-  }
-  /**
-   * This test is identical to testSmallJobs but sets assignMultiple to
-   * true so that multiple tasks can be assigned per heartbeat.
-   */
-  public void testSmallJobsWithAssignMultiple() throws IOException {
-    setUpCluster(1, 2, true);
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 2, 1);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,    info1.mapSchedulable.getDemand());
-    assertEquals(1,    info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
-    verifyMetrics();
-    
-    // Advance time before submitting another job j2, to make j1 run before j2
-    // deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 2);
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,    info1.mapSchedulable.getDemand());
-    assertEquals(1,    info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(1,    info2.mapSchedulable.getDemand());
-    assertEquals(2,    info2.reduceSchedulable.getDemand());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    verifyMetrics();
-    
-    // Assign tasks and check that jobs alternate in filling slots
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1",
-                           "attempt_test_0001_r_000000_0 on tt1",
-                           "attempt_test_0002_m_000000_0 on tt1",
-                           "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2",
-                           "attempt_test_0002_r_000001_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // Check that the scheduler has started counting the tasks as running
-    // as soon as it launched them.
-    assertEquals(2,  info1.mapSchedulable.getRunningTasks());
-    assertEquals(1,  info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2,  info1.mapSchedulable.getDemand());
-    assertEquals(1,  info1.reduceSchedulable.getDemand());
-    assertEquals(1,  info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info2.reduceSchedulable.getRunningTasks());
-    assertEquals(1, info2.mapSchedulable.getDemand());
-    assertEquals(2, info2.reduceSchedulable.getDemand());
-    verifyMetrics();
-  }
-  
-  /**
-   * This test begins by submitting two jobs with 10 maps and reduces each.
-   * The first job is submitted 100ms after the second, to make it get slots
-   * first deterministically. We then assign a wave of tasks and check that
-   * they are given alternately to job1, job2, job1, job2, etc. We finish
-   * these tasks and assign a second wave, which should continue to be
-   * allocated in this manner.
-   */
-  public void testLargeJobs() throws IOException {
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    
-    // Advance time before submitting another job j2, to make j1 run before j2
-    // deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info2.mapSchedulable.getDemand());
-    assertEquals(10,   info2.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    
-    // Check that tasks are filled alternately by the jobs
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    
-    // Check that no new tasks can be launched once the tasktrackers are full
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // Check that the scheduler has started counting the tasks as running
-    // as soon as it launched them.
-    assertEquals(2,  info1.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,  info1.mapSchedulable.getDemand());
-    assertEquals(10,  info1.reduceSchedulable.getDemand());
-    assertEquals(2,  info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info2.reduceSchedulable.getRunningTasks());
-    assertEquals(10, info2.mapSchedulable.getDemand());
-    assertEquals(10, info2.reduceSchedulable.getDemand());
-    
-    // Finish up the tasks and advance time again. Note that we must finish
-    // the task since FakeJobInProgress does not properly maintain running
-    // tasks, so the scheduler will always get an empty task list from
-    // the JobInProgress's getTasks(TaskType.MAP)/getTasks(TaskType.REDUCE) and 
-    // think they finished.
-    taskTrackerManager.finishTask("tt1", "attempt_test_0001_m_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_m_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0001_r_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_r_000000_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0001_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0001_r_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_r_000001_0");
-    advanceTime(200);
-    assertEquals(0,   info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info1.reduceSchedulable.getRunningTasks());
-    assertEquals(0,   info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info2.reduceSchedulable.getRunningTasks());
-
-    // Check that tasks are filled alternately by the jobs
-    checkAssignment("tt1", "attempt_test_0001_m_000002_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000002_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_m_000002_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000002_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000003_0 on tt2");
-    
-    // Check scheduler variables; the demands should now be 8 because 2 tasks
-    // of each type have finished in each job
-    assertEquals(2,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(2,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(8,   info1.mapSchedulable.getDemand());
-    assertEquals(8,   info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(2,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(8,   info2.mapSchedulable.getDemand());
-    assertEquals(8,   info2.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-  }
-  
-  /**
-   * A copy of testLargeJobs that enables the assignMultiple feature to launch
-   * multiple tasks per heartbeat. Results should be the same as testLargeJobs.
-   */
-  public void testLargeJobsWithAssignMultiple() throws IOException {
-    setUpCluster(1, 2, true);
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    
-    // Advance time before submitting another job j2, to make j1 run before j2
-    // deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check scheduler variables; the fair shares should now have been allocated
-    // equally between j1 and j2, but j1 should have (4 slots)*(100 ms) deficit
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info2.mapSchedulable.getDemand());
-    assertEquals(10,   info2.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    
-    // Check that tasks are filled alternately by the jobs
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1",
-                           "attempt_test_0001_r_000000_0 on tt1",
-                           "attempt_test_0002_m_000000_0 on tt1",
-                           "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2",
-                           "attempt_test_0001_r_000001_0 on tt2",
-                           "attempt_test_0002_m_000001_0 on tt2",
-                           "attempt_test_0002_r_000001_0 on tt2");
-    
-    // Check that no new tasks can be launched once the tasktrackers are full
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // Check that the scheduler has started counting the tasks as running
-    // as soon as it launched them.
-    assertEquals(2,  info1.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,  info1.mapSchedulable.getDemand());
-    assertEquals(10,  info1.reduceSchedulable.getDemand());
-    assertEquals(2,  info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,  info2.reduceSchedulable.getRunningTasks());
-    assertEquals(10, info2.mapSchedulable.getDemand());
-    assertEquals(10, info2.reduceSchedulable.getDemand());
-    
-    // Finish up the tasks and advance time again. Note that we must finish
-    // the task since FakeJobInProgress does not properly maintain running
-    // tasks, so the scheduler will always get an empty task list from
-    // the JobInProgress's getTasks(TaskType.MAP)/getTasks(TaskType.REDUCE) and
-    // think they finished.
-    taskTrackerManager.finishTask("tt1", "attempt_test_0001_m_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_m_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0001_r_000000_0");
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_r_000000_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0001_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0001_r_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_r_000001_0");
-    advanceTime(200);
-    assertEquals(0,   info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info1.reduceSchedulable.getRunningTasks());
-    assertEquals(0,   info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info2.reduceSchedulable.getRunningTasks());
-
-    // Check that tasks are filled alternately by the jobs
-    checkAssignment("tt1", "attempt_test_0001_m_000002_0 on tt1",
-                           "attempt_test_0001_r_000002_0 on tt1",
-                           "attempt_test_0002_m_000002_0 on tt1",
-                           "attempt_test_0002_r_000002_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2",
-                           "attempt_test_0001_r_000003_0 on tt2",
-                           "attempt_test_0002_m_000003_0 on tt2",
-                           "attempt_test_0002_r_000003_0 on tt2");
-    
-    // Check scheduler variables; the demands should now be 8 because 2 tasks
-    // of each type have finished in each job
-    assertEquals(2,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(2,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(8,   info1.mapSchedulable.getDemand());
-    assertEquals(8,   info1.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(2,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(2,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(8,   info2.mapSchedulable.getDemand());
-    assertEquals(8,   info2.reduceSchedulable.getDemand());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-  }
-
-  /**
-   * We submit two jobs such that one has 2x the priority of the other to 
-   * a cluster of 3 nodes, wait for 100 ms, and check that the weights/shares 
-   * the high-priority job gets 4 tasks while the normal-priority job gets 2.
-   */
-  public void testJobsWithPriorities() throws IOException {
-    setUpCluster(1, 3, false);
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info2 = scheduler.infos.get(job2);
-    job2.setPriority(JobPriority.HIGH);
-    scheduler.update();
-    
-    // Check scheduler variables
-    assertEquals(0,   info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,  info1.mapSchedulable.getDemand());
-    assertEquals(10,  info1.reduceSchedulable.getDemand());
-    assertEquals(2.0, info1.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(2.0, info1.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0,   info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,   info2.reduceSchedulable.getRunningTasks());
-    assertEquals(10,  info2.mapSchedulable.getDemand());
-    assertEquals(10,  info2.reduceSchedulable.getDemand());
-    assertEquals(4.0, info2.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(4.0, info2.reduceSchedulable.getFairShare(), 0.1);
-    
-    // Advance time
-    advanceTime(100);
-    
-    // Assign tasks and check that j2 gets 2x more tasks than j1. In addition,
-    // whenever the jobs' runningTasks/weight ratios are tied, j1 should get
-    // the new task first because it started first; thus the tasks of each
-    // type should be handed out alternately to 1, 2, 2, 1, 2, 2, etc.
-    System.out.println("HEREEEE");
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt3", "attempt_test_0002_m_000002_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0002_r_000002_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0002_m_000003_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0002_r_000003_0 on tt3");
-  }
-  
-  /**
-   * This test starts by submitting three large jobs:
-   * - job1 in the default pool, at time 0
-   * - job2 in poolA, with an allocation of 1 map / 2 reduces, at time 200
-   * - job3 in poolB, with an allocation of 2 maps / 1 reduce, at time 300
-   * 
-   * We then assign tasks to all slots. The maps should be assigned in the
-   * order job2, job3, job 3, job1 because jobs 3 and 2 have guaranteed slots
-   * (1 and 2 respectively). Job2 comes before job3 when they are both at 0
-   * slots because it has an earlier start time. In a similar manner,
-   * reduces should be assigned as job2, job3, job2, job1.
-   */
-  public void testLargeJobsWithPools() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a minimum of 1 map, 2 reduces
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>1</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("</pool>");
-    // Give pool B a minimum of 2 maps, 1 reduce
-    out.println("<pool name=\"poolB\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>1</minReduces>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool defaultPool = scheduler.getPoolManager().getPool("default");
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-    Pool poolB = scheduler.getPoolManager().getPool("poolB");
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    
-    // Advance time 200ms and submit jobs 2 and 3
-    advanceTime(200);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    advanceTime(100);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    JobInfo info3 = scheduler.infos.get(job3);
-    
-    // Check that minimum and fair shares have been allocated
-    assertEquals(0,    defaultPool.getMapSchedulable().getMinShare());
-    assertEquals(0,    defaultPool.getReduceSchedulable().getMinShare());
-    assertEquals(1.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(1,    poolA.getMapSchedulable().getMinShare());
-    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    assertEquals(2,    poolB.getMapSchedulable().getMinShare());
-    assertEquals(1,    poolB.getReduceSchedulable().getMinShare());
-    assertEquals(2.0,  info3.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info3.reduceSchedulable.getFairShare());
-    
-    // Advance time 100ms
-    advanceTime(100);
-    
-    // Assign tasks and check that slots are first given to needy jobs
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0003_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000000_0 on tt2");
-  }
-
-  /**
-   * This test starts by submitting three large jobs:
-   * - job1 in the default pool, at time 0
-   * - job2 in poolA, with an allocation of 2 maps / 2 reduces, at time 200
-   * - job3 in poolA, with an allocation of 2 maps / 2 reduces, at time 300
-   * 
-   * After this, we start assigning tasks. The first two tasks of each type
-   * should be assigned to job2 and job3 since they are in a pool with an
-   * allocation guarantee, but the next two slots should be assigned to job 3
-   * because the pool will no longer be needy.
-   */
-  public void testLargeJobsWithExcessCapacity() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a minimum of 2 maps, 2 reduces
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    
-    // Advance time 200ms and submit job 2
-    advanceTime(200);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check that minimum and fair shares have been allocated
-    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
-    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    
-    // Advance time 100ms and submit job 3
-    advanceTime(100);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info3 = scheduler.infos.get(job3);
-    
-    // Check that minimum and fair shares have been allocated
-    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
-    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info2.reduceSchedulable.getFairShare());
-    assertEquals(1.0,  info3.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info3.reduceSchedulable.getFairShare());
-    
-    // Advance time
-    advanceTime(100);
-    
-    // Assign tasks and check that slots are first given to needy jobs, but
-    // that job 1 gets two tasks after due to having a larger share.
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-  }
-  
-  /**
-   * A copy of testLargeJobsWithExcessCapacity that enables assigning multiple
-   * tasks per heartbeat. Results should match testLargeJobsWithExcessCapacity.
-   */
-  public void testLargeJobsWithExcessCapacityAndAssignMultiple() 
-      throws Exception {
-    setUpCluster(1, 2, true);
-    
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a minimum of 2 maps, 2 reduces
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    
-    // Advance time 200ms and submit job 2
-    advanceTime(200);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check that minimum and fair shares have been allocated
-    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
-    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    
-    // Advance time 100ms and submit job 3
-    advanceTime(100);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info3 = scheduler.infos.get(job3);
-    
-    // Check that minimum and fair shares have been allocated
-    assertEquals(2,    poolA.getMapSchedulable().getMinShare());
-    assertEquals(2,    poolA.getReduceSchedulable().getMinShare());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info2.reduceSchedulable.getFairShare());
-    assertEquals(1.0,  info3.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info3.reduceSchedulable.getFairShare());
-    
-    // Advance time
-    advanceTime(100);
-    
-    // Assign tasks and check that slots are first given to needy jobs, but
-    // that job 1 gets two tasks after due to having a larger share.
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1",
-                           "attempt_test_0002_r_000000_0 on tt1",
-                           "attempt_test_0003_m_000000_0 on tt1",
-                           "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000000_0 on tt2",
-                           "attempt_test_0001_r_000000_0 on tt2",
-                           "attempt_test_0001_m_000001_0 on tt2",
-                           "attempt_test_0001_r_000001_0 on tt2");
-  }
-  
-  /**
-   * This test starts by submitting two jobs at time 0:
-   * - job1 in the default pool
-   * - job2, with 1 map and 1 reduce, in poolA, which has an alloc of 4
-   *   maps and 4 reduces
-   * 
-   * When we assign the slots, job2 should only get 1 of each type of task.
-   */
-  public void testSmallJobInLargePool() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a minimum of 4 maps, 4 reduces
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>4</minMaps>");
-    out.println("<minReduces>4</minReduces>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 1, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    
-    // Check scheduler variables
-    assertEquals(0,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(10,   info1.mapSchedulable.getDemand());
-    assertEquals(10,   info1.reduceSchedulable.getDemand());
-    assertEquals(3.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(3.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0,    info2.mapSchedulable.getRunningTasks());
-    assertEquals(0,    info2.reduceSchedulable.getRunningTasks());
-    assertEquals(1,    info2.mapSchedulable.getDemand());
-    assertEquals(1,    info2.reduceSchedulable.getDemand());
-    assertEquals(1.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(1.0,  info2.reduceSchedulable.getFairShare());
-    
-    // Assign tasks and check that slots are first given to needy jobs
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-  }
-  
-  /**
-   * This test starts by submitting four jobs in the default pool. However, the
-   * maxRunningJobs limit for this pool has been set to two. We should see only
-   * the first two jobs get scheduled, each with half the total slots.
-   */
-  public void testPoolMaxJobs() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"default\">");
-    out.println("<maxRunningJobs>2</maxRunningJobs>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
-    assertTrue(((FakeJobInProgress)job1).inited());
-    job1.getStatus().setRunState(JobStatus.RUNNING);
-    JobInfo info1 = scheduler.infos.get(job1);
-    advanceTime(10);
-    JobInProgress job2 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
-    assertTrue(((FakeJobInProgress)job2).inited());
-    job2.getStatus().setRunState(JobStatus.RUNNING);
-    JobInfo info2 = scheduler.infos.get(job2);
-    advanceTime(10);
-    JobInProgress job3 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
-    JobInfo info3 = scheduler.infos.get(job3);
-    advanceTime(10);
-    JobInProgress job4 = submitJobNotInitialized(JobStatus.PREP, 10, 10);
-    JobInfo info4 = scheduler.infos.get(job4);
-    
-    // Only two of the jobs should be initialized.
-    assertTrue(((FakeJobInProgress)job1).inited());
-    assertTrue(((FakeJobInProgress)job2).inited());
-    assertFalse(((FakeJobInProgress)job3).inited());
-    assertFalse(((FakeJobInProgress)job4).inited());
-    
-    // Check scheduler variables
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(2.0,  info2.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info2.reduceSchedulable.getFairShare());
-    assertEquals(0.0,  info3.mapSchedulable.getFairShare());
-    assertEquals(0.0,  info3.reduceSchedulable.getFairShare());
-    assertEquals(0.0,  info4.mapSchedulable.getFairShare());
-    assertEquals(0.0,  info4.reduceSchedulable.getFairShare());
-    
-    // Assign tasks and check that only jobs 1 and 2 get them
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-  }
-
-  /**
-   * This test starts by submitting two jobs by user "user1" to the default
-   * pool, and two jobs by "user2". We set user1's job limit to 1. We should
-   * see one job from user1 and two from user2. 
-   */
-  public void testUserMaxJobs() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<user name=\"user1\">");
-    out.println("<maxRunningJobs>1</maxRunningJobs>");
-    out.println("</user>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    job1.getJobConf().set(JobContext.USER_NAME, "user1");
-    JobInfo info1 = scheduler.infos.get(job1);
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
-    job2.getJobConf().set(JobContext.USER_NAME, "user1");
-    JobInfo info2 = scheduler.infos.get(job2);
-    advanceTime(10);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10);
-    job3.getJobConf().set(JobContext.USER_NAME, "user2");
-    JobInfo info3 = scheduler.infos.get(job3);
-    advanceTime(10);
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10);
-    job4.getJobConf().set(JobContext.USER_NAME, "user2");
-    JobInfo info4 = scheduler.infos.get(job4);
-    
-    // Check scheduler variables
-    assertEquals(1.33,  info1.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(1.33,  info1.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.0,   info2.mapSchedulable.getFairShare());
-    assertEquals(0.0,   info2.reduceSchedulable.getFairShare());
-    assertEquals(1.33,  info3.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(1.33,  info3.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(1.33,  info4.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(1.33,  info4.reduceSchedulable.getFairShare(), 0.1);
-    
-    // Assign tasks and check that slots are given only to jobs 1, 3 and 4
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0004_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0004_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-  }
-  
-  /**
-   * Test a combination of pool job limits and user job limits, the latter
-   * specified through both the userMaxJobsDefaults (for some users) and
-   * user-specific &lt;user&gt; elements in the allocations file. 
-   */
-  public void testComplexJobLimits() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"poolA\">");
-    out.println("<maxRunningJobs>1</maxRunningJobs>");
-    out.println("</pool>");
-    out.println("<user name=\"user1\">");
-    out.println("<maxRunningJobs>1</maxRunningJobs>");
-    out.println("</user>");
-    out.println("<user name=\"user2\">");
-    out.println("<maxRunningJobs>10</maxRunningJobs>");
-    out.println("</user>");
-    out.println("<userMaxJobsDefault>2</userMaxJobsDefault>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    
-    // Two jobs for user1; only one should get to run
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    job1.getJobConf().set(JobContext.USER_NAME, "user1");
-    JobInfo info1 = scheduler.infos.get(job1);
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10);
-    job2.getJobConf().set(JobContext.USER_NAME, "user1");
-    JobInfo info2 = scheduler.infos.get(job2);
-    advanceTime(10);
-    
-    // Three jobs for user2; all should get to run
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10);
-    job3.getJobConf().set(JobContext.USER_NAME, "user2");
-    JobInfo info3 = scheduler.infos.get(job3);
-    advanceTime(10);
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10);
-    job4.getJobConf().set(JobContext.USER_NAME, "user2");
-    JobInfo info4 = scheduler.infos.get(job4);
-    advanceTime(10);
-    JobInProgress job5 = submitJob(JobStatus.RUNNING, 10, 10);
-    job5.getJobConf().set(JobContext.USER_NAME, "user2");
-    JobInfo info5 = scheduler.infos.get(job5);
-    advanceTime(10);
-    
-    // Three jobs for user3; only two should get to run
-    JobInProgress job6 = submitJob(JobStatus.RUNNING, 10, 10);
-    job6.getJobConf().set(JobContext.USER_NAME, "user3");
-    JobInfo info6 = scheduler.infos.get(job6);
-    advanceTime(10);
-    JobInProgress job7 = submitJob(JobStatus.RUNNING, 10, 10);
-    job7.getJobConf().set(JobContext.USER_NAME, "user3");
-    JobInfo info7 = scheduler.infos.get(job7);
-    advanceTime(10);
-    JobInProgress job8 = submitJob(JobStatus.RUNNING, 10, 10);
-    job8.getJobConf().set(JobContext.USER_NAME, "user3");
-    JobInfo info8 = scheduler.infos.get(job8);
-    advanceTime(10);
-    
-    // Two jobs for user4, in poolA; only one should get to run
-    JobInProgress job9 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    job9.getJobConf().set(JobContext.USER_NAME, "user4");
-    JobInfo info9 = scheduler.infos.get(job9);
-    advanceTime(10);
-    JobInProgress job10 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    job10.getJobConf().set(JobContext.USER_NAME, "user4");
-    JobInfo info10 = scheduler.infos.get(job10);
-    advanceTime(10);
-    
-    // Check scheduler variables. The jobs in poolA should get half
-    // the total share, while those in the default pool should get
-    // the other half. This works out to 2 slots each for the jobs
-    // in poolA and 1/3 each for the jobs in the default pool because
-    // there are 2 runnable jobs in poolA and 6 jobs in the default pool.
-    assertEquals(0.33,   info1.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info1.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.0,    info2.mapSchedulable.getFairShare());
-    assertEquals(0.0,    info2.reduceSchedulable.getFairShare());
-    assertEquals(0.33,   info3.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info3.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info4.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info4.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info5.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info5.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info6.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info6.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info7.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(0.33,   info7.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.0,    info8.mapSchedulable.getFairShare());
-    assertEquals(0.0,    info8.reduceSchedulable.getFairShare());
-    assertEquals(2.0,    info9.mapSchedulable.getFairShare(), 0.1);
-    assertEquals(2.0,    info9.reduceSchedulable.getFairShare(), 0.1);
-    assertEquals(0.0,    info10.mapSchedulable.getFairShare());
-    assertEquals(0.0,    info10.reduceSchedulable.getFairShare());
-  }
-  
-  public void testSizeBasedWeight() throws Exception {
-    scheduler.sizeBasedWeight = true;
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 2, 10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 20, 1);
-    assertTrue(scheduler.infos.get(job2).mapSchedulable.getFairShare() >
-               scheduler.infos.get(job1).mapSchedulable.getFairShare());
-    assertTrue(scheduler.infos.get(job1).reduceSchedulable.getFairShare() >
-               scheduler.infos.get(job2).reduceSchedulable.getFairShare());
-  }
-
-  /**
-   * This test submits jobs in three pools: poolA, which has a weight
-   * of 2.0; poolB, which has a weight of 0.5; and the default pool, which
-   * should have a weight of 1.0. It then checks that the map and reduce
-   * fair shares are given out accordingly. We then submit a second job to
-   * pool B and check that each gets half of the pool (weight of 0.25).
-   */
-  public void testPoolWeights() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"poolA\">");
-    out.println("<weight>2.0</weight>");
-    out.println("</pool>");
-    out.println("<pool name=\"poolB\">");
-    out.println("<weight>0.5</weight>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    JobInfo info1 = scheduler.infos.get(job1);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    JobInfo info3 = scheduler.infos.get(job3);
-    advanceTime(10);
-    
-    assertEquals(1.14,  info1.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(1.14,  info1.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(2.28,  info2.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(2.28,  info2.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(0.57,  info3.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(0.57,  info3.reduceSchedulable.getFairShare(), 0.01);
-    
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    JobInfo info4 = scheduler.infos.get(job4);
-    advanceTime(10);
-    
-    assertEquals(1.14,  info1.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(1.14,  info1.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(2.28,  info2.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(2.28,  info2.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(0.28,  info3.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(0.28,  info3.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(0.28,  info4.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(0.28,  info4.reduceSchedulable.getFairShare(), 0.01);
-    verifyMetrics();    
-  }
-
-  /**
-   * This test submits jobs in two pools, poolA and poolB. None of the
-   * jobs in poolA have maps, but this should not affect their reduce
-   * share.
-   */
-  public void testPoolWeightsWhenNoMaps() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"poolA\">");
-    out.println("<weight>2.0</weight>");
-    out.println("</pool>");
-    out.println("<pool name=\"poolB\">");
-    out.println("<weight>1.0</weight>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 0, 10, "poolA");
-    JobInfo info1 = scheduler.infos.get(job1);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 0, 10, "poolA");
-    JobInfo info2 = scheduler.infos.get(job2);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    JobInfo info3 = scheduler.infos.get(job3);
-    advanceTime(10);
-    
-    /*
-    assertEquals(0,     info1.mapWeight, 0.01);
-    assertEquals(1.0,   info1.reduceWeight, 0.01);
-    assertEquals(0,     info2.mapWeight, 0.01);
-    assertEquals(1.0,   info2.reduceWeight, 0.01);
-    assertEquals(1.0,   info3.mapWeight, 0.01);
-    assertEquals(1.0,   info3.reduceWeight, 0.01);
-    */
-    
-    assertEquals(0,     info1.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(1.33,  info1.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(0,     info2.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(1.33,  info2.reduceSchedulable.getFairShare(), 0.01);
-    assertEquals(4,     info3.mapSchedulable.getFairShare(), 0.01);
-    assertEquals(1.33,  info3.reduceSchedulable.getFairShare(), 0.01);
-  }
-
-  public void testPoolMaxMapsReduces() throws Exception {
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Pool with upper bound
-    out.println("<pool name=\"poolLimited\">");
-    out.println("<weight>1.0</weight>");
-    out.println("<maxMaps>2</maxMaps>");
-    out.println("<maxReduces>1</maxReduces>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    // Create two jobs with ten maps
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 5, "poolLimited");
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 5);
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0002_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000002_0 on tt2");
-
-    Pool limited = scheduler.getPoolManager().getPool("poolLimited");
-    assertEquals(2, limited.getSchedulable(TaskType.MAP).getRunningTasks());
-    assertEquals(1, limited.getSchedulable(TaskType.REDUCE).getRunningTasks());
-    Pool defaultPool = scheduler.getPoolManager().getPool("default");
-    assertEquals(2, defaultPool.getSchedulable(TaskType.MAP).getRunningTasks());
-    assertEquals(3, defaultPool.getSchedulable(TaskType.REDUCE)
-        .getRunningTasks());
-    assertEquals(2, job1.runningMapTasks);
-    assertEquals(1, job1.runningReduceTasks);
-    assertEquals(2, job2.runningMapTasks);
-    assertEquals(3, job2.runningReduceTasks);
-  }
-
-  /**
-   * Tests that max-running-tasks per node are set by assigning load
-   * equally accross the cluster in CapBasedLoadManager.
-   */
-  public void testCapBasedLoadManager() {
-    CapBasedLoadManager loadMgr = new CapBasedLoadManager();
-    // Arguments to getCap: totalRunnableTasks, nodeCap, totalSlots
-    // Desired behavior: return ceil(nodeCap * min(1, runnableTasks/totalSlots))
-    assertEquals(1, loadMgr.getCap(1, 1, 100));
-    assertEquals(1, loadMgr.getCap(1, 2, 100));
-    assertEquals(1, loadMgr.getCap(1, 10, 100));
-    assertEquals(1, loadMgr.getCap(200, 1, 100));
-    assertEquals(1, loadMgr.getCap(1, 5, 100));
-    assertEquals(3, loadMgr.getCap(50, 5, 100));
-    assertEquals(5, loadMgr.getCap(100, 5, 100));
-    assertEquals(5, loadMgr.getCap(200, 5, 100));
-  }
-
-  /**
-   * This test starts by launching a job in the default pool that takes
-   * all the slots in the cluster. We then submit a job in a pool with
-   * min share of 2 maps and 1 reduce task. After the min share preemption
-   * timeout, this pool should be allowed to preempt tasks. 
-   */
-  public void testMinSharePreemption() throws Exception {
-    // Enable preemption in scheduler
-    scheduler.preemptionEnabled = true;
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a min share of 2 maps and 1 reduce, and a preemption
-    // timeout of 1 minute
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>1</minReduces>");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    
-    // Ten seconds later, submit job 2.
-    advanceTime(10000);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    
-    // Ten seconds later, check that job 2 is not able to preempt tasks.
-    advanceTime(10000);
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Advance time by 49 more seconds, putting us at 59s after the
-    // submission of job 2. It should still not be able to preempt.
-    advanceTime(49000);
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Advance time by 2 seconds, putting us at 61s after the submission
-    // of job 2. It should now be able to preempt 2 maps and 1 reduce.
-    advanceTime(2000);
-    assertEquals(2, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(1, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Test that the tasks actually get preempted and we can assign new ones
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(2, job1.runningMaps());
-    assertEquals(3, job1.runningReduces());
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000000_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-
-  /**
-   * This test starts by launching a job in the default pool that takes
-   * all the slots in the cluster. We then submit a job in a pool with
-   * min share of 3 maps and 3 reduce tasks, but which only actually
-   * needs 1 map and 2 reduces. We check that this pool does not prempt
-   * more than this many tasks despite its min share being higher. 
-   */
-  public void testMinSharePreemptionWithSmallJob() throws Exception {
-    // Enable preemption in scheduler
-    scheduler.preemptionEnabled = true;
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a min share of 2 maps and 1 reduce, and a preemption
-    // timeout of 1 minute
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>3</minMaps>");
-    out.println("<minReduces>3</minReduces>");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    
-    // Ten seconds later, submit job 2.
-    advanceTime(10000);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 1, 2, "poolA");
-    
-    // Advance time by 59 seconds and check that no preemption occurs.
-    advanceTime(59000);
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Advance time by 2 seconds, putting us at 61s after the submission
-    // of job 2. Job 2 should now preempt 1 map and 2 reduces.
-    advanceTime(2000);
-    assertEquals(1, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-
-    // Test that the tasks actually get preempted and we can assign new ones
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(3, job1.runningMaps());
-    assertEquals(2, job1.runningReduces());
-    checkAssignment("tt2", "attempt_test_0002_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-
-  /**
-   * This test runs on a 4-node (8-slot) cluster to allow 3 pools with fair
-   * shares greater than 2 slots to coexist (which makes the half-fair-share 
-   * of each pool more than 1 so that fair share preemption can kick in). 
-   * 
-   * The test first starts job 1, which takes 6 map slots and 6 reduce slots,
-   * in pool 1.  We then submit job 2 in pool 2, which takes 2 slots of each
-   * type. Finally, we submit a third job, job 3 in pool3, which gets no slots. 
-   * At this point the fair share of each pool will be 8/3 ~= 2.7 slots. 
-   * Pool 1 will be above its fair share, pool 2 will be below it but at half
-   * fair share, and pool 3 will be below half fair share. Therefore pool 3 
-   * should preempt a task (after a timeout) but pools 1 and 2 shouldn't. 
-   */
-  public void testFairSharePreemption() throws Exception {
-    // Create a bigger cluster than normal (4 tasktrackers instead of 2)
-    setUpCluster(1, 4, false);
-    // Enable preemption in scheduler
-    scheduler.preemptionEnabled = true;
-    // Set up pools file with a fair share preemtion timeout of 1 minute
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<fairSharePreemptionTimeout>60</fairSharePreemptionTimeout>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Grab pools (they'll be created even though they're not in the alloc file)
-    Pool pool1 = scheduler.getPoolManager().getPool("pool1");
-    Pool pool2 = scheduler.getPoolManager().getPool("pool2");
-    Pool pool3 = scheduler.getPoolManager().getPool("pool3");
-
-    // Submit job 1. We advance time by 100 between each task tracker
-    // assignment stage to ensure that the tasks from job1 on tt3 are the ones
-    // that are deterministically preempted first (being the latest launched
-    // tasks in an over-allocated job).
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 6, 6, "pool1");
-    advanceTime(100);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    advanceTime(100);
-    checkAssignment("tt3", "attempt_test_0001_m_000004_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0001_r_000004_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0001_m_000005_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0001_r_000005_0 on tt3");
-    advanceTime(100);
-    
-    // Submit job 2. It should get the last 2 slots.
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "pool2");
-    advanceTime(100);
-    checkAssignment("tt4", "attempt_test_0002_m_000000_0 on tt4");
-    checkAssignment("tt4", "attempt_test_0002_r_000000_0 on tt4");
-    checkAssignment("tt4", "attempt_test_0002_m_000001_0 on tt4");
-    checkAssignment("tt4", "attempt_test_0002_r_000001_0 on tt4");
-    
-    // Submit job 3.
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "pool3");
-    
-    // Check that after 59 seconds, neither pool can preempt
-    advanceTime(59000);
-    assertEquals(0, scheduler.tasksToPreempt(pool2.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(pool2.getReduceSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(pool3.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(pool3.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Wait 2 more seconds, so that job 3 has now been in the system for 61s.
-    // Now pool 3 should be able to preempt 2 tasks (its share of 2.7 rounded
-    // down to its floor), but pool 2 shouldn't.
-    advanceTime(2000);
-    assertEquals(0, scheduler.tasksToPreempt(pool2.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(pool2.getReduceSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(pool3.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(pool3.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Test that the tasks actually get preempted and we can assign new ones
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(4, job1.runningMaps());
-    assertEquals(4, job1.runningReduces());
-    checkAssignment("tt3", "attempt_test_0003_m_000000_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0003_r_000000_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0003_m_000001_0 on tt3");
-    checkAssignment("tt3", "attempt_test_0003_r_000001_0 on tt3");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    assertNull(scheduler.assignTasks(tracker("tt3")));
-    assertNull(scheduler.assignTasks(tracker("tt4")));
-  }
-  
-  /**
-   * This test runs on a 3-node (6-slot) cluster to allow 3 pools with fair
-   * shares equal 2 slots to coexist (which makes the half-fair-share 
-   * of each pool equal to 1 so that fair share preemption can kick in). 
-   * 
-   * The test first starts job 1, which takes 3 map slots and 0 reduce slots,
-   * in pool 1.  We then submit job 2 in pool 2, which takes 3 map slots and zero
-   * reduce slots. Finally, we submit a third job, job 3 in pool3, which gets no slots. 
-   * At this point the fair share of each pool will be 6/3 = 2 slots. 
-   * Pool 1 and 2 will be above their fair share and pool 3 will be below half fair share. 
-   * Therefore pool 3 should preempt tasks from both pool 1 & 2 (after a timeout) but 
-   * pools 1 and 2 shouldn't. 
-   */
-  public void testFairSharePreemptionFromMultiplePools() throws Exception {
-	// Create a bigger cluster than normal (3 tasktrackers instead of 2)
-	setUpCluster(1, 3, false);
-	// Enable preemption in scheduler
-	scheduler.preemptionEnabled = true;
-	// Set up pools file with a fair share preemtion timeout of 1 minute
-	PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-	out.println("<?xml version=\"1.0\"?>");
-	out.println("<allocations>");
-	out.println("<fairSharePreemptionTimeout>60</fairSharePreemptionTimeout>");
-	out.println("</allocations>");
-	out.close();
-	scheduler.getPoolManager().reloadAllocs();
-	 
-	// Grab pools (they'll be created even though they're not in the alloc file)
-	Pool pool1 = scheduler.getPoolManager().getPool("pool1");
-	Pool pool2 = scheduler.getPoolManager().getPool("pool2");
-	Pool pool3 = scheduler.getPoolManager().getPool("pool3");
-
-	// Submit job 1. We advance time by 100 between each task tracker
-	// assignment stage to ensure that the tasks from job1 on tt3 are the ones
-	// that are deterministically preempted first (being the latest launched
-	// tasks in an over-allocated job).
-	JobInProgress job1 = submitJob(JobStatus.RUNNING, 12, 0, "pool1");
-	advanceTime(100);
-	checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-	checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-	advanceTime(100);
-	checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-	advanceTime(100);
-	    
-	// Submit job 2. It should get the last 3 slots.
-	JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 0, "pool2");
-	advanceTime(100);
-	checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-	checkAssignment("tt3", "attempt_test_0002_m_000001_0 on tt3");
-	advanceTime(100);
-	checkAssignment("tt3", "attempt_test_0002_m_000002_0 on tt3");
-
-	advanceTime(100);
-	    
-	// Submit job 3.
-	JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 0, "pool3");
-	    
-	// Check that after 59 seconds, neither pool can preempt
-	advanceTime(59000);
-	assertEquals(0, scheduler.tasksToPreempt(pool2.getMapSchedulable(),
-			clock.getTime()));
-	assertEquals(0, scheduler.tasksToPreempt(pool2.getReduceSchedulable(),
-	        clock.getTime()));
-	assertEquals(0, scheduler.tasksToPreempt(pool3.getMapSchedulable(),
-	        clock.getTime()));
-	assertEquals(0, scheduler.tasksToPreempt(pool3.getReduceSchedulable(),
-	        clock.getTime()));
-	    
-	// Wait 2 more seconds, so that job 3 has now been in the system for 61s.
-	// Now pool 3 should be able to preempt 2 tasks (its share of 2 rounded
-	// down to its floor), but pool 1 & 2 shouldn't.
-	advanceTime(2000);
-	assertEquals(0, scheduler.tasksToPreempt(pool2.getMapSchedulable(),
-	        clock.getTime()));
-	assertEquals(0, scheduler.tasksToPreempt(pool2.getReduceSchedulable(),
-	        clock.getTime()));
-	assertEquals(2, scheduler.tasksToPreempt(pool3.getMapSchedulable(),
-	        clock.getTime()));
-	assertEquals(0, scheduler.tasksToPreempt(pool3.getReduceSchedulable(),
-	        clock.getTime()));
-	    
-	// Test that the tasks actually get preempted and we can assign new ones.
-	// This should preempt one task each from pool1 and pool2
-	scheduler.preemptTasksIfNecessary();
-	scheduler.update();
-	assertEquals(2, job2.runningMaps());  
-	assertEquals(2, job1.runningMaps());  
-	checkAssignment("tt2", "attempt_test_0003_m_000000_0 on tt2");
-	checkAssignment("tt3", "attempt_test_0003_m_000001_0 on tt3");
-	assertNull(scheduler.assignTasks(tracker("tt1")));
-	assertNull(scheduler.assignTasks(tracker("tt2")));
-	assertNull(scheduler.assignTasks(tracker("tt3")));
-  }
-  
-  
-  /**
-   * This test submits a job that takes all 4 slots, and then a second job in
-   * a pool that has both a min share of 2 slots with a 60s timeout and a
-   * fair share timeout of 60s. After 60 seconds, this pool will be starved
-   * of both min share (2 slots of each type) and fair share (2 slots of each
-   * type), and we test that it does not kill more than 2 tasks of each type
-   * in total.
-   */
-  public void testMinAndFairSharePreemption() throws Exception {
-    // Enable preemption in scheduler
-    scheduler.preemptionEnabled = true;
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a min share of 2 maps and 1 reduce, and a preemption
-    // timeout of 1 minute
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("<fairSharePreemptionTimeout>60</fairSharePreemptionTimeout>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    
-    // Ten seconds later, submit job 2.
-    advanceTime(10000);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    
-    // Ten seconds later, check that job 2 is not able to preempt tasks.
-    advanceTime(10000);
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Advance time by 49 more seconds, putting us at 59s after the
-    // submission of job 2. It should still not be able to preempt.
-    advanceTime(49000);
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-    
-    // Advance time by 2 seconds, putting us at 61s after the submission
-    // of job 2. It should now be able to preempt 2 maps and 1 reduce.
-    advanceTime(2000);
-    assertEquals(2, scheduler.tasksToPreempt(poolA.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(poolA.getReduceSchedulable(),
-        clock.getTime()));
-
-    // Test that the tasks actually get preempted and we can assign new ones
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(2, job1.runningMaps());
-    assertEquals(2, job1.runningReduces());
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-  
-  /**
-   * This is a copy of testMinAndFairSharePreemption that turns preemption
-   * off and verifies that no tasks get killed.
-   */
-  public void testNoPreemptionIfDisabled() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a min share of 2 maps and 1 reduce, and a preemption
-    // timeout of 1 minute
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("<fairSharePreemptionTimeout>60</fairSharePreemptionTimeout>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    
-    // Ten seconds later, submit job 2.
-    advanceTime(10000);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    
-    // Advance time by 61s, putting us past the preemption timeout,
-    // and check that no tasks get preempted.
-    advanceTime(61000);
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(4, job1.runningMaps());
-    assertEquals(4, job1.runningReduces());
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-
-  /**
-   * This is a copy of testMinAndFairSharePreemption that turns preemption
-   * on but also turns on mapred.fairscheduler.preemption.only.log (the
-   * "dry run" parameter for testing out preemption) and verifies that no
-   * tasks get killed.
-   */
-  public void testNoPreemptionIfOnlyLogging() throws Exception {
-    // Turn on preemption, but for logging only
-    scheduler.preemptionEnabled = true;
-    scheduler.onlyLogPreemption = true;
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    // Give pool A a min share of 2 maps and 1 reduce, and a preemption
-    // timeout of 1 minute
-    out.println("<pool name=\"poolA\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("<fairSharePreemptionTimeout>60</fairSharePreemptionTimeout>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-    
-    // Ten seconds later, submit job 2.
-    advanceTime(10000);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    
-    // Advance time by 61s, putting us past the preemption timeout,
-    // and check that no tasks get preempted.
-    advanceTime(61000);
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(4, job1.runningMaps());
-    assertEquals(4, job1.runningReduces());
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-
-  /**
-   * This test exercises delay scheduling at the node level. We submit a job
-   * with data on rack1.node2 and check that it doesn't get assigned on earlier
-   * nodes. A second job with no locality info should get assigned instead.
-   * 
-   * TaskTracker names in this test map to nodes as follows:
-   * - tt1 = rack1.node1
-   * - tt2 = rack1.node2
-   * - tt3 = rack2.node1
-   * - tt4 = rack2.node2
-   */
-  public void testDelaySchedulingAtNodeLevel() throws IOException {
-    setUpCluster(2, 2, true);
-    scheduler.assignMultiple = true;
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 1, 0, "pool1",
-        new String[][] {
-          {"rack2.node2"}
-        }, true);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Advance time before submitting another job j2, to make j1 be ahead
-    // of j2 in the queue deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 0);
-    
-    // Assign tasks on nodes 1-3 and check that j2 gets them
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1", 
-                           "attempt_test_0002_m_000001_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000002_0 on tt2",
-                           "attempt_test_0002_m_000003_0 on tt2");
-    checkAssignment("tt3", "attempt_test_0002_m_000004_0 on tt3",
-                           "attempt_test_0002_m_000005_0 on tt3");
-    
-    // Assign a task on node 4 now and check that j1 gets it. The other slot
-    // on the node should be given to j2 because j1 will be out of tasks.
-    checkAssignment("tt4", "attempt_test_0001_m_000000_0 on tt4",
-                           "attempt_test_0002_m_000006_0 on tt4");
-    
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.NODE);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-  }
-  
-  /**
-   * This test submits a job and causes it to exceed its node-level delay,
-   * and thus to go on to launch a rack-local task. We submit one job with data
-   * on rack2.node4 and check that it does not get assigned on any of the other
-   * nodes until 10 seconds (the delay configured in setUpCluster) pass.
-   * Finally, after some delay, we let the job assign local tasks and check
-   * that it has returned to waiting for node locality.
-   * 
-   * TaskTracker names in this test map to nodes as follows:
-   * - tt1 = rack1.node1
-   * - tt2 = rack1.node2
-   * - tt3 = rack2.node1
-   * - tt4 = rack2.node2
-   */
-  public void testDelaySchedulingAtRackLevel() throws IOException {
-    setUpCluster(2, 2, true);
-    scheduler.assignMultiple = true;
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 4, 0, "pool1",
-        new String[][] {
-          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}
-        }, true);
-    JobInfo info1 = scheduler.infos.get(job1);
-    
-    // Advance time before submitting another job j2, to make j1 be ahead
-    // of j2 in the queue deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 20, 0);
-    
-    // Assign tasks on nodes 1-3 and check that j2 gets them
-    checkAssignment("tt1", "attempt_test_0002_m_000000_0 on tt1", 
-                           "attempt_test_0002_m_000001_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000002_0 on tt2",
-                           "attempt_test_0002_m_000003_0 on tt2");
-    checkAssignment("tt3", "attempt_test_0002_m_000004_0 on tt3",
-                           "attempt_test_0002_m_000005_0 on tt3");
-    
-    // Advance time by 6 seconds to put us past the 5-second node locality delay
-    advanceTime(6000);
-    
-    // Finish some tasks on each node
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_m_000000_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_m_000002_0");
-    taskTrackerManager.finishTask("tt3", "attempt_test_0002_m_000004_0");
-    advanceTime(100);
-    
-    // Check that job 1 is only assigned on node 3 (which is rack-local)
-    checkAssignment("tt1", "attempt_test_0002_m_000006_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000007_0 on tt2");
-    checkAssignment("tt3", "attempt_test_0001_m_000000_0 on tt3");
-    
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.RACK);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-    
-    // Also give job 1 some tasks on node 4. Its lastMapLocalityLevel
-    // should go back to 0 after it gets assigned these.
-    checkAssignment("tt4", "attempt_test_0001_m_000001_0 on tt4",
-                           "attempt_test_0001_m_000002_0 on tt4");
-    
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.NODE);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-    
-    // Check that job 1 no longer assigns tasks in the same rack now
-    // that it has obtained a node-local task
-    taskTrackerManager.finishTask("tt1", "attempt_test_0002_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0002_m_000003_0");
-    taskTrackerManager.finishTask("tt3", "attempt_test_0002_m_000005_0");
-    advanceTime(100);
-    checkAssignment("tt1", "attempt_test_0002_m_000008_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000009_0 on tt2");
-    checkAssignment("tt3", "attempt_test_0002_m_000010_0 on tt3");
-    advanceTime(100);
-    
-    // However, job 1 should still be able to launch tasks on node 4
-    taskTrackerManager.finishTask("tt4", "attempt_test_0001_m_000001_0");
-    advanceTime(100);
-    checkAssignment("tt4", "attempt_test_0001_m_000003_0 on tt4");
-  }
-  
-  /**
-   * This test submits a job and causes it to exceed its node-level delay,
-   * then its rack-level delay. It should then launch tasks off-rack.
-   * However, once the job gets a rack-local slot it should stay in-rack,
-   * and once it gets a node-local slot it should stay in-node.
-   * For simplicity, we don't submit a second job in this test.
-   * 
-   * TaskTracker names in this test map to nodes as follows:
-   * - tt1 = rack1.node1
-   * - tt2 = rack1.node2
-   * - tt3 = rack2.node1
-   * - tt4 = rack2.node2
-   */
-  public void testDelaySchedulingOffRack() throws IOException {
-    setUpCluster(2, 2, true);
-    scheduler.assignMultiple = true;
-    
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 8, 0, "pool1",
-        new String[][] {
-          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"},
-          {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"}, {"rack2.node2"},
-        }, true);
-    JobInfo info1 = scheduler.infos.get(job1);
-    advanceTime(100);
-    
-    // Check that nothing is assigned on trackers 1-3
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    assertNull(scheduler.assignTasks(tracker("tt3")));
-    
-    // Advance time by 6 seconds to put us past the 5-sec node locality delay
-    advanceTime(6000);
-
-    // Check that nothing is assigned on trackers 1-2; the job would assign
-    // a task on tracker 3 (rack1.node2) so we skip that one 
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // Repeat to see that receiving multiple heartbeats works
-    advanceTime(100);
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    advanceTime(100);
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.NODE);
-    assertEquals(info1.timeWaitedForLocalMap, 6200);
-    assertEquals(info1.skippedAtLastHeartbeat, true);
-    
-    // Advance time by 11 seconds to put us past the 10-sec rack locality delay
-    advanceTime(11000);
-    
-    // Now the job should be able to assign tasks on tt1 and tt2
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1",
-                           "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2",
-                           "attempt_test_0001_m_000003_0 on tt2");
-
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.ANY);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-    
-    // Now assign a task on tt3. This should make the job stop assigning
-    // on tt1 and tt2 (checked after we finish some tasks there)
-    checkAssignment("tt3", "attempt_test_0001_m_000004_0 on tt3",
-                           "attempt_test_0001_m_000005_0 on tt3");
-
-    // Check that delay scheduling info is properly set
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.RACK);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-    
-    // Check that j1 no longer assigns tasks on rack 1 now
-    taskTrackerManager.finishTask("tt1", "attempt_test_0001_m_000001_0");
-    taskTrackerManager.finishTask("tt2", "attempt_test_0001_m_000003_0");
-    advanceTime(100);
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    
-    // However, tasks on rack 2 should still be assigned
-    taskTrackerManager.finishTask("tt3", "attempt_test_0001_m_000004_0");
-    advanceTime(100);
-    checkAssignment("tt3", "attempt_test_0001_m_000006_0 on tt3");
-    
-    // Now assign a task on node 4
-    checkAssignment("tt4", "attempt_test_0001_m_000007_0 on tt4");
-
-    // Check that delay scheduling info is set so we are looking for node-local
-    // tasks at this point
-    assertEquals(info1.lastMapLocalityLevel, LocalityLevel.NODE);
-    assertEquals(info1.timeWaitedForLocalMap, 0);
-    assertEquals(info1.skippedAtLastHeartbeat, false);
-  }
-  
-  /**
-   * This test submits two jobs with 4 maps and 3 reduces in total to a
-   * 4-node cluster. Although the cluster has 2 map slots and 2 reduce
-   * slots per node, it should only launch one map and one reduce on each
-   * node to balance the load. We check that this happens even if
-   * assignMultiple is set to true so the scheduler has the opportunity
-   * to launch multiple tasks per heartbeat.
-   */
-  public void testAssignMultipleWithUnderloadedCluster() throws IOException {
-    setUpCluster(1, 4, true);
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 2, 2);
-    
-    // Advance to make j1 be scheduled before j2 deterministically.
-    advanceTime(100);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 2, 1);
-    
-    // Assign tasks and check that at most one map and one reduce slot is used
-    // on each node, and that no tasks are assigned on subsequent heartbeats
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1",
-                           "attempt_test_0001_r_000000_0 on tt1");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2",
-                           "attempt_test_0002_r_000000_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-    checkAssignment("tt3", "attempt_test_0001_m_000001_0 on tt3",
-                           "attempt_test_0001_r_000001_0 on tt3");
-    assertNull(scheduler.assignTasks(tracker("tt3")));
-    checkAssignment("tt4", "attempt_test_0002_m_000001_0 on tt4");
-    assertNull(scheduler.assignTasks(tracker("tt4")));
-  }
-  
-  /**
-   * This test submits four jobs in the default pool, which is set to FIFO mode:
-   * - job1, with 1 map and 1 reduce
-   * - job2, with 3 maps and 3 reduces
-   * - job3, with 1 map, 1 reduce, and priority set to HIGH
-   * - job4, with 3 maps and 3 reduces
-   * 
-   * We check that the scheduler assigns tasks first to job3 (because it is
-   * high priority), then to job1, then to job2.
-   */
-  public void testFifoPool() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"default\">");
-    out.println("<schedulingMode>fifo</schedulingMode>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 1, 1);
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 3, 3);
-    advanceTime(10);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 1, 1);
-    job3.setPriority(JobPriority.HIGH);
-    advanceTime(10);
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 3, 3);
-    
-    // Assign tasks and check that they're given first to job3 (because it is
-    // high priority), then to job1, then to job2.
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-  }
-  
-  /**
-   * This test submits 2 large jobs each to 2 pools, which are both set to FIFO
-   * mode through the global defaultPoolSchedulingMode setting. We check that
-   * the scheduler assigns tasks only to the first job within each pool, but
-   * alternates between the pools to give each pool a fair share.
-   */
-  public void testMultipleFifoPools() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<defaultPoolSchedulingMode>fifo</defaultPoolSchedulingMode>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    advanceTime(10);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    advanceTime(10);
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    
-    // Assign tasks and check that they alternate between jobs 1 and 3, the
-    // head-of-line jobs in their respective pools.
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0003_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0003_r_000001_0 on tt2");
-  }
-  
-  /**
-   * This test submits 2 large jobs each to 2 pools, one of which is set to FIFO
-   * mode through the global defaultPoolSchedulingMode setting, and one of which
-   * is set to fair mode. We check that the scheduler assigns tasks only to the
-   * first job in the FIFO pool but to both jobs in the fair sharing pool.
-   */
-  public void testFifoAndFairPools() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<defaultPoolSchedulingMode>fifo</defaultPoolSchedulingMode>");
-    out.println("<pool name=\"poolB\">");
-    out.println("<schedulingMode>fair</schedulingMode>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    
-    // Submit jobs, advancing time in-between to make sure that they are
-    // all submitted at distinct times.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    advanceTime(10);
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "poolA");
-    advanceTime(10);
-    JobInProgress job3 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    advanceTime(10);
-    JobInProgress job4 = submitJob(JobStatus.RUNNING, 10, 10, "poolB");
-    
-    // Assign tasks and check that only job 1 gets tasks in pool A, but
-    // jobs 3 and 4 both get tasks in pool B.
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0003_r_000000_0 on tt1");
-    checkAssignment("tt2", "attempt_test_0001_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0004_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0004_r_000000_0 on tt2");
-  }
-
-  /**
-   * This test uses the mapred.fairscheduler.pool property to assign jobs to pools.
-   */
-  public void testPoolAssignment() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"default\">");
-    out.println("<schedulingMode>fair</schedulingMode>");
-    out.println("</pool>");
-    out.println("<pool name=\"poolA\">");
-    out.println("<schedulingMode>fair</schedulingMode>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool defaultPool = scheduler.getPoolManager().getPool("default");
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
- 
-    // Submit a job to the default pool.  All specifications take default values.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 1, 3);
-
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(0,    poolA.getMapSchedulable().getDemand());
-    assertEquals(0,    poolA.getReduceSchedulable().getDemand());
-
-    // Submit a job to the default pool and move it to poolA using setPool.
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 5, 7);
-
-    assertEquals(6,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(10,   defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(0,    poolA.getMapSchedulable().getDemand());
-    assertEquals(0,    poolA.getReduceSchedulable().getDemand());
-
-    scheduler.getPoolManager().setPool(job2, "poolA");
-    assertEquals("poolA", scheduler.getPoolManager().getPoolName(job2));
-
-    defaultPool.getMapSchedulable().updateDemand();
-    defaultPool.getReduceSchedulable().updateDemand();
-    poolA.getMapSchedulable().updateDemand();
-    poolA.getReduceSchedulable().updateDemand();
-
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(5,    poolA.getMapSchedulable().getDemand());
-    assertEquals(7,    poolA.getReduceSchedulable().getDemand());
-
-    // Submit a job to poolA by specifying mapred.fairscheduler.pool
-    JobConf jobConf = new JobConf(conf);
-    jobConf.setNumMapTasks(11);
-    jobConf.setNumReduceTasks(13);
-    jobConf.set(POOL_PROPERTY, "nonsense"); // test that this is overridden
-    jobConf.set(EXPLICIT_POOL_PROPERTY, "poolA");
-    JobInProgress job3 = new FakeJobInProgress(jobConf, taskTrackerManager,
-        null, UtilsForTests.getJobTracker());
-    job3.initTasks();
-    job3.getStatus().setRunState(JobStatus.RUNNING);
-    taskTrackerManager.submitJob(job3);
-
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(16,   poolA.getMapSchedulable().getDemand());
-    assertEquals(20,   poolA.getReduceSchedulable().getDemand());
-
-    // Submit a job to poolA by specifying pool and not mapred.fairscheduler.pool
-    JobConf jobConf2 = new JobConf(conf);
-    jobConf2.setNumMapTasks(17);
-    jobConf2.setNumReduceTasks(19);
-    jobConf2.set(POOL_PROPERTY, "poolA");
-    JobInProgress job4 = new FakeJobInProgress(jobConf2, taskTrackerManager,
-        null, UtilsForTests.getJobTracker());
-    job4.initTasks();
-    job4.getStatus().setRunState(JobStatus.RUNNING);
-    taskTrackerManager.submitJob(job4);
-
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(33,   poolA.getMapSchedulable().getDemand());
-    assertEquals(39,   poolA.getReduceSchedulable().getDemand());
-  }
-
-
-  /**
-   * Test switching a job from one pool to another, then back to the original
-   * one. This is a regression test for a bug seen during development of
-   * MAPREDUCE-2323 (fair scheduler metrics).
-   */
-  public void testSetPoolTwice() throws Exception {
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<pool name=\"default\">");
-    out.println("<schedulingMode>fair</schedulingMode>");
-    out.println("</pool>");
-    out.println("<pool name=\"poolA\">");
-    out.println("<schedulingMode>fair</schedulingMode>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool defaultPool = scheduler.getPoolManager().getPool("default");
-    Pool poolA = scheduler.getPoolManager().getPool("poolA");
-
-    // Submit a job to the default pool.  All specifications take default values.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 1, 3);
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(0,    poolA.getMapSchedulable().getDemand());
-    assertEquals(0,    poolA.getReduceSchedulable().getDemand());
-
-    // Move job to poolA and make sure demand moves with it
-    scheduler.getPoolManager().setPool(job1, "poolA");
-    assertEquals("poolA", scheduler.getPoolManager().getPoolName(job1));
-
-    defaultPool.getMapSchedulable().updateDemand();
-    defaultPool.getReduceSchedulable().updateDemand();
-    poolA.getMapSchedulable().updateDemand();
-    poolA.getReduceSchedulable().updateDemand();
-
-    assertEquals(0,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(0,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(1,    poolA.getMapSchedulable().getDemand());
-    assertEquals(3,    poolA.getReduceSchedulable().getDemand());
-
-    // Move back to default pool and make sure demand goes back
-    scheduler.getPoolManager().setPool(job1, "default");
-    assertEquals("default", scheduler.getPoolManager().getPoolName(job1));
-
-    defaultPool.getMapSchedulable().updateDemand();
-    defaultPool.getReduceSchedulable().updateDemand();
-    poolA.getMapSchedulable().updateDemand();
-    poolA.getReduceSchedulable().updateDemand();
-
-    assertEquals(1,    defaultPool.getMapSchedulable().getDemand());
-    assertEquals(3,    defaultPool.getReduceSchedulable().getDemand());
-    assertEquals(0,    poolA.getMapSchedulable().getDemand());
-    assertEquals(0,    poolA.getReduceSchedulable().getDemand());
-  }
-  
-  private void advanceTime(long time) {
-    clock.advance(time);
-    scheduler.update();
-  }
-
-  protected TaskTracker tracker(String taskTrackerName) {
-    return taskTrackerManager.getTaskTracker(taskTrackerName);
-  }
-  
-  protected void checkAssignment(String taskTrackerName,
-      String... expectedTasks) throws IOException {
-    List<Task> tasks = scheduler.assignTasks(tracker(taskTrackerName));
-    assertNotNull(tasks);
-    System.out.println("Assigned tasks:");
-    for (int i = 0; i < tasks.size(); i++)
-      System.out.println("- " + tasks.get(i));
-    assertEquals(expectedTasks.length, tasks.size());
-    for (int i = 0; i < tasks.size(); i++)
-      assertEquals("assignment " + i, expectedTasks[i], tasks.get(i).toString());
-  }
-
-  /**
-   * This test submits a job that takes all 2 slots in a pool has both a min
-   * share of 2 slots with minshare timeout of 5s, and then a second job in
-   * default pool with a fair share timeout of 5s. After 60 seconds, this pool
-   * will be starved of fair share (2 slots of each type), and we test that it
-   * does not kill more than 2 tasks of each type.
-   */
-  public void testFairSharePreemptionWithShortTimeout() throws Exception {
-    // Enable preemption in scheduler
-    scheduler.preemptionEnabled = true;
-    // Set up pools file
-    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-    out.println("<?xml version=\"1.0\"?>");
-    out.println("<allocations>");
-    out.println("<fairSharePreemptionTimeout>5</fairSharePreemptionTimeout>");
-    out.println("<pool name=\"pool1\">");
-    out.println("<minMaps>2</minMaps>");
-    out.println("<minReduces>2</minReduces>");
-    out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
-    out.println("</pool>");
-    out.println("</allocations>");
-    out.close();
-    scheduler.getPoolManager().reloadAllocs();
-    Pool pool1 = scheduler.getPoolManager().getPool("pool1");
-    Pool defaultPool = scheduler.getPoolManager().getPool("default");
-
-    // Submit job 1 and assign all slots to it. Sleep a bit before assigning
-    // tasks on tt1 and tt2 to ensure that the ones on tt2 get preempted first.
-    JobInProgress job1 = submitJob(JobStatus.RUNNING, 10, 10, "pool1");
-    JobInfo info1 = scheduler.infos.get(job1);
-    checkAssignment("tt1", "attempt_test_0001_m_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000000_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_m_000001_0 on tt1");
-    checkAssignment("tt1", "attempt_test_0001_r_000001_0 on tt1");
-    advanceTime(100);
-    checkAssignment("tt2", "attempt_test_0001_m_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000002_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_m_000003_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0001_r_000003_0 on tt2");
-
-    advanceTime(10000);
-    assertEquals(4,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(4,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(4.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(4.0,  info1.reduceSchedulable.getFairShare());
-    // Ten seconds later, submit job 2.
-    JobInProgress job2 = submitJob(JobStatus.RUNNING, 10, 10, "default");
-
-    // Advance time by 6 seconds without update the scheduler.
-    // This simulates the time gap between update and task preemption.
-    clock.advance(6000);
-    assertEquals(4,    info1.mapSchedulable.getRunningTasks());
-    assertEquals(4,    info1.reduceSchedulable.getRunningTasks());
-    assertEquals(2.0,  info1.mapSchedulable.getFairShare());
-    assertEquals(2.0,  info1.reduceSchedulable.getFairShare());
-    assertEquals(0, scheduler.tasksToPreempt(pool1.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(0, scheduler.tasksToPreempt(pool1.getReduceSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(defaultPool.getMapSchedulable(),
-        clock.getTime()));
-    assertEquals(2, scheduler.tasksToPreempt(defaultPool.getReduceSchedulable(),
-        clock.getTime()));
-
-    // Test that the tasks actually get preempted and we can assign new ones
-    scheduler.preemptTasksIfNecessary();
-    scheduler.update();
-    assertEquals(2, job1.runningMaps());
-    assertEquals(2, job1.runningReduces());
-    checkAssignment("tt2", "attempt_test_0002_m_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000000_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_m_000001_0 on tt2");
-    checkAssignment("tt2", "attempt_test_0002_r_000001_0 on tt2");
-    assertNull(scheduler.assignTasks(tracker("tt1")));
-    assertNull(scheduler.assignTasks(tracker("tt2")));
-  }
-  
-  
-  /**
-   * Ask scheduler to update metrics and then verify that they're all
-   * correctly published to the metrics context
-   */
-  private void verifyMetrics() {
-    scheduler.updateMetrics();
-    verifyPoolMetrics();
-    verifyJobMetrics();
-  }
-  
-  /**
-   * Verify that pool-level metrics match internal data
-   */
-  private void verifyPoolMetrics() {
-    MetricsContext ctx = MetricsUtil.getContext("fairscheduler");
-    Collection<OutputRecord> records = ctx.getAllRecords().get("pools");
-
-    try {
-      assertEquals(scheduler.getPoolSchedulables(TaskType.MAP).size() * 2,
-          records.size());
-    } catch (Error e) {
-      for (OutputRecord rec : records) {
-        System.err.println("record:");
-        System.err.println(" name: " + rec.getTag("name"));
-        System.err.println(" type: " + rec.getTag("type"));
-      }
-
-      throw e;
-    }
-    
-    Map<String, OutputRecord> byPoolAndType =
-      new HashMap<String, OutputRecord>();
-    for (OutputRecord rec : records) {
-      String pool = (String)rec.getTag("name");
-      String type = (String)rec.getTag("taskType");
-      assertNotNull(pool);
-      assertNotNull(type);
-      byPoolAndType.put(pool + "_" + type, rec);
-    }
-    
-    List<PoolSchedulable> poolScheds = new ArrayList<PoolSchedulable>();
-    poolScheds.addAll(scheduler.getPoolSchedulables(TaskType.MAP));
-    poolScheds.addAll(scheduler.getPoolSchedulables(TaskType.REDUCE));
-    
-    for (PoolSchedulable pool : poolScheds) {
-      String poolName = pool.getName();
-      OutputRecord metrics = byPoolAndType.get(
-          poolName + "_" + pool.getTaskType().toString());
-      assertNotNull("Need metrics for " + pool, metrics);
-      
-      verifySchedulableMetrics(pool, metrics);
-    }
-    
-  }
-  
-  /**
-   * Verify that the job-level metrics match internal data
-   */
-  private void verifyJobMetrics() {
-    MetricsContext ctx = MetricsUtil.getContext("fairscheduler");
-    Collection<OutputRecord> records = ctx.getAllRecords().get("jobs");
-    
-    System.out.println("Checking job metrics...");
-    Map<String, OutputRecord> byJobIdAndType =
-      new HashMap<String, OutputRecord>();
-    for (OutputRecord rec : records) {
-      String jobId = (String)rec.getTag("name");
-      String type = (String)rec.getTag("taskType");
-      assertNotNull(jobId);
-      assertNotNull(type);
-      byJobIdAndType.put(jobId + "_" + type, rec);
-      System.out.println("Got " + type + " metrics for job: " + jobId);
-    }
-    assertEquals(scheduler.infos.size() * 2, byJobIdAndType.size());
-    
-    for (Map.Entry<JobInProgress, JobInfo> entry :
-            scheduler.infos.entrySet()) {
-      JobInfo info = entry.getValue();
-      String jobId = entry.getKey().getJobID().toString();
-      
-      OutputRecord mapMetrics = byJobIdAndType.get(jobId + "_MAP");
-      assertNotNull("Job " + jobId + " should have map metrics", mapMetrics);
-      verifySchedulableMetrics(info.mapSchedulable, mapMetrics);
-      
-      OutputRecord reduceMetrics = byJobIdAndType.get(jobId + "_REDUCE");
-      assertNotNull("Job " + jobId + " should have reduce metrics", reduceMetrics);
-      verifySchedulableMetrics(info.reduceSchedulable, reduceMetrics);
-    }
-  }
-
-  /**
-   * Verify that the metrics for a given Schedulable are correct
-   */
-  private void verifySchedulableMetrics(
-      Schedulable sched, OutputRecord metrics) {
-    assertEquals(sched.getRunningTasks(), metrics.getMetric("runningTasks"));
-    assertEquals(sched.getDemand(), metrics.getMetric("demand"));
-    assertEquals(sched.getFairShare(),
-        metrics.getMetric("fairShare").doubleValue(), .001);
-    assertEquals(sched.getWeight(),
-        metrics.getMetric("weight").doubleValue(), .001);
-  }
-}

hadoop-mapreduce-project/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java

@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.mapreduce.SleepJob;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.conf.Configuration;
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.HttpURLConnection;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.TimeUnit;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Test;
-import org.junit.BeforeClass;
-import org.junit.AfterClass;
-import static org.junit.Assert.*;
-
-/**
- * System tests for the fair scheduler. These run slower than the
- * mock-based tests in TestFairScheduler but have a better chance
- * of catching synchronization bugs with the real JT.
- *
- * This test suite will often be run inside JCarder in order to catch
- * deadlock bugs which have plagued the scheduler in the past - hence
- * it is a bit of a "grab-bag" of system tests, since it's important
- * that they all run as part of the same JVM instantiation.
- */
-public class TestFairSchedulerSystem {
-  static final int NUM_THREADS=2;
-
-  static MiniMRCluster mr;
-  static JobConf conf;
-
-  @BeforeClass
-  public static void setUp() throws Exception {
-    conf = new JobConf();
-    final int taskTrackers = 1;
-
-    // Bump up the frequency of preemption updates to test against
-    // deadlocks, etc.
-    conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
-    conf.set("mapred.fairscheduler.update.interval", "1");
-    conf.set("mapred.fairscheduler.preemption.interval", "1");
-    conf.set("mapred.fairscheduler.preemption", "true");
-    conf.set("mapred.fairscheduler.eventlog.enabled", "true");
-    conf.set("mapred.fairscheduler.poolnameproperty", "group.name");
-    conf.set(JTConfig.JT_PERSIST_JOBSTATUS, "false");
-    mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, conf);
-  }
-
-  @AfterClass
-  public static void tearDown() throws Exception {
-    if (mr != null) {
-      mr.shutdown();
-    }
-  }
-
-  private void runSleepJob(JobConf conf) throws Exception {
-    String[] args = { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
-    ToolRunner.run(conf, new SleepJob(), args);
-  }
-
-  /**
-   * Submit some concurrent sleep jobs, and visit the scheduler servlet
-   * while they're running.
-   */
-  @Test
-  public void testFairSchedulerSystem() throws Exception {
-    ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
-    List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
-    for (int i = 0; i < NUM_THREADS; i++) {
-      futures.add(exec.submit(new Callable<Void>() {
-            public Void call() throws Exception {
-              JobConf jobConf = mr.createJobConf();
-              runSleepJob(jobConf);
-              return null;
-            }
-          }));
-    }
-
-    JobClient jc = new JobClient(mr.createJobConf(null));
-
-    // Wait for the tasks to finish, and visit the scheduler servlet
-    // every few seconds while waiting.
-    for (Future<Void> future : futures) {
-      while (true) {
-        try {
-          future.get(3, TimeUnit.SECONDS);
-          break;
-        } catch (TimeoutException te) {
-          // It's OK
-        }
-        checkServlet(true);
-        checkServlet(false);
-
-        JobStatus jobs[] = jc.getAllJobs();
-        if (jobs == null) {
-          System.err.println("No jobs running, not checking tasklog servlet");
-          continue;
-        }
-        for (JobStatus j : jobs) {
-          System.err.println("Checking task graph for " + j.getJobID());
-          try {
-            checkTaskGraphServlet(j.getJobID());
-          } catch (AssertionError err) {
-            // The task graph servlet will be empty if the job has retired.
-            // This is OK.
-            RunningJob rj = jc.getJob(j.getJobID());
-            if (!rj.isRetired()) {
-              throw err;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Check the fair scheduler servlet for good status code and smoke test
-   * for contents.
-   */
-  private void checkServlet(boolean advanced) throws Exception {
-    String jtURL = "http://localhost:" +
-      mr.getJobTrackerRunner().getJobTrackerInfoPort();
-    URL url = new URL(jtURL + "/scheduler" +
-                      (advanced ? "?advanced" : ""));
-    HttpURLConnection connection = (HttpURLConnection)url.openConnection();
-    connection.setRequestMethod("GET");
-    connection.connect();
-    assertEquals(200, connection.getResponseCode());
-
-    // Just to be sure, slurp the content and make sure it looks like the scheduler
-    BufferedReader reader = new BufferedReader(
-      new InputStreamReader(connection.getInputStream()));
-    StringBuilder sb = new StringBuilder();
-
-    String line = null;
-    while ((line = reader.readLine()) != null) {
-      sb.append(line).append('\n');
-    }
-
-    String contents = sb.toString();
-    assertTrue("Bad contents for fair scheduler servlet: " + contents,
-      contents.contains("Fair Scheduler Administration"));
-
-    String userGroups[] = UserGroupInformation.getCurrentUser().getGroupNames();
-    String primaryGroup = ">" + userGroups[0] + "<";
-    assertTrue(contents.contains(primaryGroup));
-  }
-
-  private void checkTaskGraphServlet(JobID job) throws Exception {
-    String jtURL = "http://localhost:" +
-      mr.getJobTrackerRunner().getJobTrackerInfoPort();
-    URL url = new URL(jtURL + "/taskgraph?jobid=" + job.toString() + "&type=map");
-    HttpURLConnection connection = (HttpURLConnection)url.openConnection();
-    connection.setRequestMethod("GET");
-    connection.connect();
-    assertEquals(200, connection.getResponseCode());
-
-    // Just to be sure, slurp the content and make sure it looks like the scheduler
-    String contents = slurpContents(connection);
-    assertTrue("Bad contents for job " + job + ":\n" + contents,
-      contents.contains("</svg>"));
-  }
-
-  private String slurpContents(HttpURLConnection connection) throws Exception {
-    BufferedReader reader = new BufferedReader(
-      new InputStreamReader(connection.getInputStream()));
-    StringBuilder sb = new StringBuilder();
-
-    String line = null;
-    while ((line = reader.readLine()) != null) {
-      sb.append(line).append('\n');
-    }
-
-    return sb.toString();
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/bin/mumak.sh

@@ -1,180 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# resolve links - $0 may be a softlink
-project=mumak
-HADOOP_VERSION= 
-
-this="$0"
-while [ -h "$this" ]; do
-  ls=`ls -ld "$this"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '.*/.*' > /dev/null; then
-    this="$link"
-  else
-    this=`dirname "$this"`/"$link"
-  fi
-done
-
-# convert relative path to absolute path
-bin=`dirname "$this"`
-bin=`cd "$bin"; pwd`
-script=`basename $this`
-this="$bin/$script"
-
-MUMAK_HOME=`dirname $bin`
-if [ -d "$MUMAK_HOME/../../../build/classes" ]; then
-  HADOOP_PREFIX=`cd $MUMAK_HOME/../../.. ; pwd`
-  IN_RELEASE=0
-else
-  HADOOP_PREFIX=`cd $MUMAK_HOME/../.. ; pwd`
-  IN_RELEASE=1
-  
-  MAPRED_JAR=$HADOOP_PREFIX/hadoop-mapred-${HADOOP_VERSION}.jar
-  if [ ! -e $MAPRED_JAR ]; then
-    echo "Error: Cannot find $MAPRED_JAR."
-    exit 1
-  fi
-fi
-
-# parse command line option
-if [ $# -gt 1 ]
-then
-  if [ "--config" = "$1" ]
-  then
-    shift
-    confdir=$1
-    shift
-    HADOOP_CONF_DIR=$confdir
-  fi
-fi
-
-# Allow alternate conf dir location.
-HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/conf}"
-
-if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
-  . "${HADOOP_CONF_DIR}/hadoop-env.sh"
-fi
-
-# Define HADOOP_PREFIX
-if [ "$HADOP_CORE_HOME" = "" ]; then
-  HADOOP_PREFIX=$HADOOP_PREFIX
-fi
-
-if [ "$JAVA_HOME" = "" ]; then
-  echo "Error: JAVA_HOME is not set."
-  exit 1
-fi
-
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1200m 
-
-# Setting classpath
-# Mumak needs to have the followinw classes and resources in place (roughly in this
-# order):
-# Mumak's conf directory (log4j.properties), must override Hadoop's conf dir.
-# Hadoop's conf directory
-# Mumak classes (including aspectj-generated classes) (or mumak jar), must
-#     override MapReduce project classes or jar..
-# MapReduce project classes (mapred jar)
-# MapReduce webapps files (included in mapred jar)
-# MapReduce tools classes (or mapred-tools jar)
-# Hadoop Common jar
-# Hadoop Common test jar
-# Depending 3rd party jars
-CLASSPATH=${MUMAK_HOME}/conf:${HADOOP_CONF_DIR}:$JAVA_HOME/lib/tools.jar
-
-if [ $IN_RELEASE = 0 ]; then
-  CLASSPATH=${CLASSPATH}:${HADOOP_PREFIX}/build/contrib/${project}/classes
-  CLASSPATH=${CLASSPATH}:${HADOOP_PREFIX}/build/classes
-  CLASSPATH=${CLASSPATH}:${HADOOP_PREFIX}/build
-  CLASSPATH=${CLASSPATH}:${HADOOP_PREFIX}/build/tools
-  # add libs to CLASSPATH
-  for f in $HADOOP_PREFIX/lib/hadoop-core-*.jar; do
-    CLASSPATH=${CLASSPATH}:$f;
-  done
-
-  for f in $HADOOP_PREFIX/build/ivy/lib/${project}/common/*.jar; do
-    CLASSPATH=${CLASSPATH}:$f;
-  done
-
-  for f in $HADOOP_PREFIX/build/ivy/lib/${project}/test/*.jar; do
-    CLASSPATH=${CLASSPATH}:$f;
-  done
-else
-  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX;
-  for f in $HADOOP_PREFIX/lib/*.jar; do
-    CLASSPATH=${CLASSPATH}:$f;
-  done
-  CLASSPATH=${CLASSPATH}:$MUMAK_HOME/hadoop-${HADOOP_VERSION}-${project}.jar
-  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/hadoop-mapred-${HADOOP_VERSION}.jar
-  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/hadoop-mapred-tools-${HADOOP_VERSION}.jar
-fi
-
-# check envvars which might override default args
-if [ "$HADOOP_HEAPSIZE" != "" ]; then
-  #echo "run with heapsize $HADOOP_HEAPSIZE"
-  JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
-  #echo $JAVA_HEAP_MAX
-fi
-
-# default log directory & file
-if [ "$HADOOP_LOG_DIR" = "" ]; then
-  HADOOP_LOG_DIR="$HADOOP_PREFIX/logs"
-fi
-
-# default policy file for service-level authorization
-if [ "$HADOOP_POLICYFILE" = "" ]; then
-  HADOOP_POLICYFILE="hadoop-policy.xml"
-fi
-
-# setup 'java.library.path' for native-hadoop code if necessary
-JAVA_LIBRARY_PATH=''
-if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/lib/native" ]; then
-  JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
-  
-  if [ -d "$HADOOP_PREFIX/build/native" ]; then
-    JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/build/native/${JAVA_PLATFORM}/lib
-  fi
-  
-  if [ -d "${HADOOP_PREFIX}/lib/native" ]; then
-    if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
-      JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/lib/native/${JAVA_PLATFORM}
-    else
-      JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${JAVA_PLATFORM}
-    fi
-  fi
-fi
-
-HADOOP_OPTS="$HADOOP_OPTS -Dmumak.log.dir=$HADOOP_LOG_DIR"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.tmp.dir=$HADOOP_LOG_DIR/tmp"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
-  HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi  
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
-
-function print_usage(){
-  echo "Usage: $script [--config dir] trace.json topology.json"
-}
-
-if [ $# <= 2 ]; then
-  print_usage
-  exit
-fi
-
-exec "$JAVA" -enableassertions $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" org.apache.hadoop.mapred.SimulatorEngine -conf=${MUMAK_HOME}/conf/${project}.xml "$@"

hadoop-mapreduce-project/src/contrib/mumak/build.xml

@@ -1,120 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- 
-Before you can run these subtargets directly, you need 
-to call at top-level: ant deploy-contrib compile-core-test
--->
-<project name="mumak" default="jar" basedir="."
-   xmlns:ivy="antlib:org.apache.ivy.ant"> 
-
-  <property name="main-class"  value="org.apache.hadoop.mapred.SimulatorEngine"/>
-  <property name="version" value="0.21.0-dev"/>
-  <property name="javac.args" value=""/>
-  <property name="javac.version" value="1.6"/>
-  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
-  <import file="../build-contrib.xml"/>
-  <property name="mumak.stamp.file" value="${build.dir}/mumak.uptodate.stamp"/>
-
-  <path id="mumak-classpath">
-    <pathelement location="${hadoop.root}/build/contrib/capacity-scheduler/classes"/>
-    <path refid="contrib-classpath"/>
-  </path>
-
-  <target name="compile-java-sources" depends="init, ivy-retrieve-common" unless="skip.contrib">
-    <echo message="contrib: ${name}"/>
-    <javac
-    encoding="${build.encoding}"
-    srcdir="${src.dir}"
-    includes="**/*.java"
-    destdir="${build.classes}"
-    debug="${javac.debug}"
-    deprecation="${javac.deprecation}">
-    <classpath refid="mumak-classpath"/>
-    </javac>
-  </target>
-
-  <!-- Weaving aspects in place -->
-  <target name="compile-aspects" depends="check.aspects,compile-java-sources"
-      unless="build.unnecessary"> 
-    <!-- AspectJ task definition -->
-    <taskdef
-        resource="org/aspectj/tools/ant/taskdefs/aspectjTaskdefs.properties">
-      <classpath>
-        <pathelement location="${common.ivy.lib.dir}/aspectjtools-1.6.5.jar"/>
-      </classpath>
-    </taskdef>
-    <echo message="${javac.version}"/>
-    <echo message="Start weaving aspects in place"/>
-    <iajc
-      encoding="${build.encoding}" 
-      srcdir="${hadoop.root}/src/java/;${hadoop.root}/src/contrib/capacity-scheduler/src/java/;${hadoop.root}/build/src/;${src.dir}"
-      includes="org/apache/hadoop/**/*.java, org/apache/hadoop/**/*.aj"
-      destDir="${build.classes}"
-      debug="${javac.debug}"
-      target="${javac.version}"
-      source="${javac.version}"
-      fork="yes"
-      deprecation="${javac.deprecation}">
-      <classpath refid="mumak-classpath"/>
-    </iajc>
-    <touch file="${mumak.stamp.file}" mkdirs="true" />
-    <echo message="Weaving of aspects is finished"/>
-  </target>
-
-  <target name="compile" depends="compile-aspects" unless="skip.contrib"/>
-
-  <target name="package" depends="jar">
-    <mkdir dir="${dist.dir}/contrib/${name}" />
-    <mkdir dir="${dist.dir}/contrib/${name}/bin" />
-    <mkdir dir="${dist.dir}/contrib/${name}/conf" />
-    <copy todir="${dist.dir}/contrib/${name}" includeEmptyDirs="false">
-      <fileset dir="${build.dir}">
-        <include name="${dest.jar}" />
-      </fileset>
-    </copy>
-    <copy todir="${dist.dir}/contrib/${name}/bin" includeEmptyDirs="true"
-		overwrite="true">
-      <fileset dir="${root}/bin" />
-    </copy>
-    <copy todir="${dist.dir}/contrib/${name}/conf" includeEmptyDirs="true">
-      <fileset dir="${root}/conf" />
-    </copy>
-    <exec executable="sed">
-      <arg value="-i"/>
-      <arg value="-e"/>
-      <arg value="s/^HADOOP_VERSION=/HADOOP_VERSION=${version}/"/>
-      <arg value="${dist.dir}/contrib/${name}/bin/mumak.sh"/>
-    </exec>
-    <chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*.sh" />
-  </target>
-
-  <target name="check.aspects">
-    <uptodate property="build.unnecessary"
-        targetfile="${mumak.stamp.file}">
-      <srcfiles dir="${hadoop.root}/src/java/"
-        includes="org/apache/hadoop/**/*.java" />
-      <srcfiles dir="${hadoop.root}/src/webapps/"
-        includes="**/*.jsp" />
-      <srcfiles dir="${src.dir}" includes="org/apache/hadoop/**/*.aj" />
-    </uptodate>
-  </target>
-
-</project>
-

hadoop-mapreduce-project/src/contrib/mumak/conf/log4j.properties

@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# Define some default values that can be overridden by system properties
-#
-
-mumak.root.logger=INFO,console,mumak
-mumak.log.dir=.
-mumak.log.file=mumak.log
-mumak.log.layout=org.apache.log4j.PatternLayout
-mumak.log.layout.pattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# null == NullAppender
-#
-
-log4j.appender.null=org.apache.log4j.varia.NullAppender
-
-#
-# console == ConsoleAppender
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=${mumak.log.layout}
-log4j.appender.console.layout.ConversionPattern=${mumak.log.layout.pattern}
-
-#
-# general mumak output goes here
-#
-log4j.appender.mumak=org.apache.log4j.FileAppender
-log4j.appender.mumak.File=${mumak.log.dir}/${mumak.log.file}
-log4j.appender.mumak.layout=${mumak.log.layout}
-log4j.appender.mumak.layout.ConversionPattern=${mumak.log.layout.pattern}
-
-#
-# job summary output (commenting/uncommenting the following block
-# to disable/enable the separate output of such information)
-#
-mumak.jsa.log.dir=${mumak.log.dir}
-mumak.jsa.log.file=mumak-jobs-summary.log
-mumak.jsa.logger=INFO,jsa
-log4j.appender.jsa=org.apache.log4j.FileAppender
-log4j.appender.jsa.File=${mumak.jsa.log.dir}/${mumak.jsa.log.file}
-log4j.appender.jsa.layout=org.apache.log4j.PatternLayout
-log4j.appender.jsa.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${mumak.jsa.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${mumak.root.logger}
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-# Custom Logging levels tuned for mumak
-
-log4j.logger.org.apache.hadoop.net.NetworkTopology=WARN
-log4j.logger.org.apache.hadoop.mapred.JobTracker=WARN
-log4j.logger.org.apache.hadoop.mapred.ResourceEstimator=WARN
-log4j.logger.org.apache.hadoop.mapred.Counters=ERROR
-log4j.logger.org.apache.hadoop.io.compress.CodecPool=WARN
-log4j.logger.org.apache.hadoop.mapred.CompletedJobStatusStore=WARN
-log4j.logger.org.apache.hadoop.mapred.EagerTaskInitializationListener=WARN
-log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
-# set the following level to WARN/ERROR to show/ignore situation where task
-# info is  missing in the trace
-log4j.logger.org.apache.hadoop.tools.rumen.ZombieJob=ERROR
-# set the following level to WARN/ERROR to show/ignore false alarms where tasks
-# complete after job failed.
-log4j.logger.org.apache.hadoop.mapred.JobInProgress=ERROR
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG

hadoop-mapreduce-project/src/contrib/mumak/conf/mumak.xml

@@ -1,44 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<configuration>
-
-<property>
-  <name>mumak.scale.racklocal</name>
-  <value>1.5</value>
-  <description>Scaling factor for task attempt runtime of rack-local over
-  node-local</description>
-</property>
-
-<property>
-  <name>mumak.scale.rackremote</name>
-  <value>3.0</value>
-  <description>Scaling factor for task attempt runtime of rack-remote over
-  node-local</description>
-</property>
-
-<property>
-  <name>mumak.job-submission.policy</name>
-  <value>REPLAY</value>
-  <description>Job submission policy
-  </description>
-</property>
-
-</configuration>

hadoop-mapreduce-project/src/contrib/mumak/ivy.xml

@@ -1,144 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<ivy-module version="1.0" xmlns:m="http://ant.apache.org/ivy/maven">
-  <info organisation="org.apache.hadoop" module="${ant.project.name}">
-    <license name="Apache 2.0"/>
-    <description>
-        Mumak
-    </description>
-  </info>
-  <configurations defaultconfmapping="default">
-    <!--these match the Maven configurations-->
-    <conf name="default" extends="master,runtime"/>
-    <conf name="master" description="contains the artifact but no dependencies"/>
-    <conf name="runtime" description="runtime but not the artifact" />
-
-    <conf name="common" visibility="private" 
-      extends="runtime"
-      description="artifacts needed to compile/test the application"/>
-    <conf name="test" visibility="private" extends="master,common,runtime"/>
-  </configurations>
-
-  <publications>
-    <!--get the artifact from our module name-->
-    <artifact conf="master"/>
-  </publications>
-  <dependencies>
-    <dependency org="org.apache.hadoop" 
-      name="hadoop-annotations"
-      rev="${hadoop-common.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-common" 
-                rev="${hadoop-common.version}" conf="common->default">
-      <artifact name="hadoop-common" ext="jar"/>
-      <artifact name="hadoop-common" type="test" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs" 
-                rev="${hadoop-hdfs.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
-                rev="${hadoop-hdfs.version}" conf="test->default">
-      <artifact name="hadoop-hdfs" type="test" ext="jar" m:classifier="tests"/>
-    </dependency>
-    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" 
-               rev="${yarn.version}" conf="common->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-yarn-common"
-               rev="${yarn.version}" conf="common->default"/>
-<dependency org="commons-logging"
-      name="commons-logging"
-      rev="${commons-logging.version}"
-      conf="common->default"/>
-    <dependency org="log4j"
-      name="log4j"
-      rev="${log4j.version}"
-      conf="common->master"/>
-    <dependency org="org.codehaus.jackson"
-      name="jackson-mapper-asl"
-      rev="${jackson.version}"
-      conf="common->default"/>
-    <dependency org="org.codehaus.jackson"
-      name="jackson-core-asl"
-      rev="${jackson.version}"
-      conf="common->default"/>
-    <dependency org="junit"
-      name="junit"
-      rev="${junit.version}"
-      conf="common->default"/>
-    <dependency org="org.aspectj"
-      name="aspectjrt"
-      rev="${aspectj.version}"
-      conf="common->default">
-    </dependency>
-    <dependency org="org.aspectj"
-      name="aspectjtools"
-      rev="${aspectj.version}"
-      conf="common->default">
-    </dependency>
-    <!-- necessary for Mini*Clusters -->
-    <dependency org="commons-httpclient"
-      name="commons-httpclient"
-      rev="${commons-httpclient.version}"
-      conf="common->master"/>
-    <dependency org="commons-codec"
-      name="commons-codec"
-      rev="${commons-codec.version}"
-      conf="common->default"/>
-    <dependency org="commons-net"
-      name="commons-net"
-      rev="${commons-net.version}"
-      conf="common->default"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jetty-util"
-      rev="${jetty-util.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-api-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="jsp-2.1"
-      rev="${jetty.version}"
-      conf="common->master"/>
-    <dependency org="org.mortbay.jetty"
-      name="servlet-api-2.5"
-      rev="${servlet-api-2.5.version}"
-      conf="common->master"/>
-    <dependency org="commons-cli"
-      name="commons-cli"
-      rev="${commons-cli.version}"
-      conf="common->default"/>
-    <dependency org="org.apache.avro"
-      name="avro"
-	  rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
-
-   <!-- Exclusions for transitive dependencies pulled in by log4j -->
-   <exclude org="com.sun.jdmk"/>
-   <exclude org="com.sun.jmx"/>
-   <exclude org="javax.jms"/> 
-   <exclude org="javax.mail"/> 
-
-  </dependencies>
-</ivy-module>

hadoop-mapreduce-project/src/contrib/mumak/ivy/libraries.properties

@@ -1,23 +0,0 @@
-#   Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  See the NOTICE file distributed with
-#   this work for additional information regarding copyright ownership.
-#   The ASF licenses this file to You under the Apache License, Version 2.0
-#   (the "License"); you may not use this file except in compliance with
-#   the License.  You may obtain a copy of the License at
-#  
-#       http://www.apache.org/licenses/LICENSE-2.0
-#    
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#This properties file lists the versions of the various artifacts used by streaming.
-#It drives ivy and the generation of a maven POM
-
-#Please list the dependencies name with version if they are different from the ones 
-#listed in the global libraries.properties file (in alphabetical order)
-
-jackson.version=1.0.1
-aspectj.version=1.6.5

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/AllMapsCompletedTaskAction.java

@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * This class is used for notifying a SimulatorTaskTracker running a reduce task
- * that all map tasks of the job are done. A SimulatorJobTracker notifies a
- * SimulatorTaskTracker by sending this TaskTrackerAction in response to a
- * heartbeat(). Represents a directive to start running the user code of the
- * reduce task.
- * 
- * We introduced this extra 'push' mechanism so that we don't have to implement
- * the corresponding, more complicated 'pull' part of the InterTrackerProtocol.
- * We do not use proper simulation Events for signaling, and hack heartbeat()
- * instead, since the job tracker does not emit Events and does not know the
- * recipient task tracker _Java_ object.
- */
-class AllMapsCompletedTaskAction extends TaskTrackerAction {
-  /** Task attempt id of the reduce task that can proceed. */
-  private final org.apache.hadoop.mapreduce.TaskAttemptID taskId;
-
-  /**
-   * Constructs an AllMapsCompletedTaskAction object for a given
-   * {@link org.apache.hadoop.mapreduce.TaskAttemptID}.
-   * 
-   * @param taskId
-   *          {@link org.apache.hadoop.mapreduce.TaskAttemptID} of the reduce
-   *          task that can proceed
-   */
-  public AllMapsCompletedTaskAction(
-      org.apache.hadoop.mapreduce.TaskAttemptID taskId) {
-    super(ActionType.LAUNCH_TASK);
-    this.taskId = taskId;
-  }
-
-  /**
-   * Get the task attempt id of the reduce task.
-   * 
-   * @return the {@link org.apache.hadoop.mapreduce.TaskAttemptID} of the
-   *         task-attempt.
-   */
-  public org.apache.hadoop.mapreduce.TaskAttemptID getTaskID() {
-    return taskId;
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    taskId.write(out);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    taskId.readFields(in);
-  }
-
-  @Override
-  public String toString() {
-    return "AllMapsCompletedTaskAction[taskID=" + taskId + "]";
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/DeterministicCollectionAspects.aj

@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.concurrent.ConcurrentHashMap;
-
-// HashSet and HashMap do not guarantee that the oder of iteration is 
-// determinstic. We need the latter for the deterministic replay of 
-// simulations. These iterations are heavily used in the JobTracker, e.g. when 
-// looking for non-local map tasks. Not all HashSet and HashMap instances
-// are iterated over, but to be safe and simple we replace all with 
-// LinkedHashSet and LinkedHashMap whose iteration order is deterministic.
-
-public privileged aspect DeterministicCollectionAspects {
-
-  // Fortunately the Java runtime type of generic classes do not contain
-  // the generic parameter. We can catch all with a single base pattern using
-  // the erasure of the generic type.
-
-  HashSet around() : call(HashSet.new()) {
-    return new LinkedHashSet();
-  }
-
-  HashMap around() : call(HashMap.new()) {
-    return new LinkedHashMap();
-  }
-
-  ConcurrentHashMap around() : call(ConcurrentHashMap.new()) {
-    return new FakeConcurrentHashMap();
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/EagerTaskInitializationListenerAspects.aj

@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-public aspect EagerTaskInitializationListenerAspects {
-
-  pointcut overrideJobAdded (JobInProgressListener listener, JobInProgress job) :
-    call (void JobInProgressListener.jobAdded(JobInProgress)) &&
-    target (listener) &&
-    args (job);
-  
-  void around(JobInProgressListener listener, JobInProgress job) : 
-    overrideJobAdded (listener, job) {
-    if (listener instanceof EagerTaskInitializationListener) {
-      ((EagerTaskInitializationListener)listener).ttm.initJob(job);
-    } else {
-      proceed(listener, job);
-    }
-  }
-
-  pointcut overrideJobRemoved (JobInProgressListener listener, JobInProgress job) :
-    call (void JobInProgressListener.jobRemoved(JobInProgress)) &&
-    target (listener) &&
-    args(job);
-  
-  void around(JobInProgressListener listener, JobInProgress job) : 
-    overrideJobRemoved (listener, job) {
-    if (listener instanceof EagerTaskInitializationListener) {
-      // no-op
-    } else {
-      proceed(listener, job);
-    }
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/FakeConcurrentHashMap.java

@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.util.Collection;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * A fake ConcurrentHashMap implementation that maintains the insertion order of
- * entries when traversing through iterator. The class is to support
- * deterministic replay of Mumak and not meant to be used as a ConcurrentHashMap
- * replacement with multiple threads.
- */
-public class FakeConcurrentHashMap<K, V> extends ConcurrentHashMap<K, V> {
-  private final Map<K, V> map;
-  
-  public FakeConcurrentHashMap() {
-    map = new LinkedHashMap<K, V>();
-  }
-  
-  @Override
-  public V putIfAbsent(K key, V value) {
-    if (!containsKey(key)) {
-      return put(key, value);
-    } else {
-      return get(key);
-    }
-  }
-
-  @Override
-  public boolean remove(Object key, Object value) {
-    if (!containsKey(key)) return false;
-    Object oldValue = get(key);
-    if ((oldValue == null) ? value == null : oldValue.equals(value)) {
-      remove(key);
-      return true;
-    }
-    return false;
-  }
-
-  @Override
-  public V replace(K key, V value) {
-    if (containsKey(key)) {
-      return put(key, value);
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  public boolean replace(K key, V oldValue, V newValue) {
-    if (!containsKey(key)) return false;
-    Object origValue = get(key);
-    if ((origValue == null) ? oldValue == null : origValue.equals(oldValue)) {
-      put(key, newValue);
-      return true;
-    }
-    return false;
-  }
-
-  @Override
-  public void clear() {
-    map.clear();
-  }
-
-  @Override
-  public boolean containsKey(Object key) {
-    return map.containsKey(key);
-  }
-
-  @Override
-  public boolean containsValue(Object value) {
-    return map.containsValue(value);
-  }
-
-  @Override
-  public Set<Map.Entry<K, V>> entrySet() {
-    return map.entrySet();
-  }
-
-  @Override
-  public V get(Object key) {
-    return map.get(key);
-  }
-
-  @Override
-  public boolean isEmpty() {
-    return map.isEmpty();
-  }
-
-  @Override
-  public Set<K> keySet() {
-    return map.keySet();
-  }
-
-  @Override
-  public V put(K key, V value) {
-    return map.put(key, value);
-  }
-
-  @Override
-  public void putAll(Map<? extends K, ? extends V> t) {
-    map.putAll(t);
-  }
-
-  @Override
-  public V remove(Object key) {
-    return map.remove(key);
-  }
-
-  @Override
-  public int size() {
-    return map.size();
-  }
-
-  @Override
-  public Collection<V> values() {
-    return map.values();
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/HeartbeatEvent.java

@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * This class is used by {@link SimulatorTaskTracker}s for signaling themselves
- * when the next hearbeat() call to the JobTracker is due.
- */
-class HeartbeatEvent extends SimulatorEvent {
-  /**
-   * Constructor.
-   * 
-   * @param listener
-   *          the {@link SimulatorTaskTracker} this event should be delivered to
-   * @param timestamp
-   *          the time when this event is to be delivered
-   */
-  public HeartbeatEvent(SimulatorEventListener listener, long timestamp) {
-    super(listener, timestamp);
-  }
-
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/JobCompleteEvent.java

@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * {@link JobCompleteEvent} is created by {@link SimulatorJobTracker} when a job
- * is completed. {@link SimulatorJobClient} picks up the event, and mark the job
- * as completed. When all jobs are completed, the simulation is terminated.
- */
-public class JobCompleteEvent extends SimulatorEvent {
-
-  private SimulatorEngine engine;
-  private JobStatus jobStatus;
-
-  public JobCompleteEvent(SimulatorJobClient jc, long timestamp, 
-                          JobStatus jobStatus, SimulatorEngine engine) {
-    super(jc, timestamp);
-    this.engine = engine;
-    this.jobStatus = jobStatus;
-  }
-
-  public SimulatorEngine getEngine() {
-    return engine;
-  }
-
-  public JobStatus getJobStatus() {
-    return jobStatus;
-  }
-
-  @Override
-  protected String realToString() {
-    return super.realToString()+", status=("+jobStatus.toString()+")";
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/JobInitializationPollerAspects.aj

@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * This aspect is used for NOT starting the job initialization threads of the capacity scheduler.
- * We schedule the body of these threads manually from the SimulatorJobTracker according to 
- * simulation time.
- */
- 
-public aspect JobInitializationPollerAspects {
-
-  pointcut overrideInitializationThreadStarts () : 
-    (target (JobInitializationPoller) || 
-     target (JobInitializationPoller.JobInitializationThread)) && 
-    call (public void start());
-  
-  void around() : overrideInitializationThreadStarts () {
-    // no-op
-  }
-  
-  
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/JobSubmissionEvent.java

@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-import org.apache.hadoop.tools.rumen.JobStory;
-
-/**
- * {@link SimulatorEvent} for trigging the submission of a job to the job tracker.
- */
-public class JobSubmissionEvent extends SimulatorEvent {
-  private final JobStory job;
-  
-  public JobSubmissionEvent(SimulatorEventListener listener, long timestamp,  
-                            JobStory job) {
-    super(listener, timestamp);
-    this.job = job;
-  }
-  
-  public JobStory getJob() {
-    return job;
-  }
-
-  @Override
-  protected String realToString() {
-    return super.realToString() + ", jobID=" + job.getJobID();
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/LoadProbingEvent.java

@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * {@link LoadProbingEvent} is created by {@link SimulatorJobTracker} when the
- * {@link SimulatorJobSubmissionPolicy} is STRESS. {@link SimulatorJobClient}
- * picks up the event, and would check whether the system load is stressed. If
- * not, it would submit the next job.
- */
-public class LoadProbingEvent extends SimulatorEvent {
-  public LoadProbingEvent(SimulatorJobClient jc, long timestamp) {
-    super(jc, timestamp);
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorCSJobInitializationThread.java

@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobInitializationPoller.JobInitializationThread;
-
-public class SimulatorCSJobInitializationThread implements SimulatorEventListener {
-
-  long lastCalled;
-  CapacityTaskScheduler taskScheduler;
-  JobInitializationPoller jobPoller;
-  private final String queue;
-  final long sleepInterval;
-  /** The log object to send our messages to; only used for debugging. */
-  private static final Log LOG = LogFactory.getLog(SimulatorCSJobInitializationThread.class);
-
-  public SimulatorCSJobInitializationThread(TaskScheduler taskScheduler, 
-      String queue) {
-    this.taskScheduler = (CapacityTaskScheduler) taskScheduler;
-    jobPoller = this.taskScheduler.getInitializationPoller(); 
-    sleepInterval = jobPoller.getSleepInterval();
-    this.queue = queue;
-  }
-
-  @Override
-  public List<SimulatorEvent> accept(SimulatorEvent event) throws IOException {
-
-    SimulatorThreadWakeUpEvent e;
-    if(event instanceof SimulatorThreadWakeUpEvent) {
-      e = (SimulatorThreadWakeUpEvent) event;
-    }
-    else {
-      throw new IOException("Received an unexpected type of event in " + 
-      "SimThrdCapSchedJobInit");
-    }
-    jobPoller.cleanUpInitializedJobsList();
-    jobPoller.selectJobsToInitialize();
-    JobInitializationThread thread = 
-      jobPoller.getThreadsToQueueMap().get(this.queue);
-    thread.initializeJobs();   	
-    lastCalled = e.getTimeStamp();
-    List<SimulatorEvent> returnEvents = 
-      Collections.<SimulatorEvent>singletonList(
-          new SimulatorThreadWakeUpEvent(this,
-              lastCalled + sleepInterval));
-    return returnEvents;
-  }
-
-  @Override
-  public List<SimulatorEvent> init(long when) throws IOException {
-
-    return Collections.<SimulatorEvent>singletonList(
-        new SimulatorThreadWakeUpEvent(this,
-            when + sleepInterval));
-  }
-
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorClock.java

@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-/**
- * A clock class - can be mocked out for testing.
- */
-class SimulatorClock extends Clock {
-
-  long currentTime;
-
-  SimulatorClock (long now) {
-	  super();
-	  currentTime = now;
-  }
-  void setTime(long now) {
-    currentTime = now;
-  }
-
-  @Override
-  long getTime() {
-    return currentTime;
-  }
-}

hadoop-mapreduce-project/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorEngine.java

@@ -1,422 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.Map;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.SimulatorEvent;
-import org.apache.hadoop.mapred.SimulatorEventQueue;
-import org.apache.hadoop.mapred.JobCompleteEvent;
-import org.apache.hadoop.mapred.SimulatorJobClient;
-import org.apache.hadoop.mapred.SimulatorJobTracker;
-import org.apache.hadoop.mapred.SimulatorTaskTracker;
-import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.StaticMapping;
-import org.apache.hadoop.tools.rumen.ClusterStory;
-import org.apache.hadoop.tools.rumen.ClusterTopologyReader;
-import org.apache.hadoop.tools.rumen.JobStoryProducer;
-import org.apache.hadoop.tools.rumen.LoggedNetworkTopology;
-import org.apache.hadoop.tools.rumen.MachineNode;
-import org.apache.hadoop.tools.rumen.RackNode;
-import org.apache.hadoop.tools.rumen.ZombieCluster;
-import org.apache.hadoop.tools.rumen.RandomSeedGenerator;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * {@link SimulatorEngine} is the main class of the simulator. To launch the
- * simulator, user can either run the main class directly with two parameters,
- * input trace file and corresponding topology file, or use the script
- * "bin/mumak.sh trace.json topology.json". Trace file and topology file are
- * produced by rumen.
- */
-public class SimulatorEngine extends Configured implements Tool {
-  public static final List<SimulatorEvent> EMPTY_EVENTS = new ArrayList<SimulatorEvent>();
-  /** Default number of milliseconds required to boot up the entire cluster. */
-  public static final int DEFAULT_CLUSTER_STARTUP_DURATION = 100*1000;
-  protected final SimulatorEventQueue queue = new SimulatorEventQueue();
-  String traceFile;
-  String topologyFile;
-  SimulatorJobTracker jt;
-  SimulatorJobClient jc;
-  boolean shutdown = false;
-  long terminateTime = Long.MAX_VALUE;
-  long currentTime;
-  /** The HashSet for storing all the simulated threads useful for 
-   * job initialization for capacity scheduler.
-   */
-  HashSet<SimulatorCSJobInitializationThread> threadSet;
-  /** The log object to send our messages to; only used for debugging. */
-  private static final Log LOG = LogFactory.getLog(SimulatorEngine.class);
-  
-  /** 
-   * Master random seed read from the configuration file, if present.
-   * It is (only) used for creating sub seeds for all the random number 
-   * generators.
-   */
-  long masterRandomSeed;
-                                                                                                                                            
-  /**
-   * Start simulated task trackers based on topology.
-   * @param clusterStory the cluster topology.
-   * @param jobConf configuration object.
-   * @param now
-   *    time stamp when the simulator is started, {@link SimulatorTaskTracker}s
-   *    are started uniformly randomly spread in [now,now+startDuration).
-   * @return time stamp by which the entire cluster is booted up and all task
-   *    trackers are sending hearbeats in their steady rate.
-   */
-  long startTaskTrackers(ClusterStory cluster, JobConf jobConf, long now) {
-    /** port assigned to TTs, incremented by 1 for each TT */
-    int port = 10000;
-    int numTaskTrackers = 0;
-
-    Random random = new Random(RandomSeedGenerator.getSeed(
-       "forStartTaskTrackers()", masterRandomSeed));
-
-    final int startDuration = jobConf.getInt("mumak.cluster.startup.duration",
-        DEFAULT_CLUSTER_STARTUP_DURATION);
-    
-    for (MachineNode node : cluster.getMachines()) {
-      jobConf.set("mumak.tasktracker.host.name", node.getName());
-      jobConf.set("mumak.tasktracker.tracker.name",
-          "tracker_" + node.getName() + ":localhost/127.0.0.1:" + port);
-      long subRandomSeed = RandomSeedGenerator.getSeed(
-         "forTaskTracker" + numTaskTrackers, masterRandomSeed);
-      jobConf.setLong("mumak.tasktracker.random.seed", subRandomSeed);
-      numTaskTrackers++;
-      port++;
-      SimulatorTaskTracker tt = new SimulatorTaskTracker(jt, jobConf);
-      long firstHeartbeat = now + random.nextInt(startDuration);
-      queue.addAll(tt.init(firstHeartbeat));
-    }
-    
-    // In startDuration + heartbeat interval of the full cluster time each 
-    // TT is started up and told on its 2nd heartbeat to beat at a rate 
-    // corresponding to the steady state of the cluster    
-    long clusterSteady = now + startDuration + jt.getNextHeartbeatInterval();
-    return clusterSteady;
-  }
-
-  /**
-   * Reads a positive long integer from a configuration.
-   *
-   * @param Configuration conf configuration objects
-   * @param String propertyName name of the property
-   * @return time
-   */
-  long getTimeProperty(Configuration conf, String propertyName,
-                       long defaultValue) 
-      throws IllegalArgumentException {
-    // possible improvement: change date format to human readable ?
-    long time = conf.getLong(propertyName, defaultValue);
-    if (time <= 0) {
-      throw new IllegalArgumentException(propertyName + "time must be positive: "
-          + time);
-    }
-    return time;
-  }
-   
-  /**
-   * Creates the configuration for mumak simulation. This is kept modular mostly for 
-   * testing purposes. so that the standard configuration can be modified before passing
-   * it to the init() function.
-   * @return JobConf: the configuration for the SimulatorJobTracker 
-   */
-    
-  JobConf createMumakConf() {
-    JobConf jobConf = new JobConf(getConf());
-    jobConf.setClass("topology.node.switch.mapping.impl",
-        StaticMapping.class, DNSToSwitchMapping.class);
-    jobConf.set("fs.default.name", "file:///");
-    jobConf.set("mapred.job.tracker", "localhost:8012");
-    jobConf.setInt("mapred.jobtracker.job.history.block.size", 512);
-    jobConf.setInt("mapred.jobtracker.job.history.buffer.size", 512);
-    jobConf.setLong("mapred.tasktracker.expiry.interval", 5000);
-    jobConf.setInt("mapred.reduce.copy.backoff", 4);
-    jobConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
-    jobConf.setUser("mumak");
-    jobConf.set("mapred.system.dir", 
-        jobConf.get("hadoop.log.dir", "/tmp/hadoop-"+jobConf.getUser()) + "/mapred/system");
-    
-    return jobConf;
-  }
-
-  /**
-   * Initialize components in the simulation.
-   * @throws InterruptedException
-   * @throws IOException if trace or topology files cannot be opened.
-   */
-  void init() throws InterruptedException, IOException {
-    
-    JobConf jobConf = createMumakConf();
-    init(jobConf);
-  }
-    
-  /**
-   * Initiate components in the simulation. The JobConf is
-   * create separately and passed to the init().
-   * @param JobConf: The configuration for the jobtracker.
-   * @throws InterruptedException
-   * @throws IOException if trace or topology files cannot be opened.
-   */
-  @SuppressWarnings("deprecation")
-  void init(JobConf jobConf) throws InterruptedException, IOException {
-    
-    FileSystem lfs = FileSystem.getLocal(getConf());
-    Path logPath =
-      new Path(System.getProperty("hadoop.log.dir")).makeQualified(lfs);
-    jobConf.set("mapred.system.dir", logPath.toString());
-    jobConf.set("hadoop.job.history.location", (new Path(logPath, "history")
-        .toString()));
-    
-    // start time for virtual clock
-    // possible improvement: set default value to sth more meaningful based on
-    // the 1st job
-    long now = getTimeProperty(jobConf, "mumak.start.time", 
-                               System.currentTimeMillis());
-
-    jt = SimulatorJobTracker.startTracker(jobConf, now, this);
-    jt.offerService();
-    
-    masterRandomSeed = jobConf.getLong("mumak.random.seed", System.nanoTime()); 
-    
-    // max Map/Reduce tasks per node
-    int maxMaps = getConf().getInt(
-        "mapred.tasktracker.map.tasks.maximum",
-        SimulatorTaskTracker.DEFAULT_MAP_SLOTS);
-    int maxReduces = getConf().getInt(
-        "mapred.tasktracker.reduce.tasks.maximum",
-    
-      SimulatorTaskTracker.DEFAULT_REDUCE_SLOTS);
-
-    MachineNode defaultNode = new MachineNode.Builder("default", 2)
-        .setMapSlots(maxMaps).setReduceSlots(maxReduces).build();
-            
-    LoggedNetworkTopology topology = new ClusterTopologyReader(new Path(
-        topologyFile), jobConf).get();
-    // Setting the static mapping before removing numeric IP hosts.
-    setStaticMapping(topology);
-    if (getConf().getBoolean("mumak.topology.filter-numeric-ips", true)) {
-      removeIpHosts(topology);
-    }
-    ZombieCluster cluster = new ZombieCluster(topology, defaultNode);
-    
-    // create TTs based on topology.json  
-    long firstJobStartTime = startTaskTrackers(cluster, jobConf, now);
-
-    long subRandomSeed = RandomSeedGenerator.getSeed("forSimulatorJobStoryProducer",
-                                                     masterRandomSeed);
-    JobStoryProducer jobStoryProducer = new SimulatorJobStoryProducer(
-        new Path(traceFile), cluster, firstJobStartTime, jobConf, subRandomSeed);
-
-    final SimulatorJobSubmissionPolicy submissionPolicy = SimulatorJobSubmissionPolicy
-        .getPolicy(jobConf);
-
-    jc = new SimulatorJobClient(jt, jobStoryProducer, submissionPolicy);
-    queue.addAll(jc.init(firstJobStartTime));
-
-    //if the taskScheduler is CapacityTaskScheduler start off the JobInitialization
-    //threads too
-    if (jobConf.get("mapred.jobtracker.taskScheduler").equals
-       (CapacityTaskScheduler.class.getName())) {
-      LOG.info("CapacityScheduler used: starting simulatorThreads");
-      startSimulatorThreadsCapSched(now);
-    }
-    terminateTime = getTimeProperty(jobConf, "mumak.terminate.time",
-                                    Long.MAX_VALUE);
-  }
-  
-  /**
-   * In this function, we collect the set of leaf queues from JobTracker, and 
-   * for each of them creates a simulated thread that performs the same
-   * check as JobInitializationPoller.JobInitializationThread in Capacity Scheduler.  
-   * @param now
-   * @throws IOException
-   */
-  private void startSimulatorThreadsCapSched(long now) throws IOException {
-    
-    Set<String> queueNames = jt.getQueueManager().getLeafQueueNames();
-    TaskScheduler taskScheduler = jt.getTaskScheduler();
-    threadSet = new HashSet<SimulatorCSJobInitializationThread>();
-    // We create a different thread for each queue and hold a 
-    //reference to  each of them 
-    for (String jobQueue: queueNames) {
-      SimulatorCSJobInitializationThread capThread = 
-        new SimulatorCSJobInitializationThread(taskScheduler,jobQueue);   
-      threadSet.add(capThread);
-      queue.addAll(capThread.init(now));
-    }
-  }
-  
-  /**
-   * The main loop of the simulation. First call init() to get objects ready,
-   * then go into the main loop, where {@link SimulatorEvent}s are handled removed from
-   * the {@link SimulatorEventQueue}, and new {@link SimulatorEvent}s are created and inserted
-   * into the {@link SimulatorEventQueue}.
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  void run() throws IOException, InterruptedException {
-    init();
-    
-    for (SimulatorEvent next = queue.get(); next != null
-        && next.getTimeStamp() < terminateTime && !shutdown; next = queue.get()) {
-      currentTime = next.getTimeStamp();
-      assert(currentTime == queue.getCurrentTime());
-      SimulatorEventListener listener = next.getListener();
-      List<SimulatorEvent> response = listener.accept(next);
-      queue.addAll(response);
-    }
-    
-    summary(System.out);
-  }
-  
-  /**
-   * Run after the main loop.
-   * @param out stream to output information about the simulation
-   */
-  void summary(PrintStream out) {
-    out.println("Done, total events processed: " + queue.getEventCount());
-  }
-  
-  public static void main(String[] args) throws Exception {
-    int res = ToolRunner.run(new Configuration(), new SimulatorEngine(), args);
-    System.exit(res);
-  }
-  
-  @Override
-  public int run(String[] args) throws Exception {
-    parseParameters(args);
-    try {
-      run();
-      return 0;
-    } finally {
-      if (jt != null) {
-        jt.getTaskScheduler().terminate();
-      }
-    }
-  }
-
-  void parseParameters(String[] args) {
-    if (args.length != 2) {
-      throw new IllegalArgumentException("Usage: java ... SimulatorEngine trace.json topology.json");
-    }
-    traceFile = args[0];
-    topologyFile = args[1];
-  }
-
-  /**
-   * Called when a job is completed. Insert a {@link JobCompleteEvent} into the
-   * {@link SimulatorEventQueue}. This event will be picked up by
-   * {@link SimulatorJobClient}, which will in turn decide whether the
-   * simulation is done.
-   * @param jobStatus final status of a job, SUCCEEDED or FAILED
-   * @param timestamp time stamp when the job is completed
-   */
-  void markCompletedJob(JobStatus jobStatus, long timestamp) {
-    queue.add(new JobCompleteEvent(jc, timestamp, jobStatus, this));
-  }
-
-  /**
-   * Called by {@link SimulatorJobClient} when the simulation is completed and
-   * should be stopped.
-   */
-  void shutdown() {
-    shutdown = true;
-  }
-  
-  /**
-   * Get the current virtual time of the on-going simulation. It is defined by
-   * the time stamp of the last event handled.
-   * @return the current virtual time
-   */
-  long getCurrentTime() {
-    return currentTime;
-  }
-  
-  // Due to HDFS-778, a node may appear in job history logs as both numeric
-  // ips and as host names. We remove them from the parsed network topology
-  // before feeding it to ZombieCluster.
-  static void removeIpHosts(LoggedNetworkTopology topology) {
-    for (Iterator<LoggedNetworkTopology> rackIt = topology.getChildren()
-        .iterator(); rackIt.hasNext();) {
-      LoggedNetworkTopology rack = rackIt.next();
-      List<LoggedNetworkTopology> nodes = rack.getChildren();
-      for (Iterator<LoggedNetworkTopology> it = nodes.iterator(); it.hasNext();) {
-        LoggedNetworkTopology node = it.next();
-        if (isIPAddress(node.getName())) {
-          it.remove();
-        }
-      }
-      if (nodes.isEmpty()) {
-        rackIt.remove();
-      }
-    }
-  }
-
-  static Pattern IP_PATTERN;
-  
-  static {
-    // 0-255
-    String IPV4BK1 = "(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)";
-    // .b.c.d - where b/c/d are 0-255, and optionally adding two more
-    // backslashes before each period
-    String IPV4BKN = "(?:\\\\?\\." + IPV4BK1 + "){3}";
-    String IPV4_PATTERN = IPV4BK1 + IPV4BKN;
-    
-    // first hexadecimal number
-    String IPV6BK1 = "(?:[0-9a-fA-F]{1,4})";
-    // remaining 7 hexadecimal numbers, each preceded with ":".
-    String IPV6BKN = "(?::" + IPV6BK1 + "){7}";
-    String IPV6_PATTERN = IPV6BK1 + IPV6BKN;
-
-    IP_PATTERN = Pattern.compile(
-        "^(?:" + IPV4_PATTERN + "|" + IPV6_PATTERN + ")$");
-  }
-
- 
-  static boolean isIPAddress(String hostname) {
-    return IP_PATTERN.matcher(hostname).matches();
-  }
-  
-  static void setStaticMapping(LoggedNetworkTopology topology) {
-    for (LoggedNetworkTopology rack : topology.getChildren()) {
-      for (LoggedNetworkTopology node : rack.getChildren()) {
-        StaticMapping.addNodeToRack(node.getName(), 
-            new RackNode(rack.getName(), 1).getName());
-      }
-    }
-  }
-}