Preview of final shape of MAPREDUCE-279 into trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-279-merge@1158935 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce/.eclipse.templates/.launches/AllMapredTests.launch

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
+<stringAttribute key="bad_container_name" value="/@PROJECT@/.l"/>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/@PROJECT@/src/test/mapred"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="2"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value="=@PROJECT@/src\/test\/mapred"/>
+<booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
+<stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
+<stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
+<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value=""/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms256m -Xmx512m -Dtest.build.data=${workspace_loc:@PROJECT@}/build/test -Dtest.cache.data=${workspace_loc:@PROJECT@}/build/test/cache -Dtest.debug.data=${workspace_loc:@PROJECT@}/build/test/debug -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/build/test/log -Dtest.src.dir=${workspace_loc:@PROJECT@}/build/test/src/mapred -Dtest.build.extraconf=${workspace_loc:@PROJECT@}/build/test/extraconf -Dhadoop.policy.file=hadoop-policy.xml"/>
+</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/JobTracker.launch

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/@PROJECT@/src/mapred/org/apache/hadoop/mapred/JobTracker.java"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="1"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+</listAttribute>
+<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/conf}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.mapred.JobTracker"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx1000m -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=${workspace_loc:@PROJECT@} -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/logs"/>
+</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/SpecificTestTemplate.launch

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
+<stringAttribute key="bad_container_name" value="/@PROJECT@/.l"/>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/@PROJECT@"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="4"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value=""/>
+<booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
+<stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
+<stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
+<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.TestNameHere"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms256m -Xmx512m -Dtest.build.data=${workspace_loc:@PROJECT@}/build/test -Dtest.cache.data=${workspace_loc:@PROJECT@}/build/test/cache -Dtest.debug.data=${workspace_loc:@PROJECT@}/build/test/debug -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/build/test/log -Dtest.src.dir=${workspace_loc:@PROJECT@}/build/test/src -Dtest.build.extraconf=${workspace_loc:@PROJECT@}/build/test/extraconf -Dhadoop.policy.file=hadoop-policy.xml"/>
+</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/TaskTracker.launch

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/@PROJECT@/src/mapred/org/apache/hadoop/mapred/TaskTracker.java"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="1"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
+<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+</listAttribute>
+<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/conf}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.mapred.TaskTracker"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx1000m -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=${workspace_loc:@PROJECT@} -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/logs"/>
+</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/README.txt

@@ -0,0 +1,6 @@
+This directory contains templates for generating Eclipse files to configure
+Eclipse for Hadoop development.
+
+For further information please consult
+
+http://wiki.apache.org/hadoop/EclipseEnvironment 

hadoop-mapreduce/.gitignore

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*~
+.classpath
+.project
+.launches
+.settings
+*.iml
+*.ipr
+*.iws
+.idea
+.svn
+build/
+build-fi/
+build.properties
+conf/masters
+conf/slaves
+conf/core-site.xml
+conf/hdfs-site.xml
+conf/hadoop-env.sh
+conf/hadoop-site.xml
+conf/mapred-site.xml
+conf/hadoop-policy.xml
+conf/capacity-scheduler.xml
+conf/fair-scheduler.xml
+conf/mapred-queue-acls.xml
+conf/mapred-queue-acls.xml.template
+conf/mapred-queues.xml
+docs/api/
+logs/
+src/contrib/capacity-scheduler/src/java/mapred-queues.xml
+src/contrib/index/conf/index-config.xml
+src/docs/build
+src/docs/cn/build
+src/docs/cn/src/documentation/sitemap.xmap
+src/docs/cn/uming.conf

hadoop-mapreduce/CHANGES.txt

@@ -0,0 +1,2526 @@
+Hadoop MapReduce Change Log
+
+Trunk (unreleased changes)
+
+  INCOMPATIBLE CHANGES
+
+    MAPREDUCE-2455. Remove deprecated JobTracker.State in favour of
+    JobTrackerStatus. (tomwhite)
+
+    MAPREDUCE-2430. Remove mrunit contrib. (nigel via eli)
+
+    MAPREDUCE-2606. Remove IsolationRunner. (Alejandro Abdelnur via eli)
+
+  NEW FEATURES
+
+    MAPREDUCE-2682. Add "mapred classpath" command to print classpath
+    for MR applications. (vinodkv via acmurthy) 
+
+    MAPREDUCE-2107. [Gridmix] Total heap usage emulation in Gridmix.
+    (Amar Kamat and Ravi Gummadi via amarrk)
+
+    MAPREDUCE-2106. [Gridmix] Cumulative CPU usage emulation in Gridmix. 
+    (amarrk)
+
+    MAPREDUCE-2543. [Gridmix] High-Ram feature emulation in Gridmix. (amarrk)
+
+    MAPREDUCE-2408. [Gridmix] Compression emulation in Gridmix. (amarrk)
+
+    MAPREDUCE-2473. Add "mapred groups" command to query the server-side groups
+    resolved for a user. (Aaron T. Myers via todd)
+
+    MAPREDUCE-461. Enable ServicePlugins for the JobTracker.
+    (Fredrik Hedberg via tomwhite)
+
+    MAPREDUCE-2521. Create RPM and Debian packages for MapReduce. Changes 
+    deployment layout to be consistent across the binary tgz, rpm, and deb.
+    (Eric Yang via omalley)
+
+    MAPREDUCE-2323. Add metrics to the fair scheduler. (todd)
+
+    MAPREDUCE-2037. Capture intermediate progress, CPU and memory usage for
+    tasks. (Dick King via acmurthy) 
+
+  IMPROVEMENTS
+
+    MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
+    acmurthy) 
+
+    MAPREDUCE-2365. Add counters to track bytes (read,written) via 
+    File(Input,Output)Format. (Siddharth Seth via acmurthy)
+ 
+    MAPREDUCE-2680. Display queue name in job client CLI. (acmurthy) 
+ 
+    MAPREDUCE-2679. Minor changes to sync trunk with MR-279 branch. (acmurthy) 
+ 
+    MAPREDUCE-2400. Remove Cluster's dependency on JobTracker via a 
+    ServiceProvider for the actual implementation. (tomwhite via acmurthy) 
+ 
+    MAPREDUCE-2596. [Gridmix] Summarize Gridmix runs. (amarrk)
+
+    MAPREDUCE-2563. [Gridmix] Add High-Ram emulation system tests to 
+    Gridmix. (Vinay Kumar Thota via amarrk)
+
+    MAPREDUCE-2104. [Rumen] Add Cpu, Memory and Heap usages to 
+    TraceBuilder's output. (amarrk)
+
+    MAPREDUCE-2554. [Gridmix]  Add distributed cache emulation system tests 
+    to Gridmix. (Vinay Kumar Thota via amarrk)
+ 
+    MAPREDUCE-2543. [Gridmix] High-Ram feature emulation testcase. (amarrk)
+
+    MAPREDUCE-2469. Task counters should also report the total heap usage of 
+    the task. (Ravi Gummadi and Amar Ramesh Kamat via amarrk)
+
+    MAPREDUCE-2544. [Gridmix] Add compression emulation system tests to 
+    Gridmix. (Vinay Kumar Thota via amarrk)
+
+    MAPREDUCE-2517. [Gridmix] Add system tests to Gridmix. 
+    (Vinay Kumar Thota via amarrk)
+
+    MAPREDUCE-2492. The new MapReduce API should make available task's
+    progress to the task. (amarrk)
+
+    MAPREDUCE-2153. Bring in more job configuration properties in to the trace 
+    file. (Rajesh Balamohan via amarrk)
+
+    MAPREDUCE-1461. Feature to instruct rumen-folder utility to skip jobs worth 
+    of specific duration. (Rajesh Balamohan via amarrk)
+
+    MAPREDUCE-2172. Added test-patch.properties required by test-patch.sh 
+    (nigel)
+
+    MAPREDUCE-2156. Raid-aware FSCK. (Patrick Kling via dhruba)
+
+    MAPREDUCE-2215. A more elegant FileSystem#listCorruptFileBlocks API
+    (RAID changes) (Patrick Kling via hairong)
+
+    MAPREDUCE-1831. BlockPlacement policy for HDFS-RAID.
+    (Scott Chen via dhruba)
+
+    MAPREDUCE-1906. Lower minimum heartbeat interval for TaskTracker
+    (Scott Carey and Todd Lipcon via todd)
+
+    MAPREDUCE-1382. MRAsyncDiscService should tolerate missing local.dir.
+    (Zheng Shao and tomwhite via tomwhite)
+
+    MAPREDUCE-2263. MapReduce side of HADOOP-6904: RPC compatibility.
+    (hairong)
+
+    MAPREDUCE-1706. Log RAID recoveries on HDFS. (schen)
+
+    MAPREDUCE-2334. Update BlockPlacementPolicyRaid for the new method
+    in BlockPlacementPolicy.  (szetszwo)
+
+    MAPREDUCE-2254. Allow setting of end-of-record delimiter for
+    TextInputFormat (Ahmed Radwan via todd)
+
+    MAPREDUCE-1927. Unit test for HADOOP-6835 (concatenated gzip support).
+    (Greg Roelofs via tomwhite)
+
+    MAPREDUCE-2206. The task-cleanup tasks should be optional. (schen)
+
+    MAPREDUCE-2225. MultipleOutputs should not require the use of 'Writable'.
+    (Harsh J Chouraria via tomwhite)
+
+    MAPREDUCE-1811. Job.monitorAndPrintJob() should print status of the job
+    at completion. (Harsh J Chouraria via tomwhite)
+
+    MAPREDUCE-993. bin/hadoop job -events <jobid> <from-event-#> <#-of-events>
+    help message is confusing. (Harsh J Chouraria via tomwhite)
+
+    MAPREDUCE-2302. Add static factory methods in GaloisField. (schen)
+
+    MAPREDUCE-2351. mapred.job.tracker.history.completed.location should
+    support an arbitrary filesystem URI. (tomwhite)
+
+    MAPREDUCE-2239. BlockPlacementPolicyRaid should call getBlockLocations
+    only when necessary. (schen)
+
+    MAPREDUCE-2331. Add coverage of task graph servlet to fair scheduler system
+    test. (todd)
+
+    MAPREDUCE-2367. Allow using a file to exclude certain tests from build.
+    (todd)
+
+    MAPREDUCE-2202. Generalize CLITest structure and interfaces to faciliate
+    upstream adoption (e.g. for web or system testing). (cos)
+
+    MAPREDUCE-2420. JobTracker should be able to renew delegation token over 
+    HTTP (Boris Shkolnik via jitendra)
+
+    MAPREDUCE-2474. Add docs to the new API Partitioner on how to access the
+    Job Configuration. (Harsh J Chouraria via todd)
+    
+    MAPREDUCE-2475. Disable IPV6 for junit tests. (suresh srinivas via mahadev)
+
+    MAPREDUCE-2422. Removed unused internal methods from DistributedCache.
+    (tomwhite)
+
+    MAPREDUCE-2456. Log the reduce taskID and associated TaskTrackers with
+    failed fetch notifications in the JobTracker log.
+    (Jeffrey Naisbitt via cdouglas)
+
+    MAPREDUCE-869. Documentation for config to set map/reduce task environment
+    (Alejandro Abdelnur via todd)
+
+    MAPREDUCE-2410. Add entry to streaming FAQ about how streaming reducers
+    receive keys. (Harsh J Chouraria via todd)
+
+    MAPREDUCE-2499. MR part of HADOOP-7291. (eli)
+
+    MAPREDUCE-2497. Missing spaces in error messages. (eli)
+
+    MAPREDUCE-2502. JobSubmitter should use mapreduce.job.maps instead of
+    its deprecated equivalent. (eli via todd)
+
+    MAPREDUCE-2381. JobTracker instrumentation not consistent about error
+    handling. (Philip Zeyliger via tomwhite)
+
+    MAPREDUCE-2449. Allow for command line arguments when performing
+    "Run on Hadoop" action in Eclipse plugin. (Jeff Zemerick via todd)
+
+    MAPREDUCE-2483. Remove duplication of jars between Hadoop subprojects
+    from build artifacts. (Eric Yang via omalley)
+
+    MAPREDUCE-2372. TaskLogAppender mechanism shouldn't be set up in
+    log4j.properties (todd)
+
+    MAPREDUCE-2516. Rename webinterface.private.actions to
+    mapreduce.jobtracker.webinterface.trusted (Ari Rabkin via todd)
+
+    MAPREDUCE-2459. Cache HAR filesystem metadata. (Mac Yang via mahadev)
+
+    HADOOP-7259. Contrib modules should include the build.properties from
+    the enclosing hadoop directory. (omalley)
+
+    MAPREDUCE-2494. Order distributed cache deletions by LRU. (Robert Joseph
+    Evans via cdouglas)
+
+    MAPREDUCE-2452. Makes the cancellation of delegation tokens happen in a 
+    separate thread. (ddas)
+
+    HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects.
+    (todd, nigel)
+
+    MAPREDUCE-2249. Check the reflexive property of Counters objects when
+    comparing equality. (Devaraj K via todd)
+
+    MAPREDUCE-2623. Update ClusterMapReduceTestCase to use
+    MiniDFSCluster.Builder (Harsh J Chouraria via eli)
+
+    MAPREDUCE-2602. Allow setting of end-of-record delimiter for
+    TextInputFormat for the old API. (Ahmed Radwan via todd)
+
+    MAPREDUCE-2705. Permits parallel multiple task launches. 
+    (Thomas Graves via ddas)
+
+    MAPREDUCE-2489. Jobsplits with random hostnames can make the queue 
+    unusable (jeffrey naisbit via mahadev)
+
+  OPTIMIZATIONS
+
+    MAPREDUCE-2026. Make JobTracker.getJobCounters() and
+    JobInProgress.getCounters() aquire locks in a shorter time period.
+    (Joydeep Sen Sarma via schen)
+
+    MAPREDUCE-2740. MultipleOutputs in new API creates needless
+    TaskAttemptContexts. (todd)
+
+    MAPREDUCE-901. Efficient framework counters. (llu via acmurthy)
+
+  BUG FIXES
+
+    MAPREDUCE-2603. Disable High-Ram emulation in system tests. 
+    (Vinay Kumar Thota via amarrk)
+
+    MAPREDUCE-2539. Fixed NPE in getMapTaskReports in JobClient. (Robert Evans via
+    acmurthy) 
+
+    MAPREDUCE-2531. Fixed jobcontrol to downgrade JobID. (Robert Evans via
+    acmurthy) 
+
+    MAPREDUCE-1978. Rumen TraceBuilder should provide recursive
+    input folder scanning.
+
+    MAPREDUCE-2416. Remove the restriction of specifying group names in
+    users-list file for Gridmix in RoundRobinUserResolver mode.
+
+    MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to
+    map testing/proxy users to unique users in a trace.
+
+    MAPREDUCE-2307. Exception thrown in Jobtracker logs, when the Scheduler
+    configured is FairScheduler. (Devaraj K via matei)
+
+    MAPREDUCE-2199. build is broken 0.22 branch creation. (cos)
+
+    MAPREDUCE-1752. Implement getFileBlockLocations in HarFilesystem.
+    (Patrick Kling via dhruba)
+
+    MAPREDUCE-2155. RaidNode should optionally use the mapreduce jobs to 
+    fix missing blocks.  (Patrick Kling via dhruba)
+
+    MAPREDUCE-1334. Fix TestIndexUpdater by ignoring _SUCCESS file in HDFS.
+    (Kay Kay via yhemanth)
+
+    MAPREDUCE-2232. Add missing methods to TestMapredGroupMappingServiceRefresh.
+    (Todd Lipcon via eli)
+
+    MAPREDUCE-2271. Fix TestSetupTaskScheduling failure on trunk.
+    (Liyin Liang via todd)
+
+    MAPREDUCE-2290. Fix compilation error in TestTaskCommit. (eli)
+
+    MAPREDUCE-2294. Fix compilation error in mumak. (todd)
+
+    MAPREDUCE-2300. Fix TestUmbilicalProtocolWithJobToken on trunk after
+    HADOOP-6904. (todd)
+
+    MAPREDUCE-2296. Fix references to misspelled method name
+    getProtocolSigature (todd)
+
+    MAPREDUCE-2311. Fix TestFairScheduler failure (schen)
+
+    MAPREDUCE-1996. API: Reducer.reduce() method detail misstatement.
+    (Harsh J Chouraria via tomwhite)
+
+    MAPREDUCE-2203. Wrong javadoc for TaskRunner's appendJobJarClasspaths
+    method. (Jingguo Yao via tomwhite)
+
+    MAPREDUCE-2074. Task should fail when symlink creation fails.
+    (Priyo Mustafi via tomwhite)
+
+    MAPREDUCE-1242. Chain APIs error misleading.
+    (Harsh J Chouraria via tomwhite)
+
+    MAPREDUCE-2379. Adds missing DistributedCache configurations in 
+    mapred-default.xml (Todd Lipcon via amareshwari)
+
+    MAPREDUCE-2348. Disable mumak tests on trunk since they currently time out
+    (todd)
+
+    MAPREDUCE-2395. TestBlockFixer timing out on trunk. (Ramkumar Vadali via
+    todd)
+
+    MAPREDUCE-2426. Make TestFairSchedulerSystem fail with more verbose output
+    (todd)
+
+    MAPREDUCE-2448. NoSuchMethodError:
+    org.apache.hadoop.hdfs.TestDatanodeBlockScanner.corruptReplica(..) (eli)
+
+    MAPREDUCE-2460. Fix flaky test TestFairSchedulerSystem. (todd)
+
+    MAPREDUCE-2451. Log the details from health check script at the
+    JobTracker. (Thomas Graves via cdouglas)
+
+    MAPREDUCE-2467. HDFS-1052 changes break the raid contrib module in 
+    MapReduce. (suresh srinivas via mahadev)
+
+    MAPREDUCE-2258. IFile reader closes stream and compressor in wrong order.
+    (todd via tomwhite)
+
+    MAPREDUCE-2518. The t flag is missing in distcp help message.  (Wei Yongjun
+    via szetszwo)
+
+    MAPREDUCE-2514. Fix typo in TaskTracker ReinitTrackerAction log message.
+    (Jonathan Eagles via cdouglas)
+
+    MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
+    diagnosis of related issues. (Jonathan Eagles via cdouglas)
+
+    MAPREDUCE-2495. exit() the TaskTracker when the distributed cache cleanup
+    thread dies. (Robert Joseph Evans via cdouglas)
+
+    MAPREDUCE-2470. Fix NPE in RunningJobs::getCounters. (Robert Joseph Evans
+    via cdouglas)
+
+    MAPREDUCE-2536. Update FsShell -mv command usage in TestMRCLI.  (Daryn
+    Sharp via szetszwo)
+
+    MAPREDUCE-2529. Add support for regex-based shuffle metric counting
+    exceptions. (Thomas Graves via cdouglas)
+
+    MAPREDUCE-2559. ant binary fails due to missing c++ lib dir. (eli)
+
+    MAPREDUCE-2573. Fix new findbugs warning introduced by MAPREDUCE-2494.
+    (Robert Joseph Evans via todd)
+
+    MAPREDUCE-2581. Spelling errors in log messages. (Tim Sell via eli)
+
+    MAPREDUCE-2588. Change raid to the new DataTransferProtocol API.  (szetszwo)
+
+    MAPREDUCE-2576. Typo in comment in SimulatorLaunchTaskAction.java.
+    (Tim Sell via jghoman)
+
+    MAPREDUCE-2550. Fix bin/mapred to work properly from within a source
+    checkout (Eric Yang via todd)
+
+    MAPREDUCE-2620. Update RAID for HDFS-2087.  (szetszwo)
+
+    MAPREDUCE-2624. Update RAID for HDFS-2107.  (szetszwo)
+
+    MAPREDUCE-2670. Fixing spelling mistake in FairSchedulerServlet.java. (eli)
+
+    MAPREDUCE-2710. Update JobSubmitter.printTokens(..) for HDFS-2161.
+    (szetszwo)
+
+    MAPREDUCE-2409. DistributedCache maps files and archives to the same path,
+    despite semantic incompatibility. (Siddharth Seth via cdouglas)
+
+    MAPREDUCE-2575. TestMiniMRDFSCaching fails if test.build.dir is set 
+    to something other than build/test (Thomas Graves via mahadev)
+
+    MAPREDUCE-2622. Remove the last remaining reference to the deprecated
+    configuration "io.sort.mb". (Harsh J Chouraria via todd)
+
+    MAPREDUCE-2732. Remove directly accessing FSNamesystem.LOG from
+    TestCopyFiles and TestDistCh.  (szetszwo)
+
+    MAPREDUCE-2463. Job history files are not moved to done folder when job
+    history location is hdfs.  (Devaraj K via szetszwo)
+
+    MAPREDUCE-2243. Close streams propely in a finally-block to avoid leakage
+    in CompletedJobStatusStore, TaskLog, EventWriter and TotalOrderPartitioner.
+    (Devaraj K via szetszwo)
+
+    MAPREDUCE-2741. Make ant build system work with hadoop-common JAR
+    generated by Maven. (Alejandro Abdelnur via tomwhite)
+
+    MAPREDUCE-2760. mapreduce.jobtracker.split.metainfo.maxsize typoed
+    in mapred-default.xml. (todd via eli)
+
+    MAPREDUCE-2797. Update mapreduce tests and RAID for HDFS-2239.  (szetszwo)
+
+    MAPREDUCE-2805. Update RAID for HDFS-2241.  (szetszwo)
+    
+    MAPREDUCE-2837. Ported bug fixes from y-merge to prepare for MAPREDUCE-279
+    merge. (acmurthy) 
+
+    MAPREDUCE-2541. Fixed a race condition in IndexCache.removeMap. (Binglin
+    Chang via acmurthy) 
+
+    MAPREDUCE-2839. Fixed TokenCache to get delegation tokens using both new
+    and old apis. (Siddharth Seth via acmurthy)
+
+    MAPREDUCE-2727. Fix divide-by-zero error in SleepJob for sleepCount equals
+    0. (Jeffrey Naisbitt via acmurthy)
+
+
+Release 0.22.0 - Unreleased
+
+  INCOMPATIBLE CHANGES
+
+    MAPREDUCE-1866. Removes deprecated class
+    org.apache.hadoop.streaming.UTF8ByteArrayUtils. (amareshwari)
+
+    MAPREDUCE-1664. Changes the behaviour of the combination of job-acls
+    when they function together with queue-acls. (Ravi Gummadi via vinodkv)
+
+  NEW FEATURES
+
+    MAPREDUCE-1804. Stress-test tool for HDFS introduced in HDFS-708.
+    (Joshua Harlow via shv)
+
+    MAPREDUCE-220. Collect cpu and memory statistics per task. (Scott Chen via
+    acmurthy)
+
+    MAPREDUCE-1970.  Reed-Solomon code implementation for HDFS RAID.
+    (Scott Chen via dhruba)
+
+    MAPREDUCE-2169. Integrated Reed-Solomon code with RaidNode. (Ramkumar
+    Vadali via schen)
+
+  IMPROVEMENTS
+
+    MAPREDUCE-2141. Add an "extra data" field to Task for use by Mesos. (matei)
+
+    MAPREDUCE-2140. Regenerate fair scheduler design doc PDF. (matei)
+
+    MAPREDUCE-1546. Redirect all job pages to corresponding history page
+    if job is not in JT memory. (Scott Chen via sharad)
+
+    MAPREDUCE-1092. Enable assertions for unit tests. (Eli Collins via
+    cdouglas)
+
+    MAPREDUCE-1680. Add a metric recording JobTracker heartbeats processed.
+    (Dick King via cdouglas)
+
+    MAPREDUCE-1761. FairScheduler allows separate configuration of node
+    and rack locality wait time (Scott Chen via dhruba)
+
+    MAPREDUCE-1539. authorization checks for inter-server protocol
+    (based on HADOOP-6600) (Boris Shkolnik via shv)
+
+    MAPREDUCE-1798. Names the configuration keys for the Kerberos
+    principals better. (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1773. streaming doesn't support jobclient.output.filter.
+    (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1785. Add streaming config option for not emitting the key.
+    (Eli Collins via sharad)
+
+    MAPREDUCE-572. If #link is missing from uri format of -cacheArchive
+    then streaming does not throw error. (Amareshwari Sriramadasu via
+    vinodkv)
+
+    MAPREDUCE-1545. Add timestamps for first task type launched in job summary.
+    (Luke Lu via cdouglas)
+
+    MAPREDUCE-1543. Add an audit log for authentication events. (Amar Kamat and
+    Luke Lu via cdouglas)
+
+    MAPREDUCE-1762. Add ability to set values of task counters. (Scott Chen via
+    cdouglas)
+
+    MAPREDUCE-1533. Reduce overhead of logging and string manipulation during
+    heartbeat processing. (Amar Kamat and Dick King via cdouglas)
+
+    MAPREDUCE-1516. JobTracker issues delegation tokens only if the user's
+    authentication is Kerberos. (Jitendra Pandey via ddas)
+
+    MAPREDUCE-647. Update distcp forrest documentation to reflect the changes
+    of HADOOP-5472, MAPREDUCE-642 and HADOOP-5620.  (Rodrigo Schmidt via
+    szetszwo)
+
+    MAPREDUCE-1851. Documents configuration parameters in streaming.
+    (amareshwari)
+    
+    MAPREDUCE-1868. Add a read and connection timeout to JobClient while
+    pulling tasklogs. (Krishna Ramachandran via acmurthy)
+
+    MAPREDUCE-1778. Ensure failure to setup CompletedJobStatusStore is not
+    silently ignored by the JobTracker. (Krishna Ramachandran via acmurthy)
+
+    MAPREDUCE-1850. Includes job submit host information (name and ip) in
+    jobconf and jobdetails display (Krishna Ramachandran via amareshwari)
+
+    MAPREDUCE-1893. Slive with multiple reducers. (shv)
+
+    MAPREDUCE-1248. Fixes redudant memory copying in StreamKeyValUtil.
+    (Ruibang He via amareshwari)
+
+    MAPREDUCE-1840. Enhancements to Gridmix benchmark simulating user
+    diversity, queue replay, and task duration for JobTracker load testing.
+    Also includes compatibility with security enhancements, and scalability
+    improvements. (Amar Kamat, Rahul Singh, Hong Tang, and cdouglas)
+
+    MAPREDUCE-1848. Put number of speculative, data local, rack local 
+    tasks in JobTracker metrics. (Scott Chen via dhruba)
+
+    MAPREDUCE-1935. Makes the Distcp to work in a secure environment.
+    (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1945. The MapReduce component for HADOOP-6632.
+    (Kan Zhang & Jitendra Pandey via ddas)
+
+    MAPREDUCE-1936. Modify Gridmix3 to support more tunable parameters for
+    stress submission and sleep jobs. (Hong Tang via cdouglas)
+
+    MAPREDUCE-1733. Makes pipes applications secure. (Jitendra Pandey via ddas)
+
+    MAPREDUCE-1566. Adds a configuration attribute using which job clients can
+    specify a credentials file. The tokens from there will be passed to the job.
+    (Jitendra Pandey and Owen O'Malley via ddas)
+
+    MAPREDUCE-1624. Documents the job credentials and associated details to do 
+    with delegation tokens (on the client side). 
+    (Jitendra Pandey and Devaraj Das via ddas)
+
+    MAPREDUCE-1834. TestSimulatorDeterministicReplay timesout on trunk.
+    (Hong Tang via mahadev)
+
+    MAPREDUCE-1993. Fixes test failure
+    TestTrackerDistributedCacheManagerWithLinuxTaskController. (Devaraj Das
+    via amareshwari)
+
+    MAPREDUCE-1523. Making Mumak work with Capacity-Scheduler (Anirban Das
+    via mahadev)
+
+    MAPREDUCE-1920. Enables completed jobstatus store by default. (Tom White
+    via amareshwari)
+
+    MAPREDUCE-1881. Improve TaskTrackerInstrumentation to enable collection of
+    advanced metrics. (Matei Zaharia via acmurthy)
+
+    MAPREDUCE-1548. Hadoop archives preserve times and other properties from 
+    original files. (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1517. Supports streaming job to run in the background. (Bochun Bai
+    via amareshwari)
+
+    MAPREDUCE-1819. RaidNode is now smarter in submitting Raid jobs. (Ramkumar
+    Vadali via schen)
+
+    MAPREDUCE-2132. A command line option in RaidShell to fix blocks using raid
+
+    MAPREDUCE-2147. Remove redundant lines in JobInProgress ctor.
+    (Harsh J Chouraria via cutting)
+   
+    HADOOP-7007. Update the hudson-test-patch ant target to work with the
+    latest test-patch.sh script (gkesavan)
+
+    MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar
+    Vadali via schen)
+
+    MAPREDUCE-2051. Contribute a fair scheduler preemption system test.
+    (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-1892. RaidNode can allow layered policies more efficiently.
+    (Ramkumar Vadali via schen)
+
+    MAPREDUCE-1592. Generate Eclipse's .classpath file from Ivy config. 
+    (tomwhite via nigel)
+
+    MAPREDUCE-2073. TestTrackerDistributedCacheManager should be up-front
+    about requirements on build environment. (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-2093. Herriot JT and TT clients should vend statistics. (cos)
+
+    MAPREDUCE-2167. Faster directory traversal for raid node. (Ramkumar Vadali
+    via schen)
+
+    MAPREDUCE-1931. Gridmix forrest documentation . (Ranjit Mathew via vinodkv).
+
+    MAPREDUCE-2184. Port DistRaid.java to new mapreduce API. (Ramkumar Vadali
+    via schen)
+
+    MAPREDUCE-1878. Add MRUnit documentation. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test (todd)
+
+    MAPREDUCE-2250. Fix logging in raid code. (Ramkumar Vadali via schen)
+
+    MAPREDUCE-2260. Remove auto-generated native build files. (rvs via eli)
+
+    MAPREDUCE-2314. configure files that are generated as part of the released
+    tarball need to have executable bit set (rvs via cos)
+
+    MAPREDUCE-1159.  Limit Job name in web UI to be 80 char long.  (Harsh J
+    Chouraria via szetszwo)
+
+    MAPREDUCE-2337. Remove dependence of public MapReduce API on classes in
+    server package. (tomwhite)
+
+    MAPREDUCE-2383. Improve documentation of DistributedCache methods (Harsh J
+    Chouraria via todd)
+
+    MAPREDUCE-2222. Ivy resolve force mode should be turned off by default.
+    (Luke Lu via tomwhite)
+
+    MAPREDUCE-2103. task-controller shouldn't require o-r permissions.
+    (todd via eli)
+
+    MAPREDUCE-2505. Explain how to use ACLs in the fair scheduler.
+    (matei via eli)
+
+  OPTIMIZATIONS
+
+    MAPREDUCE-1354. Enhancements to JobTracker for better performance and
+    scalability. (Arun C. Murthy & Richard King via acmurthy) 
+
+    MAPREDUCE-1829. JobInProgress.findSpeculativeTask should use min() to
+    find the candidate instead of sort(). (Scott Chen via vinodkv)
+
+  BUG FIXES
+
+    MAPREDUCE-1845. FairScheduler.tasksToPreempt() can return negative number.
+    (Scott Chen via matei)
+
+    MAPREDUCE-1707. TaskRunner can get NPE in getting ugi from TaskTracker.
+    (Vinod Kumar Vavilapalli)
+    
+    MAPREDUCE-1532. Ensures that delegation tokens is obtained as the
+    actual user when the proxy-user is used for submitting jobs. Also
+    refactors the DelegationTokenToRenew class. (ddas)
+
+    MAPREDUCE-1558. Fixes MRAdmin to look up the principal of the 
+    JobTracker and use that in the RefreshUserToGroupsMapping protocol and
+    RefreshAuthorizationPolicyProtocol. (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1662. Remove unused methods from TaskRunner. (Amareshwari
+    Sriramadasu via cdouglas)
+
+    MAPREDUCE-1617. Use IPv4 stack for unit tests. (Amar Kamat and Luke Lu via
+    cdouglas)
+
+    MAPREDUCE-1599. Fixes MRBench so that it reuses tokens across jobs
+    correctly. (Jitendra Nath Pandey via ddas)
+
+    MAPREDUCE-1836. Refresh for proxy superuser config (mr part for HDFS-1096).
+    (Boris Shkolnik via shv)
+
+    MAPREDUCE-1505. Create RPC client on job submission, not in cstr of Job
+    instance. (Dick King via cdouglas)
+
+    MAPREDUCE-1813. NPE in PipeMapred.MRErrorThread. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1225. Fixes DistributedCache to check if the file is fresh or not,
+    for the first localization also. (Zhong Wang via amareshwari)
+
+    MAPREDUCE-1559. Fixes the token renewer to use the JobTracker's 
+    credentials for talking to the NameNode. (ddas)
+
+    MAPREDUCE-1492. Delete  obsolete har files used on the parity files
+    of hdfs raid. (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1857. Removes unused configuration parameters in streaming.
+    (amareshwari)
+
+    MAPREDUCE-1887. MRAsyncDiskService now properly absolutizes volume root
+    paths. (Aaron Kimball via zshao)
+
+    MAPREDUCE-1863. Fix NPE in Rumen when processing null CDF for failed task
+    attempts. (Amar Kamat via cdouglas)
+
+    MAPREDUCE-1864. Removes uninitialized/unused variables in
+    org.apache.hadoop.streaming.PipeMapRed. (amareshwari)
+
+    MAPREDUCE-1888. Fixes Streaming to override output key and value types, 
+    only if mapper/reducer is a command. (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-577. Fixes duplicate records in StreamXmlRecordReader.
+    (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-1894. Fixed a bug in DistributedRaidFileSystem.readFully() 
+    that was causing it to loop infinitely. (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-1838. Reduce the time needed for raiding a bunch of files
+    by randomly assigning files to map tasks. (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-1820. Fix InputSampler to clone sampled keys. (Alex Kozlov via
+    cdouglas)
+
+    MAPREDUCE-1528. Incorporates the changes to the credentials API done in
+    HADOOP-6845. Also, introduces Credentials in JobConf, and in JobContext.
+    (Jitendra Pandey and Arun Murthy via ddas)
+
+    MAPREDUCE-1865. Rumen should also support jobhistory files generated using
+    trunk. (Amar Kamat via amareshwari)
+
+    MAPREDUCE-1621. Fixes NPE in TextOutputReader.getLastOutput if it has never
+    read any output. (amareshwari)
+
+    MAPREDUCE-1911. Fixes errors in -info message in streaming. (amareshwari) 
+
+    MAPREDUCE-1772. Corrects errors in streaming documentation in forrest.
+    (amareshwari)
+
+    MAPREDUCE-1925. Fix failing TestRumenJobTraces. (Ravi Gummadi via cdouglas)
+
+    MAPREDUCE-1718. Fixes a bug in the construction of jobconf key for the
+    mapping that the tasks use at runtime for looking up delegation tokens.
+    (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1701. Fixes a problem to do with exception handling in 
+    delegation-token renewals. (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1686. Fixes StreamUtil.goodClassOrNull to find classes without
+    package names. (Paul Burkhardt via amareshwari)
+
+    MAPREDUCE-1288. Fixes TrackerDistributedCacheManager to take into account
+    the owner of the localized file in the mapping from cache URIs to
+    CacheStatus objects. (ddas)
+
+    MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either 
+    of configuration properties "mapreduce.job.name" and "mapred.job.name".
+    (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-1958. The MapReduce part corresponding to the HADOOP-6873.
+    (Boris Shkolnik & Owen O'Malley via ddas)
+
+    MAPREDUCE-1900. TaskTracker and JobTracker closes FileSystems, opened on
+    behalf of users that it no longer requires. (Kan Zhang and ddas via ddas)
+
+    MAPREDUCE-1992. Fixes a problem to do with bringing up the JobTracker in
+    unsecure mode. (Kan Zhang via ddas)
+
+    MAPREDUCE-1999. Fixes ClientProtocol to use the correct 
+    DelegationTokenSelector. (Jitendra Pandey via ddas)
+
+    MAPREDUCE-1780. AccessControlList.toString() is used for serialization of
+    ACL in JobStatus.java. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1961. Fix ConcurrentModificationException in Gridmix during
+    shutdown. (Hong Tang via cdouglas)
+
+    MAPREDUCE-2000. Fix parsing of JobHistory lines in Rumen when quotes are
+    escaped. (Hong Tang via cdouglas)
+
+    MAPREDUCE-2022. Fixes compilation errors in TestSubmitJob. (amareshwari)
+
+    MAPREDUCE-1670. RAID policies should not scan their own destination path.
+    (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-1668. RaidNode Hars a directory only if all its parity files 
+    have been created. (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-2021. Fixes duplicate hostnames in CombineFileInputFormat's
+    split locations. (amareshwari)
+
+    MAPREDUCE-1375. Fixes flaky test TestFileArgs. (Todd Lipcon via
+    amareshwari)
+
+    MAPREDUCE-2023. TestDFSIO should not stop reading if curSize != bufferSize.
+    (Hong Tang via szetszwo)
+
+    MAPREDUCE-2031. Fixes test failures TestTaskLauncher and
+    TestTaskTrackerLocalization. (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-2046. Fixes CombineFileInputFormat to allow splits with size
+    less than DFS block size. (dhruba borthakur via amareshwari)
+
+    MAPREDUCE-1975. Fixes unnecessary InterruptedException log in gridmix.
+    (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-1597. Fixes CombineFileInputFormat to work with non-splittable
+    files. (amareshwari)
+
+    MAPREDUCE-2032. Fixes TestJobCleanup to cleanup test directory in
+    tearDown. (Dick King via amareshwari)
+
+    MAPREDUCE-1979. Fixes "Output directory already exists" error in gridmix
+    when gridmix.output.directory is not defined. (Ravi Gummadi via
+    amareshwari)
+
+    MAPREDUCE-1918. Adds documentation to Rumen. (Amar Kamat via amareshwari)
+
+    MAPREDUCE-2078. Fixes TraceBuilder to generate traces when a globbed job
+    history path is given. (Amar Kamat via amareshwari)
+
+    MAPREDUCE-1989. Fixes error message in gridmix when user resolver is set
+    and no user list is given. (Ravi Gummadi via amareshwari)
+
+    MAPREDUCE-2067.  Distinct minicluster services (e.g. NN and JT) overwrite
+    each other's service policies.  (Aaron T. Myers via tomwhite)
+
+    MAPREDUCE-2029. DistributedRaidFileSystem removes itself from FileSystem
+    cache when it is closed. (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable 
+    partfile size. (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-2082. Fixes Pipes to create the jobtoken file in the right
+    place. (Jitendra Pandey via ddas)
+
+    MAPREDUCE-2095. Fixes Gridmix to run from compressed traces. (Ranjit
+    Mathew via amareshwari)
+
+    MAPREDUCE-1908. DistributedRaidFileSystem now handles ChecksumException
+    correctly. (Ramkumar Vadali via schen)
+
+    MAPREDUCE-2126. JobQueueJobInProgressListener's javadoc is inconsistent
+    with source code. (Jingguo Yao via tomwhite)
+
+    MAPREDUCE-2143.  HarFileSystem is able to handle spaces in pathnames.
+    (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-1867.  Remove unused methods in
+    org.apache.hadoop.streaming.StreamUtil.  (amareshwari via tomwhite)
+
+    MAPREDUCE-2146.  Raid does not affect access time of a source file.
+    (Ramkumar Vadali via dhruba)
+
+    MAPREDUCE-2150.  RaidNode periodically fixes corrupt blocks. (Ramkumar Vadali via
+    schen)
+
+    MAPREDUCE-2099.  RaidNode recreates outdated parity HARs. (Ramkumar Vadali
+    via schen)
+
+    MAPREDUCE-2173.  Fix race condition in TestBlockFixer that was
+    causing  intermittent failure (Patrick Kling via dhruba)
+
+    MAPREDUCE-2142.  Refactor RaidNode so that the map-reduce component is
+    clearly separated out. (Patrick Kling via dhruba)
+
+    MAPREDUCE-2179. Fix RaidBlockSender compilation failure. (Ramkumar Vadali
+    via schen)
+
+    MAPREDUCE-2034. TestSubmitJob triggers NPE instead of permissions error.
+    (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-2195. New property for local conf directory in
+    system-test-mapreduce.xml file. (cos)
+
+    MAPREDUCE-1783. FairScheduler initializes tasks only when the job can be
+    run. (Ramkumar Vadali via schen)
+
+    MAPREDUCE-2224. Fix synchronization bugs in JvmManager. (todd)
+
+    MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux (todd)
+
+    MAPREDUCE-2096. Secure local filesystem IO from symlink vulnerabilities (todd)
+
+    MAPREDUCE-2234. If Localizer can't create task log directory, it should fail
+    on the spot. (todd)
+
+    MAPREDUCE-2219. JobTracker should not try to remove mapred.system.dir
+    during startup. (todd)
+
+    MAPREDUCE-2207. Task-cleanup task should not be scheduled on the node that
+    the task just failed. (Liyin Liang via schen)
+
+    MAPREDUCE-2084. Remove deprecate annotation for package file.  The package
+    classes themselves are already deprecated. This removes an Eclipse error.
+    (tomwhite via nigel)
+
+    MAPREDUCE-2248. DistributedRaidFileSystem should unraid only the corrupt
+    block (Ramkumar Vadali via schen)
+
+    MAPREDUCE-1085. For tasks, "ulimit -v -1" is being run when user doesn't
+    specify a ulimit (todd)
+
+    MAPREDUCE-2282. Fix TestMRServerPorts for the changes in
+    TestHDFSServerPorts.  (shv via szetszwo)
+
+    MAPREDUCE-2238. Fix permissions handling to avoid leaving undeletable
+    directories in local dirs. (todd)
+
+    MAPREDUCE-2277. TestCapacitySchedulerWithJobTracker needs to wait for jobs
+    to complete before testing status. (todd)
+
+    MAPREDUCE-2253. Servlets should specify content type (todd)
+
+    MAPREDUCE-2283. Add timeout for Raid Tests (Ramkumar Vadali via schen)
+
+    MAPREDUCE-1754. Replace mapred.persmissions.supergroup with an 
+    acl : mapreduce.cluster.administrators (Amareshwari Sriramadasu via shv)
+
+    MAPREDUCE-2256. FairScheduler fairshare preemption from multiple pools may
+    preempt all tasks from one pool causing that pool to go below fairshare.
+    (Priyo Mustafi via shv)
+
+    MAPREDUCE-2281. MR part of HADOOP-6642. (Chris Douglas, Po Cheung via shv)
+
+    MAPREDUCE-2200. TestUmbilicalProtocolWithJobToken is failing without Krb
+    evironment: needs to be conditional. (cos)
+
+    MAPREDUCE-2077. Resolve name clash in the deprecated 
+    o.a.h.util.MemoryCalculatorPlugin (Luke Lu via shv)
+
+    MAPREDUCE-2188. The new API MultithreadedMapper doesn't initialize
+    RecordReader. (Owen O'Malley via shv)
+
+    MAPREDUCE-1915. Fix IndexOutOfBoundsException in IndexCache.
+    (Priyo Mustafi via shv)
+
+    MAPREDUCE-1974. Fix multiple preemtions of the same task in FairScheduler.
+    (Scott Chen via shv)
+
+    MAPREDUCE-2304. Fix TestMRCLI to allow hostname with a hyphen (-).
+    (Priyo Mustafi via shv)
+
+    MAPREDUCE-1825. jobqueue_details.jsp and FairSchedulerServelet should not
+    call finishedMaps and finishedReduces when job is not initialized.
+    (Scott Chen via shv)
+
+    MAPREDUCE-2285. MiniMRCluster does not start after ant test-patch
+    (todd)
+
+    MAPREDUCE-2315. javadoc is failing in nightly build (todd)
+
+    MAPREDUCE-2054. Hierarchical queue implementation broke dynamic queue
+    addition in Dynamic Scheduler. (Thomas Sandholm via tomwhite)
+
+    MAPREDUCE-2272. Job ACL file should not be executable
+    (Harsh J Chouraria via todd)
+
+    MAPREDUCE-2241. ClusterWithLinuxTaskController should accept relative path
+    on the command line. (todd)
+
+    MAPREDUCE-2251. Remove unused mapreduce.job.userhistorylocation config.
+    (Harsh J Chouraria via todd)
+
+    MAPREDUCE-2284. TestLocalRunner.testMultiMaps times out (todd)
+
+    MAPREDUCE-2336. Tool-related packages should be in the Tool javadoc group.
+    (tomwhite)
+    
+    MAPREDUCE-2394. JUnit output format doesn't propagate into raid contrib
+    build. (todd)
+
+    MAPREDUCE-2392. TaskTracker shutdown in the tests sometimes take 60s.
+    (tomwhite)
+
+    MAPREDUCE-2437. SLive uses only part* files to generating the final report.
+    (shv)
+
+    MAPREDUCE-2428. start-mapred.sh script fails if HADOOP_HOME is not set.
+    (tomwhite via eli)
+
+    MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test.
+    (todd)
+
+    MAPREDUCE-2457. Job submission should inject group.name on the JobTracker
+    (Alejandro Abdelnur via todd)
+
+    MAPREDUCE-2472. Extra whitespace in mapred.child.java.opts breaks JVM
+    initialization. (Aaron T. Myers via todd)
+
+    MAPREDUCE-2222. Ivy resolve force mode should be turned off by default.
+    (Luke Lu via tomwhite)
+
+    MAPREDUCE-2486. Incorrect snapshot dependency published in .pom files
+    (todd)
+
+    MAPREDUCE-2327. MapTask doesn't need to put username information in
+    SpillRecord. (todd via tomwhite)
+
+    MAPREDUCE-2515. MapReduce code references some deprecated options
+    (Ari Rabkin via todd)
+
+    MAPREDUCE-2487. ChainReducer uses MAPPER_BY_VALUE instead of
+    REDUCER_BY_VALUE. (Devaraj K via todd)
+
+    MAPREDUCE-2185. Fix infinite loop at creating splits using
+    CombineFileInputFormat. (Ramkumar Vadali via schen)
+
+    MAPREDUCE-2571. CombineFileInputFormat.getSplits throws a
+    java.lang.ArrayStoreException. (Bochun Bai via todd)
+
+Release 0.21.1 - Unreleased
+
+  NEW FEATURES
+
+    MAPREDUCE-2040. Forrest Documentation for Dynamic Priority Scheduler.
+    (Thomas Sandholm via tomwhite)
+
+  BUG FIXES
+
+    MAPREDUCE-1897. trunk build broken on compile-mapred-test (cos)
+
+    MAPREDUCE-1280. Update Eclipse plugin to the new eclipse.jdt API.
+    (Alex Kozlov via szetszwo)
+
+    MAPREDUCE-1984. herriot TestCluster fails because exclusion is not there
+    (Balaji Rajagopalan via cos)
+
+    MAPREDUCE-2090. Clover build doesn't generate per-test coverage. (cos)
+
+    MAPREDUCE-2134. ant binary-system is broken in mapreduce project. (cos)
+
+    MAPREDUCE-1905. Fixes Context.setStatus() and progress() apis.
+    (amareshwari)
+
+    MAPREDUCE-1809. Ant build changes for Streaming system tests in contrib
+    projects. (Vinay Kumar Thota via amareshwari)
+
+    MAPREDUCE-2223. TestMRCLI might fail on Ubuntu with default /etc/hosts
+    (cos)
+
+    MAPREDUCE-2228. Remove java5 dependencies from build. (cos)
+
+    MAPREDUCE-1929. Allow artifacts to be published to the staging Apache Nexus
+    Maven Repository. (tomwhite)
+
+    MAPREDUCE-2317. Fix a NPE in HadoopArchives.  (Devaraj K via szetszwo)
+
+    MAPREDUCE-2127. mapreduce trunk builds are filing on hudson. 
+    (Bruno Mahé via eli)
+
+Release 0.21.0 - 2010-08-13
+
+  INCOMPATIBLE CHANGES
+
+    MAPREDUCE-516. Fix the starvation problem in the Capacity Scheduler 
+    when running High RAM Jobs. (Arun Murthy via yhemanth)
+
+    MAPREDUCE-358. Change org.apache.hadoop.examples. AggregateWordCount 
+    and org.apache.hadoop.examples.AggregateWordHistogram to use new 
+    mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-245. Change Job and jobcontrol classes to use the List interface
+    rather than ArrayList in APIs. (Tom White via cdouglas)
+
+    MAPREDUCE-766. Enhanced list-blacklisted-trackers to display reasons
+    for blacklisting a node. (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-817. Add a cache for retired jobs with minimal job info and 
+    provide a way to access history file url. (sharad)
+
+    MAPREDUCE-711. Moved Distributed Cache from Common to Map/Reduce
+    project. (Vinod Kumar Vavilapalli via yhemanth)
+
+    MAPREDUCE-895. Per the contract elucidated in HADOOP-6201, throw
+    FileNotFoundException from FileSystem::listStatus rather than returning
+    null. (Jakob Homan via cdouglas)
+
+    MAPREDUCE-479. Provide full task id to map output servlet rather than the
+    reduce id, only. (Jiaqi Tan via cdouglas)
+
+    MAPREDUCE-873. Simplify job recovery. Incomplete jobs are resubmitted on 
+    jobtracker restart. Removes a public constructor in JobInProgress. (sharad)
+
+    HADOOP-6230. Moved process tree and memory calculator related classes from
+    Common to Map/Reduce. (Vinod Kumar Vavilapalli via yhemanth)
+
+    MAPREDUCE-157. Refactor job history APIs and change the history format to 
+    JSON. (Jothi Padmanabhan via sharad)
+
+    MAPREDUCE-849. Rename configuration properties. (Amareshwari Sriramadasu 
+    via sharad)
+
+    MAPREDUCE-1287. Only call the partitioner with more than one reducer.
+    (cdouglas)
+
+    MAPREDUCE-1385. Use the new UserGroupInformation from HADOOP-6299.
+    (ddas via omalley)
+
+    MAPREDUCE-1493. Authorization for job-history pages. (vinodkv)
+
+    MAPREDUCE-1607. Task controller may not set permissions for a
+    task cleanup attempt's log directory (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1683. Remove JNI calls from ClusterStatus cstr. (Arun Murthy and
+    Luke Lu via cdouglas)
+
+    MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users) 
+    independent of the client side configuration. (Boris Shkolnik via ddas)
+
+  NEW FEATURES
+
+    MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth
+    Ramakrishnan, Konstantin Boudnik, at all via cos)
+
+    MAPREDUCE-706. Support for FIFO pools in the fair scheduler.
+    (Matei Zaharia)
+
+    MAPREDUCE-546. Provide sample fair scheduler config file in conf/ and use
+    it by default if no other config file is specified. (Matei Zaharia)
+
+    MAPREDUCE-551. Preemption support in the Fair Scheduler. (Matei Zaharia)
+
+    MAPREDUCE-567. Add a new example MR that always fails. (Philip Zeyliger
+    via tomwhite)
+
+    MAPREDUCE-211. Provides ability to run a health check script on the
+    tasktracker nodes and blacklist nodes if they are unhealthy.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-637. Add an example, distbbp, which able to compute the n th bit
+    of Pi for some large n.  (szetszwo)
+
+    MAPREDUCE-532. Provide a way to limit the number of used slots 
+    per queue in the capacity scheduler.
+    (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-467. Provide ability to collect statistics about total tasks 
+    and succeeded tasks in different time windows. (sharad)
+
+    MAPREDUCE-740. Log a job-summary at the end of a job, while allowing it
+    to be configured to use a custom appender if desired. (acmurthy)
+
+    MAPREDUCE-814. Provide a way to configure completed job history files 
+    to be on HDFS. (sharad)
+
+    MAPREDUCE-800. MRUnit should support the new API. (Aaron Kimball via
+    tomwhite)
+
+    MAPREDUCE-798. MRUnit should be able to test a succession of MapReduce
+    passes. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-768. Provide an option to dump jobtracker configuration in JSON
+    format to standard output. (V.V.Chaitanya Krishna via yhemanth)
+
+    MAPREDUCE-824. Add support for a hierarchy of queues in the capacity 
+    scheduler. (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-751. Add Rumen, a tool for extracting statistics from job tracker
+    logs and generating job traces for simulation and analysis. (Dick King via
+    cdouglas)
+
+    MAPREDUCE-830. Add support for splittable compression to TextInputFormats.
+    (Abdul Qadeer via cdouglas)
+
+    MAPREDUCE-861. Add support for hierarchical queues in the Map/Reduce
+    framework. (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-776. Add Gridmix, a benchmark processing Rumen traces to simulate
+    a measured mix of jobs on a cluster. (cdouglas)
+
+    MAPREDUCE-862. Enhance JobTracker UI to display hierarchical queues.
+    (V.V.Chaitanya Krishna via yhemanth)
+
+    MAPREDUCE-777. Brand new apis to track and query jobs as a
+    replacement for JobClient. (Amareshwari Sriramadasu via acmurthy)
+
+    MAPREDUCE-775. Add native and streaming support for Vertica as an input
+    or output format taking advantage of parallel read and write properties of 
+    the DBMS. (Omer Trajman via ddas)
+
+    MAPREDUCE-679. XML-based metrics as JSP servlet for JobTracker.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-980. Modify JobHistory to use Avro for serialization. (cutting)
+
+    MAPREDUCE-728. Add Mumak, a Hadoop map/reduce simulator. (Arun C Murthy,
+    Tamas Sarlos, Anirban Dasgupta, Guanying Wang, and Hong Tang via cdouglas)
+
+    MAPREDUCE-1383. Automates fetching of delegation tokens in File*Formats
+    Distributed Cache and Distcp. Also, provides a config 
+    mapreduce.job.hdfs-servers that the jobs can populate with a comma
+    separated list of namenodes. The job client automatically fetches
+    delegation tokens from those namenodes. (Boris Shkolnik via ddas)    
+
+    MAPREDUCE-698. Per-pool task limits for the fair scheduler.
+    (Kevin Peterson via matei)
+
+    MAPREDUCE-1026. Does mutual authentication of the shuffle
+    transfers using a shared JobTracker generated key.
+    (Boris Shkolnik via ddas)
+
+    MAPREDUCE-744. Introduces the notion of a public distributed cache.
+    (Devaraj Das)
+
+    MAPREDUCE-1338. Introduces the notion of token cache using which
+    tokens and secrets can be sent by the Job client to the JobTracker.
+    (Boris Shkolnik via ddas)
+
+    HDFS-503. This patch implements an optional layer over HDFS that 
+    implements offline erasure-coding.  It can be used to reduce the 
+    total storage requirements of HDFS.  (dhruba)
+
+    MAPREDUCE-1432. Adds hooks in the jobtracker and tasktracker 
+    for loading the tokens in the user's ugi. This is required
+    for the copying of files from the hdfs. (ddas)
+
+    MAPREDUCE-1335. Adds SASL Kerberos/Digest authentication in MapReduce.
+    (Kan Zhang via ddas)
+
+    MAPREDUCE-1464. Makes a compatible change in JobTokenIdentifier to
+    account for HADOOP-6510. (Jitendra Nath Pandey via ddas)
+
+    MAPREDUCE-1433. Add a delegation token for MapReduce. (omalley)
+
+    MAPREDUCE-1307. Introduces the Job level ACLs feature. 
+    (Vinod Kumar Vavilapalli via ddas)
+
+    MAPREDUCE-1430. JobTracker automatically renews delegation tokens for jobs.
+    (Boris Shkolnik via ddas)
+
+    MAPREDUCE-1455. Introduces job-level authorization for mapreduce servlets.
+    (Ravi Gummadi via vinodkv)
+
+  IMPROVEMENTS
+
+    MAPREDUCE-463. Makes job setup and cleanup tasks as optional.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-502. Allow jobtracker to be configured with zero completed jobs
+    in memory. (Amar Kamat via sharad)
+
+    MAPREDUCE-416. Moves the history file to a "done" folder whenever a job 
+    completes. (Amar Kamat via ddas)
+
+    MAPREDUCE-646. Increase srcfilelist replication number in dictcp job.
+    (Ravi Gummadi via szetszwo)
+   
+    HADOOP-6106. Updated hadoop-core and test jars from hudson trunk 
+    build #12. (Giridharan Kesavan)
+
+    MAPREDUCE-642. A option to distcp that allows preserving the full
+    source path of a file in the specified destination directory.
+    (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-686. Move TestSpeculativeExecution.Fake* into a separate class
+    so that it can be used by other tests. (Jothi Padmanabhan via sharad)
+
+    MAPREDUCE-625. Modify TestTaskLimits to improve execution time.
+    (Jothi Padmanabhan via sharad)
+
+    MAPREDUCE-465. Deprecate o.a.h.mapred.lib.MultithreadedMapRunner and add
+    test for o.a.h.mapreduce.lib.MultithreadedMapper.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-701. Improves the runtime of the TestRackAwareTaskPlacement
+    by making it a unit test.  (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-371. Change KeyFieldBasedComparator and KeyFieldBasedPartitioner
+    to use new api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-623. Resolve javac warnings in mapreduce. (Jothi Padmanabhan
+    via sharad)
+
+    MAPREDUCE-655. Change KeyValueLineRecordReader and KeyValueTextInputFormat
+    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-632. Merge TestCustomOutputCommitter with 
+    TestCommandLineJobSubmission. (Jothi Padmanabhan via sharad)
+
+    MAPREDUCE-627. Improves execution time of TestTrackerBlacklistAcrossJobs.
+    (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-630. Improves execution time of TestKillCompletedJob.
+    (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-626. Improves the execution time of TestLostTracker.
+    (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-353. Makes the shuffle read and connection timeouts
+    configurable. (Ravi Gummadi via ddas)
+
+    MAPREDUCE-739. Allow relative paths to be created in archives. (Mahadev
+    Konar via cdouglas)
+
+    MAPREDUCE-772. Merge HADOOP-4010 changes to LineRecordReader into mapreduce
+    package. (Abdul Qadeer via cdouglas)
+
+    MAPREDUCE-785. Separate sub-test of TestReduceFetch to be included in
+    MR-670. (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-784. Modify TestUserDefinedCounters to use LocalJobRunner 
+    instead of MiniMR. (Jothi Padmanabhan via sharad)
+
+    HADOOP-6160. Fix releaseaudit target to run on specific directories. 
+    (gkesavan)
+
+    MAPREDUCE-782. Use PureJavaCrc32 in SpillRecord.  (Todd Lipcon via
+    szetszwo)
+
+    MAPREDUCE-369. Change org.apache.hadoop.mapred.lib.MultipleInputs to 
+    use new api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-373. Change org.apache.hadoop.mapred.lib.FieldSelectionMapReduce
+    to use new api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-628. Improves the execution time of TestJobInProgress.
+    (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-793. Creates a new test that consolidates a few tests to
+    include in the commit-test list. (Jothi Padmanabhan via ddas)
+    
+    MAPREDUCE-797. Adds combiner support to MRUnit MapReduceDriver.
+    (Aaron Kimball via johan)    
+
+    MAPREDUCE-656. Change org.apache.hadoop.mapred.SequenceFile* classes
+    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-670. Creates ant target for 10 mins patch test build.
+    (Jothi Padmanabhan via gkesavan)
+
+    MAPREDUCE-375. Change org.apache.hadoop.mapred.lib.NLineInputFormat 
+    and org.apache.hadoop.mapred.MapFileOutputFormat to use new api.
+    (Amareshwari Sriramadasu via ddas)
+
+    MAPREDUCE-779. Added node health failure counts into 
+    JobTrackerStatistics. (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-842. Setup secure permissions for localized job files,
+    intermediate outputs and log files on tasktrackers.
+    (Vinod Kumar Vavilapalli via yhemanth)
+    
+    MAPREDUCE-478. Allow map and reduce jvm parameters, environment variables
+    and ulimit to be set separately.
+    Configuration changes:
+      add mapred.map.child.java.opts
+      add mapred.reduce.child.java.opts
+      add mapred.map.child.env
+      add mapred.reduce.child.ulimit
+      add mapred.map.child.env
+      add mapred.reduce.child.ulimit
+      deprecated mapred.child.java.opts
+      deprecated mapred.child.env
+      deprecated mapred.child.ulimit
+    (acmurthy)
+
+    MAPREDUCE-767. Remove the dependence on the CLI 2.0 snapshot.
+    (Amar Kamat via omalley)
+
+    MAPREDUCE-712. Minor efficiency tweaks to RandomTextWriter. (cdouglas)
+
+    MAPREDUCE-870. Remove the job retire thread and the associated 
+    config parameters. (sharad)
+
+    MAPREDUCE-874. Rename the PiEstimator example to QuasiMonteCarlo.
+    (szetszwo)
+
+    MAPREDUCE-336. Allow logging level of map/reduce tasks to be configurable. 
+    Configuration changes:
+      add mapred.map.child.log.level 
+      add mapred.reduce.child.log.level 
+    (acmurthy)
+
+    MAPREDUCE-355. Update mapred.join package to use the new API. (Amareshwari
+    Sriramadasu via cdouglas)
+
+    HADOOP-6184. Updated hadoop common and test jars to get the new API
+    in Configuration for dumping in JSON format from Hudson trunk build #68.
+    (yhemanth)
+
+    MAPREDUCE-476. Extend DistributedCache to work locally (LocalJobRunner).
+    (Philip Zeyliger via tomwhite)
+
+    MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests
+    in local mode. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-910. Support counters in MRUnit. (Aaron Kimball via cdouglas)
+
+    MAPREDUCE-788. Update gridmix2 to use the new API (Amareshwari Sriramadasu
+    via cdouglas)
+
+    MAPREDUCE-875. Make DBRecordReader execute queries lazily. (Aaron Kimball 
+    via enis)
+
+    MAPREDUCE-318. Modularizes the shuffle code. (Jothi Padmanabhan and 
+    Arun Murthy via ddas)
+
+    MAPREDUCE-936. Allow a load difference for fairshare scheduler.
+    (Zheng Shao via dhruba)
+
+    MAPREDUCE-370. Update MultipleOutputs to use the API, merge funcitonality
+    of MultipleOutputFormat. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-898. Changes DistributedCache to use the new API.
+    (Amareshwari Sriramadasu via ddas)
+
+    MAPREDUCE-144. Includes dump of the process tree in task diagnostics when 
+    a task is killed due to exceeding memory limits.
+    (Vinod Kumar Vavilapalli via yhemanth)
+
+    MAPREDUCE-945. Modifies MRBench and TestMapRed to use ToolRunner so that
+    options such as queue name can be passed via command line.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-963. Deprecate o.a.h.mapred.FileAlreadyExistsException and
+    replace it with o.a.h.fs.FileAlreadyExistsException.  (Boris Shkolnik
+    via szetszwo)
+
+    MAPREDUCE-960. Remove an unnecessary intermediate copy and obsolete API
+    from KeyValueLineRecordReader. (cdouglas)
+
+    MAPREDUCE-930. Modify Rumen to resolve paths in the canonical way, rather
+    than defaulting to the local filesystem. (cdouglas)
+
+    MAPREDUCE-944. Extend the LoadManager API of the fair-share scheduler
+    to support regulating tasks for a job based on resources currently in use
+    by that job. (dhruba)
+
+    MAPREDUCE-973. Move FailJob and SleepJob from examples to test. (cdouglas 
+    via omalley)
+
+    MAPREDUCE-966. Modify Rumen to clean up interfaces and simplify integration
+    with other tools. (Hong Tang via cdouglas)
+
+    MAPREDUCE-856. Setup secure permissions for distributed cache files.
+    (Vinod Kumar Vavilapalli via yhemanth)
+
+    MAPREDUCE-885. More efficient SQL queries for DBInputFormat. (Aaron Kimball 
+    via enis)
+
+    MAPREDUCE-284. Enables ipc.client.tcpnodelay in Tasktracker's Child.
+    (Ravi Gummadi via sharad)
+
+    MAPREDUCE-916. Split the documentation to match the project split.
+    (Corinne Chandel via omalley)
+
+    MAPREDUCE-649. Validate a copy by comparing the source and destination
+    checksums in distcp. Also adds an intra-task retry mechanism for errors
+    detected during the copy. (Ravi Gummadi via cdouglas)
+
+    MAPREDUCE-654. Add a -dryrun option to distcp printing a summary of the
+    file data to be copied, without actually performing the copy. (Ravi Gummadi
+    via cdouglas)
+
+    MAPREDUCE-664. Display the number of files deleted by distcp when the
+    -delete option is specified. (Ravi Gummadi via cdouglas)
+
+    MAPREDUCE-781. Let the name of distcp jobs be configurable. (Venkatesh S
+    via cdouglas)
+
+    MAPREDUCE-975. Add an API in job client to get the history file url for 
+    a given job id. (sharad)
+
+    MAPREDUCE-905. Add Eclipse launch tasks for MapReduce. (Philip Zeyliger
+    via tomwhite)
+
+    MAPREDUCE-277. Makes job history counters available on the job history
+    viewers. (Jothi Padmanabhan via ddas)
+
+    MAPREDUCE-893. Provides an ability to refresh queue configuration
+    without restarting the JobTracker.
+    (Vinod Kumar Vavilapalli and Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-1011. Add build.properties to svn and git ignore. (omalley)
+
+    MAPREDUCE-954. Change Map-Reduce context objects to be interfaces.
+    (acmurthy) 
+
+    MAPREDUCE-639. Change Terasort example to reflect the 2009 updates. 
+    (omalley)
+
+    MAPREDUCE-1063. Document gridmix benchmark. (cdouglas)
+
+    MAPREDUCE-931. Use built-in interpolation classes for making up task
+    runtimes in Rumen. (Dick King via cdouglas)
+
+    MAPREDUCE-1012. Mark Context interfaces as public evolving. (Tom White via
+    cdouglas)
+
+    MAPREDUCE-971. Document use of distcp when copying to s3, managing timeouts
+    in particular. (Aaron Kimball via cdouglas)
+
+    HDFS-663. DFSIO for append. (shv)
+
+    HDFS-641. Move all of the components that depend on map/reduce to 
+    map/reduce. (omalley)
+
+    HADOOP-5107. Use Maven ant tasks to publish artifacts. (Giridharan Kesavan
+    via omalley)
+
+    MAPREDUCE-1229. Allow customization of job submission policy in Mumak.
+    (Hong Tang via cdouglas)
+
+    MAPREDUCE-1317. Reduce the memory footprint of Rumen objects by interning
+    host Strings. (Hong Tang via cdouglas)
+
+    MAPREDUCE-1097. Add support for Vertica 3.5 to its contrib module. (Omer
+    Trajman via cdouglas)
+
+    MAPREDUCE-1627. HadoopArchives should not uses a method in DistCp.
+    (szetszwo)
+
+    MAPREDUCE-1198. Alternatively schedule different types of tasks in
+    fair share scheduler. (Scott Chen via matei)
+
+    MAPREDUCE-707. Provide a jobconf property for explicitly assigning a job to 
+    a pool in the Fair Scheduler. (Alan Heirich via matei)
+
+    MAPREDUCE-947. Added commitJob and abortJob apis to OutputCommitter.
+    Enhanced FileOutputCommitter to create a _SUCCESS file for successful
+    jobs. (Amar Kamat & Jothi Padmanabhan via acmurthy) 
+
+    MAPREDUCE-1103. Added more metrics to Jobtracker. (sharad) 
+
+    MAPREDUCE-1048. Add occupied/reserved slot usage summary on jobtracker UI.
+    (Amareshwari Sriramadasu and Hemanth Yamijala via sharad)
+
+    MAPREDUCE-1090. Modified log statement in TaskMemoryManagerThread to
+    include task attempt id. (yhemanth)
+
+    MAPREDUCE-1189. Reduce ivy console output to ovservable level (cos)
+
+    MAPREDUCE-1167. ProcfsBasedProcessTree collects rss memory information.
+    (Scott Chen via dhruba)
+
+    MAPREDUCE-1231. Added a new DistCp option, -skipcrccheck, so that the CRC
+    check during setup can be skipped.  (Jothi Padmanabhan via szetszwo)
+
+    MAPREDUCE-1190. Add package documentation for BBP example.
+    (Tsz Wo (Nicholas) Sze via cdouglas)
+
+    MAPREDUCE-1119. When tasks fail to report status, show tasks's stack dump
+    before killing. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1185. Redirect running job url to history url if job is already 
+    retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
+
+    MAPREDUCE-1050. Introduce a mock object testing framework. (tomwhite)
+
+    MAPREDUCE-1084. Implementing aspects development and fault injeciton
+    framework for MapReduce. (Sreekanth Ramakrishnan via cos)
+
+    MAPREDUCE-1209. Move common specific part of the test TestReflectionUtils
+    out of mapred into common. (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-967. TaskTracker does not need to fully unjar job jars.
+    (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-1083. Changes in MapReduce so that group information of users 
+    can be refreshed in the JobTracker via command line. 
+    (Boris Shkolnik via ddas)
+
+    MAPREDUCE-181. Changes the job submission process to be secure.
+    (Devaraj Das)
+
+    MAPREDUCE-1250. Refactors the JobToken to use Common's Token interface.
+    (Kan Zhang via ddas)
+
+    MAPREDUCE-896. Enhance tasktracker to cleanup files that might have
+    been created by user tasks with non-writable permissions.
+    (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-372. Change org.apache.hadoop.mapred.lib.ChainMapper/Reducer 
+    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-1295. Add a tool in Rumen for folding and manipulating job
+    traces. (Dick King via cdouglas)
+
+    MAPREDUCE-1302. TrackerDistributedCacheManager deletes file
+    asynchronously, thus reducing task initialization delays.
+    (Zheng Shao via dhruba)
+
+    MAPREDUCE-1218. TaskTrackers send cpu and memory usage of
+    node to JobTracker. (Scott Chen via dhruba)
+
+    MAPREDUCE-847. Fix Releaseaudit warning count to zero
+    (Giridharan Kesavan)
+
+    MAPREDUCE-1337. Use generics in StreamJob to improve readability of that
+    class. (Kay Kay via cdouglas)
+
+    MAPREDUCE-361. Port terasort example to the new mapreduce API. (Amareshwari
+    Sriramadasu via cdouglas)
+
+    MAPREDUCE-1367. LocalJobRunner should support parallel mapper execution.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-64. Eliminate io.sort.record.percent from MapTask configuration.
+    (cdouglas)
+
+    MAPREDUCE-1440. Replace the long user name in MapReduce with the local
+    name. (omalley)
+
+    MAPREDUCE-1470. Move delegation tokens from HDFS to Common so that 
+    MapReduce can use them too. (omalley)
+
+    MAPREDUCE-1425. Reduce memory usage by archive. (mahadev via szetszwo)
+
+    MAPREDUCE-1441. Trim whitespace from directory lists pulled from the
+    configuration. (Todd Lipcon via cdouglas)
+
+    MAPREDUCE-1309. Refactor Rumen trace generator to improve code structure
+    and add extensible support for log formats. (Dick King via cdouglas)
+
+    MAPREDUCE-1503. Delegation token renewing and cancelling should provide
+    meaningful exceptions when there are failures instead of returning 
+    false. (omalley)
+
+    HADOOP-6579. Upgrade commons-codec library to 1.4. (omalley)
+
+    MAPREDUCE-1423. Improve performance of CombineFileInputFormat when multiple
+    pools are configured. (Dhruba Borthakur via zshao)
+
+    MAPREDUCE-1454. Quote user supplied strings in Tracker servlets. (cdouglas)
+
+    MAPREDUCE-1408. Add customizable job submission policies to Gridmix. (Rahul
+    Singh via cdouglas)
+
+    MAPREDUCE-1527. Better warning logged when mapred.queue.names is
+    overshadowed by mapred-queues.xml. (Hong Tang via acmurthy)
+
+    MAPREDUCE-1403. Save the size and number of distributed cache artifacts in
+    the configuration. (Arun Murthy via cdouglas)
+
+    MAPREDUCE-1482. Truncate state string and diagnostic information in
+    TaskStatus. (Amar Kamat via szetszwo)
+
+    MAPREDUCE-1593.  [Rumen] Improvements to random seed generation (tamas via
+    mahadev)
+
+    MAPREDUCE-1460. Oracle support in DataDrivenDBInputFormat.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1569. Pass configuration through mocked contexts in MRUnit.
+    (Chris White via cdouglas)
+
+    MAPREDUCE-1590. Move HarFileSystem from Hadoop Common to Mapreduce tools.
+    (mahadev)
+
+    MAPREDUCE-1629. Get rid of fakeBlockLocations() on HarFileSystem, since
+    it's not used (mahadev)
+
+    MAPREDUCE-1489. DataDrivenDBInputFormat should not query the database
+    when generating only one split. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1514.  Add documentation on replication, permissions, new options,
+    limitations and internals of har.  (mahadev via szetszwo)
+
+    MAPREDUCE-1428.  Make block size and the size of archive created files
+    configurable.  (mahadev via szetszwo)
+
+    MAPREDUCE-1656. JobStory should provide queue info. (hong via mahadev)
+
+    MAPREDUCE-1466. Record number of files processed in FileInputFormat in the
+    Configuration for offline analysis. (Luke Lu and Arun Murthy via cdouglas)
+
+    MAPREDUCE-1538. TrackerDistributedCacheManager manages the
+    number of files. (Scott Chen via dhruba)
+
+    MAPREDUCE-1673. Scripts to start and stop RaidNode.
+    (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1659. RaidNode writes temp files on configured tmp directory and
+    add random numbers to their names to avoid conflicts
+    (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1221. Allow admins to control physical memory limits per-task
+    and per-node. (Scott Chen via acmurthy) 
+
+    MAPREDUCE-1065. Update mapred tutorial to use the new API. (Aaron Kimball
+    via cdouglas)
+
+    MAPREDUCE-1304. Add a task counter tracking time spent in GC. (Aaron
+    Kimball via cdouglas)
+
+    MAPREDUCE-1570. Add grouping comparators to MRUnit. (Chris White via
+    cdouglas)
+
+    MAPREDUCE-1650. Exclude Private elements from generated MapReduce
+    Javadoc. (tomwhite)
+
+    MAPREDUCE-1625. Improve grouping of packages in Javadoc. (tomwhite)
+
+    MAPREDUCE-1417. Forrest documentation should be updated to reflect 
+    the changes in MAPREDUCE-744. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1568. TrackerDistributedCacheManager should clean up cache
+    in a background thread. (Scott Chen via zshao)
+
+    MAPREDUCE-1749. Move configuration strings out of JobContext so that it
+    can be made public stable. (omalley)
+
+    MAPREDUCE-1623. Apply audience and stability notations to Hadoop
+    Map-Reduce. (tomwhite via acmurthy) 
+
+    MAPREDUCE-1751. Change MapReduce to depend on Hadoop 'common' artifacts
+    instead of 'core'. (tomwhite)
+
+    MAPREDUCE-1535.  Replace usage of FileStatus#isDir().  (Eli Collins via
+    tomwhite)
+
+    MAPREDUCE-1832. Allow file sizes less than 1MB in DFSIO benchmark. (shv)
+
+    MAPREDUCE-1404.  Move Cluster-Setup and Single-Node-Setup Docs from
+    MapReduce to Common.  (tomwhite)
+
+    MAPREDUCE-1697. Document the behavior of -file option and deprecate it
+    in favour of -files option in streaming. (Amareshwari Sriramadasu
+    via vinodkv)
+
+    MAPREDUCE-1033. Resolve location of scripts and configuration files after
+    project split. (tomwhite)
+
+    MAPREDUCE-1018. Document changes to the memory management and scheduling
+    model. (Hemanth Yamijala via vinodkv)
+
+    MAPREDUCE-1896. [Herriot] New property for multi user list. (Vinay Thota
+    via cos)
+
+    MAPREDUCE-1812. New properties for suspend and resume process. (Vinay
+    Thota via cos)
+
+  OPTIMIZATIONS
+
+    MAPREDUCE-270. Fix the tasktracker to optionally send an out-of-band
+    heartbeat on task-completion for better job-latency. (acmurthy) 
+    Configuration changes:
+      add mapreduce.tasktracker.outofband.heartbeat 
+
+    MAPREDUCE-1186. Modified code in distributed cache to set permissions
+    only on required set of localized paths.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1501. FileInputFormat supports multi-level, recursive 
+    directory listing.  (Zheng Shao via dhruba)
+
+    MAPREDUCE-1556. upgrade to Avro 1.3.0. (cutting via tomwhite)
+
+    MAPREDUCE-1613. Install/deploy source jars to Maven repo 
+    (Patrick Angeles via ddas)
+
+    MAPREDUCE-1610. Forrest documentation should be updated to reflect
+    the changes in MAPREDUCE-856. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1853. Adds caching for TaskAttemptContext in MultipleOutputs.
+    (Torsten Curdt via amareshwari)
+
+  BUG FIXES
+
+    MAPREDUCE-878. Rename fair scheduler design doc to 
+    fair-scheduler-design-doc.tex and add Apache license header (matei)
+
+    HADOOP-4687. MapReduce is split from Hadoop Core. It is a subproject under 
+    Hadoop (Owen O'Malley)
+
+    HADOOP-6096. Fix Eclipse project and classpath files following project
+    split. (tomwhite)
+
+    MAPREDUCE-419. Reconcile mapred.userlog.limit.kb defaults in configuration
+    and code. (Philip Zeyliger via cdouglas)
+
+    MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
+    keys. (Amar Kamat via sharad)
+
+    MAPREDUCE-130. Delete the jobconf copy from the log directory of the 
+    JobTracker when the job is retired. (Amar Kamat via sharad)
+
+    MAPREDUCE-657. Fix hardcoded filesystem problem in CompletedJobStatusStore.
+    (Amar Kamat via sharad)
+
+    MAPREDUCE-179. Update progress in new RecordReaders. (cdouglas)
+
+    MAPREDUCE-658. Replace NPE in distcp with a meaningful error message when
+    the source path does not exist. (Ravi Gummadi via cdouglas)
+
+    MAPREDUCE-671. Update ignore list to include untracked, generated
+    build artifacts and config files. (cdouglas)
+
+    MAPREDUCE-433. Use more reliable counters in TestReduceFetch. (cdouglas)
+
+    MAPREDUCE-124. Fix a bug in failure handling of abort task of 
+    OutputCommiter. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-694. Fix to add jsp-api jars to capacity-scheduler classpath.
+    (Giridharan Kesavan)
+    
+    MAPREDUCE-702. Fix eclipse-plugin jar target (Giridharan Kesavan) 
+
+    MAPREDUCE-522. Replace TestQueueCapacities with simpler test case to
+    test integration between capacity scheduler and MR framework.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-683. Fixes an initialization problem in the JobHistory. 
+    The initialization of JobHistoryFilesManager is now done in the 
+    JobHistory.init call. (Amar Kamat via ddas)
+
+    MAPREDUCE-708. Fixes a bug to allow updating the reason for
+    blacklisting a node on the JobTracker UI.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-709. Fixes message displayed for a blacklisted node where
+    the reason for blacklisting is due to the health check script
+    timing out. (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-676. Existing diagnostic rules fail for MAP ONLY jobs.
+    (Suhas Gogate via tomwhite)
+
+    MAPREDUCE-722. Fixes a bug with tasktracker reservations for
+    high memory jobs in capacity scheduler.
+    (Vinod Kumar Vavilapalli via yhemanth)
+
+    HADOOP-6090. Updates gridmix script to use new mapreduce api output 
+    format. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-732. Removed spurious log statements in the node
+    blacklisting logic. (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-734. Fix a ConcurrentModificationException in unreserving
+    unused reservations for a job when it completes.
+    (Arun Murthy and Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-733. Fix a RuntimeException while unreserving trackers
+    that are blacklisted for a job.
+    (Arun Murthy and Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-677. Fix timeout in TestNodeRefresh. (Amar Kamat via 
+    sharad)
+
+    MAPREDUCE-153. Fix timeout in TestJobInProgressListener. (Amar 
+    Kamat via sharad)
+
+    MAPREDUCE-742. Fix output messages and java comments in the Pi related
+    examples.  (szetszwo)
+
+    MAPREDUCE-565. Fix partitioner to work with new API. (Owen O'Malley via
+    cdouglas)
+    
+    MAPREDUCE-680. Fix so MRUnit can handle reuse of Writable objects.
+    (Aaron Kimball via johan)
+
+    MAPREDUCE-18. Puts some checks for cross checking whether a reduce
+    task gets the correct shuffle data. (Ravi Gummadi via ddas)
+
+    MAPREDUCE-771. Fix scheduling of setup and cleanup tasks to use
+    free slots instead of tasks for scheduling. (yhemanth)
+
+    MAPREDUCE-717. Fixes some corner case issues in speculative 
+    execution heuristics. (Devaraj Das)
+
+    MAPREDUCE-716. Make DBInputFormat work with Oracle. (Aaron Kimball
+    via tomwhite)
+
+    MAPREDUCE-735. Fixes a problem in the KeyFieldHelper to do with 
+    the end index for some inputs (Amar Kamat via ddas)
+
+    MAPREDUCE-682. Removes reservations on tasktrackers which are
+    blacklisted. (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-743. Fixes a problem to do with progress reporting
+    in the map phase. (Ravi Gummadi via ddas)
+
+    MAPREDUCE-765. Eliminate the deprecated warnings introduced by H-5438.
+    (He Yongqiang via szetszwo)
+
+    MAPREDUCE-383. Fix a bug in Pipes combiner due to bytes count not 
+    getting reset after the spill. (Christian Kunz via sharad)
+
+    MAPREDUCE-809. Fix job-summary logs to correctly record status of FAILED
+    and KILLED jobs.  (acmurthy)
+
+    MAPREDUCE-792. Fix unchecked warnings in DBInputFormat.  (Aaron Kimball
+    via szetszwo)
+
+    MAPREDUCE-760. Fix a timing issue in TestNodeRefresh. (Amar Kamat via 
+    sharad)
+
+    MAPREDUCE-40. Keep memory management backwards compatible for job
+    configuration parameters and limits. (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-587. Fixes a OOM issue in TestStreamingExitStatus.
+    (Amar Kamat via ddas) 
+
+    MAPREDUCE-408. Fixes an assertion problem in TestKillSubProcesses
+    (Ravi Gummadi via ddas)
+     
+    MAPREDUCE-659. Fix gridmix2 compilation. (Giridharan Kesavan)
+
+    MAPREDUCE-796. Fixes a ClassCastException in an exception log in
+    MultiThreadedMapRunner. (Amar Kamat via ddas)
+
+    MAPREDUCE-808. Fixes a serialization problem in TypedBytes.
+    (Klaas Bosteels via ddas)
+
+    MAPREDUCE-845. Fix a findbugs heap size problem in build.xml and add
+    a new property findbugs.heap.size.  (Lee Tucker via szetszwo)
+
+    MAPREDUCE-838. Fixes a problem in the way commit of task outputs
+    happens. The bug was that even if commit failed, the task would
+    be declared as successful. (Amareshwari Sriramadasu via ddas)
+
+    MAPREDUCE-813. Updates Streaming and M/R tutorial documents.
+    (Corinne Chandel via ddas)
+
+    MAPREDUCE-805. Fixes some deadlocks in the JobTracker due to the fact
+    the JobTracker lock hierarchy wasn't maintained in some JobInProgress
+    method calls. (Amar Kamat via ddas)
+    
+    MAPREDUCE-799. Fixes so all of the MRUnit self-tests run.
+    (Aaron Kimball via johan)    
+
+    MAPREDUCE-848. Fixes a problem to do with TestCapacityScheduler
+    failing (Amar Kamat via ddas)
+
+    MAPREDUCE-840. DBInputFormat leaves open transaction.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-859. Adds Avro and its dependencies required by Hadoop 
+    common. (Ravi Gummadi via sharad)
+    
+    MAPREDUCE-867. Fix ivy conf to look for avro jar from maven repo.	
+    (Giridharan Kesavan)
+
+    MAPREDUCE-877. Added avro as a dependency to contrib ivy settings.
+    (Tsz Wo (Nicholas) Sze via yhemanth)
+
+    MAPREDUCE-852. In build.xml, remove the Main-Class, which is incorrectly
+    set in tools, and rename the target "tools-jar" to "tools".  (szetszwo)
+
+    MAPREDUCE-773. Sends progress reports for compressed gzip inputs in maps.
+    Fixes a native direct buffer leak in LineRecordReader classes.
+    (Hong Tang and ddas)
+
+    MAPREDUCE-832. Reduce number of warning messages printed when
+    deprecated memory variables are used. (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-745. Fixes a testcase problem to do with generation of JobTracker
+    IDs. (Amar Kamat via ddas)
+
+    MAPREDUCE-834. Enables memory management on tasktrackers when old
+    memory management parameters are used in configuration.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-818. Fixes Counters#getGroup API. (Amareshwari Sriramadasu 
+    via sharad)
+
+    MAPREDUCE-807. Handles the AccessControlException during the deletion of
+    mapred.system.dir in the JobTracker. The JobTracker will bail out if it
+    encounters such an exception. (Amar Kamat via ddas)
+
+    MAPREDUCE-430. Fix a bug related to task getting stuck in case of 
+    OOM error. (Amar Kamat via ddas)
+
+    MAPREDUCE-871. Fix ownership of Job/Task local files to have correct 
+    group ownership according to the egid of the tasktracker.
+    (Vinod Kumar Vavilapalli via yhemanth) 
+
+    MAPREDUCE-911. Fix a bug in TestTaskFail related to speculative 
+    execution. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-687. Fix an assertion in TestMiniMRMapRedDebugScript.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-924. Fixes the TestPipes testcase to use Tool.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-903. Add Avro jar to eclipse classpath.
+    (Philip Zeyliger via tomwhite)
+
+    MAPREDUCE-943. Removes a testcase in TestNodeRefresh that doesn't make 
+    sense in the new Job recovery model. (Amar Kamat via ddas)
+
+    MAPREDUCE-764. TypedBytesInput's readRaw() does not preserve custom type
+    codes. (Klaas Bosteels via tomwhite)
+
+    HADOOP-6243. Fixes a NullPointerException in handling deprecated keys.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-968. NPE in distcp encountered when placing _logs directory on
+    S3FileSystem. (Aaron Kimball via tomwhite)
+ 
+    MAPREDUCE-826. harchive doesn't use ToolRunner / harchive returns 0 even
+    if the job fails with exception (koji Noguchi via mahadev)
+
+    MAPREDUCE-839. unit test TestMiniMRChildTask fails on mac os-x (hong tang
+    via mahadev)
+
+    MAPREDUCE-112. Add counters for reduce input, output records to the new API.
+    (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-648. Fix two distcp bugs: (1) it should not launch a job if all
+    src paths are directories, and (2) it does not skip copying when updating
+    a single file.  (Ravi Gummadi via szetszwo)
+
+    MAPREDUCE-946. Fix a regression in LineRecordReader where the
+    maxBytesToConsume parameter is not set correctly. (cdouglas)
+
+    MAPREDUCE-977. Missing jackson jars from Eclipse template. (tomwhite)
+
+    MAPREDUCE-988. Fix a packaging issue in the contrib modules. (Hong Tang via
+    cdouglas)
+
+    MAPREDUCE-971. distcp does not always remove distcp.tmp.dir. (Aaron Kimball
+    via tomwhite)
+
+    MAPREDUCE-995. Fix a bug in JobHistory where tasks completing after the job
+    is closed cause a NPE. (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-953. Fix QueueManager to dump queue configuration in JSON format.
+    (V.V. Chaitanya Krishna via yhemanth)
+
+    MAPREDUCE-645. Prevent distcp from running a job when the destination is a
+    file, but the source is not. (Ravi Gummadi via cdouglas)
+
+    MAPREDUCE-1002. Flushed writer in JobQueueClient so queue information is
+    printed correctly. (V.V. Chaitanya Krishna via yhemanth)
+
+    MAPREDUCE-1003. Fix compilation problem in eclipse plugin when
+    eclipse.home is set. (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-941. Vaidya script fails on Solaris. (Chad Metcalf
+    via tomwhite)
+
+    MAPREDUCE-912. Add and standardize Apache license headers. (Chad Metcalf
+    via cdouglas)
+
+    MAPREDUCE-1022. Fix compilation of vertica testcases. (Vinod Kumar 
+    Vavilapalli via acmurthy)
+
+    MAPREDUCE-1000. Handle corrupt history files in JobHistory.initDone().
+    (Jothi Padmanabhan via sharad)
+
+    MAPREDUCE-1028. Fixed number of slots occupied by cleanup tasks to one
+    irrespective of slot size for the job.
+    (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-964. Fixed start and finish times of TaskStatus to be
+    consistent, thereby fixing inconsistencies in metering tasks.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-1076. Deprecate ClusterStatus and add javadoc in ClusterMetrics.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return
+    values of new configuration variables when deprecated variables are
+    disabled. (Sreekanth Ramakrishnan via yhemanth)
+   
+    MAPREDUCE-1030. Modified scheduling algorithm to return a map and reduce
+    task per heartbeat in the capacity scheduler.
+    (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-1071. Use DataInputStream rather than FSDataInputStream in the
+    JobHistory EventReader. (Hong Tang via cdouglas)
+
+    MAPREDUCE-986. Fix Rumen to work with truncated task lines. (Dick King via
+    cdouglas)
+
+    MAPREDUCE-1029. Fix failing TestCopyFiles by restoring the unzipping of
+    HDFS webapps from the hdfs jar. (Aaron Kimball and Jothi Padmanabhan via
+    cdouglas)
+
+    MAPREDUCE-769. Make findbugs and javac warnings to zero.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-1104. Initialize RecoveryManager in JobTracker cstr called by
+    Mumak. (Hong Tang via cdouglas)
+
+    MAPREDUCE-1061. Add unit test validating byte specifications for gridmix
+    jobs. (cdouglas)
+
+    MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
+    successful. (Dick King via cdouglas)
+
+    MAPREDUCE-1041. Make TaskInProgress::taskStatuses map package-private.
+    (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
+    (Todd Lipcon via cdouglas)
+
+    MAPREDUCE-1086. Setup Hadoop logging environment for tasks to point to
+    task related parameters. (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-1105. Remove max limit configuration in capacity scheduler in
+    favor of max capacity percentage thus allowing the limit to go over
+    queue capacity. (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-1016.  Make the job history log format JSON.  (cutting)
+
+    MAPREDUCE-1038. Weave Mumak aspects only if related files have changed.
+    (Aaron Kimball via cdouglas)
+
+    MAPREDUCE-1163. Remove unused, hard-coded paths from libhdfs. (Allen
+    Wittenauer via cdouglas)
+
+    MAPREDUCE-962. Fix a NullPointerException while killing task process 
+    trees. (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-1177. Correct setup/cleanup inversion in
+    JobTracker::getTaskReports. (Vinod Kumar Vavilapalli via cdouglas)
+
+    MAPREDUCE-1178. Fix ClassCastException in MultipleInputs by adding 
+    a DelegatingRecordReader. (Amareshwari Sriramadasu and Jay Booth 
+    via sharad)
+
+    MAPREDUCE-1068. Fix streaming job to show proper message if file is 
+    is not present. (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-1147. Add map output counters to new API. (Amar Kamat via
+    cdouglas)
+
+    MAPREDUCE-915. The debug scripts are run as the job user. (ddas)
+
+    MAPREDUCE-1007. Fix NPE in CapacityTaskScheduler.getJobs(). 
+    (V.V.Chaitanya Krishna via sharad)
+
+    MAPREDUCE-28. Refactor TestQueueManager and fix default ACLs.
+    (V.V.Chaitanya Krishna and Rahul K Singh via sharad)
+
+    MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed the
+    configured threshold. (cdouglas)
+
+    MAPREDUCE-1239. Fix contrib components build dependencies. 
+    (Giridharan Kesavan and omalley) 
+
+    MAPREDUCE-787. Fix JobSubmitter to honor user given symlink path.
+    (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-1249. Update config default value for socket read timeout to
+    match code default. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1161. Remove ineffective synchronization in NotificationTestCase.
+    (Owen O'Malley via cdouglas)
+
+    MAPREDUCE-1244. Fix eclipse-plugin's build dependencies. (gkesavan)
+
+    MAPREDUCE-1075. Fix JobTracker to not throw an NPE for a non-existent
+    queue. (V.V.Chaitanya Krishna via yhemanth)
+
+    MAPREDUCE-754. Fix NPE in expiry thread when a TT is lost. (Amar Kamat 
+    via sharad)
+
+    MAPREDUCE-1074. Document Reducer mark/reset functionality. (Jothi
+    Padmanabhan via cdouglas)
+
+    MAPREDUCE-1267. Fix typo in mapred-default.xml. (Todd Lipcon via cdouglas)
+
+    MAPREDUCE-952. Remove inadvertently reintroduced Task.Counter enum. (Jothi
+    Padmanabhan via cdouglas)
+
+    MAPREDUCE-1230. Fix handling of null records in VerticaInputFormat. (Omer
+    Trajman via cdouglas)
+
+    MAPREDUCE-1171. Allow shuffle retries and read-error reporting to be
+    configurable. (Amareshwari Sriramadasu via acmurthy)
+
+    MAPREDUCE-879. Fix broken unit test TestTaskTrackerLocalization on MacOS.
+    (Sreekanth Ramakrishnan via yhemanth)
+
+    MAPREDUCE-1124. Fix imprecise byte counts in Gridmix. (cdouglas)
+
+    MAPREDUCE-1222. Add an option to exclude numeric IP addresses in topologies
+    processed by Mumak. (Hong Tang via cdouglas)
+
+    MAPREDUCE-1284. Fix fts_open() call in task-controller that was failing
+    LinuxTaskController unit tests. (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-1143. Fix running task counters to be updated correctly
+    when speculative attempts are running for a TIP.
+    (Rahul Kumar Singh via yhemanth)
+
+    MAPREDUCE-1241. Use a default queue configuration in JobTracker when
+    mapred-queues.xml is unavailable. (Todd Lipcon via cdouglas)
+
+    MAPREDUCE-1301. Fix set up of permission checking script used in 
+    localization tests. (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1286. Remove quoting from client opts in TaskRunner. (Yuri
+    Pradkin via cdouglas)
+
+    MAPREDUCE-1059. Use distcp.bytes.per.map when adding sync markers in
+    distcp. (Aaron Kimball via cdouglas)
+
+    MAPREDUCE-1009. Update forrest documentation describing hierarchical
+    queues. (Vinod Kumar Vavilapalli via yhemanth)
+
+    MAPREDUCE-1342. Fixed deadlock in global blacklisting of tasktrackers.
+    (Amareshwari Sriramadasu via acmurthy)
+
+    MAPREDUCE-1316. Fixes a memory leak of TaskInProgress instances in
+    the jobtracker. (Amar Kamat via yhemanth)
+
+    MAPREDUCE-1359. TypedBytes TestIO doesn't mkdir its test dir first.
+    (Anatoli Fomenko via cos)
+
+    MAPREDUCE-1314. Correct errant mapreduce.x.mapreduce.x replacements from
+    bulk change. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1365. Restore accidentally renamed test in
+    TestTaskTrackerBloacklisting. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1406. Fix spelling of JobContext.MAP_COMBINE_MIN_SPILLS.
+    (cdouglas)
+
+    MAPREDUCE-1369. JUnit tests should never depend on anything in conf
+    (Anatoli Fomenko via cos)
+
+    MAPREDUCE-1412. Fix timer granularity issue causing failures in
+    TestTaskTrackerBlacklisting. (cdouglas)
+
+    MAPREDUCE-1448. Respect --config option in Mumak script. (Hong Tang via
+    cdouglas)
+
+    MAPREDUCE-1251. c++ utils doesn't compile. (Eli Collins via tomwhite)
+
+    MAPREDUCE-1522. FileInputFormat may use the default FileSystem for the
+    input path. (Tsz Wo (Nicholas), SZE via cdouglas)
+
+    MAPREDUCE-1407. Update javadoc in mapreduce.{Mapper,Reducer} to match
+    actual usage. (Benoit Sigoure via cdouglas)
+
+    MAPREDUCE-1258. Fix fair scheduler event log not logging job info.
+    (matei)
+
+    MAPREDUCE-1089. Fix NPE in fair scheduler preemption when tasks are
+    scheduled but not running. (Todd Lipcon via matei)
+
+    MAPREDUCE-1014. Fix the libraries for common and hdfs. (omalley)
+
+    MAPREDUCE-1111. JT Jetty UI not working if we run mumak.sh 
+    off packaged distribution directory. (hong tang via mahadev)
+
+    MAPREDUCE-1133. Eclipse .classpath template has outdated jar files and is
+    missing some new ones. (cos)
+
+    MAPREDUCE-1098. Fixed the distributed-cache to not do i/o while holding a
+    global lock. (Amareshwari Sriramadasu via acmurthy)
+
+    MAPREDUCE-1158. Fix JT running maps and running reduces metrics.
+    (sharad)
+
+    MAPREDUCE-1160. Reduce verbosity of log lines in some Map/Reduce classes
+    to avoid filling up jobtracker logs on a busy cluster.
+    (Ravi Gummadi and Hong Tang via yhemanth)
+
+    MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned.
+    (sharad)
+
+    MAPREDUCE-1128. Fix MRUnit to prohibit iterating over values twice. (Aaron
+    Kimball via cdouglas)
+
+    MAPREDUCE-665. Move libhdfs to HDFS subproject. (Eli Collins via dhruba)
+
+    MAPREDUCE-1196. Fix FileOutputCommitter to use the deprecated cleanupJob
+    api correctly. (acmurthy)
+   
+    MAPREDUCE-1244. Fix eclipse-plugin's build dependencies. (gkesavan)
+
+    MAPREDUCE-1140. Fix DistributedCache to not decrement reference counts for
+    unreferenced files in error conditions.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1245. Fix TestFairScheduler failures by instantiating lightweight
+    Jobtracker. (sharad)
+
+    MAPREDUCE-1260. Update Eclipse configuration to match changes to Ivy
+    configuration. (Edwin Chan via cos)
+
+    MAPREDUCE-1152. Distinguish between failed and killed tasks in
+    JobTrackerInstrumentation. (Sharad Agarwal via cdouglas)
+
+    MAPREDUCE-1285. In DistCp.deleteNonexisting(..), get class from the
+    parameter instead of using FileStatus.class.  (Peter Romianowski via
+    szetszwo)
+
+    MAPREDUCE-1294. Build fails to pull latest hadoop-core-* artifacts (cos)
+
+    MAPREDUCE-1213. TaskTrackers restart is faster because it deletes
+    distributed cache directory asynchronously. (Zheng Shao via dhruba)
+
+    MAPREDUCE-1265. The task attempt error log prints the name of the 
+    tasktracker machine. (Scott Chen via dhruba)
+
+    MAPREDUCE-1201. ProcfsBasedProcessTree collects CPU usage information.
+    (Scott Chen via dhruba)
+
+    MAPREDUCE-1326. fi tests don't use fi-site.xml (cos)
+
+    MAPREDUCE-1165. Replace non-portable function name with C99 equivalent.
+    (Allen Wittenauer via cdouglas)
+
+    MAPREDUCE-1331. Fixes a typo in a testcase (Devaraj Das)
+
+    MAPREDUCE-1293. AutoInputFormat doesn't work with non-default FileSystems.
+    (Andrew Hitchcock via tomwhite)
+
+    MAPREDUCE-1131. Using profilers other than hprof can cause JobClient to
+    report job failure. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1155. Streaming tests swallow exceptions.
+    (Todd Lipcon via tomwhite)
+
+    MAPREDUCE-1212. Mapreduce contrib project ivy dependencies are not included
+    in binary target. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1388. Move the HDFS RAID package from HDFS to MAPREDUCE.
+    (Eli Collins via dhruba)
+
+    MAPREDUCE-1322. Defines default value for staging directory to be user
+    based fixing a failing streaming test.
+    (Devaraj Das and Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-899. Modified LinuxTaskController to check that task-controller
+    has right permissions and ownership before performing any actions.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1443. DBInputFormat can leak connections.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1457. Fixes JobTracker to get the FileSystem object within 
+    getStagingAreaDir within a privileged block. Fixes Child.java to use the
+    appropriate UGIs while getting the TaskUmbilicalProtocol proxy and 
+    while executing the task. (Jakob Homan via ddas)
+
+    MAPREDUCE-1399. The archive command shows a null error message (nicholas
+    via mahadev)
+
+    MAPREDUCE-1305. Improve efficiency of distcp -delete. (Peter Romianowski
+    via cdouglas)
+
+    MAPREDUCE-1474. Update forrest documentation for Hadoop Archives. (Mahadev
+    Konar via cdouglas)
+
+    MAPREDUCE-1400. Use tr rather than sed to effect literal substitution in
+    the build script. (Allen Wittenauer via cdouglas)
+
+    MAPREDUCE-1358. Avoid false positives in OutputLogFilter. (Todd Lipcon via
+    cdouglas)
+
+    MAPREDUCE-1490. Fix a NullPointerException that could occur during 
+    instantiation and initialization of the DistributedRaidFileSystem. 
+    (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1476. Fix the M/R framework to not call commit for special
+    tasks like job setup/cleanup and task cleanup.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1398. Fix TaskLauncher to stop waiting for slots on a TIP that
+    is killed / failed.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1491. The parity files created by the RAID are combined
+    using Hadoop Archive Files (HAR).  (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1378. URL encode link in jobhistory.jsp to avoid errors caused by
+    unescaped characters. (E. Sammer via cdouglas)
+
+    MAPREDUCE-1519. RaidNode fails to create new parity file 
+    if an older version already exists. (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1537. Fixes a compilation problem in a testcase after commit
+    HDFS-984. (Jitendra Nath Pandey via ddas)
+
+    MAPREDUCE-1537. The patch makes the job client call the getDelegationToken
+    only when security is enabled. (Jitendra Nath Pandey via ddas)
+
+    MAPREDUCE-1510. RAID should regenerate parity files if they get deleted.
+    (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1421. Fix the LinuxTaskController tests failing on trunk after
+    the commit of MAPREDUCE-1385. (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1520. Fix TestMiniMRLocalFS failure caused by regression in
+    getting user working dir. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1512. RAID uses HarFileSystem directly instead of
+    FileSystem.get (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1435. Fix symlink handling in task work directory when
+    cleaning up, essentially to avoid following links.
+    (Ravi Gummadi via yhemanth)
+
+    MAPREDUCE-1518. RaidNode does not run the deletion check on the
+    directory that stores the parity files.  (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1573. TestStreamingAsDifferentUser fails if run as tt_user.
+    (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-927. Cleanup of task-logs should happen in TaskTracker instead
+    of the Child. (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1578. Decouple HadoopArchives vesrion from HarFileSystem version.
+    (Rodrigo Schmidt via szetszwo)
+
+    MAPREDUCE-1422. Fix cleanup of localized job directory to work if files
+    with non-deletable permissions are created within it.
+    (Amar Kamat via yhemanth)
+
+    MAPREDUCE-1306. Randomize the arrival of heartbeat responses in Mumak.
+    (Tamas Sarlos via cdouglas)
+
+    MAPREDUCE-1579. archive: check and possibly replace the space charater
+    in source paths.  (szetszwo)
+
+    MAPREDUCE-1536. DataDrivenDBInputFormat does not split date columns correctly.
+    (Aaron Kimball via enis)
+
+    MAPREDUCE-890. After HADOOP-4491, the user who started mapred system is 
+    not able to run job. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1615. Fix compilation of TestSubmitJob. (cdouglas)
+
+    MAPREDUCE-1508. Protect against NPE in TestMultipleLevelCaching. (Aaron
+    Kimball via cdouglas)
+
+    MAPREDUCE-1497. Suppress spurious findbugs warning about IndexCache
+    synchronization. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1420. Fix TestTTResourceReporting failure. (Scott Chen via
+    cdouglas)
+
+    MAPREDUCE-1480. Correctly initialize child RecordReaders in
+    CombineFileInputFormat. (Aaron Kimball via cdouglas)
+
+    MAPREDUCE-1348. Fix block forensics packaging. (Tom White via cdouglas)
+
+    MAPREDUCE-1628. HarFileSystem shows incorrect replication numbers and
+    permissions.  (szetszwo via mahadev)
+
+    MAPREDUCE-1602. Fix the error message for the case that src does not
+    exist.  (szetszwo)
+
+    MAPREDUCE-1585. Create Hadoop Archives version 2 with filenames
+    URL-encoded (rodrigo via mahadev)
+
+    MAPREDUCE-1523. Sometimes rumen trace generator fails to extract the job
+    finish time. (dick king via mahadev)
+
+    MAPREDUCE-1635. ResourceEstimator does not work after MAPREDUCE-842.
+    (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-889. binary communication formats added to Streaming by
+    HADOOP-1722 should be documented. (Klaas Bosteels via tomwhite)
+
+    MAPREDUCE-1031. ant tar target doens't seem to compile tests in contrib
+    projects. (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1692. Removed unused testcase TestStreamedMerge.
+    (Sreekanth Ramakrishnan and Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1062. Fix ReliabilityTest to work with retired jobs. (Sreekanth
+    Ramakrishnan via cdouglas)
+
+    MAPREDUCE-1409. IOExceptions thrown from FIleOutputCommitter::abortTask
+    should cause the task to fail. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev
+    targets and also fix existing warnings. (Hong Tang via yhemanth)
+
+    MAPREDUCE-1494. Ensure TestJobDirCleanup verifies the correct paths.
+    (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1622. Include missing slf4j dependencies. (cdouglas)
+
+    MAPREDUCE-1515. Accept java5.home from build.properties, not only from the
+    command line when building forrest docs. (Al Thompson via cdouglas)
+
+    MAPREDUCE-1618. Add missing javadoc to JobStatus::*JobAcls. (Amareshwari
+    Sriramadasu via cdouglas)
+
+    MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease 
+    undue load on jobtracker. (Sreekanth Ramakrishnan via sharad)
+
+    MAPREDUCE-1604. Add Forrest documentation for Job ACLs.
+    (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1705. Archiving and Purging of HDFS parity files should 
+    handle globbed policies accurately. (Rodrigo Schmidt via dhruba)
+
+    MAPREDUCE-1612. job conf file is not accessible from job history web page.
+    (Ravi Gummadi and Sreekanth Ramakrishnan via vinodkv)
+
+    MAPREDUCE-1397. NullPointerException observed during task failures.
+    (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1728. Oracle timezone strings do not match Java.
+    (Aaron Kimball via tomwhite)
+
+    MAPREDUCE-1609. TaskTracker.localizeJob should not set permissions on 
+    job log directory recursively. (Amareshwari Sriramadasu via vinodkv)
+
+    MAPREDUCE-1657. After task logs directory is deleted, tasklog servlet 
+    displays wrong error message about job ACLs. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1727. TestJobACLs fails after HADOOP-6686. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1611. Refresh nodes and refresh queues doesnt work with service
+    authorization enabled. (Amar Kamat via vinodkv)
+
+    MAPREDUCE-1276. Correct flaws in the shuffle related to connection setup
+    and failure attribution. (Amareshwari Sriramadasu via cdouglas)
+
+    MAPREDUCE-1372. ConcurrentModificationException in JobInProgress.
+    (Dick King and Amareshwari Sriramadasu via tomwhite)
+
+    MAPREDUCE-118. Fix Job.getJobID(). (Amareshwari Sriramadasu via sharad)
+
+    MAPREDUCE-913. TaskRunner crashes with NPE resulting in held up slots,
+    UNINITIALIZED tasks and hung TaskTracker. (Amareshwari Sriramadasu and
+    Sreekanth Ramakrishnan via vinodkv)
+
+    MAPREDUCE-1725.  Fix MapReduce API incompatibilities between 0.20 and 0.21.
+    (tomwhite)
+
+    MAPREDUCE-1606. TestJobACLs may timeout as there are no slots for launching
+    JOB_CLEANUP task. (Ravi Gummadi via vinodkv)
+
+    MAPREDUCE-1765. Correct streaming documentation for StreamXmlRecordReader.
+    (Corinne Chandel via amareshwari)
+
+    MAPREDUCE-1880. Fix BigDecimal.divide(..) in the pi example.  (szetszwo)
+
+    MAPREDUCE-1885. Revert FileSystem create method that takes CreateFlags
+    (MapReduce part of HADOOP-6826). (Ravi Gummadi via tomwhite)
+
+    MAPREDUCE-1870. Harmonize MapReduce JAR library versions with Common and
+    HDFS. (tomwhite)
+
+    MAPREDUCE-1791. Remote cluster control functionality needs JavaDocs
+    improvement (Konstantin Boudnik)
+
+    MAPREDUCE-1942. 'compile-fault-inject' should never be called directly.
+    (Konstantin Boudnik)
+
+    MAPREDUCE-1876. Fixes TaskAttemptStartedEvent to correctly log event type
+    for all task types. (Amar Kamat via amareshwari)
+
+    MAPREDUCE-1926. MapReduce distribution is missing build-utils.xml.
+    (tomwhite)
+
+    MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk.
+    (Amareshwari Sriramadasu via tomwhite)
+
+    MAPREDUCE-1980. Fixes TaskAttemptUnsuccessfulCompletionEvent and
+    TaskAttemptFinishedEvent to correctly log event type for all task types.
+    (Amar Kamat via amareshwari)
+
+    MAPREDUCE-1856. Extract a subset of tests for smoke (DOA) validation (cos)
+

hadoop-mapreduce/INSTALL

@@ -0,0 +1,89 @@
+To compile  Hadoop Mapreduce next following, do the following:
+
+Step 1) Install dependencies for yarn
+
+See http://svn.apache.org/repos/asf/hadoop/common/branches/MR-279/mapreduce/yarn/README
+Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib
+
+Step 2) Checkout
+
+svn checkout http://svn.apache.org/repos/asf/hadoop/common/branches/MR-279/
+
+Step 3) Build common
+
+Go to common directory
+ant veryclean mvn-install 
+
+Step 4) Build HDFS 
+
+Go to hdfs directory
+ant veryclean mvn-install -Dresolvers=internal 
+
+Step 5) Build yarn and mapreduce
+
+Go to mapreduce directory
+export MAVEN_OPTS=-Xmx512m
+
+mvn clean install assembly:assembly
+ant veryclean jar jar-test  -Dresolvers=internal 
+
+In case you want to skip the tests run:
+
+mvn clean install assembly:assembly -DskipTests
+ant veryclean jar jar-test  -Dresolvers=internal 
+
+You will see a tarball in
+ls target/hadoop-mapreduce-1.0-SNAPSHOT-bin.tar.gz  
+
+Step 6) Untar the tarball in a clean and different directory.
+say HADOOP_YARN_INSTALL
+
+To run Hadoop Mapreduce next applications :
+
+Step 7) cd $HADOOP_YARN_INSTALL
+
+Step 8) export the following variables:
+
+HADOOP_MAPRED_HOME=
+HADOOP_COMMON_HOME=
+HADOOP_HDFS_HOME=
+YARN_HOME=directory where you untarred yarn
+HADOOP_CONF_DIR=
+YARN_CONF_DIR=$HADOOP_CONF_DIR
+
+Step 9) bin/yarn-daemon.sh start resourcemanager
+
+Step 10) bin/yarn-daemon.sh start nodemanager
+
+Step 11) bin/yarn-daemon.sh start historyserver
+
+Step 12) Create the following symlinks in hadoop-common/lib 
+
+ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar .	
+ln -s $HADOOP_YARN_INSTALL/modules/yarn-api-1.0-SNAPSHOT.jar .
+ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-common-1.0-SNAPSHOT.jar .	
+ln -s $HADOOP_YARN_INSTALL/modules/yarn-common-1.0-SNAPSHOT.jar .
+ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-core-1.0-SNAPSHOT.jar .	
+ln -s $HADOOP_YARN_INSTALL/modules/yarn-server-common-1.0-SNAPSHOT.jar .
+ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar .
+ln -s $HADOOP_YARN_INSTALL/lib/protobuf-java-2.4.0a.jar .
+
+Step 13) Yarn daemons are up! But for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
+    <property>
+      <name>nodemanager.auxiluary.services</name>
+      <value>mapreduce.shuffle</value>
+    </property>
+
+    <property>
+      <name>nodemanager.aux.service.mapreduce.shuffle.class</name>
+      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+    </property>
+
+Step 14) You are all set, an example on how to run a mapreduce job is:
+
+cd $HADOOP_MAPRED_HOME
+ant examples -Dresolvers=internal
+$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapred-examples-0.22.0-SNAPSHOT.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $HADOOP_YARN_INSTALL/hadoop-mapreduce-1.0-SNAPSHOT/modules/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar output 
+
+The output on the command line should be almost similar to what you see in the JT/TT setup (Hadoop 0.20/0.21)
+

hadoop-mapreduce/LICENSE.txt

@@ -0,0 +1,244 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+APACHE HADOOP SUBCOMPONENTS:
+
+The Apache Hadoop project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses. 
+
+For the org.apache.hadoop.util.bloom.* classes:
+
+/**
+ *
+ * Copyright (c) 2005, European Commission project OneLab under contract
+ * 034819 (http://www.one-lab.org)
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or 
+ * without modification, are permitted provided that the following 
+ * conditions are met:
+ *  - Redistributions of source code must retain the above copyright 
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright 
+ *    notice, this list of conditions and the following disclaimer in 
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the name of the University Catholique de Louvain - UCL
+ *    nor the names of its contributors may be used to endorse or 
+ *    promote products derived from this software without specific prior 
+ *    written permission.
+ *    
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+ * POSSIBILITY OF SUCH DAMAGE.
+ */

hadoop-mapreduce/NOTICE.txt

@@ -0,0 +1,2 @@
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

hadoop-mapreduce/assembly/all.xml

@@ -0,0 +1,101 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+  <id>all</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <includeBaseDirectory>true</includeBaseDirectory>
+  <!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
+  <fileSets>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes>
+        <include>container-executor</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>bin</directory>
+      <outputDirectory>bin</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/conf</directory>
+      <outputDirectory>conf</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>sources</outputDirectory>
+      <excludes>
+        <exclude>**/*.jar</exclude>
+        <exclude>**/target/**</exclude>
+        <!-- scripts to include later for setting fileMode -->
+        <exclude>**/bin/*</exclude>
+        <exclude>**/scripts/*</exclude>
+        <!-- images that we don't need (and cause problems for our tools) -->
+        <exclude>**/dt-*/images/**</exclude>
+        <!-- until the code that does this is fixed -->
+        <exclude>**/file:/**</exclude>
+        <exclude>**/SecurityAuth.audit*</exclude>
+      </excludes>
+      <includes>
+        <include>assembly/**</include>
+        <include>pom.xml</include>
+        <include>build*.xml</include>
+        <include>ivy.xml</include>
+        <include>ivy/**</include>
+        <include>INSTALL</include>
+        <include>LICENSE.txt</include>
+        <include>mr-client/**</include>
+        <include>hadoop-yarn/**</include>
+        <include>src/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>sources</outputDirectory>
+      <includes>
+        <include>**/bin/*</include>
+        <include>**/scripts/*</include>
+      </includes>
+      <fileMode>0755</fileMode>
+    </fileSet>
+  </fileSets>
+  <moduleSets>
+    <moduleSet>
+      <excludes>
+        <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
+      </excludes>
+      <binaries>
+        <outputDirectory>modules</outputDirectory>
+        <includeDependencies>false</includeDependencies>
+        <unpack>false</unpack>
+      </binaries>
+    </moduleSet>
+  </moduleSets>
+  <dependencySets>
+    <dependencySet>
+      <useProjectArtifact>false</useProjectArtifact>
+      <outputDirectory>/lib</outputDirectory>
+      <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
+      <excludes>
+        <exclude>org.apache.hadoop:hadoop-common</exclude>
+        <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+      </excludes>
+    </dependencySet>
+  </dependencySets>
+</assembly>

hadoop-mapreduce/bin/mapred

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin=`which $0`
+bin=`dirname ${bin}`
+bin=`cd "$bin"; pwd`
+
+if [ -e $bin/../libexec/mapred-config.sh ]; then
+  . $bin/../libexec/mapred-config.sh
+else
+  . "$bin/mapred-config.sh"
+fi
+
+function print_usage(){
+  echo "Usage: mapred [--config confdir] COMMAND"
+  echo "       where COMMAND is one of:"
+  echo "  mradmin              run a Map-Reduce admin client"
+  echo "  jobtracker           run the MapReduce job Tracker node" 
+  echo "  tasktracker          run a MapReduce task Tracker node" 
+  echo "  pipes                run a Pipes job"
+  echo "  job                  manipulate MapReduce jobs"
+  echo "  queue                get information regarding JobQueues"
+  echo "  classpath            prints the class path needed for running"
+  echo "                       mapreduce subcommands"
+  echo "  groups               get the groups which users belong to"
+  echo ""
+  echo "Most commands print help when invoked w/o parameters."
+}
+
+if [ $# = 0 ]; then
+  print_usage
+  exit
+fi
+
+COMMAND=$1
+shift
+
+if [ "$COMMAND" = "mradmin" ] ; then
+  CLASS=org.apache.hadoop.mapred.tools.MRAdmin
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "jobtracker" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobTracker
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
+elif [ "$COMMAND" = "tasktracker" ] ; then
+  CLASS=org.apache.hadoop.mapred.TaskTracker
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
+elif [ "$COMMAND" = "job" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobClient
+elif [ "$COMMAND" = "queue" ] ; then
+  CLASS=org.apache.hadoop.mapred.JobQueueClient
+elif [ "$COMMAND" = "pipes" ] ; then
+  CLASS=org.apache.hadoop.mapred.pipes.Submitter
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "sampler" ] ; then
+  CLASS=org.apache.hadoop.mapred.lib.InputSampler
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "classpath" ] ; then
+  echo -n 
+elif [ "$COMMAND" = "groups" ] ; then
+  CLASS=org.apache.hadoop.mapred.tools.GetGroups
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+else
+  echo $COMMAND - invalid command
+  print_usage
+  exit
+fi
+
+# for developers, add mapred classes to CLASSPATH
+if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
+fi
+if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
+fi
+if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
+fi
+if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
+fi
+
+# for releases, add core mapred jar & webapps to CLASSPATH
+if [ -d "$HADOOP_PREFIX/share/hadoop/mapreduce/webapps" ]; then
+  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/mapreduce
+fi
+for f in $HADOOP_MAPRED_HOME/share/hadoop-mapreduce/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+# add libs to CLASSPATH
+for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
+  CLASSPATH=${CLASSPATH}:$f;
+done
+
+if $cygwin; then
+  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
+fi
+
+if [ "$COMMAND" = "classpath" ] ; then
+  echo $CLASSPATH
+  exit
+fi
+
+export CLASSPATH
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"

hadoop-mapreduce/bin/mapred-config.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# included in all the mapred scripts with source command
+# should not be executed directly
+
+bin=`which "$0"`
+bin=`dirname "${bin}"`
+bin=`cd "$bin"; pwd`
+
+if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
+  . "$bin/../libexec/hadoop-config.sh"
+elif [ -e "${HADOOP_COMMON_HOME}/bin/hadoop-config.sh" ]; then
+  . "$HADOOP_COMMON_HOME"/bin/hadoop-config.sh
+elif [ -e "${HADOOP_HOME}/bin/hadoop-config.sh" ]; then
+  . "$HADOOP_HOME"/bin/hadoop-config.sh
+elif [ -e "${HADOOP_MAPRED_HOME}/bin/hadoop-config.sh" ]; then
+  . "$HADOOP_MAPRED_HOME"/bin/hadoop-config.sh
+else
+  echo "Hadoop common not found."
+  exit
+fi

hadoop-mapreduce/bin/start-mapred.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start hadoop map reduce daemons.  Run this on master node.
+
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=`cd "$bin"; pwd`
+
+if [ -e $bin/../libexec/mapred-config.sh ]; then
+  . $bin/../libexec/mapred-config.sh
+else
+  . "$bin/mapred-config.sh"
+fi
+
+
+# start mapred daemons
+# start jobtracker first to minimize connection errors at startup
+"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker
+"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker

hadoop-mapreduce/bin/stop-mapred.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stop hadoop map reduce daemons.  Run this on master node.
+
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=`cd "$bin"; pwd`
+
+if [ -e $bin/../libexec/mapred-config.sh ]; then
+  . $bin/../libexec/mapred-config.sh
+else
+  . "$bin/mapred-config.sh"
+fi
+
+"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker
+"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker

hadoop-mapreduce/build-utils.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!--
+Contains utilities that are common for the main and contrib builds.
+-->
+<project name="build-utils">
+
+  <!-- Load properties from build properties file, if available -->
+  <dirname property="build-utils.basedir" file="${ant.file.build-utils}"/>
+  <property file="${build-utils.basedir}/build.properties"/>
+
+  <target name="forrest.check" unless="forrest.home">
+    <fail message="'forrest.home' is not defined. Please pass -Dforrest.home=&lt;base of Apache Forrest installation&gt; to Ant on the command-line, or set forest.home in build properties file." />
+  </target>
+  
+</project>

hadoop-mapreduce/build.xml

@@ -0,0 +1,2514 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="Hadoop" default="compile" 
+   xmlns:artifact="urn:maven-artifact-ant"
+   xmlns:ivy="antlib:org.apache.ivy.ant"> 
+
+  <import file="build-utils.xml" />
+
+  <!-- Load all the default properties, and any the user wants    -->
+  <!-- to contribute (without having to type -D or edit this file -->
+  <property file="${user.home}/build.properties" />
+  <property file="${basedir}/build.properties" />
+
+  <property name="module" value="mapreduce"/> 
+  <property name="Name" value="Hadoop-Mapred"/>
+  <property name="name" value="hadoop-${module}"/>
+  <!-- Need to change aop.xml project.version prop. synchronously -->
+  <property name="_version" value="0.23.0"/>
+  <property name="version" value="${_version}-SNAPSHOT"/>
+  <property name="final.name" value="${name}-${version}"/>
+  <property name="test.final.name" value="${name}-test-${version}"/>
+  <property name="examples.final.name" value="${name}-examples-${version}"/>
+  <property name="tools.final.name" value="${name}-tools-${version}"/>
+  <property name="year" value="2009"/>
+  <property name="package.release" value="1"/>
+
+  <property name="src.dir" value="${basedir}/src"/>  	
+  <property name="mapred.src.dir" value="${src.dir}/java"/> 
+  <property name="examples.dir" value="${basedir}/src/examples"/>
+  <property name="lib.dir" value="${basedir}/lib"/>
+  <property name="conf.dir" value="${basedir}/conf"/>
+  <property name="contrib.dir" value="${basedir}/src/contrib"/>
+  <property name="docs.src" value="${basedir}/src/docs"/>
+  <property name="changes.src" value="${docs.src}/changes"/>
+  <property name="c++.src" value="${basedir}/src/c++"/>
+  <property name="c++.utils.src" value="${c++.src}/utils"/>
+  <property name="c++.pipes.src" value="${c++.src}/pipes"/>
+  <property name="c++.examples.pipes.src" value="${examples.dir}/pipes"/>
+  <property name="librecordio.src" value="${c++.src}/librecordio"/>
+  <property name="tools.src" value="${basedir}/src/tools"/>
+
+  <property name="xercescroot" value=""/> 
+  <property name="build.dir" value="${basedir}/build"/>
+  <property name="build.classes" value="${build.dir}/classes"/>
+  <property name="build.src" value="${build.dir}/src"/>
+  <property name="build.tools" value="${build.dir}/tools"/>
+  <property name="build.webapps" value="${build.dir}/webapps"/>
+  <property name="build.examples" value="${build.dir}/examples"/>
+  <property name="build.librecordio" value="${build.dir}/librecordio"/>
+
+  <!-- convert spaces to _ so that mac os doesn't break things -->
+  <exec executable="tr" inputstring="${os.name}" 
+        outputproperty="nonspace.os">
+     <arg value="[:space:]"/>
+     <arg value="_"/>
+  </exec>
+  <property name="build.platform" 
+            value="${nonspace.os}-${os.arch}-${sun.arch.data.model}"/>
+  <property name="jvm.arch" 
+            value="${sun.arch.data.model}"/>
+  <property name="build.native" value="${build.dir}/native/${build.platform}"/>
+  <property name="build.c++" value="${build.dir}/c++-build/${build.platform}"/>
+  <property name="build.c++.utils" value="${build.c++}/utils"/>
+  <property name="build.c++.pipes" value="${build.c++}/pipes"/>
+  <property name="build.c++.examples.pipes" 
+            value="${build.c++}/examples/pipes"/>
+  <property name="build.docs" value="${build.dir}/docs"/>
+  <property name="build.javadoc" value="${build.docs}/api"/>
+  <property name="build.javadoc.timestamp" value="${build.javadoc}/index.html" />
+  <property name="build.javadoc.dev" value="${build.docs}/dev-api"/>
+  <property name="build.encoding" value="ISO-8859-1"/>
+  <property name="install.c++" value="${build.dir}/c++/${build.platform}"/>
+  <property name="install.c++.examples" 
+            value="${build.dir}/c++-examples/${build.platform}"/>
+
+  <property environment="env"/>
+  <property name="test.src.dir" value="${basedir}/src/test"/>
+  <property name="test.lib.dir" value="${basedir}/src/test/lib"/>
+  <property name="test.build.dir" value="${build.dir}/test"/>
+  <property name="test.generated.dir" value="${test.build.dir}/src"/>
+  <property name="test.build.data" value="${test.build.dir}/data"/>
+  <property name="test.cache.data" value="${test.build.dir}/cache"/>
+  <property name="test.concat.data" value="${test.build.dir}/concat"/>
+  <property name="test.debug.data" value="${test.build.dir}/debug"/>
+  <property name="test.log.dir" value="${test.build.dir}/logs"/>
+  <property name="test.build.classes" value="${test.build.dir}/classes"/>
+  <property name="test.mapred.build.testjar" value="${test.build.dir}/mapred/testjar"/>
+  <property name="test.mapred.build.testshell" value="${test.build.dir}/mapred/testshell"/>
+  <property name="test.build.extraconf" value="${test.build.dir}/extraconf"/>
+  <property name="test.build.javadoc" value="${test.build.dir}/docs/api"/>
+  <property name="test.build.javadoc.dev" value="${test.build.dir}/docs/dev-api"/>
+  <property name="test.include" value="Test*"/>
+  <property name="test.classpath.id" value="test.classpath"/>
+  <property name="test.output" value="no"/>
+  <property name="test.timeout" value="900000"/>
+  <property name="test.junit.output.format" value="plain"/>
+  <property name="test.junit.fork.mode" value="perTest" />
+  <property name="test.junit.printsummary" value="yes" />
+  <property name="test.junit.haltonfailure" value="no" />
+  <property name="test.junit.maxmemory" value="512m" />
+  <property name="test.junit.jvmargs" value="-ea" />
+  <property name="test.tmp.dir" value="${env.TMP}" />
+  <property name="test.temp.dir" value="${env.TEMP}" />
+  <property name="test.conf.dir" value="${build.dir}/test/conf" />
+
+  <property name="test.tools.input.dir" value="${basedir}/src/test/tools/data" />
+
+  <property name="test.mapred.build.classes" value="${test.build.dir}/mapred/classes"/>
+  <property name="test.mapred.commit.tests.file" value="${test.src.dir}/commit-tests" />
+  <property name="test.mapred.smoke.tests.file" value="${test.src.dir}/smoke-tests" />
+  <property name="test.mapred.all.tests.file" value="${test.src.dir}/all-tests" />
+  <property name="test.exclude.file" value="${test.src.dir}/empty-file" />
+
+  <property name="librecordio.test.dir" value="${test.build.dir}/librecordio"/>
+  <property name="web.src.dir" value="${basedir}/src/web"/>
+  <property name="src.webapps" value="${basedir}/src/webapps"/>
+
+  <property name="javadoc.link.java"
+	    value="http://java.sun.com/javase/6/docs/api/"/>
+  <property name="javadoc.packages" value="org.apache.hadoop.*"/>
+  <property name="javadoc.maxmemory" value="512m" />
+
+  <property name="dist.dir" value="${build.dir}/${final.name}"/>
+
+  <property name="javac.debug" value="on"/>
+  <property name="javac.optimize" value="on"/>
+  <property name="javac.deprecation" value="off"/>
+  <property name="javac.version" value="1.6"/>
+  <property name="javac.args" value=""/>
+  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+
+  <property name="clover.db.dir" location="${build.dir}/test/clover/db"/>
+  <property name="clover.report.dir" location="${build.dir}/test/clover/reports"/>
+
+  <property name="rat.reporting.classname" value="rat.Report"/>
+
+  <property name="jdiff.build.dir" value="${build.docs}/jdiff"/>
+  <property name="jdiff.xml.dir" value="${lib.dir}/jdiff"/>
+  <property name="jdiff.stability" value="-unstable"/>
+  <property name="jdiff.compatibility" value=""/>
+  <property name="jdiff.stable" value="0.21.0"/>
+  <property name="jdiff.stable.javadoc" 
+            value="http://hadoop.apache.org/core/docs/r${jdiff.stable}/api/"/>
+
+  <property name="scratch.dir" value="${user.home}/tmp"/>
+  <property name="svn.cmd" value="svn"/>
+  <property name="grep.cmd" value="grep"/>
+  <property name="patch.cmd" value="patch"/>
+  <property name="make.cmd" value="make"/>
+
+  <property name="findbugs.heap.size" value="512M"/>
+  <!-- task-controller properties set here -->
+  <!-- Source directory from where configure is run and files are copied
+  -->
+	
+  <property name="c++.task-controller.src" 
+    value="${basedir}/src/c++/task-controller" />
+  <!-- directory where autoconf files + temporary files and src is 
+    stored for compilation -->
+  <property name="build.c++.task-controller" 
+    value="${build.c++}/task-controller" />
+  <!-- the default install dir is build directory override it using
+   -Dtask-controller.install.dir=$HADOOP_HOME/bin -->
+  <property name="task-controller.install.dir" value="${dist.dir}/bin" />
+  <!-- end of task-controller properties -->
+	
+  <!-- IVY properteis set here -->
+  <property name="ivy.dir" location="ivy" />
+  <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
+  <property name="mvn.repo" value="http://repo2.maven.org/maven2"/>
+  <property name="ivy.jar" location="${ivy.dir}/ivy-${ivy.version}.jar"/>
+  <property name="asfrepo" value="https://repository.apache.org"/>
+  <property name="asfsnapshotrepo" value="${asfrepo}/content/repositories/snapshots"/>
+  <property name="asfstagingrepo"
+    value="${asfrepo}/service/local/staging/deploy/maven2"/>
+  <property name="ant_task.jar" location="${ivy.dir}/maven-ant-tasks-${ant-task.version}.jar"/>
+  <property name="ant_task_repo_url" value="${mvn.repo}/org/apache/maven/maven-ant-tasks/${ant-task.version}/maven-ant-tasks-${ant-task.version}.jar"/>
+  <property name="ivy_repo_url" value="${mvn.repo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>
+  <property name="ivysettings.xml" location="${ivy.dir}/ivysettings.xml" />
+  <property name="ivy.org" value="org.apache.hadoop"/>
+  <property name="build.dir" location="build" />
+  <property name="dist.dir" value="${build.dir}/${final.name}"/>
+  <property name="build.ivy.dir" location="${build.dir}/ivy" />
+  <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
+  <property name="common.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/common"/>
+  <property name="mapred.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/mapred"/>
+  <property name="build.ivy.report.dir" location="${build.ivy.dir}/report" />
+  <property name="build.ivy.maven.dir" location="${build.ivy.dir}/maven" />
+  <property name="build.ivy.maven.pom" location="${build.ivy.maven.dir}/hadoop-mapred-${version}.pom" />
+  <property name="build.ivy.maven.jar" location="${build.ivy.maven.dir}/hadoop-mapred-${version}.jar" />
+  <property name="hadoop-mapred.pom" location="${ivy.dir}/hadoop-mapred.xml"/>
+  <property name="hadoop-mapred-test.pom" location="${ivy.dir}/hadoop-mapred-test.xml"/>
+  <property name="hadoop-mapred-examples.pom" location="${ivy.dir}/hadoop-mapred-examples.xml"/>
+  <property name="hadoop-mapred-tools.pom" location="${ivy.dir}/hadoop-mapred-tools.xml"/>
+
+  <!--this is the naming policy for artifacts we want pulled down-->
+  <property name="ivy.artifact.retrieve.pattern" value="${ant.project.name}/[conf]/[artifact]-[revision].[ext]"/>
+
+  <!--this is how artifacts that get built are named-->
+  <property name="ivy.publish.pattern" value="hadoop-[revision]-core.[ext]"/>
+  <property name="hadoop-mapred.jar" location="${build.dir}/${final.name}.jar" />
+  <property name="hadoop-mapred-test.jar" location="${build.dir}/${test.final.name}.jar" />
+  <property name="hadoop-mapred-examples.jar" location="${build.dir}/${examples.final.name}.jar" />
+  <property name="hadoop-mapred-tools.jar" location="${build.dir}/${tools.final.name}.jar" />
+  <property name="hadoop-mapred-fi.jar" location="${build.dir}/${final.name}-fi.jar" />
+  <property name="hadoop-mapred-sources.jar" location="${build.dir}/${final.name}-sources.jar" />
+  <property name="hadoop-mapred-test-sources.jar" location="${build.dir}/${test.final.name}-sources.jar" />
+  <property name="hadoop-mapred-examples-sources.jar" location="${build.dir}/${examples.final.name}-sources.jar" />
+  <property name="hadoop-mapred-tools-sources.jar" location="${build.dir}/${tools.final.name}-sources.jar" />
+
+  <!-- jdiff.home property set -->
+  <property name="jdiff.home" value="${build.ivy.lib.dir}/${ant.project.name}/jdiff"/>
+  <property name="jdiff.jar" value="${jdiff.home}/jdiff-${jdiff.version}.jar"/>
+  <property name="xerces.jar" value="${jdiff.home}/xerces-${xerces.version}.jar"/>
+	
+  <!-- Eclipse properties -->
+  <property name="build.dir.eclipse" value="${build.dir}/eclipse"/>
+  <property name="build.dir.eclipse-main-classes" value="${build.dir.eclipse}/classes-main"/>
+  <property name="build.dir.eclipse-main-generated-classes" value="${build.dir.eclipse}/classes-main-generated"/>
+  <property name="build.dir.eclipse-test-classes" value="${build.dir.eclipse}/classes-test"/>
+  <property name="build.dir.eclipse-example-classes" value="${build.dir.eclipse}/classes-example"/>
+  <property name="build.dir.eclipse-tools-classes" value="${build.dir.eclipse}/classes-tools"/>
+  <property name="build.dir.eclipse-contrib-classes" value="${build.dir.eclipse}/classes-contrib"/>
+
+  <property name="clover.jar" location="${clover.home}/lib/clover.jar"/>
+  <available property="clover.present" file="${clover.jar}" />
+
+  <!-- check if clover reports should be generated -->
+  <condition property="clover.enabled">
+    <and>
+        <isset property="run.clover"/>
+        <isset property="clover.present"/>
+    </and>
+  </condition>
+
+  <condition property="staging">
+    <equals arg1="${repo}" arg2="staging"/>
+  </condition>
+
+  <!-- packaging properties -->
+  <property name="package.prefix" value="/usr"/>
+  <property name="package.conf.dir" value="/etc/hadoop"/>
+  <property name="package.log.dir" value="/var/log/hadoop/mapred"/>
+  <property name="package.pid.dir" value="/var/run/hadoop"/>
+  <property name="package.var.dir" value="/var/lib/hadoop"/>
+  <property name="package.share.dir" value="/share/hadoop/${module}"/>
+  <!-- Use fixed path to build rpm for avoiding rpmbuild conflict with dash path names -->
+  <property name="package.buildroot" value="/tmp/hadoop_mapred_package_build_${user.name}"/>
+  <property name="package.build.dir" value="/tmp/hadoop_mapred_package_build_${user.name}/BUILD"/>
+
+  <!-- the normal classpath -->
+  <path id="classpath">
+    <pathelement location="${build.classes}"/>
+    <pathelement location="${conf.dir}"/>
+    <path refid="ivy-mapred.classpath"/>
+  </path>
+
+  <path id="test.classpath">
+    <pathelement location="${test.build.extraconf}"/>
+    <pathelement location="${test.mapred.build.classes}" />
+    <pathelement location="${test.src.dir}"/>
+    <pathelement location="${build.dir}"/>
+    <pathelement location="${build.examples}"/>
+    <pathelement location="${build.tools}"/>
+    <pathelement path="${clover.jar}"/>
+    <pathelement location="${build.classes}"/>
+    <pathelement location="${test.conf.dir}"/>
+    <path refid="ivy-common.classpath"/>
+    <path refid="ivy-mapred.classpath"/>
+    <pathelement location="${test.mapred.build.classes}" />
+    <path refid="ivy-test.classpath"/>
+  </path>
+
+  <!-- the cluster test classpath: uses conf.dir for configuration -->
+  <path id="test.cluster.classpath">
+    <path refid="classpath"/>
+    <pathelement location="${test.build.classes}" />
+    <pathelement location="${test.src.dir}"/>
+    <pathelement location="${build.dir}"/>
+  </path>
+
+
+  <!-- ====================================================== -->
+  <!-- Macro definitions                                      -->
+  <!-- ====================================================== -->
+  <macrodef name="macro_tar" description="Worker Macro for tar">
+    <attribute name="param.destfile"/>
+    <element name="param.listofitems"/>
+    <sequential>
+      <tar compression="gzip" longfile="gnu"
+      destfile="@{param.destfile}">
+      <param.listofitems/>
+      </tar>
+    </sequential>
+  </macrodef>
+
+  <!-- ====================================================== -->
+  <!-- Stuff needed by all targets                            -->
+  <!-- ====================================================== -->
+  <target name="init" depends="ivy-retrieve-common,ivy-retrieve-mapred">
+    <mkdir dir="${build.dir}"/>
+    <mkdir dir="${build.classes}"/>
+    <mkdir dir="${build.tools}"/>
+    <mkdir dir="${build.src}"/>
+    <mkdir dir="${build.webapps}/task/WEB-INF"/>
+    <mkdir dir="${build.webapps}/job/WEB-INF"/>
+    <mkdir dir="${build.examples}"/>
+    <mkdir dir="${build.dir}/c++"/>
+ 
+    <mkdir dir="${test.build.dir}"/>
+    <mkdir dir="${test.build.classes}"/>
+    <mkdir dir="${test.build.extraconf}"/>
+    <tempfile property="touch.temp.file" destDir="${java.io.tmpdir}"/>
+    <touch millis="0" file="${touch.temp.file}">
+      <fileset dir="${conf.dir}" includes="**/*.template"/>
+      <fileset dir="${contrib.dir}" includes="**/*.template"/>
+    </touch>
+    <delete file="${touch.temp.file}"/>
+
+    <!-- copy all of the jsp and static files -->
+    <copy todir="${build.webapps}">
+      <fileset dir="${src.webapps}">
+        <exclude name="**/*.jsp" />
+        <exclude name="**/*.jspx" />
+      </fileset>
+    </copy>
+
+     <unzip src="${common.ivy.lib.dir}/hadoop-hdfs-${hadoop-hdfs.version}.jar"
+         dest="${build.dir}">
+       <patternset>
+         <include name="webapps/hdfs/**"/>
+         <include name="webapps/datanode/**"/>
+         <include name="webapps/secondary/**"/>
+       </patternset>
+     </unzip>
+
+    <copy todir="${conf.dir}" verbose="true">
+      <fileset dir="${conf.dir}" includes="**/*.template"/>
+      <mapper type="glob" from="*.template" to="*"/>
+    </copy>
+
+    <mkdir dir="${test.conf.dir}"/>
+    <copy todir="${test.conf.dir}" verbose="true">
+      <fileset dir="${conf.dir}" includes="**/*.template"/>
+      <mapper type="glob" from="*.template" to="*"/>
+    </copy>
+
+    <copy todir="${contrib.dir}" verbose="true">
+      <fileset dir="${contrib.dir}" includes="**/*.template"/>
+      <mapper type="glob" from="*.template" to="*"/>
+    </copy>
+
+  </target>
+
+  <target name="avro-generate" depends="init">
+    <mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/>
+    <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask">
+      <classpath refid="classpath" />
+    </taskdef>
+    <protocol destdir="${build.src}">
+      <fileset dir="${mapred.src.dir}">
+	<include name="**/*.avpr" />
+      </fileset>
+    </protocol>
+  </target>
+
+  <target name="compile-mapred-classes" depends="init,avro-generate">
+    <taskdef classname="org.apache.jasper.JspC" name="jsp-compile" >
+       <classpath refid="classpath"/>
+    </taskdef>
+    <jsp-compile
+     uriroot="${src.webapps}/task"
+     outputdir="${build.src}"
+     package="org.apache.hadoop.mapred"
+     webxml="${build.webapps}/task/WEB-INF/web.xml">
+    </jsp-compile>
+
+    <jsp-compile
+     uriroot="${src.webapps}/job"
+     outputdir="${build.src}"
+     package="org.apache.hadoop.mapred"
+     webxml="${build.webapps}/job/WEB-INF/web.xml">
+    </jsp-compile>
+
+    <!-- Compile Java files (excluding JSPs) checking warnings -->
+    <javac 
+     encoding="${build.encoding}" 
+     srcdir="${mapred.src.dir};${build.src}" 
+     includes="org/apache/hadoop/**/*.java"
+     destdir="${build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args} ${javac.args.warnings}" />
+      <classpath refid="classpath"/>
+    </javac>   
+    
+    <copy todir="${build.classes}">
+      <fileset dir="${mapred.src.dir}" includes="**/*.properties"/>
+      <fileset dir="${mapred.src.dir}" includes="**/META-INF/services/*"/>
+      <fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/>
+      <fileset dir="${mapred.src.dir}" includes="mapred-queues-default.xml"/>
+    </copy>
+  </target>
+
+  <target name="compile-tools" depends="init">
+    <javac 
+     encoding="${build.encoding}" 
+     srcdir="${tools.src}"
+     includes="org/apache/hadoop/**/*.java"
+     destdir="${build.tools}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args} ${javac.args.warnings}" />
+      <classpath refid="classpath"/>
+    </javac>   
+  	
+    <copy todir="${build.tools}">
+      <fileset 
+        dir="${tools.src}" 
+        includes="**/*.properties"
+      />
+    </copy>
+  </target>
+
+  <target name="compile-core" depends="clover, compile-mapred-classes, compile-c++" description="Compile core only"/> 
+
+  <target name="compile-contrib" depends="compile-core,tools">
+     <subant target="compile">
+        <property name="version" value="${version}"/>
+        <property name="hadoop-common.version" value="${hadoop-common.version}"/>
+        <fileset file="${contrib.dir}/build.xml"/>
+     </subant>  	
+  </target>
+  
+  <target name="compile" depends="compile-core, compile-contrib, compile-tools" description="Compile core, contrib">
+  </target>
+
+  <target name="compile-examples" 
+          depends="compile-core,compile-tools,compile-c++-examples">
+    <javac 
+     encoding="${build.encoding}" 
+     srcdir="${examples.dir}"
+     includes="org/apache/hadoop/**/*.java"
+     destdir="${build.examples}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args} ${javac.args.warnings}" />
+      <classpath>
+        <path refid="classpath"/>
+        <pathelement location="${build.tools}"/>
+      </classpath>
+    </javac>    
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Make hadoop-mapred.jar                                          -->
+  <!-- ================================================================== -->
+  <!--                                                                    -->
+  <!-- ================================================================== -->
+  <target name="jar" depends="compile-core" description="Make hadoop-mapered.jar">
+    <jar jarfile="${hadoop-mapred.jar}"
+         basedir="${build.classes}">
+      <manifest>
+        <section name="org/apache/hadoop">
+          <attribute name="Implementation-Title" value="${ant.project.name}"/>
+          <attribute name="Implementation-Version" value="${version}"/>
+          <attribute name="Implementation-Vendor" value="Apache"/>
+        </section>
+      </manifest>
+     <!-- <fileset file="${conf.dir}/commons-logging.properties"/>
+      <fileset file="${conf.dir}/log4j.properties"/>
+      <fileset file="${conf.dir}/hadoop-metrics.properties"/> -->
+      <zipfileset dir="${build.webapps}" prefix="webapps"/>
+      <fileset file="${jar.extra.properties.list}" />
+    </jar>
+
+    <jar jarfile="${hadoop-mapred-sources.jar}">
+      <fileset dir="${mapred.src.dir}" includes="org/apache/hadoop/**/*.java" />
+      <fileset dir="${build.src}" includes="org/apache/hadoop/**/*.java" />
+    </jar>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Make the Hadoop examples jar.                                      -->
+  <!-- ================================================================== -->
+  <!--                                                                    -->
+  <!-- ================================================================== -->
+  <target name="examples" depends="jar, compile-examples" description="Make the Hadoop examples jar.">
+    <jar jarfile="${build.dir}/${examples.final.name}.jar"
+         basedir="${build.examples}">
+      <manifest>
+        <attribute name="Main-Class" 
+                   value="org/apache/hadoop/examples/ExampleDriver"/>
+      </manifest>
+    </jar>
+
+    <jar jarfile="${hadoop-mapred-examples-sources.jar}">
+      <fileset dir="${examples.dir}" includes="org/apache/hadoop/**/*.java" />
+    </jar>
+  </target>
+
+  <target name="tools" depends="jar, compile-tools" 
+          description="Make the Hadoop tools jar.">
+    <jar jarfile="${build.dir}/${tools.final.name}.jar"
+         basedir="${build.tools}">
+    </jar>
+
+    <jar jarfile="${hadoop-mapred-tools-sources.jar}">
+      <fileset dir="${tools.src}" includes="org/apache/hadoop/**/*.java" />
+    </jar>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Compile test code                                                  --> 
+  <!-- ================================================================== -->
+
+  <target name="compile-mapred-test" depends="compile-mapred-classes, compile-examples, ivy-retrieve-test">
+
+    <mkdir dir="${test.mapred.build.classes}"/>
+    <mkdir dir="${test.mapred.build.testjar}"/>
+    <mkdir dir="${test.mapred.build.testshell}"/>
+
+    <macro-compile-test
+      source.dir="${test.src.dir}/mapred;${test.src.dir}/unit"
+      dest.dir="${test.mapred.build.classes}"
+      classpath="test.classpath"/>
+
+    <javac
+      encoding="${build.encoding}"
+      srcdir="${test.src.dir}/mapred/testjar"
+      includes="*.java"
+      destdir="${test.mapred.build.testjar}"
+      debug="${javac.debug}"
+      optimize="${javac.optimize}"
+      target="${javac.version}"
+      source="${javac.version}"
+      deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args} ${javac.args.warnings}" />
+      <classpath refid="test.classpath"/>
+    </javac>
+
+    <delete file="${test.mapred.build.testjar}/testjob.jar"/> 
+    <jar jarfile="${test.mapred.build.testjar}/testjob.jar"
+       basedir="${test.mapred.build.testjar}">
+    </jar>
+
+    <javac 
+     encoding="${build.encoding}"
+     srcdir="${test.src.dir}/mapred/testshell"
+     includes="*.java"
+     destdir="${test.mapred.build.testshell}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args} ${javac.args.warnings}"/>
+      <classpath refid="test.classpath"/>
+    </javac>
+    <delete file="${test.mapred.build.testshell}/testshell.jar"/>
+    <jar jarfile="${test.mapred.build.testshell}/testshell.jar"
+      basedir="${test.mapred.build.testshell}">
+    </jar>
+
+     <delete dir="${test.cache.data}"/>
+     <mkdir dir="${test.cache.data}"/>
+     <delete dir="${test.concat.data}"/>
+     <mkdir dir="${test.concat.data}"/>
+     <delete dir="${test.debug.data}"/>
+     <mkdir dir="${test.debug.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testscript.txt" todir="${test.debug.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.txt" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.jar" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.zip" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tar" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tgz" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tar.gz" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/cli/testMRConf.xml" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/cli/data60bytes" todir="${test.cache.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/concat.bz2" todir="${test.concat.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/concat.gz" todir="${test.concat.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.bz2" todir="${test.concat.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.gz" todir="${test.concat.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.bz2" todir="${test.concat.data}"/>
+     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.gz" todir="${test.concat.data}"/>
+  </target>
+
+  <macrodef name="macro-compile-test">
+    <attribute name="source.dir"/>
+    <attribute name="dest.dir"/>
+    <attribute name="classpath"/>
+    <sequential>
+      <mkdir dir="@{dest.dir}"/>
+      <javac
+        encoding="${build.encoding}"
+        srcdir="@{source.dir}"
+        includes="org/apache/hadoop/**/*.java"
+        destdir="@{dest.dir}"
+        debug="${javac.debug}"
+        optimize="${javac.optimize}"
+        target="${javac.version}"
+        source="${javac.version}"
+        deprecation="${javac.deprecation}">
+        <compilerarg line="${javac.args} ${javac.args.warnings}"/>
+        <classpath refid="@{classpath}"/>
+      </javac>
+    </sequential>
+  </macrodef>
+
+  <!-- ================================================================== -->
+  <!-- Make hadoop-mapred-test.jar                                        -->
+  <!-- ================================================================== -->
+  <!--                                                                    -->
+  <!-- ================================================================== -->
+  <target name="jar-test" depends="compile-mapred-test" description="Make hadoop-mapred-test.jar">
+    <copy todir="${test.build.classes}">
+      <fileset dir="${test.mapred.build.classes}"/>
+    </copy>
+    <subant buildpath="build.xml" target="-do-jar-test"/>
+    <jar jarfile="${hadoop-mapred-test-sources.jar}">
+      <fileset dir="${test.src.dir}/mapred" includes="org/apache/hadoop/**/*.java" />
+      <fileset dir="${test.src.dir}/unit" includes="org/apache/hadoop/**/*.java" />
+      <fileset dir="${test.src.dir}/mapred/testjar" includes="*.java" />
+      <fileset dir="${test.src.dir}/mapred/testshell" includes="*.java" />
+    </jar>
+  </target>
+
+  <target  name="-do-jar-test">
+    <jar jarfile="${build.dir}/${test.final.name}.jar"
+         basedir="${test.build.classes}">
+         <manifest>
+           <attribute name="Main-Class"
+                      value="org/apache/hadoop/test/MapredTestDriver"/>
+          <section name="org/apache/hadoop">
+            <attribute name="Implementation-Title" value="${ant.project.name}"/>
+            <attribute name="Implementation-Version" value="${version}"/>
+            <attribute name="Implementation-Vendor" value="Apache"/>
+          </section>
+         </manifest>
+    </jar>
+  </target>
+  <!-- ================================================================== -->
+  <!-- Define exclude lists for different kinds of testing                -->
+  <!-- ================================================================== -->
+  <patternset id="empty.exclude.list.id" />
+  <patternset id="commit.smoke.exclude.list.id">
+    <excludesfile name="${test.mapred.commit.tests.file}"/>
+    <excludesfile name="${test.mapred.smoke.tests.file}"/>
+  </patternset>
+
+  <macrodef name="macro-test-runner">
+    <attribute name="test.file" />
+    <attribute name="suite.type" />
+    <attribute name="classpath" />
+    <attribute name="test.dir" />
+    <attribute name="fileset.dir" />
+    <attribute name="hadoop.conf.dir.deployed" default="" />
+    <attribute name="exclude.list.id" default="empty.exclude.list.id" />
+    <sequential>
+      <delete dir="@{test.dir}/data"/>
+      <mkdir dir="@{test.dir}/data"/>
+      <delete dir="@{test.dir}/logs"/>
+      <mkdir dir="@{test.dir}/logs"/>
+      <copy file="${test.src.dir}/hadoop-policy.xml" 
+        todir="@{test.dir}/extraconf" />
+      <copy file="${test.src.dir}/fi-site.xml"
+        todir="@{test.dir}/extraconf" />
+      <junit showoutput="${test.output}"
+        printsummary="${test.junit.printsummary}"
+        haltonfailure="${test.junit.haltonfailure}"
+        fork="yes"
+        forkmode="${test.junit.fork.mode}"
+        maxmemory="${test.junit.maxmemory}"
+        dir="${basedir}" timeout="${test.timeout}"
+        errorProperty="tests.failed" failureProperty="tests.failed">
+        <jvmarg value="${test.junit.jvmargs}" />
+        <sysproperty key="java.net.preferIPv4Stack" value="true"/>
+        <sysproperty key="test.build.data" value="@{test.dir}/data"/>
+        <sysproperty key="java.security.krb5.conf"
+          value="${test.src.dir}/krb5.conf"/>
+        <sysproperty key="test.tools.input.dir" value = "${test.tools.input.dir}"/>
+        <sysproperty key="test.cache.data" value="${test.cache.data}"/>     
+        <sysproperty key="test.concat.data" value="${test.concat.data}"/>
+        <sysproperty key="test.debug.data" value="${test.debug.data}"/>
+        <sysproperty key="hadoop.log.dir" value="@{test.dir}/logs"/>
+        <sysproperty key="test.src.dir" value="@{fileset.dir}"/>
+        <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
+        <sysproperty key="taskcontroller-ugi" value="${taskcontroller-ugi}"/>
+        <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
+        <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
+        <sysproperty key="java.library.path"
+          value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
+        <sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
+        <syspropertyset dynamic="no">
+          <propertyref name="hadoop.tmp.dir"/>
+        </syspropertyset>
+        <!-- set compile.c++ in the child jvm only if it is set -->
+        <syspropertyset dynamic="no">
+          <propertyref name="compile.c++"/>
+        </syspropertyset>
+        
+        <!-- Pass probability specifications to the spawn JVM -->
+        <syspropertyset id="FaultProbabilityProperties">
+          <propertyref regex="fi.*"/>
+        </syspropertyset>
+        <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
+                     value="@{hadoop.conf.dir.deployed}" />
+        <classpath refid="@{classpath}"/>
+        <formatter type="${test.junit.output.format}" />
+        <batchtest todir="@{test.dir}" unless="testcase">
+          <fileset dir="@{fileset.dir}/@{suite.type}"
+            excludes="**/${test.exclude}.java aop/** system/**">
+            <patternset>
+              <includesfile name="@{test.file}"/>
+              <excludesfile name="${test.exclude.file}" />
+            </patternset>
+            <patternset refid="@{exclude.list.id}"/>
+          </fileset>
+        </batchtest>
+        <batchtest todir="@{test.dir}" if="testcase">
+          <fileset dir="@{fileset.dir}/mapred" includes="**/${testcase}.java"/>
+          <fileset dir="@{fileset.dir}/unit" includes="**/${testcase}.java"/>
+          <fileset dir="@{fileset.dir}/system/test" includes="**/${testcase}.java"/>
+        </batchtest>
+        <!--batch test to test all the testcases in aop folder with fault 
+        injection-->
+        <batchtest todir="@{test.dir}" if="tests.notestcase.fi">
+          <fileset dir="@{fileset.dir}/aop"
+            includes="**/${test.include}.java"
+            excludes="**/${test.exclude}.java"
+            excludesfile="${test.exclude.file}" />
+        </batchtest>
+        <!-- batch test for testing a single test case in aop folder with
+        fault injection-->
+        <batchtest todir="@{test.dir}" if="tests.testcase.fi">
+         <fileset dir="@{fileset.dir}/aop" includes="**/${testcase}.java"/>
+        </batchtest>
+         <!--The following batch is for very special occasions only when
+         a non-FI tests are needed to be executed against FI-environment -->
+         <batchtest todir="@{test.dir}" if="tests.testcaseonly.fi">
+          <fileset dir="@{fileset.dir}/mapred" 
+            includes="**/${testcase}.java"/>
+        </batchtest>
+      </junit>
+      <antcall target="checkfailure"/>
+    </sequential>
+  </macrodef>
+
+  <target name="run-test-mapred" depends="run-commit-test, run-smoke-test,
+    run-test-mapred-excluding-commit-and-smoke, run-test-mapred-all-withtestcaseonly"
+    description="Run mapred functional and system tests">
+  </target>
+
+  <target name="run-test-mapred-all-withtestcaseonly"
+    depends="compile-mapred-test" if="testcase">
+    <macro-test-runner
+      test.file="${test.mapred.all.tests.file}"
+      suite.type="mapred"
+      classpath="${test.classpath.id}"
+      test.dir="${test.build.dir}"
+      fileset.dir="${test.src.dir}"/>
+  </target>
+
+  <target name="run-test-mapred-excluding-commit-and-smoke"
+    depends="compile-mapred-test" unless="testcase">
+    <macro-test-runner
+      test.file="${test.mapred.all.tests.file}"
+      suite.type="mapred"
+      classpath="${test.classpath.id}"
+      test.dir="${test.build.dir}"
+      fileset.dir="${test.src.dir}"
+      exclude.list.id="commit.smoke.exclude.list.id"
+    />
+   </target>
+
+   <target name="run-commit-test" depends="compile-mapred-test"
+     description="Run approximate 10-minute set of unit tests prior to commiting"
+     unless="testcase">
+     <macro-test-runner test.file="${test.mapred.commit.tests.file}" suite.type="mapred"
+      classpath="${test.classpath.id}"
+      test.dir="${test.build.dir}"
+      fileset.dir="${test.src.dir}"/>
+   </target>
+ 
+   <target name="run-smoke-test" depends="compile-mapred-test"
+     description="Run approximate 30-minute set of functional tests to guarantee viability of the MapReduce"
+     unless="testcase">
+     <macro-test-runner
+       test.file="${test.mapred.smoke.tests.file}"
+       suite.type="mapred"
+       classpath="${test.classpath.id}"
+       test.dir="${test.build.dir}"
+       fileset.dir="${test.src.dir}"/>
+   </target>
+
+   <target name="run-test-unit" depends="compile-mapred-test" description="Run unit tests">
+     <macro-test-runner test.file="${test.mapred.all.tests.file}" suite.type="unit"
+      classpath="${test.classpath.id}"
+      test.dir="${test.build.dir}"
+      fileset.dir="${test.src.dir}"/>
+   </target>
+
+
+  <target name="checkfailure" if="tests.failed">
+    <touch file="${test.build.dir}/testsfailed"/>
+    <fail unless="continueOnFailure">Tests failed!</fail>
+  </target>
+
+  <target name="test-contrib" depends="compile,compile-tools,compile-mapred-test" description="Run contrib unit tests">
+    <subant target="test">
+       <property name="version" value="${version}"/>
+       <property name="clover.jar" value="${clover.jar}"/>
+       <fileset file="${contrib.dir}/build.xml"/>
+    </subant> 
+  </target>
+
+  <target name="test-core" description="Run core, hdfs and mapred unit tests">
+    <delete file="${test.build.dir}/testsfailed"/>
+    <property name="continueOnFailure" value="true"/>
+    <antcall target="run-test-mapred"/>
+    <available file="${test.build.dir}/testsfailed" property="testsfailed"/>
+    <fail if="testsfailed">Tests failed!</fail>
+  </target>
+
+  <target name="test" depends="jar-test, test-core" description="Run all unit tests">
+    <subant target="test-contrib">
+      <fileset file="${basedir}/build.xml"/>
+     </subant>
+  </target>
+
+  <!-- Run all unit tests, not just Test*, and use non-test configuration. -->
+  <target name="test-cluster" description="Run all unit tests, not just Test*, and use non-test configuration.">
+    <antcall target="test">
+      <param name="test.include" value="*"/>
+      <param name="test.classpath.id" value="test.cluster.classpath"/>
+    </antcall>
+  </target>
+
+  <target name="nightly" depends="test, tar">
+  </target>
+	
+  <!-- ================================================================== -->
+  <!-- Run optional third-party tool targets                              --> 
+  <!-- ================================================================== -->
+  <target name="checkstyle" depends="ivy-retrieve-checkstyle,check-for-checkstyle" if="checkstyle.present" description="Run optional third-party tool targets">
+       <taskdef resource="checkstyletask.properties">
+         <classpath refid="checkstyle-classpath"/>
+       </taskdef>
+  
+	<mkdir dir="${test.build.dir}"/>
+  	
+  	<checkstyle config="${test.src.dir}/checkstyle.xml"
+  		failOnViolation="false">
+      <fileset dir="${mapred.src.dir}" includes="**/*.java" excludes="**/generated/**"/>
+      <formatter type="xml" toFile="${test.build.dir}/checkstyle-errors.xml"/>
+  	</checkstyle>
+  	
+  	<xslt style="${test.src.dir}/checkstyle-noframes-sorted.xsl"
+        in="${test.build.dir}/checkstyle-errors.xml"
+        out="${test.build.dir}/checkstyle-errors.html"/>
+  </target>
+	
+  <target name="check-for-checkstyle">
+    <available property="checkstyle.present" resource="checkstyletask.properties">
+       <classpath refid="checkstyle-classpath"/>
+    </available>  	
+  </target>
+
+ <target name="all-jars" depends="tools,examples,jar-test">
+   <subant target="jar">
+      <property name="version" value="${version}"/>
+      <property name="dist.dir" value="${dist.dir}"/>
+      <fileset file="${contrib.dir}/capacity-scheduler/build.xml"/>
+      <fileset file="${contrib.dir}/streaming/build.xml"/>
+      <fileset file="${contrib.dir}/gridmix/build.xml"/>
+      <fileset file="${contrib.dir}/mumak/build.xml"/>
+    </subant>
+ </target>
+
+ <property name="findbugs.home" value=""/>
+  <target name="findbugs" depends="check-for-findbugs, all-jars" if="findbugs.present" description="Run findbugs if present">
+    <property name="findbugs.out.dir" value="${test.build.dir}/findbugs"/>
+    <property name="findbugs.exclude.file" value="${test.src.dir}/findbugsExcludeFile.xml"/>
+    <property name="findbugs.report.htmlfile" value="${findbugs.out.dir}/hadoop-findbugs-report.html"/>
+    <property name="findbugs.report.xmlfile" value="${findbugs.out.dir}/hadoop-findbugs-report.xml"/>
+    <taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
+        classpath="${findbugs.home}/lib/findbugs-ant.jar" />
+
+        <mkdir dir="${findbugs.out.dir}"/>
+
+    <findbugs home="${findbugs.home}" output="xml:withMessages"
+        outputFile="${findbugs.report.xmlfile}" effort="max"
+        excludeFilter="${findbugs.exclude.file}" jvmargs="-Xmx${findbugs.heap.size}">
+      <auxClasspath>
+        <fileset dir="${lib.dir}">
+          <include name="**/*.jar"/>
+        </fileset>
+        <fileset dir="${build.ivy.lib.dir}/${ant.project.name}/common">
+          <include name="**/*.jar"/>
+        </fileset>
+      </auxClasspath>
+      <sourcePath path="${mapred.src.dir}"/>
+      <sourcePath path="${examples.dir}" />
+      <sourcePath path="${tools.src}" />
+      <sourcePath path="${basedir}/src/contrib/capacity-scheduler/src/java" />
+      <sourcePath path="${basedir}/src/contrib/streaming/src/java" />
+      <sourcePath path="${basedir}/src/contrib/gridmix/src/java" />
+      <sourcePath path="${basedir}/src/contrib/mumak/src/java" />
+      <class location="${basedir}/build/${final.name}.jar" />
+      <class location="${basedir}/build/${examples.final.name}.jar" />
+      <class location="${basedir}/build/${tools.final.name}.jar" />
+      <class location="${basedir}/build/contrib/capacity-scheduler/hadoop-${version}-capacity-scheduler.jar" />
+      <class location="${basedir}/build/contrib/streaming/hadoop-${version}-streaming.jar" />
+      <class location="${basedir}/build/contrib/gridmix/hadoop-${version}-gridmix.jar" />
+      <class location="${basedir}/build/contrib/mumak/hadoop-${version}-mumak.jar" />
+    </findbugs>
+
+        <xslt style="${findbugs.home}/src/xsl/default.xsl"
+        in="${findbugs.report.xmlfile}"
+        out="${findbugs.report.htmlfile}"/>
+  </target>
+	
+  <target name="check-for-findbugs">
+    <available property="findbugs.present"
+        file="${findbugs.home}/lib/findbugs.jar" />
+  </target>
+
+
+  <!-- ================================================================== -->
+  <!-- Documentation                                                      -->
+  <!-- ================================================================== -->
+  
+  <target name="docs" depends="forrest.check" description="Generate forrest-based documentation. To use, specify -Dforrest.home=&lt;base of Apache Forrest installation&gt; on the command line." if="forrest.home">
+  	<!-- The template file may not exist if building from a tarball -->
+  	<copy file="${conf.dir}/mapred-queues.xml.template"
+  	      tofile="${build.docs}/mapred-queues.xml" failonerror="false"/>
+    <exec dir="${docs.src}" executable="${forrest.home}/bin/forrest"
+	  failonerror="true">
+    </exec>
+    <copy todir="${build.docs}">
+      <fileset dir="${docs.src}/build/site/" />
+    </copy>
+    <copy file="${docs.src}/releasenotes.html" todir="${build.docs}"/>
+    <style basedir="${mapred.src.dir}" destdir="${build.docs}"
+           includes="mapred-default.xml" style="conf/configuration.xsl"/>
+    <antcall target="changes-to-html"/>
+    <subant target="docs">
+       <property name="build.docs" value="${build.docs}"/>
+       <fileset file="${contrib.dir}/build.xml"/>
+    </subant> 
+  </target>
+
+  <target name="javadoc-dev" depends="compile, ivy-retrieve-javadoc" description="Generate javadoc for hadoop developers">
+    <mkdir dir="${build.javadoc.dev}"/>
+    <javadoc
+      overview="${mapred.src.dir}/overview.html"
+      packagenames="org.apache.hadoop.*"
+      destdir="${build.javadoc.dev}"
+      author="true"
+      version="true"
+      use="true"
+      windowtitle="${Name} ${version} API"
+      doctitle="${Name} ${version} Developer API"
+      bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
+      maxmemory="${javadoc.maxmemory}"
+      >
+        <packageset dir="${mapred.src.dir}"/>
+    	<packageset dir="${examples.dir}"/>
+    	<packageset dir="${tools.src}"/>
+
+       <packageset dir="src/contrib/data_join/src/java"/>
+       <packageset dir="src/contrib/gridmix/src/java"/>
+       <packageset dir="src/contrib/index/src/java"/>
+       <packageset dir="src/contrib/mumak/src/java"/>
+       <packageset dir="src/contrib/streaming/src/java"/>
+       <packageset dir="src/contrib/vaidya/src/java"/>
+       <packageset dir="src/contrib/vertica/src/java"/>
+
+        <link href="${javadoc.link.java}"/>
+
+        <classpath >
+          <path refid="classpath" />
+          <fileset dir="src/contrib/">
+            <include name="*/lib/*.jar" />
+          </fileset>
+          <path refid="javadoc-classpath"/>
+          <pathelement path="${java.class.path}"/>
+          <pathelement path="${lib.dir}/hadoop-common-test-${hadoop-common.version}.jar"/>
+          <pathelement location="${build.tools}"/>
+        </classpath>
+
+       <group title="Packages" packages="org.apache.*"/>
+       <group title="Libraries" packages="org.apache.hadoop.mapred.lib*:org.apache.hadoop.mapreduce.lib*"/>
+       <group title="Tools" packages="org.apache.hadoop.tools"/>
+       <group title="Examples" packages="org.apache.hadoop.examples*"/>
+
+       <group title="contrib: DataJoin" packages="org.apache.hadoop.contrib.utils.join*"/>
+       <group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
+       <group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
+       <group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
+       <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
+       <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
+    </javadoc>
+  </target>	
+
+  <target name="javadoc-uptodate" depends="compile, ivy-retrieve-common, ivy-retrieve-javadoc">
+    <uptodate property="javadoc.is.uptodate">
+      <srcfiles dir="${src.dir}">
+        <include name="**/*.java" />
+        <include name="**/*.html" />
+      </srcfiles>
+      <mapper type="merge" to="${build.javadoc.timestamp}" />
+    </uptodate>
+  </target>
+ 
+  <target name="javadoc" description="Generate javadoc" depends="javadoc-uptodate"
+       unless="javadoc.is.uptodate">
+    <mkdir dir="${build.javadoc}"/>
+    <javadoc
+      overview="${mapred.src.dir}/overview.html"
+      packagenames="org.apache.hadoop.*"
+      destdir="${build.javadoc}"
+      author="true"
+      version="true"
+      use="true"
+      windowtitle="${Name} ${version} API"
+      doctitle="${Name} ${version} API"
+      bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
+      maxmemory="${javadoc.maxmemory}"
+      >
+        <packageset dir="${mapred.src.dir}"/>
+    	<packageset dir="${examples.dir}"/>
+
+       <!-- Don't include contrib modules that use the same packages as core
+       MapReduce. This includes capacity-scheduler, dynamic-scheduler,
+       fairscheduler, mumak. See also the javadoc-dev target. -->
+       <packageset dir="src/contrib/data_join/src/java"/>
+       <packageset dir="src/contrib/gridmix/src/java"/>
+       <packageset dir="src/contrib/index/src/java"/>
+       <packageset dir="src/contrib/streaming/src/java"/>
+       <packageset dir="src/contrib/vaidya/src/java"/>
+       <packageset dir="src/contrib/vertica/src/java"/>
+       <packageset dir="${tools.src}"/>
+	
+        <link href="${javadoc.link.java}"/>
+
+        <classpath >
+          <path refid="classpath" />
+          <fileset dir="src/contrib/">
+            <include name="*/lib/*.jar" />
+          </fileset>
+          <path refid="javadoc-classpath"/>
+          <pathelement path="${java.class.path}"/>
+          <pathelement path="${lib.dir}/hadoop-common-test-${hadoop-common.version}.jar"/>
+          <pathelement location="${build.tools}"/>
+        </classpath>
+
+       <group title="Packages" packages="org.apache.*"/>
+       <group title="Libraries" packages="org.apache.hadoop.mapred.lib*:org.apache.hadoop.mapreduce.lib*"/>
+       <group title="Tools" packages="org.apache.hadoop.fs*:org.apache.hadoop.tools*:org.apache.hadoop.mapred.tool*:org.apache.hadoop.mapreduce.tool*"/>
+       <group title="Examples" packages="org.apache.hadoop.examples*"/>
+
+       <group title="contrib: DataJoin" packages="org.apache.hadoop.contrib.utils.join*"/>
+       <group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
+       <group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
+       <group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
+       <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
+       <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
+
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet"
+         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar"/>
+    </javadoc>
+  </target>	
+
+  <target name="api-xml" depends="ivy-retrieve-jdiff,javadoc,write-null">
+    <javadoc maxmemory="${javadoc.maxmemory}">
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
+         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar:${jdiff.jar}:${xerces.jar}">
+         <param name="-apidir" value="${jdiff.xml.dir}"/>
+         <param name="-apiname" value="hadoop-mapred ${version}"/>
+         <param name="${jdiff.stability}"/>
+       </doclet>
+       <packageset dir="src/java"/>
+       <packageset dir="src/tools"/>
+       <packageset dir="${tools.src}"/>
+       <classpath >
+         <path refid="classpath" />
+         <path refid="jdiff-classpath" />
+         <pathelement path="${java.class.path}"/>
+       </classpath>
+    </javadoc>
+  </target>
+	
+  <target name="write-null">
+	<exec executable="touch">
+	   <arg value="${jdiff.home}/Null.java"/>
+        </exec>
+  </target> 
+
+  <target name="api-report" depends="ivy-retrieve-jdiff,api-xml">
+    <mkdir dir="${jdiff.build.dir}"/>
+    <javadoc destdir="${jdiff.build.dir}"
+	     sourceFiles="${jdiff.home}/Null.java"
+	     maxmemory="${javadoc.maxmemory}">
+       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
+         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar:${jdiff.jar}:${xerces.jar}">
+         <param name="-oldapi" value="hadoop-mapred ${jdiff.stable}"/>
+         <param name="-newapi" value="hadoop-mapred ${version}"/>
+         <param name="-oldapidir" value="${jdiff.xml.dir}"/>
+         <param name="-newapidir" value="${jdiff.xml.dir}"/>
+         <param name="-javadocold" value="${jdiff.stable.javadoc}"/>
+         <param name="-javadocnew" value="../../api/"/>
+         <param name="-stats"/>
+         <param name="${jdiff.stability}"/>
+         <param name="${jdiff.compatibility}"/>
+       </doclet>
+       <packageset dir="src/java"/>
+       <packageset dir="src/tools"/>
+       <packageset dir="${tools.src}"/>
+       <classpath >
+         <path refid="classpath" />
+         <path refid="jdiff-classpath"/>
+         <pathelement path="${java.class.path}"/>
+       </classpath>
+    </javadoc>
+  </target>
+	
+  <target name="changes-to-html" description="Convert CHANGES.txt into an html file">
+    <mkdir dir="${build.docs}"/>
+    <exec executable="perl" input="CHANGES.txt" output="${build.docs}/changes.html" failonerror="true">
+      <arg value="${changes.src}/changes2html.pl"/>
+    </exec>
+    <copy todir="${build.docs}">
+      <fileset dir="${changes.src}" includes="*.css"/>
+    </copy>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- D I S T R I B U T I O N                                            -->
+  <!-- ================================================================== -->
+  <!--                                                                    -->
+  <!-- ================================================================== -->
+  <target name="package" depends="compile, jar, javadoc, docs, api-report, examples, tools, jar-test, package-librecordio"
+	  description="Build distribution">
+    <mkdir dir="${dist.dir}"/>
+    <mkdir dir="${dist.dir}/lib"/>
+    <mkdir dir="${dist.dir}/contrib"/>
+    <mkdir dir="${dist.dir}/bin"/>
+    <mkdir dir="${dist.dir}/docs"/>
+    <mkdir dir="${dist.dir}/docs/api"/>
+    <mkdir dir="${dist.dir}/docs/jdiff"/>
+
+    <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
+      <fileset dir="${mapred.ivy.lib.dir}"/>
+    </copy>
+
+    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
+      <fileset dir="lib">
+        <exclude name="**/native/**"/>
+      </fileset>
+    </copy>
+
+    <subant target="compile-test">
+      <!--Pass down the version in case its needed again and the target
+      distribution directory so contribs know where to install to.-->
+      <property name="version" value="${version}"/>
+      <property name="dist.dir" value="${dist.dir}"/>
+      <fileset file="${contrib.dir}/build.xml"/>
+    </subant>  	
+    <subant target="package">
+      <!--Pass down the version in case its needed again and the target
+      distribution directory so contribs know where to install to.-->
+      <property name="version" value="${version}"/>
+      <property name="dist.dir" value="${dist.dir}"/>
+      <fileset file="${contrib.dir}/build.xml"/>
+    </subant>  	
+
+    <copy todir="${dist.dir}/webapps">
+      <fileset dir="${build.webapps}"/>
+    </copy>
+
+    <copy todir="${dist.dir}"> 
+      <fileset file="${build.dir}/${final.name}*.jar"/>
+      <fileset file="${build.dir}/${test.final.name}.jar"/>
+      <fileset file="${build.dir}/${examples.final.name}.jar"/>
+      <fileset file="${build.dir}/${tools.final.name}.jar"/> 
+    </copy>
+    
+    <copy todir="${dist.dir}/bin">
+      <fileset dir="bin"/>
+    </copy>
+
+    <copy todir="${dist.dir}/conf">
+      <fileset dir="${conf.dir}" excludes="**/*.template"/>
+    </copy>
+
+    <copy todir="${dist.dir}/docs">
+      <fileset dir="${build.docs}"/>
+    </copy>
+
+    <copy file="ivy.xml" tofile="${dist.dir}/ivy.xml"/>
+
+    <copy todir="${dist.dir}/ivy">
+      <fileset dir="ivy"/>
+    </copy>
+
+    <copy todir="${dist.dir}">
+      <fileset dir=".">
+        <include name="*.txt" />
+      </fileset>
+    </copy>
+
+    <copy todir="${dist.dir}/src" includeEmptyDirs="true">
+      <fileset dir="src" excludes="**/*.template **/docs/build/**/*"/>
+    </copy>
+  	
+    <copy todir="${dist.dir}/c++" includeEmptyDirs="false">
+      <fileset dir="${build.dir}/c++"/>
+    </copy>
+
+    <copy todir="${dist.dir}/" file="build.xml"/>
+    <copy todir="${dist.dir}/" file="build-utils.xml"/>
+
+    <chmod perm="ugo+x" type="file" parallel="false">
+        <fileset file="${dist.dir}/src/examples/pipes/configure"/>
+        <fileset file="${dist.dir}/src/c++/utils/configure"/>
+        <fileset file="${dist.dir}/src/c++/pipes/configure"/>
+        <fileset file="${dist.dir}/src/c++/task-controller/configure"/>
+    </chmod>
+    <chmod perm="ugo+x" type="file" parallel="false">
+        <fileset dir="${dist.dir}/bin"/>
+        <fileset dir="${dist.dir}/src/contrib/">
+          <include name="*/bin/*" />
+        </fileset>
+    </chmod>
+    <chmod perm="ugo+x" type="file">
+        <fileset dir="${dist.dir}/src/c++/pipes/debug"/>
+    </chmod>
+
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Make release tarball                                               -->
+  <!-- ================================================================== -->
+  <target name="tar" depends="package" description="Make release tarball">
+    <macro_tar param.destfile="${build.dir}/${final.name}.tar.gz">
+      <param.listofitems>
+        <tarfileset dir="${build.dir}" mode="664">
+          <exclude name="${final.name}/bin/*" />
+          <exclude name="${final.name}/contrib/*/bin/*" />
+          <exclude name="${final.name}/src/examples/pipes/configure"/>
+          <exclude name="${final.name}/src/c++/utils/configure"/>
+          <exclude name="${final.name}/src/c++/pipes/configure"/>
+          <exclude name="${final.name}/src/c++/task-controller/configure"/>
+          <include name="${final.name}/**" />
+        </tarfileset>
+        <tarfileset dir="${build.dir}" mode="755">
+          <include name="${final.name}/bin/*" />
+          <include name="${final.name}/contrib/*/bin/*" />
+          <include name="${final.name}/src/examples/pipes/configure"/>
+          <include name="${final.name}/src/c++/utils/configure"/>
+          <include name="${final.name}/src/c++/pipes/configure"/>
+          <include name="${final.name}/src/c++/task-controller/configure"/>
+        </tarfileset>
+      </param.listofitems>
+    </macro_tar>
+  </target>
+
+  <target name="bin-package" depends="compile, jar, examples, tools, jar-test, package-librecordio" 
+		description="assembles artifacts for binary target">
+    <mkdir dir="${dist.dir}"/>
+    <mkdir dir="${dist.dir}/include"/>
+    <mkdir dir="${dist.dir}/lib"/>
+    <mkdir dir="${dist.dir}/${package.share.dir}/contrib"/>
+    <mkdir dir="${dist.dir}/${package.share.dir}/lib"/>
+    <mkdir dir="${dist.dir}/${package.share.dir}/templates"/>
+    <mkdir dir="${dist.dir}/bin"/>
+    <mkdir dir="${dist.dir}/sbin"/>
+
+    <!-- enable this if there is mapred specific dependencies
+    <copy todir="${dist.dir}/${package.share.dir}/lib" includeEmptyDirs="false" flatten="true">
+      <fileset dir="${mapred.ivy.lib.dir}"/>
+    </copy> -->
+
+    <copy todir="${dist.dir}/include" includeEmptyDirs="false">
+      <fileset dir="${build.dir}/c++/${build.platform}/include" 
+               erroronmissingdir="false">
+        <include name="**"/>
+      </fileset>
+    </copy>
+
+    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
+      <fileset dir="${build.dir}/c++/${build.platform}/lib"
+               erroronmissingdir="false">
+        <include name="**"/>
+      </fileset>
+    </copy>
+
+    <subant target="package">
+      <!--Pass down the version in case its needed again and the target
+      distribution directory so contribs know where to install to.-->
+      <property name="version" value="${version}"/>
+      <property name="dist.dir" value="${dist.dir}/${package.share.dir}"/>
+      <fileset file="${contrib.dir}/build.xml"/>
+    </subant>  	
+
+    <copy todir="${dist.dir}/${package.share.dir}"> 
+      <fileset file="${build.dir}/${final.name}*.jar"/>
+      <fileset file="${build.dir}/${test.final.name}.jar"/>
+      <fileset file="${build.dir}/${examples.final.name}.jar"/>
+      <fileset file="${build.dir}/${tools.final.name}.jar"/>
+    </copy>
+
+    <copy todir="${dist.dir}/bin">
+      <fileset dir="bin">
+        <include name="mapred"/>
+      </fileset>
+    </copy>
+
+    <copy todir="${dist.dir}/libexec">
+      <fileset dir="bin">
+        <include name="mapred-config.sh"/>
+      </fileset>
+    </copy>
+
+    <copy todir="${dist.dir}/sbin">
+      <fileset dir="bin">
+        <include name="start-*.sh"/>
+        <include name="stop-*.sh"/>
+      </fileset>
+    </copy>
+
+    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
+    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.redhat"/>
+    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.redhat"/>
+    <copy file="${basedir}/src/packages/deb/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.debian"/>
+    <copy file="${basedir}/src/packages/deb/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.debian"/>
+    
+    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
+      	
+    <copy todir="${dist.dir}/etc/hadoop">
+      <fileset dir="${conf.dir}" excludes="**/*.template"/>
+    </copy>
+
+    <copy todir="${dist.dir}/${package.share.dir}/templates">
+      <fileset dir="${basedir}/src/packages/templates/conf" includes="*"/>
+    </copy>
+
+    <copy todir="${dist.dir}/${package.share.dir}/webapps">
+      <fileset dir="${build.webapps}"/>
+    </copy>
+
+    <copy todir="${dist.dir}/share/doc/hadoop/${module}">
+      <fileset dir=".">
+        <include name="*.txt" />
+      </fileset>
+    </copy>
+
+    <chmod perm="ugo+x" type="file" parallel="false">
+        <fileset dir="${dist.dir}/bin"/>
+        <fileset dir="${dist.dir}/sbin"/>
+    </chmod>  	
+  </target>
+
+  <target name="binary-system" depends="bin-package, jar-system, jar-test-system"
+     description="make system test package for deployment">
+    <!--TODO!!!!! fix this shit...-->
+    <copy todir="${system-test-build-dir}/${final.name}">
+      <fileset dir="${dist.dir}">
+      </fileset>
+    </copy>
+    <copy todir="${system-test-build-dir}/${final.name}/conf">
+      <fileset dir="${test.src.dir}/system/conf/"/>
+    </copy>
+    <copy todir="${system-test-build-dir}">
+      <fileset dir="${build.dir}">
+        <include name="${test.final.name}.jar"/>
+        <include name="${examples.final.name}.jar"/>
+      </fileset>
+    </copy>
+    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-common-${hadoop-common.version}.jar"
+      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-common-${herriot.suffix}-${hadoop-common.version}.jar"
+      overwrite="true"/>
+    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-hdfs-${version}.jar"
+      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-hdfs-${herriot.suffix}-${version}.jar"
+      overwrite="true"/>
+    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}.jar"
+      file="${system-test-build-dir}/${instrumented.final.name}.jar" overwrite="true"/>
+    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}-sources.jar"
+      file="${system-test-build-dir}/${instrumented.final.name}-sources.jar" overwrite="true"/>
+    <macro_tar 
+      param.destfile="${system-test-build-dir}/${final.name}-bin.${herriot.suffix}.tar.gz">
+        <param.listofitems>
+          <tarfileset dir="${system-test-build-dir}" mode="664">
+            <exclude name="${final.name}/bin/*" />
+            <exclude name="${final.name}/src/**" />
+            <exclude name="${final.name}/docs/**" />
+            <include name="${final.name}/**" />
+          </tarfileset>
+        </param.listofitems>
+      </macro_tar>
+  </target>
+  
+  <target name="binary" depends="bin-package" description="Make tarball without source and documentation">
+    <macro_tar param.destfile="${build.dir}/${final.name}-bin.tar.gz">
+      <param.listofitems>
+        <tarfileset dir="${build.dir}" mode="664">
+          <exclude name="${final.name}/bin/*" />
+          <exclude name="${final.name}/src/**" />
+          <exclude name="${final.name}/docs/**" />
+          <include name="${final.name}/**" />
+        </tarfileset>
+        <tarfileset dir="${build.dir}" mode="755">
+          <include name="${final.name}/bin/*" />
+        </tarfileset>
+      </param.listofitems>
+    </macro_tar>
+  </target>
+
+  <target name="rpm" depends="binary" description="Make rpm package">
+    <mkdir dir="${package.buildroot}/BUILD" />
+    <mkdir dir="${package.buildroot}/RPMS" />
+    <mkdir dir="${package.buildroot}/SRPMS" />
+    <mkdir dir="${package.buildroot}/SOURCES" />
+    <mkdir dir="${package.buildroot}/SPECS" />
+    <copy todir="${package.buildroot}/SOURCES">
+      <fileset dir="${build.dir}">
+        <include name="${final.name}-bin.tar.gz" />
+      </fileset>
+    </copy>
+    <copy file="${src.dir}/packages/rpm/spec/hadoop-mapred.spec" todir="${package.buildroot}/SPECS">
+      <filterchain>
+        <replacetokens>
+          <token key="final.name" value="${final.name}" />
+          <token key="version" value="${_version}" />
+          <token key="package.release" value="${package.release}" />
+          <token key="package.build.dir" value="${package.build.dir}" />
+          <token key="package.prefix" value="${package.prefix}" />
+          <token key="package.conf.dir" value="${package.conf.dir}" />
+          <token key="package.log.dir" value="${package.log.dir}" />
+          <token key="package.pid.dir" value="${package.pid.dir}" />
+          <token key="package.var.dir" value="${package.var.dir}" />
+        </replacetokens>
+      </filterchain>
+    </copy>
+    <rpm specFile="hadoop-mapred.spec" command="-bb --target ${os.arch}" topDir="${package.buildroot}" cleanBuildDir="true" failOnError="true"/>
+    <copy todir="${build.dir}/" flatten="true">
+      <fileset dir="${package.buildroot}/RPMS">
+        <include name="**/*.rpm" />
+      </fileset>
+    </copy>
+    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
+  </target>
+
+  <target name="deb" depends="ivy-retrieve-package, binary" description="Make deb package">
+    <taskdef name="deb"
+           classname="org.vafer.jdeb.ant.DebAntTask">
+      <classpath refid="ivy-package.classpath" />
+    </taskdef>
+
+    <mkdir dir="${package.build.dir}/hadoop.control" />
+    <mkdir dir="${package.buildroot}/${package.prefix}" />
+    <copy todir="${package.buildroot}/${package.prefix}">
+      <fileset dir="${build.dir}/${final.name}">
+        <include name="**" />
+      </fileset>
+    </copy>
+    <copy todir="${package.build.dir}/hadoop.control">
+      <fileset dir="${src.dir}/packages/deb/hadoop.control">
+        <exclude name="control" />
+      </fileset>
+    </copy>
+    <copy file="${src.dir}/packages/deb/hadoop.control/control" todir="${package.build.dir}/hadoop.control">
+      <filterchain>
+        <replacetokens>
+          <token key="final.name" value="${final.name}" />
+          <token key="version" value="${_version}" />
+          <token key="package.release" value="${package.release}" />
+          <token key="package.build.dir" value="${package.build.dir}" />
+          <token key="package.prefix" value="${package.prefix}" />
+          <token key="package.conf.dir" value="${package.conf.dir}" />
+          <token key="package.log.dir" value="${package.log.dir}" />
+          <token key="package.pid.dir" value="${package.pid.dir}" />
+        </replacetokens>
+      </filterchain>
+    </copy>
+    <deb destfile="${package.buildroot}/${name}_${_version}-${package.release}_${os.arch}.deb" control="${package.build.dir}/hadoop.control">
+      <tarfileset dir="${build.dir}/${final.name}" filemode="644" prefix="${package.prefix}">
+        <exclude name="bin/*" />
+        <exclude name="${package.share.dir}/contrib/*/bin/*" />
+        <exclude name="etc" />
+        <exclude name="etc/**" />
+        <exclude name="libexec/*" />
+        <exclude name="sbin/*" />
+        <include name="**" />
+      </tarfileset>
+      <tarfileset dir="${build.dir}/${final.name}" filemode="755" prefix="${package.prefix}">
+        <include name="bin/*" />
+        <include name="sbin/*" />
+        <exclude name="sbin/*.redhat" />
+        <exclude name="sbin/*.debian" />
+        <include name="libexec/*" />
+        <include name="${package.share.dir}/contrib/*/bin/*" />
+      </tarfileset>
+      <tarfileset dir="${src.dir}/packages" filemode="755" prefix="${package.prefix}/sbin">
+        <include name="*.sh" />
+      </tarfileset>
+      <tarfileset dir="${build.dir}/${final.name}/etc/hadoop" filemode="644" prefix="${package.conf.dir}">
+        <exclude name="configuration.xsl" />
+        <exclude name="hadoop-metrics2.properties" />
+        <exclude name="core-site.xml" />
+        <exclude name="hdfs-site.xml" />
+        <exclude name="mapred-site.xml" />
+        <include name="**" />
+      </tarfileset>
+      <tarfileset dir="${basedir}/src/packages/deb/init.d" filemode="755" prefix="/etc/init.d">
+        <include name="**" />
+      </tarfileset>
+    </deb>
+    <copy todir="${build.dir}/" flatten="true">
+      <fileset dir="${package.buildroot}">
+        <include name="**/${name}*.deb" />
+      </fileset>
+    </copy>
+    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
+  </target>
+
+  <target name="ant-task-download" description="To download mvn-ant-task">
+    <get src="${ant_task_repo_url}" dest="${ant_task.jar}" usetimestamp="true"/>
+  </target>
+
+  <target name="mvn-taskdef" depends="ant-task-download">
+     <path id="mvn-ant-task.classpath" path="${ant_task.jar}"/> 
+     <typedef resource="org/apache/maven/artifact/ant/antlib.xml" 
+         uri="urn:maven-artifact-ant"
+         classpathref="mvn-ant-task.classpath"/>
+  </target>   
+
+  <target name="clean-cache" description="Clean. Delete ivy cache">
+    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-common"/>
+    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-common-test"/>
+    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-hdfs"/>
+    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-hdfs-test"/>
+  </target>
+
+  <target name="mvn-install" depends="mvn-taskdef,jar,jar-test,set-version,-mvn-system-install"
+      description="To install hadoop mapreduce and test jars to local filesystem's m2 cache">
+    <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
+    <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
+    <artifact:install file="${hadoop-mapred.jar}">
+      <pom refid="hadoop.mapred"/>
+      <attach file="${hadoop-mapred-sources.jar}" classifier="sources" />
+    </artifact:install>
+    <artifact:install file="${hadoop-mapred-test.jar}">
+      <pom refid="hadoop.mapred.test"/>
+      <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources" />
+    </artifact:install>
+  </target>
+
+  <target name="mvn-deploy" depends="mvn-taskdef, jar, jar-test,
+    jar-system, jar-test-system, set-version, signanddeploy, simpledeploy"
+    description="To deploy hadoop mapredice and test jar's to apache
+    snapshot's repository"/>
+
+   <target name="signanddeploy" if="staging" depends="sign">
+     <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
+     <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
+     <artifact:pom file="${hadoop-mapred-instrumented.pom}" 
+       id="hadoop.mapred.${herriot.suffix}"/>
+     <artifact:pom file="${hadoop-mapred-instrumented-test.pom}" 
+       id="hadoop.mapred.${herriot.suffix}.test"/>
+     <artifact:install-provider artifactId="wagon-http"
+     version="${wagon-http.version}"/>
+
+     <artifact:deploy file="${hadoop-mapred.jar}">
+       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
+       <pom refid="hadoop.mapred"/>
+       <attach file="${hadoop-mapred.jar}.asc" type="jar.asc"/>
+       <attach file="${hadoop-mapred.pom}.asc" type="pom.asc"/>
+       <attach file="${hadoop-mapred-sources.jar}.asc" type="jar.asc"
+         classifier="sources" />
+       <attach file="${hadoop-mapred-sources.jar}" classifier="sources"/>
+     </artifact:deploy>
+
+     <artifact:deploy file="${hadoop-mapred-test.jar}">
+       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
+       <pom refid="hadoop.mapred.test"/>
+       <attach file="${hadoop-mapred-test.jar}.asc" type="jar.asc"/>
+       <attach file="${hadoop-mapred-test.pom}.asc" type="pom.asc"/>
+       <attach file="${hadoop-mapred-test-sources.jar}.asc" type="jar.asc"
+         classifier="sources"/>
+       <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources"/>
+     </artifact:deploy>
+
+     <artifact:deploy file="${hadoop-mapred-instrumented.jar}">
+       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
+       <pom refid="hadoop.mapred.${herriot.suffix}"/>
+       <attach file="${hadoop-mapred-instrumented.jar}.asc" type="jar.asc"/>
+       <attach file="${hadoop-mapred-instrumented.pom}.asc" type="pom.asc"/>
+       <attach file="${hadoop-mapred-instrumented-sources.jar}.asc" 
+         type="jar.asc" classifier="sources"/>
+       <attach file="${hadoop-mapred-instrumented-sources.jar}"
+         classifier="sources"/>
+     </artifact:deploy>
+
+      <artifact:deploy file="${hadoop-mapred-instrumented-test.jar}">
+       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
+       <pom refid="hadoop.mapred.${herriot.suffix}.test"/>
+       <attach file="${hadoop-mapred-instrumented-test.jar}.asc" type="jar.asc"/>
+       <attach file="${hadoop-mapred-instrumented-test.pom}.asc" type="pom.asc"/>
+       <attach file="${hadoop-mapred-instrumented-test-sources.jar}.asc" 
+         type="jar.asc" classifier="sources"/>
+       <attach file="${hadoop-mapred-instrumented-test-sources.jar}"
+         classifier="sources"/>
+     </artifact:deploy>
+   </target>
+
+   <target name="sign" depends="clean-sign" if="staging">
+    <input message="password:>" addproperty="gpg.passphrase">
+     <handler classname="org.apache.tools.ant.input.SecureInputHandler" />
+    </input>
+    <macrodef name="sign-artifact" description="Signs the artifact">
+      <attribute name="input.file"/>
+      <attribute name="output.file" default="@{input.file}.asc"/>
+      <attribute name="gpg.passphrase"/>
+      <sequential>
+        <echo>Signing @{input.file} Sig File: @{output.file}</echo>
+        <exec executable="gpg" >
+          <arg value="--armor"/>
+          <arg value="--output"/>
+          <arg value="@{output.file}"/>
+          <arg value="--passphrase"/>
+          <arg value="@{gpg.passphrase}"/>
+          <arg value="--detach-sig"/>
+          <arg value="@{input.file}"/>
+        </exec>
+      </sequential>
+    </macrodef>
+    <sign-artifact input.file="${hadoop-mapred.jar}" 
+     output.file="${hadoop-mapred.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-test.jar}" 
+     output.file="${hadoop-mapred-test.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-sources.jar}" 
+     output.file="${hadoop-mapred-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-test-sources.jar}" 
+     output.file="${hadoop-mapred-test-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred.pom}" 
+     output.file="${hadoop-mapred.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-test.pom}" 
+     output.file="${hadoop-mapred-test.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented.jar}" 
+     output.file="${hadoop-mapred-instrumented.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented.pom}" 
+     output.file="${hadoop-mapred-instrumented.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented-sources.jar}" 
+     output.file="${hadoop-mapred-instrumented-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented-test.jar}" 
+     output.file="${hadoop-mapred-instrumented-test.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented-test.pom}" 
+     output.file="${hadoop-mapred-instrumented-test.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
+    <sign-artifact input.file="${hadoop-mapred-instrumented-test-sources.jar}" 
+     output.file="${hadoop-mapred-instrumented-test-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
+  </target>
+
+  <target name="simpledeploy" unless="staging">
+     <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
+     <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
+     <artifact:pom file="${hadoop-mapred-instrumented.pom}" 
+       id="hadoop.mapred.${herriot.suffix}"/>
+
+     <artifact:install-provider artifactId="wagon-http" version="${wagon-http.version}"/>
+     <artifact:deploy file="${hadoop-mapred.jar}">
+         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
+         <pom refid="hadoop.mapred"/>
+         <attach file="${hadoop-mapred-sources.jar}" classifier="sources" />
+     </artifact:deploy>
+
+     <artifact:deploy file="${hadoop-mapred-test.jar}">
+         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
+         <pom refid="hadoop.mapred.test"/>
+         <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources" />
+     </artifact:deploy> 
+
+     <artifact:deploy file="${hadoop-mapred-instrumented.jar}">
+         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
+         <pom refid="hadoop.mapred.${herriot.suffix}"/>
+         <attach file="${hadoop-mapred-instrumented-sources.jar}" classifier="sources" />
+     </artifact:deploy>
+  </target>
+
+  <target name="set-version">
+    <delete file="${basedir}/ivy/hadoop-mapred.xml"/>
+    <delete file="${basedir}/ivy/hadoop-mapred-test.xml"/>
+    <delete file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}.xml"/>
+    <delete file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test.xml"/>
+    <copy file="${basedir}/ivy/hadoop-mapred-template.xml" tofile="${basedir}/ivy/hadoop-mapred.xml"/>
+    <copy file="${basedir}/ivy/hadoop-mapred-test-template.xml" tofile="${basedir}/ivy/hadoop-mapred-test.xml"/>
+    <copy file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-template.xml"
+      tofile="${basedir}/ivy/hadoop-mapred-${herriot.suffix}.xml"/>
+    <copy file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test-template.xml"
+      tofile="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test.xml"/>
+    <replaceregexp byline="true">
+      <regexp pattern="@version"/>
+      <substitution expression="${version}"/>
+      <fileset dir="${basedir}/ivy">
+        <include name="hadoop-mapred.xml"/>
+        <include name="hadoop-mapred-test.xml"/>
+        <include name="hadoop-mapred-${herriot.suffix}.xml"/>
+        <include name="hadoop-mapred-${herriot.suffix}-test.xml"/>
+      </fileset>
+    </replaceregexp>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Perform audit activities for the release                           -->
+  <!-- ================================================================== -->
+  <target name="rats-taskdef" depends="ivy-retrieve-releaseaudit">
+     <typedef format="xml" resource="org/apache/rat/anttasks/antlib.xml" uri="antlib:org.apache.rat.anttasks"
+      classpathref="releaseaudit-classpath"/>
+  </target>
+
+  <!--<target name="releaseaudit" depends="package, rats-taskdef" description="Release Audit activities"> -->
+  <target name="releaseaudit" depends="package, rats-taskdef" description="Release Audit activities">
+   <rat:report xmlns:rat="antlib:org.apache.rat.anttasks">
+      <fileset dir="${dist.dir}">
+        <exclude name="CHANGES.txt"/>
+        <exclude name="**/conf/*"/>
+        <exclude name="**/docs/"/>
+        <exclude name="**/VERSION"/>
+        <exclude name="webapps/**/WEB-INF/web.xml"/>
+        <exclude name="lib/jdiff/"/>
+        <exclude name="src/test/all-tests"/>
+        <exclude name="src/test/commit-tests"/>
+        <exclude name="src/test/empty-file"/>
+        <exclude name="src/test/mapred/org/apache/hadoop/mapred/test.tgz"/>
+        <exclude name="src/test/tools/data/rumen/**/*"/>
+        <exclude name="src/test/mapred/org/apache/hadoop/mapred/*.txt"/>
+        <exclude name="src/contrib/mumak/src/test/data/*.json"/>
+        <exclude name="src/contrib/index/sample/*.txt"/>
+        <exclude name="src/test/mapred/org/apache/hadoop/cli/data60bytes"/>
+        <exclude name="src/examples/org/apache/hadoop/examples/dancing/puzzle1.dta"/>
+        <exclude name="src/contrib/eclipse-plugin/META-INF/MANIFEST.MF"/>
+        <exclude name="src/c++/librecordio/*"/>
+        <exclude name="src/c++/pipes/*"/>
+        <exclude name="src/c++/utils/*"/>
+        <exclude name="src/c++/task-controller/*"/>
+        <exclude name="src/examples/pipes/*"/>
+        <exclude name="src/c++/pipes/debug/*"/>
+      </fileset>
+    </rat:report>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Clean.  Delete the build files, and their directories              -->
+  <!-- ================================================================== -->
+  <target name="clean" depends="clean-contrib, clean-fi, clean-sign" description="Clean.  Delete the build files, and their directories">
+    <delete dir="${build.dir}"/>
+    <delete dir="${docs.src}/build"/>
+    <delete file="${hadoop-mapred.pom}"/>
+    <delete file="${hadoop-mapred-test.pom}"/>
+    <delete file="${hadoop-mapred-instrumented.pom}"/>
+    <delete file="${hadoop-mapred-instrumented-test.pom}"/>
+    <delete file="${hadoop-mapred-examples.pom}"/>
+    <delete file="${hadoop-mapred-tools.pom}"/>
+  </target>
+
+  <target name="clean-sign" description="Clean.  Delete .asc files">
+    <delete>
+      <fileset dir="." includes="**/**/*.asc"/>
+    </delete>
+  </target>
+
+  <target name="veryclean" depends="clean-cache,clean" 
+          description="veryclean.  Delete ivy and ant maven task jar">
+    <delete file="${ant_task.jar}"/>
+    <delete file="${ivy.jar}"/>
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Clean contrib target. For now, must be called explicitly           -->
+  <!-- Using subant instead of ant as a workaround for 30569              -->
+  <!-- ================================================================== -->
+  <target name="clean-contrib">
+     <subant target="clean">        
+        <fileset file="src/contrib/build.xml"/>
+     </subant>  	
+  </target>
+	
+
+<!-- ================================================================== -->
+<!-- librecordio targets.                                               -->
+<!-- ================================================================== -->		
+
+  <target name="compile-librecordio" depends="init" if="librecordio" >
+     <mkdir dir="${build.librecordio}"/>
+     <exec dir="${librecordio.src}" executable="${make.cmd}" failonerror="true">
+        <env key="XERCESCROOT" value="${xercescroot}"/>
+        <env key="LIBRECORDIO_BUILD_DIR" value="${build.librecordio}"/>
+     </exec>
+  </target>
+  	
+  <target name="test-librecordio" depends="compile-librecordio, compile-core" if="librecordio">
+    <delete dir="${librecordio.test.dir}"/>
+    <mkdir dir="${librecordio.test.dir}"/>
+    <exec dir="${librecordio.src}/test" executable="${make.cmd}" failonerror="true">
+        <env key="HADOOP_HOME" value="${basedir}"/>
+	<env key="XERCESCROOT" value="${xercescroot}"/>
+        <env key="LIBRECORDIO_BUILD_DIR" value="${build.librecordio}"/> 	
+        <env key="LIBRECORDIO_TEST_DIR" value="${librecordio.test.dir}"/>
+      		<arg value="all"/>		
+    </exec>
+  </target>
+
+  <target name="package-librecordio" depends="compile-librecordio" if="librecordio">
+    <mkdir dir="${dist.dir}/lib"/> 
+    <copy todir="${dist.dir}/lib">
+       <fileset dir="${build.librecordio}" casesensitive="yes" followsymlinks="false">
+          <exclude name="**/tests/**"/>
+          <exclude name="*.so"/> 
+          <exclude name="*.o"/>
+       </fileset>
+    </copy>
+    <chmod perm="ugo+x" type="file">
+       <fileset dir="${dist.dir}/lib"/>
+    </chmod>
+  </target>
+ 
+  <target name="check-c++-configure" depends="init" if="compile.c++">
+    <condition property="need.c++.utils.configure">
+       <not> <available file="${c++.utils.src}/configure"/> </not>
+    </condition>
+    <condition property="need.c++.pipes.configure">
+       <not> <available file="${c++.pipes.src}/configure"/> </not>
+    </condition>
+    <condition property="need.c++.examples.pipes.configure">
+       <not> <available file="${c++.examples.pipes.src}/configure"/> </not>
+    </condition>
+    <condition property="need.c++.task-controller.configure">
+       <not> <available file="${c++.task-controller.src}/configure"/> </not>
+    </condition>
+  </target>
+
+  <target name="create-c++-utils-configure" depends="check-c++-configure" 
+                                            if="need.c++.utils.configure">
+    <exec executable="autoreconf" dir="${c++.utils.src}" searchpath="yes" 
+          failonerror="yes">
+       <arg value="-i"/>
+       <arg value="-f"/>
+    </exec>
+  </target>
+
+  <target name="create-c++-pipes-configure" depends="check-c++-configure" 
+                                            if="need.c++.pipes.configure">
+    <exec executable="autoreconf" dir="${c++.pipes.src}" searchpath="yes" 
+          failonerror="yes">
+       <arg value="-i"/>
+       <arg value="-f"/>
+    </exec>
+  </target>
+
+  <target name="create-c++-examples-pipes-configure" depends="check-c++-configure" 
+                                            if="need.c++.examples.pipes.configure">
+    <exec executable="autoreconf" dir="${c++.examples.pipes.src}" 
+          searchpath="yes" failonerror="yes">
+       <arg value="-i"/>
+       <arg value="-f"/>
+    </exec>
+  </target>
+
+  <target name="create-c++-task-controller-configure" depends="check-c++-configure" 
+                                            if="need.c++.task-controller.configure">
+    <exec executable="autoreconf" dir="${c++.task-controller.src}" 
+          searchpath="yes" failonerror="yes">
+       <arg value="-i"/>
+       <arg value="-f"/>
+    </exec>
+  </target>
+
+  <target name="create-c++-configure" depends="create-c++-utils-configure,
+                                               create-c++-pipes-configure,
+                                               create-c++-examples-pipes-configure,
+                                               create-c++-task-controller-configure" 
+                                      if="compile.c++">
+  </target>
+   
+  <target name="check-c++-makefiles" depends="init" if="compile.c++">
+    <condition property="need.c++.utils.makefile">
+       <not> <available file="${build.c++.utils}/Makefile"/> </not>
+    </condition>
+    <condition property="need.c++.pipes.makefile">
+       <not> <available file="${build.c++.pipes}/Makefile"/> </not>
+    </condition>
+    <condition property="need.c++.examples.pipes.makefile">
+       <not> <available file="${build.c++.examples.pipes}/Makefile"/> </not>
+    </condition>
+  </target>
+
+  <target name="create-c++-utils-makefile" depends="check-c++-makefiles" 
+                                           if="need.c++.utils.makefile">
+    <antcall target="create-c++-utils-configure"/>
+    <mkdir dir="${build.c++.utils}"/>
+    <chmod file="${c++.utils.src}/configure" perm="ugo+x"/>
+    <exec executable="${c++.utils.src}/configure" dir="${build.c++.utils}"
+          failonerror="yes">
+      <arg value="--prefix=${install.c++}"/>
+    </exec>
+  </target>
+
+  <target name="compile-c++-utils" depends="create-c++-utils-makefile"
+                                   if="compile.c++">
+    <exec executable="${make.cmd}" dir="${build.c++.utils}" searchpath="yes" 
+          failonerror="yes">
+      <arg  value="install"/>
+    </exec>
+  </target>
+
+  <target name="create-c++-pipes-makefile" depends="check-c++-makefiles" 
+                                           if="need.c++.pipes.makefile">
+    <antcall target="create-c++-pipes-configure"/>
+    <mkdir dir="${build.c++.pipes}"/>
+    <chmod file="${c++.pipes.src}/configure" perm="ugo+x"/>
+    <exec executable="${c++.pipes.src}/configure" dir="${build.c++.pipes}"
+          failonerror="yes">
+      <arg value="--prefix=${install.c++}"/>
+    </exec>
+  </target>
+
+  <target name="compile-c++-pipes" 
+          depends="create-c++-pipes-makefile,compile-c++-utils"
+          if="compile.c++">
+    <exec executable="${make.cmd}" dir="${build.c++.pipes}" searchpath="yes" 
+          failonerror="yes">
+      <arg value="install"/>
+    </exec>
+  </target>
+
+  <target name="compile-c++" 
+          depends="compile-c++-pipes"/>
+
+  <target name="create-c++-examples-pipes-makefile" 
+          depends="check-c++-makefiles" 
+          if="need.c++.examples.pipes.makefile">
+    <antcall target="create-c++-examples-pipes-configure"/>
+    <mkdir dir="${build.c++.examples.pipes}"/>
+    <chmod file="${c++.examples.pipes.src}/configure" perm="ugo+x"/>
+    <exec executable="${c++.examples.pipes.src}/configure" 
+          dir="${build.c++.examples.pipes}"
+          failonerror="yes">
+      <arg value="--prefix=${install.c++.examples}"/>
+      <arg value="--with-hadoop-utils=${install.c++}"/>
+      <arg value="--with-hadoop-pipes=${install.c++}"/>
+    </exec>
+  </target>
+
+  <target name="compile-c++-examples-pipes" 
+          depends="create-c++-examples-pipes-makefile,compile-c++-pipes"
+          if="compile.c++">
+    <exec executable="${make.cmd}" dir="${build.c++.examples.pipes}" searchpath="yes" 
+          failonerror="yes">
+      <arg  value="install"/>
+    </exec>
+  </target>
+
+  <target name="compile-c++-examples" 
+          depends="compile-c++-examples-pipes"/>
+
+ <target name="clover" depends="clover.setup, clover.info" description="Instrument the Unit tests using Clover.  To use, specify -Dclover.home=&lt;base of clover installation&gt; -Drun.clover=true on the command line."/>
+
+<target name="clover.setup" if="clover.enabled">
+   <taskdef resource="cloverlib.xml" classpath="${clover.jar}"/>
+   <mkdir dir="${clover.db.dir}"/>
+   <clover-setup initString="${clover.db.dir}/hadoop_coverage.db">
+     <fileset dir="${src.dir}" includes="tools/**/* java/**/*"/>
+     <testsources dir="${test.src.dir}" />
+   </clover-setup>
+</target>
+
+<target name="clover.info" unless="clover.present">
+  <echo>
+     Clover not found. Code coverage reports disabled.
+  </echo>
+</target>
+
+<target name="clover.check">
+  <fail unless="clover.present">
+  ##################################################################
+   Clover not found.
+   Please specify -Dclover.home=&lt;base of clover installation&gt;
+   on the command line.
+  ##################################################################
+  </fail>
+</target>
+
+<target name="generate-clover-reports" depends="clover.check, clover">
+  <mkdir dir="${clover.report.dir}"/>
+  <clover-report>
+     <current outfile="${clover.report.dir}" title="${final.name}">
+     <format type="html"/>
+     </current>
+  </clover-report>
+  <clover-report>
+     <current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
+     <format type="xml"/>
+     </current>
+  </clover-report>
+</target>
+
+<target name="findbugs.check" depends="check-for-findbugs" unless="findbugs.present">
+  <fail message="'findbugs.home' is not defined. Please pass -Dfindbugs.home=&lt;base of Findbugs installation&gt; to Ant on the command-line." />
+</target>
+
+<target name="patch.check" unless="patch.file">
+  <fail message="'patch.file' is not defined. Please pass -Dpatch.file=&lt;location of patch file&gt; to Ant on the command-line." />
+</target>
+
+<target name="test-patch" depends="patch.check,findbugs.check,forrest.check">
+  <exec executable="bash" failonerror="true">
+    <arg value="${basedir}/src/test/bin/test-patch.sh"/>
+    <arg value="DEVELOPER"/>
+    <arg value="${patch.file}"/>
+    <arg value="${scratch.dir}"/>
+    <arg value="${svn.cmd}"/>
+    <arg value="${grep.cmd}"/>
+    <arg value="${patch.cmd}"/>
+    <arg value="${findbugs.home}"/>
+    <arg value="${forrest.home}"/>
+    <arg value="${basedir}"/>
+  </exec>
+</target>
+
+<target name="hudson-test-patch" depends="findbugs.check,forrest.check">
+  <exec executable="bash" failonerror="true">
+    <arg value="${basedir}/src/test/bin/test-patch.sh"/>
+    <arg value="HUDSON"/>
+    <arg value="${scratch.dir}"/>
+    <arg value="${support.dir}"/>
+    <arg value="${ps.cmd}"/>
+    <arg value="${wget.cmd}"/>
+    <arg value="${jiracli.cmd}"/>
+    <arg value="${svn.cmd}"/>
+    <arg value="${grep.cmd}"/>
+    <arg value="${patch.cmd}"/>
+    <arg value="${findbugs.home}"/>
+    <arg value="${forrest.home}"/>
+    <arg value="${eclipse.home}"/>
+    <arg value="${basedir}"/>
+    <arg value="${jira.passwd}"/>
+    <arg value="${curl.cmd}"/>
+    <arg value="${defect}"/>
+  </exec>
+</target>
+	
+  <condition property="ant-eclipse.jar.exists">
+    <available file="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar"/>
+  </condition>
+
+  <target name="ant-eclipse-download" unless="ant-eclipse.jar.exists"
+          description="Downloads the ant-eclipse binary.">
+    <get src="http://downloads.sourceforge.net/project/ant-eclipse/ant-eclipse/1.0/ant-eclipse-1.0.bin.tar.bz2"
+         dest="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" usetimestamp="false" />
+
+    <untar src="${build.dir}/ant-eclipse-1.0.bin.tar.bz2"
+           dest="${build.dir}" compression="bzip2">
+      <patternset>
+        <include name="lib/ant-eclipse-1.0-jvm1.2.jar"/>
+      </patternset>
+    </untar>
+    <delete file="${build.dir}/java/ant-eclipse-1.0.bin.tar.bz2" />
+  </target>
+
+  <target name="eclipse" 
+          depends="init,ant-eclipse-download,ivy-retrieve-common,ivy-retrieve-mapred,ivy-retrieve-test"
+          description="Create eclipse project files">
+       <pathconvert property="eclipse.project">
+         <path path="${basedir}"/>
+         <regexpmapper from="^.*/([^/]+)$$" to="\1" handledirsep="yes"/>
+       </pathconvert>
+    <taskdef name="eclipse"
+             classname="prantl.ant.eclipse.EclipseTask"
+             classpath="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />
+    <eclipse updatealways="true">
+      <project name="${eclipse.project}" />
+      <classpath>
+        <source path="${java.src.dir}"
+                output="${build.dir.eclipse-main-classes}" />
+        <source path="${build.src}"
+                output="${build.dir.eclipse-main-generated-classes}" />
+        <source path="${test.src.dir}/mapred"
+                output="${build.dir.eclipse-test-classes}" />
+        <source path="${test.src.dir}/aop"
+                output="${build.dir.eclipse-test-classes}" />
+        <source path="${test.src.dir}/unit"
+                output="${build.dir.eclipse-test-classes}" />
+        <source path="${examples.dir}"
+                output="${build.dir.eclipse-example-classes}" />
+        <source path="${tools.src}"
+                output="${build.dir.eclipse-tools-classes}" />
+        <source path="${contrib.dir}/block_forensics/src/java"
+                output="${build.dir.eclipse-contrib-classes}/block_forensics/main" />
+        <source path="${contrib.dir}/capacity-scheduler/src/java"
+                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/main" />
+        <source path="${contrib.dir}/capacity-scheduler/src/test"
+                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/test" />
+        <source path="${contrib.dir}/data_join/src/java"
+                output="${build.dir.eclipse-contrib-classes}/data_join/main" />
+        <source path="${contrib.dir}/data_join/src/examples"
+                output="${build.dir.eclipse-contrib-classes}/data_join/examples" />
+        <source path="${contrib.dir}/data_join/src/test"
+                output="${build.dir.eclipse-contrib-classes}/data_join/test" />
+        <source path="${contrib.dir}/dynamic-scheduler/src/java"
+                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/main" />
+        <source path="${contrib.dir}/dynamic-scheduler/src/test"
+                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/test" />
+        <source path="${contrib.dir}/fairscheduler/src/java"
+                output="${build.dir.eclipse-contrib-classes}/fairscheduler/main" />
+        <source path="${contrib.dir}/fairscheduler/src/test"
+                output="${build.dir.eclipse-contrib-classes}/fairscheduler/test" />
+        <source path="${contrib.dir}/gridmix/src/java"
+                output="${build.dir.eclipse-contrib-classes}/gridmix/main" />
+        <source path="${contrib.dir}/gridmix/src/test"
+                output="${build.dir.eclipse-contrib-classes}/gridmix/test" />
+        <source path="${contrib.dir}/mumak/src/java"
+                output="${build.dir.eclipse-contrib-classes}/mumak/main" />
+        <source path="${contrib.dir}/mumak/src/test"
+                output="${build.dir.eclipse-contrib-classes}/mumak/test" />
+        <source path="${contrib.dir}/raid/src/java"
+                output="${build.dir.eclipse-contrib-classes}/raid/main" />
+        <source path="${contrib.dir}/raid/src/test"
+                output="${build.dir.eclipse-contrib-classes}/raid/test" />
+        <source path="${contrib.dir}/streaming/src/java"
+                output="${build.dir.eclipse-contrib-classes}/streaming/main" />
+        <source path="${contrib.dir}/streaming/src/test"
+                output="${build.dir.eclipse-contrib-classes}/streaming/test" />
+        <source path="${contrib.dir}/vaidya/src/java"
+                output="${build.dir.eclipse-contrib-classes}/vaidya/main" />
+        <source path="${contrib.dir}/vertica/src/java"
+                output="${build.dir.eclipse-contrib-classes}/vertica/main" />
+        <source path="${contrib.dir}/vertica/src/test"
+                output="${build.dir.eclipse-contrib-classes}/vertica/test" />
+        <output path="${build.dir.eclipse-main-classes}" />
+        <library pathref="ivy-common.classpath" exported="true" />
+        <library pathref="ivy-mapred.classpath" exported="true" />
+        <library pathref="ivy-test.classpath" exported="false" />
+        <library path="${conf.dir}" exported="false" />
+      </classpath>
+    </eclipse>
+    <copy todir="." overwrite="true">
+      <fileset dir=".eclipse.templates">
+        <exclude name="**/README.txt"/>
+      </fileset>
+      <filterset>
+        <filter token="PROJECT" value="${eclipse.project}"/>
+      </filterset>
+    </copy>
+  </target>
+
+  <target name="ivy-init-dirs">
+    <mkdir dir="${build.ivy.dir}" />
+    <mkdir dir="${build.ivy.lib.dir}" />
+    <mkdir dir="${build.ivy.report.dir}" />
+    <mkdir dir="${build.ivy.maven.dir}" />
+  </target>
+
+  <target name="ivy-probe-antlib" >
+    <condition property="ivy.found">
+      <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
+    </condition>
+  </target>
+
+  <target name="ivy-download" description="To download ivy" unless="offline">
+    <get src="${ivy_repo_url}" dest="${ivy.jar}" usetimestamp="true"/>
+  </target>
+
+  <!--
+  To avoid Ivy leaking things across big projects, always load Ivy in the same classloader.
+  Also note how we skip loading Ivy if it is already there, just to make sure all is well.
+  -->
+  <target name="ivy-init-antlib" depends="ivy-download,ivy-init-dirs,ivy-probe-antlib" unless="ivy.found">
+    <typedef uri="antlib:org.apache.ivy.ant" onerror="fail"
+      loaderRef="ivyLoader">
+      <classpath>
+        <pathelement location="${ivy.jar}"/>
+      </classpath>
+    </typedef>
+    <fail >
+      <condition >
+        <not>
+          <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
+        </not>
+      </condition>
+      You need Apache Ivy 2.0 or later from http://ant.apache.org/
+      It could not be loaded from ${ivy_repo_url}
+    </fail>
+  </target>
+
+
+  <property name="ivyresolvelog" value="download-only"/>
+  <property name="ivyretrievelog" value="quiet"/>
+
+  <target name="ivy-init" depends="ivy-init-antlib" >
+
+    <!--Configure Ivy by reading in the settings file
+        If anyone has already read in a settings file into this settings ID, it gets priority
+    -->
+    <ivy:configure settingsid="${ant.project.name}.ivy.settings" file="${ivysettings.xml}" override='false'/>
+  </target>
+
+  <target name="ivy-resolve" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-javadoc" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="javadoc"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-releaseaudit" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="releaseaudit"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-test" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="test"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-common" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="common"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-package" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="package"
+        log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-mapred" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="mapred"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-jdiff" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="jdiff" 
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-checkstyle" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="checkstyle"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-resolve-system" depends="ivy-init">
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="system"
+      log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-retrieve" depends="ivy-resolve"
+    description="Retrieve Ivy-managed artifacts">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-retrieve-checkstyle" depends="ivy-resolve-checkstyle"
+    description="Retrieve Ivy-managed artifacts for the checkstyle configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="checkstyle-classpath" conf="checkstyle"/>
+  </target>
+
+  <target name="ivy-retrieve-jdiff" depends="ivy-resolve-jdiff"
+    description="Retrieve Ivy-managed artifacts for the javadoc configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="jdiff-classpath" conf="jdiff"/>
+  </target>
+
+  <target name="ivy-retrieve-javadoc" depends="ivy-resolve-javadoc"
+    description="Retrieve Ivy-managed artifacts for the javadoc configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="javadoc-classpath" conf="javadoc"/>
+  </target>
+
+  <target name="ivy-retrieve-test" depends="ivy-resolve-test"
+    description="Retrieve Ivy-managed artifacts for the test configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="ivy-test.classpath" conf="test"/>
+  </target>
+
+  <target name="ivy-retrieve-common" depends="ivy-resolve-common"
+    description="Retrieve Ivy-managed artifacts for the compile configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="ivy-common.classpath" conf="common"/>
+  </target>
+
+  <target name="ivy-retrieve-package" depends="ivy-resolve-package"
+    description="Retrieve Ivy-managed artifacts for the package configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+                log="${ivyretrievelog}"/>
+    <ivy:cachepath pathid="ivy-package.classpath" conf="package"/>
+  </target>
+
+  <target name="ivy-retrieve-mapred" depends="ivy-resolve-mapred"
+    description="Retrieve Ivy-managed artifacts for the mapred configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="ivy-mapred.classpath" conf="mapred"/>
+  </target>
+
+  <target name="ivy-retrieve-releaseaudit" depends="ivy-resolve-releaseaudit"
+    description="Retrieve Ivy-managed artifacts for the compile configurations">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="releaseaudit-classpath" conf="releaseaudit"/>
+  </target>
+
+  <target name="ivy-retrieve-system" depends="ivy-resolve-system"
+    description="Retrieve Ivy-managed artifacts for the system tests">
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+        log="${ivyresolvelog}"/>
+    <ivy:cachepath pathid="ivy-system.classpath" conf="system"/>
+  </target>
+
+  <target name="ivy-report" depends="ivy-resolve-releaseaudit"
+    description="Generate">
+    <ivy:report todir="${build.ivy.report.dir}" settingsRef="${ant.project.name}.ivy.settings"/>
+    <echo>
+      Reports generated:${build.ivy.report.dir}
+    </echo>
+  </target>
+
+  <!-- taskcontroller targets -->
+  <target name="init-task-controller-build">
+    <antcall target="create-c++-task-controller-configure"/>
+    <mkdir dir="${build.c++.task-controller}" />
+    <copy todir="${build.c++.task-controller}">
+      <fileset dir="${c++.task-controller.src}" includes="*.c"/>
+      <fileset dir="${c++.task-controller.src}" includes="*.h"/>
+    </copy>
+    <chmod file="${c++.task-controller.src}/configure" perm="ugo+x"/> 
+    <condition property="task-controller.conf.dir.passed">
+      <not>
+        <equals arg1="${hadoop.conf.dir}" arg2="$${hadoop.conf.dir}"/>
+      </not>
+    </condition>
+  </target>
+  <target name="configure-task-controller" depends="init,
+          init-task-controller-build,
+          task-controller-configuration-with-confdir,
+          task-controller-configuration-with-no-confdir">
+  </target>
+  <target name="task-controller-configuration-with-confdir" 
+          if="task-controller.conf.dir.passed" >
+    <exec executable="${c++.task-controller.src}/configure" 
+          dir="${build.c++.task-controller}" failonerror="yes">
+      <arg value="--prefix=${task-controller.install.dir}" />
+      <arg value="--with-confdir=${hadoop.conf.dir}" />
+    </exec>
+  </target>
+  <target name="task-controller-configuration-with-no-confdir" 
+          unless="task-controller.conf.dir.passed">
+    <exec executable="${c++.task-controller.src}/configure" 
+          dir="${build.c++.task-controller}" failonerror="yes">
+      <arg value="--prefix=${task-controller.install.dir}" />
+    </exec>
+  </target>
+  <!--
+    * Create the installation directory.
+    * Do a make install.
+   -->
+  <target name="task-controller" depends="configure-task-controller">
+    <mkdir dir="${task-controller.install.dir}" />
+    <exec executable="${make.cmd}" dir="${build.c++.task-controller}" 
+        searchpath="yes" failonerror="yes">
+      <arg value="install" />
+    </exec>
+  </target>
+  <target name="test-task-controller" depends="task-controller">
+    <copy todir="${build.c++.task-controller}" verbose="true">
+      <fileset dir="${c++.task-controller.src}" includes="tests/"/>
+    </copy>
+    <exec executable="${make.cmd}" dir="${build.c++.task-controller}" 
+        searchpath="yes" failonerror="yes">
+      <arg value="clean" />
+      <arg value="test" />
+    </exec>
+    <exec executable="${build.c++.task-controller}/tests/test-task-controller"
+        dir="${build.c++.task-controller}/tests/"
+        failonerror="yes">
+    </exec>
+  </target>
+  <!-- end of task-controller targets -->
+  
+  <!-- Begining of fault-injection targets-->
+  <import file="${test.src.dir}/aop/build/aop.xml"/>
+  
+  <!-- declaring mapred.src.dir as java.src.dir for aop.xml -->
+  <property name="java.src.dir" value="${src.dir}/java"/>
+  
+  <!-- target dependency from aop.xml -->
+  <target name="-classes-compilation" 
+    depends="compile-mapred-classes, compile-mapred-test"/>
+  
+  <target name="jar-test-fault-inject" depends="jar-mapred-test-fault-inject"
+            description="Make hadoop-mapred-test-fi.jar files"/>
+  
+  <!-- target to build test-fi.jar-->
+  <target name="jar-mapred-test-fault-inject" depends="injectfaults"
+    description="Make hadoop-mapred-test-fi.jar">
+    <macro-jar-test-fault-inject target.name="jar-test" 
+      jar.final.name="test.final.name"
+      jar.final.value="${name}-test-${version}-fi"/>
+  </target>
+  
+  <!-- target to build the hadoop-fi.jar -->
+  <target name="jar-fault-inject" depends="injectfaults"
+    description="Make hadoop-fi.jar">
+    <macro-jar-fault-inject
+      target.name="jar"
+      build.dir="${build-fi.dir}"
+      jar.final.name="final.name"
+      jar.final.value="${final.name}-fi" />
+  </target>
+  
+  <!-- target to run fault injected test cases will run entire mapred test 
+       suite-->
+  <target name="run-test-mapred-fault-inject" depends="injectfaults"
+    description="Run full suite of unit tests with fault injection">
+    <macro-run-tests-fault-inject target.name="run-test-mapred"
+      testcasesonly="false"/>
+  </target>
+
+  <!-- target to run non-FI tests in a FI environment-->
+  <target name="run-fault-inject-with-testcaseonly" depends="injectfaults">
+    <fail unless="testcase">
+    Can't run this target without -Dtestcase setting!
+    </fail>
+    <macro-run-tests-fault-inject target.name="run-test-mapred"
+      testcasesonly="true"/>
+  </target>
+  <condition property="tests.notestcase">
+    <and>
+      <isfalse value="${test.fault.inject}"/>
+      <not>
+        <isset property="testcase"/>
+      </not>
+    </and>
+  </condition>
+  <condition property="tests.notestcase.fi">
+    <and>
+      <not>
+        <isset property="testcase"/>
+      </not>
+      <istrue value="${test.fault.inject}"/>
+    </and>
+  </condition>
+  <condition property="test.testcase">
+    <and>
+      <isfalse value="${test.fault.inject}"/>
+      <isset property="testcase"/>
+    </and>
+  </condition>
+  <condition property="tests.testcaseonly.fi">
+    <istrue value="${special.fi.testcasesonly}" />
+  </condition>
+  <condition property="tests.testcase.fi">
+    <and>
+      <istrue value="${test.fault.inject}" />
+      <isset property="testcase" />
+      <isfalse value="${special.fi.testcasesonly}" />
+    </and>
+  </condition>
+  <!-- End of fault injection targets-->
+  
+</project>

hadoop-mapreduce/conf/capacity-scheduler.xml.template

@@ -0,0 +1,82 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- This is one of the configuration files for capacity-scheduler
+     (org.apache.hadoop.mapred.CapacityTaskScheduler), a TaskScheduler
+     for Map/Reduce system. The other configuration file is
+     conf/mapred-queues.xml which it shares with the framework for
+     configuring queues in the system. -->
+
+<!-- This file can be used to configure (1) job-initialization-poller
+     related properties and (2) the default values for various properties
+     for all the queues.-->
+
+<configuration>   
+  <!-- The default configuration settings for the capacity task scheduler --> 
+  <!-- The default values would be applied to all the queues which don't have -->    
+  <!-- the appropriate property for the particular queue configured in the -->       
+  <!-- queue-configuration file conf/mapred-queues.xml -->                           
+
+  <property>
+    <name>mapred.capacity-scheduler.default-supports-priority</name>
+    <value>false</value>
+    <description>If true, priorities of jobs will be taken into 
+      account in scheduling decisions by default in a job queue.
+    </description>
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
+    <value>100</value>
+    <description>The percentage of the resources limited to a particular user
+      for the job queue at any given point of time by default.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user</name>
+    <value>2</value>
+    <description>The maximum number of jobs to be pre-initialized for a user
+    of the job queue.
+    </description>
+  </property>
+
+
+  <!-- Capacity scheduler Job Initialization configuration parameters -->
+  <property>
+    <name>mapred.capacity-scheduler.init-poll-interval</name>
+    <value>5000</value>
+    <description>The amount of time in miliseconds which is used to poll 
+    the job queues for jobs to initialize.
+    </description>
+  </property>
+  <property>
+    <name>mapred.capacity-scheduler.init-worker-threads</name>
+    <value>5</value>
+    <description>Number of worker threads which would be used by
+    Initialization poller to initialize jobs in a set of queue.
+    If number mentioned in property is equal to number of job queues
+    then a single thread would initialize jobs in a queue. If lesser
+    then a thread would get a set of queues assigned. If the number
+    is greater then number of threads would be equal to number of 
+    job queues.
+    </description>
+  </property>
+
+</configuration>

hadoop-mapreduce/conf/configuration.xsl

@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+  <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+  <td><xsl:value-of select="value"/></td>
+  <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>

hadoop-mapreduce/conf/fair-scheduler.xml.template

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+
+<!--
+  This file contains pool and user allocations for the Fair Scheduler.
+  Its format is explained in the Fair Scheduler documentation at
+  http://hadoop.apache.org/core/docs/r0.21.0/fair_scheduler.html.
+  The documentation also includes a sample config file.
+-->
+
+<allocations>
+
+</allocations>

hadoop-mapreduce/conf/mapred-queues.xml.template

@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!-- This is the template for queue configuration. The format supports nesting of
+     queues within queues - a feature called hierarchical queues. All queues are
+     defined within the 'queues' tag which is the top level element for this
+     XML document. The queue acls configured here for different queues are
+     checked for authorization only if the configuration property
+     mapreduce.cluster.acls.enabled is set to true. -->
+<queues>
+
+  <!-- Configuration for a queue is specified by defining a 'queue' element. -->
+  <queue>
+
+    <!-- Name of a queue. Queue name cannot contain a ':'  -->
+    <name>default</name>
+
+    <!-- properties for a queue, typically used by schedulers,
+    can be defined here -->
+    <properties>
+    </properties>
+
+	<!-- State of the queue. If running, the queue will accept new jobs.
+         If stopped, the queue will not accept new jobs. -->
+    <state>running</state>
+
+    <!-- Specifies the ACLs to check for submitting jobs to this queue.
+         If set to '*', it allows all users to submit jobs to the queue.
+         If set to ' '(i.e. space), no user will be allowed to do this
+         operation. The default value for any queue acl is ' '.
+         For specifying a list of users and groups the format to use is
+         user1,user2 group1,group2
+
+         It is only used if authorization is enabled in Map/Reduce by setting
+         the configuration property mapreduce.cluster.acls.enabled to true.
+
+         Irrespective of this ACL configuration, the user who started the
+         cluster and cluster administrators configured via
+         mapreduce.cluster.administrators can do this operation. -->
+    <acl-submit-job> </acl-submit-job>
+
+    <!-- Specifies the ACLs to check for viewing and modifying jobs in this
+         queue. Modifications include killing jobs, tasks of jobs or changing
+         priorities.
+         If set to '*', it allows all users to view, modify jobs of the queue.
+         If set to ' '(i.e. space), no user will be allowed to do this
+         operation.
+         For specifying a list of users and groups the format to use is
+         user1,user2 group1,group2
+
+         It is only used if authorization is enabled in Map/Reduce by setting
+         the configuration property mapreduce.cluster.acls.enabled to true.
+
+         Irrespective of this ACL configuration, the user who started the
+         cluster  and cluster administrators configured via
+         mapreduce.cluster.administrators can do the above operations on all
+         the jobs in all the queues. The job owner can do all the above
+         operations on his/her job irrespective of this ACL configuration. -->
+    <acl-administer-jobs> </acl-administer-jobs>
+  </queue>
+
+  <!-- Here is a sample of a hierarchical queue configuration
+       where q2 is a child of q1. In this example, q2 is a leaf level
+       queue as it has no queues configured within it. Currently, ACLs
+       and state are only supported for the leaf level queues.
+       Note also the usage of properties for the queue q2.
+  <queue>
+    <name>q1</name>
+    <queue>
+      <name>q2</name>
+      <properties>
+        <property key="capacity" value="20"/>
+        <property key="user-limit" value="30"/>
+      </properties>
+    </queue>
+  </queue>
+ -->
+</queues>

hadoop-mapreduce/conf/mapred-site.xml.template

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>

hadoop-mapreduce/conf/taskcontroller.cfg

@@ -0,0 +1,3 @@
+mapreduce.cluster.local.dir=#configured value of mapreduce.cluster.local.dir. It can be a list of comma separated paths.
+hadoop.log.dir=#configured value of hadoop.log.dir.
+mapreduce.tasktracker.group=#configured value of mapreduce.tasktracker.group

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml

@@ -0,0 +1,103 @@
+<?xml version="1.0"?>
+<project>
+  <parent>
+    <artifactId>hadoop-mapreduce-client</artifactId>
+    <groupId>org.apache.hadoop</groupId>
+    <version>${hadoop-mapreduce.version}</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.hadoop</groupId>
+  <artifactId>hadoop-mapreduce-client-app</artifactId>
+  <name>hadoop-mapreduce-client-app</name>
+
+  <properties>
+    <install.file>${project.artifact.file}</install.file>
+    <applink.base>${project.build.directory}/${project.name}</applink.base>
+    <mr.basedir>${project.parent.parent.basedir}</mr.basedir>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-common</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <!-- local name for links -->
+    <finalName>mr-app</finalName>
+    <plugins>
+      <plugin>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+            <phase>test-compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>build-classpath</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>build-classpath</goal>
+            </goals>
+            <configuration>
+              <outputFile>target/classes/mrapp-generated-classpath</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>create-mr-app-symlinks</id>
+            <phase>package</phase>
+            <configuration>
+              <target>
+                <symlink link="${applink.base}.jar"
+                    resource="mr-app.jar" failonerror="false"/>
+                <symlink link="${applink.base}-1.0-SNAPSHOT.jar"
+                    resource="mr-app.jar" failonerror="false"/>
+              </target>
+            </configuration>
+            <goals>
+              <goal>run</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/CustomOutputCommitter.java

@@ -0,0 +1,67 @@
+package org.apache.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+
+public class CustomOutputCommitter extends OutputCommitter {
+
+  public static final String JOB_SETUP_FILE_NAME = "_job_setup";
+  public static final String JOB_COMMIT_FILE_NAME = "_job_commit";
+  public static final String JOB_ABORT_FILE_NAME = "_job_abort";
+  public static final String TASK_SETUP_FILE_NAME = "_task_setup";
+  public static final String TASK_ABORT_FILE_NAME = "_task_abort";
+  public static final String TASK_COMMIT_FILE_NAME = "_task_commit";
+
+  @Override
+  public void setupJob(JobContext jobContext) throws IOException {
+    writeFile(jobContext.getJobConf(), JOB_SETUP_FILE_NAME);
+  }
+
+  @Override
+  public void commitJob(JobContext jobContext) throws IOException {
+    super.commitJob(jobContext);
+    writeFile(jobContext.getJobConf(), JOB_COMMIT_FILE_NAME);
+  }
+
+  @Override
+  public void abortJob(JobContext jobContext, int status) 
+  throws IOException {
+    super.abortJob(jobContext, status);
+    writeFile(jobContext.getJobConf(), JOB_ABORT_FILE_NAME);
+  }
+  
+  @Override
+  public void setupTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
+  }
+
+  @Override
+  public boolean needsTaskCommit(TaskAttemptContext taskContext)
+      throws IOException {
+    return true;
+  }
+
+  @Override
+  public void commitTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
+  }
+
+  @Override
+  public void abortTask(TaskAttemptContext taskContext) throws IOException {
+    writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
+  }
+
+  private void writeFile(JobConf conf , String filename) throws IOException {
+    System.out.println("writing file ----" + filename);
+    Path outputPath = FileOutputFormat.getOutputPath(conf);
+    FileSystem fs = outputPath.getFileSystem(conf);
+    fs.create(new Path(outputPath, filename)).close();
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java

@@ -0,0 +1,455 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+/**
+ * Runs the container task locally in a thread.
+ * Since all (sub)tasks share the same local directory, they must be executed
+ * sequentially in order to avoid creating/deleting the same files/dirs.
+ */
+public class LocalContainerLauncher extends AbstractService implements
+    ContainerLauncher {
+
+  private static final File curDir = new File(".");
+  private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
+
+  private FileContext curFC = null;
+  private final HashSet<File> localizedFiles;
+  private final AppContext context;
+  private final TaskUmbilicalProtocol umbilical;
+  private Thread eventHandlingThread;
+  private BlockingQueue<ContainerLauncherEvent> eventQueue =
+      new LinkedBlockingQueue<ContainerLauncherEvent>();
+
+  public LocalContainerLauncher(AppContext context,
+                                TaskUmbilicalProtocol umbilical) {
+    super(LocalContainerLauncher.class.getName());
+    this.context = context;
+    this.umbilical = umbilical;
+        // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us  (TODO/FIXME:  pointless to use RPC to talk to self; should create LocalTaskAttemptListener or similar:  implement umbilical protocol but skip RPC stuff)
+
+    try {
+      curFC = FileContext.getFileContext(curDir.toURI());
+    } catch (UnsupportedFileSystemException ufse) {
+      LOG.error("Local filesystem " + curDir.toURI().toString()
+                + " is unsupported?? (should never happen)");
+    }
+
+    // Save list of files/dirs that are supposed to be present so can delete
+    // any extras created by one task before starting subsequent task.  Note
+    // that there's no protection against deleted or renamed localization;
+    // users who do that get what they deserve (and will have to disable
+    // uberization in order to run correctly).
+    File[] curLocalFiles = curDir.listFiles();
+    localizedFiles = new HashSet<File>(curLocalFiles.length);
+    for (int j = 0; j < curLocalFiles.length; ++j) {
+      localizedFiles.add(curLocalFiles[j]);
+    }
+
+    // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
+    // full localization info is in AppSubmissionContext passed from client to
+    // RM and then to NM for AM-container launch:  no difference between AM-
+    // localization and MapTask- or ReduceTask-localization, so can assume all
+    // OK.  Longer-term, will need to override uber-AM container-localization
+    // request ("needed resources") with union of regular-AM-resources + task-
+    // resources (and, if maps and reduces ever differ, then union of all three
+    // types), OR will need localizer service/API that uber-AM can request
+    // after running (e.g., "localizeForTask()" or "localizeForMapTask()").
+  }
+
+  public void start() {
+    eventHandlingThread = new Thread(new SubtaskRunner(), "uber-SubtaskRunner");
+    eventHandlingThread.start();
+    super.start();
+  }
+
+  public void stop() {
+    eventHandlingThread.interrupt();
+    super.stop();
+  }
+
+  @Override
+  public void handle(ContainerLauncherEvent event) {
+    try {
+      eventQueue.put(event);
+    } catch (InterruptedException e) {
+      throw new YarnException(e);  // FIXME? YarnException is "for runtime exceptions only"
+    }
+  }
+
+
+  /*
+   * Uber-AM lifecycle/ordering ("normal" case):
+   *
+   * - [somebody] sends TA_ASSIGNED
+   *   - handled by ContainerAssignedTransition (TaskAttemptImpl.java)
+   *     - creates "remoteTask" for us == real Task
+   *     - sends CONTAINER_REMOTE_LAUNCH
+   *     - TA: UNASSIGNED -> ASSIGNED
+   * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us)
+   *   - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask()
+   *   - sends TA_CONTAINER_LAUNCHED
+   *     [[ elsewhere...
+   *       - TA_CONTAINER_LAUNCHED handled by LaunchedContainerTransition
+   *         - registers "remoteTask" with TaskAttemptListener (== umbilical)
+   *         - NUKES "remoteTask"
+   *         - sends T_ATTEMPT_LAUNCHED (Task: SCHEDULED -> RUNNING)
+   *         - TA: ASSIGNED -> RUNNING
+   *     ]]
+   *   - runs Task (runSubMap() or runSubReduce())
+   *     - TA can safely send TA_UPDATE since in RUNNING state
+   *       [modulo possible TA-state-machine race noted below:  CHECK (TODO)]
+   */
+  private class SubtaskRunner implements Runnable {
+
+    private boolean doneWithMaps = false;
+    private int finishedSubMaps = 0;
+
+    SubtaskRunner() {
+    }
+
+    @Override
+    public void run() {
+      ContainerLauncherEvent event = null;
+
+      // _must_ either run subtasks sequentially or accept expense of new JVMs
+      // (i.e., fork()), else will get weird failures when maps try to create/
+      // write same dirname or filename:  no chdir() in Java
+      while (!Thread.currentThread().isInterrupted()) {
+        try {
+          event = eventQueue.take();
+        } catch (InterruptedException e) {  // mostly via T_KILL? JOB_KILL?
+          LOG.error("Returning, interrupted : " + e);
+          return;
+        }
+
+        LOG.info("Processing the event " + event.toString());
+
+        if (event.getType() == EventType.CONTAINER_REMOTE_LAUNCH) {
+
+          ContainerRemoteLaunchEvent launchEv =
+              (ContainerRemoteLaunchEvent)event;
+          TaskAttemptId attemptID = launchEv.getTaskAttemptID(); //FIXME:  can attemptID ever be null?  (only if retrieved over umbilical?)
+
+          Job job = context.getAllJobs().get(attemptID.getTaskId().getJobId());
+          int numMapTasks = job.getTotalMaps();
+          int numReduceTasks = job.getTotalReduces();
+
+          // YARN (tracking) Task:
+          org.apache.hadoop.mapreduce.v2.app.job.Task ytask =
+              job.getTask(attemptID.getTaskId());
+          // classic mapred Task:
+          org.apache.hadoop.mapred.Task remoteTask = launchEv.getRemoteTask();
+
+          // after "launching," send launched event to task attempt to move
+          // state from ASSIGNED to RUNNING (also nukes "remoteTask", so must
+          // do getRemoteTask() call first)
+          context.getEventHandler().handle(
+              new TaskAttemptEvent(attemptID,
+                  TaskAttemptEventType.TA_CONTAINER_LAUNCHED)); //FIXME:  race condition here?  or do we have same kind of lock on TA handler => MapTask can't send TA_UPDATE before TA_CONTAINER_LAUNCHED moves TA to RUNNING state?  (probably latter)
+
+          if (numMapTasks == 0) {
+            doneWithMaps = true;
+          }
+
+          try {
+            if (remoteTask.isMapOrReduce()) {
+              JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.getTaskId().getJobId());
+              jce.addCounterUpdate(JobCounter.TOTAL_LAUNCHED_UBERTASKS, 1);
+              if (remoteTask.isMapTask()) {
+                jce.addCounterUpdate(JobCounter.NUM_UBER_SUBMAPS, 1);
+              } else {
+                jce.addCounterUpdate(JobCounter.NUM_UBER_SUBREDUCES, 1);
+              }
+              context.getEventHandler().handle(jce);
+            }
+            runSubtask(remoteTask, ytask.getType(), attemptID, numMapTasks,
+                       (numReduceTasks > 0));
+            
+          } catch (RuntimeException re) {
+            JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.getTaskId().getJobId());
+            jce.addCounterUpdate(JobCounter.NUM_FAILED_UBERTASKS, 1);
+            context.getEventHandler().handle(jce);
+            // this is our signal that the subtask failed in some way, so
+            // simulate a failed JVM/container and send a container-completed
+            // event to task attempt (i.e., move state machine from RUNNING
+            // to FAIL_CONTAINER_CLEANUP [and ultimately to FAILED])
+            context.getEventHandler().handle(new TaskAttemptEvent(attemptID,
+                TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+          } catch (IOException ioe) {
+            // if umbilical itself barfs (in error-handler of runSubMap()),
+            // we're pretty much hosed, so do what YarnChild main() does
+            // (i.e., exit clumsily--but can never happen, so no worries!)
+            LOG.fatal("oopsie...  this can never happen: "
+                + StringUtils.stringifyException(ioe));
+            System.exit(-1);
+          }
+
+        } else if (event.getType() == EventType.CONTAINER_REMOTE_CLEANUP) {
+
+          // no container to kill, so just send "cleaned" event to task attempt
+          // to move us from SUCCESS_CONTAINER_CLEANUP to SUCCEEDED state
+          // (or {FAIL|KILL}_CONTAINER_CLEANUP to {FAIL|KILL}_TASK_CLEANUP)
+          context.getEventHandler().handle(
+              new TaskAttemptEvent(event.getTaskAttemptID(),
+                  TaskAttemptEventType.TA_CONTAINER_CLEANED));
+
+        } else {
+          LOG.warn("Ignoring unexpected event " + event.toString());
+        }
+
+      }
+    }
+
+    private void runSubtask(org.apache.hadoop.mapred.Task task,
+                            final TaskType taskType,
+                            TaskAttemptId attemptID,
+                            final int numMapTasks,
+                            boolean renameOutputs)
+    throws RuntimeException, IOException {
+      org.apache.hadoop.mapred.TaskAttemptID classicAttemptID =
+          TypeConverter.fromYarn(attemptID);
+
+      try {
+        JobConf conf = new JobConf(getConfig());
+
+        // mark this as an uberized subtask so it can set task counter
+        // (longer-term/FIXME:  could redefine as job counter and send
+        // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
+        // will need new Job state-machine transition and JobImpl jobCounters
+        // map to handle)
+        conf.setBoolean("mapreduce.task.uberized", true);
+
+        // META-FIXME: do we want the extra sanity-checking (doneWithMaps,
+        // etc.), or just assume/hope the state machine(s) and uber-AM work
+        // as expected?
+        if (taskType == TaskType.MAP) {
+          if (doneWithMaps) {
+            LOG.error("CONTAINER_REMOTE_LAUNCH contains a map task ("
+                      + attemptID + "), but should be finished with maps");
+            // throw new RuntimeException()  (FIXME: what's appropriate here?)
+          }
+
+          MapTask map = (MapTask)task;
+
+          //CODE-REVIEWER QUESTION: why not task.getConf() or map.getConf() instead of conf? do we need Task's localizeConfiguration() run on this first?
+          map.run(conf, umbilical);
+
+          if (renameOutputs) {
+            renameMapOutputForReduce(conf, attemptID, map.getMapOutputFile());
+          }
+          relocalize();
+
+          if (++finishedSubMaps == numMapTasks) {
+            doneWithMaps = true;
+          }
+
+        } else /* TaskType.REDUCE */ {
+
+          if (!doneWithMaps) {
+            //check if event-queue empty?  whole idea of counting maps vs. checking event queue is a tad wacky...but could enforce ordering (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):  doesn't send reduce event until maps all done]
+            LOG.error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
+                      + attemptID + "), but not yet finished with maps");
+            // throw new RuntimeException()  (FIXME) // or push reduce event back onto end of queue? (probably former)
+          }
+
+          ReduceTask reduce = (ReduceTask)task;
+
+          // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
+          conf.set(MRConfig.MASTER_ADDRESS, "local");  // bypass shuffle
+
+          reduce.run(conf, umbilical);
+          //relocalize();  // needed only if more than one reducer supported (is MAPREDUCE-434 fixed yet?)
+        }
+
+      } catch (FSError e) {
+        LOG.fatal("FSError from child", e);
+        // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
+        umbilical.fsError(classicAttemptID, e.getMessage());
+        throw new RuntimeException();
+
+      } catch (Exception exception) {
+        LOG.warn("Exception running local (uberized) 'child' : "
+            + StringUtils.stringifyException(exception));
+        try {
+          if (task != null) {
+            // do cleanup for the task
+//          if (childUGI == null) { // no need to job into doAs block
+              task.taskCleanup(umbilical);
+//          } else {
+//            final Task taskFinal = task;
+//            childUGI.doAs(new PrivilegedExceptionAction<Object>() {
+//              @Override
+//              public Object run() throws Exception {
+//                taskFinal.taskCleanup(umbilical);
+//                return null;
+//              }
+//            });
+//          }
+          }
+        } catch (Exception e) {
+          LOG.info("Exception cleaning up: "
+              + StringUtils.stringifyException(e));
+        }
+        // Report back any failures, for diagnostic purposes
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        exception.printStackTrace(new PrintStream(baos));
+//      if (classicAttemptID != null) {
+          umbilical.reportDiagnosticInfo(classicAttemptID, baos.toString());
+//      }
+        throw new RuntimeException();
+
+      } catch (Throwable throwable) {
+        LOG.fatal("Error running local (uberized) 'child' : "
+            + StringUtils.stringifyException(throwable));
+//      if (classicAttemptID != null) {
+          Throwable tCause = throwable.getCause();
+          String cause = (tCause == null)
+              ? throwable.getMessage()
+              : StringUtils.stringifyException(tCause);
+          umbilical.fatalError(classicAttemptID, cause);
+//      }
+        throw new RuntimeException();
+
+      } finally {
+/*
+FIXME:  do we need to do any of this stuff?  (guessing not since not in own JVM)
+        RPC.stopProxy(umbilical);
+        DefaultMetricsSystem.shutdown();
+        // Shutting down log4j of the child-vm...
+        // This assumes that on return from Task.run()
+        // there is no more logging done.
+        LogManager.shutdown();
+ */
+      }
+    }
+
+
+/* FIXME:  may not need renameMapOutputForReduce() anymore?  TEST!
+
+${local.dir}/usercache/$user/appcache/$appId/$contId/ == $cwd for tasks;
+contains task.sh script, which, when executed, creates symlinks and sets up env
+ "$local.dir"/usercache/$user/appcache/$appId/$contId/file.out
+ "$local.dir"/usercache/$user/appcache/$appId/$contId/file.out.idx (?)
+ "$local.dir"/usercache/$user/appcache/$appId/output/$taskId/ is where file.out* is moved after MapTask done
+
+	OHO!  no further need for this at all?  $taskId is unique per subtask
+	now => should work fine to leave alone.  TODO:  test with teragen or
+	similar
+ */
+
+    /**
+     * Within the _local_ filesystem (not HDFS), all activity takes place within
+     * a single subdir (${local.dir}/usercache/$user/appcache/$appId/$contId/),
+     * and all sub-MapTasks create the same filename ("file.out").  Rename that
+     * to something unique (e.g., "map_0.out") to avoid collisions.
+     *
+     * Longer-term, we'll modify [something] to use TaskAttemptID-based
+     * filenames instead of "file.out". (All of this is entirely internal,
+     * so there are no particular compatibility issues.)
+     */
+    private void renameMapOutputForReduce(JobConf conf, TaskAttemptId mapId,
+                                          MapOutputFile subMapOutputFile)
+    throws IOException {
+      FileSystem localFs = FileSystem.getLocal(conf);
+      // move map output to reduce input
+      Path mapOut = subMapOutputFile.getOutputFile();
+      Path reduceIn = subMapOutputFile.getInputFileForWrite(
+          TypeConverter.fromYarn(mapId).getTaskID(), localFs.getLength(mapOut));
+      if (!localFs.mkdirs(reduceIn.getParent())) {
+        throw new IOException("Mkdirs failed to create "
+            + reduceIn.getParent().toString());
+      }
+      if (!localFs.rename(mapOut, reduceIn))
+        throw new IOException("Couldn't rename " + mapOut);
+    }
+
+    /**
+     * Also within the local filesystem, we need to restore the initial state
+     * of the directory as much as possible.  Compare current contents against
+     * the saved original state and nuke everything that doesn't belong, with
+     * the exception of the renamed map outputs (see above).
+FIXME:  do we really need to worry about renamed map outputs, or already moved to output dir on commit?  if latter, fix comment
+     *
+     * Any jobs that go out of their way to rename or delete things from the
+     * local directory are considered broken and deserve what they get...
+     */
+    private void relocalize() {
+      File[] curLocalFiles = curDir.listFiles();
+      for (int j = 0; j < curLocalFiles.length; ++j) {
+        if (!localizedFiles.contains(curLocalFiles[j])) {
+          // found one that wasn't there before:  delete it
+          boolean deleted = false;
+          try {
+            if (curFC != null) {
+              // this is recursive, unlike File delete():
+              deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
+            }
+          } catch (IOException e) {
+            deleted = false;
+          }
+          if (!deleted) {
+            LOG.warn("Unable to delete unexpected local file/dir "
+                + curLocalFiles[j].getName() + ": insufficient permissions?");
+          }
+        }
+      }
+    }
+
+  } // end SubtaskRunner
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java

@@ -0,0 +1,264 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Vector;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.TaskLog.LogName;
+import org.apache.hadoop.mapreduce.ID;
+import org.apache.hadoop.util.StringUtils;
+
+public class MapReduceChildJVM {
+  private static final String SYSTEM_PATH_SEPARATOR = 
+    System.getProperty("path.separator");
+
+  private static final Log LOG = LogFactory.getLog(MapReduceChildJVM.class);
+
+  private static File getTaskLogFile(String logDir, LogName filter) {
+    return new File(logDir, filter.toString());
+  }
+
+  private static String getChildEnv(JobConf jobConf, boolean isMap) {
+    if (isMap) {
+      return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV,
+          jobConf.get(JobConf.MAPRED_TASK_ENV));
+    }
+    return jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV,
+        jobConf.get(jobConf.MAPRED_TASK_ENV));
+  }
+
+  public static void setVMEnv(Map<String, String> env,
+      List<String> classPaths, String pwd, String containerLogDir,
+      String nmLdLibraryPath, Task task, CharSequence applicationTokensFile) {
+
+    JobConf conf = task.conf;
+
+    // Add classpath.
+    CharSequence cp = env.get("CLASSPATH");
+    String classpath = StringUtils.join(SYSTEM_PATH_SEPARATOR, classPaths);
+    if (null == cp) {
+      env.put("CLASSPATH", classpath);
+    } else {
+      env.put("CLASSPATH", classpath + SYSTEM_PATH_SEPARATOR + cp);
+    }
+
+    /////// Environmental variable LD_LIBRARY_PATH
+    StringBuilder ldLibraryPath = new StringBuilder();
+
+    ldLibraryPath.append(nmLdLibraryPath);
+    ldLibraryPath.append(SYSTEM_PATH_SEPARATOR);
+    ldLibraryPath.append(pwd);
+    env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
+    /////// Environmental variable LD_LIBRARY_PATH
+
+    // for the child of task jvm, set hadoop.root.logger
+    env.put("HADOOP_ROOT_LOGGER", "DEBUG,CLA"); // TODO: Debug
+
+    // TODO: The following is useful for instance in streaming tasks. Should be
+    // set in ApplicationMaster's env by the RM.
+    String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
+    if (hadoopClientOpts == null) {
+      hadoopClientOpts = "";
+    } else {
+      hadoopClientOpts = hadoopClientOpts + " ";
+    }
+    // FIXME: don't think this is also needed given we already set java
+    // properties.
+    long logSize = TaskLog.getTaskLogLength(conf);
+    Vector<String> logProps = new Vector<String>(4);
+    setupLog4jProperties(logProps, logSize, containerLogDir);
+    Iterator<String> it = logProps.iterator();
+    StringBuffer buffer = new StringBuffer();
+    while (it.hasNext()) {
+      buffer.append(" " + it.next());
+    }
+    hadoopClientOpts = hadoopClientOpts + buffer.toString();
+    
+    env.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
+
+    // add the env variables passed by the user
+    String mapredChildEnv = getChildEnv(conf, task.isMapTask());
+    if (mapredChildEnv != null && mapredChildEnv.length() > 0) {
+      String childEnvs[] = mapredChildEnv.split(",");
+      for (String cEnv : childEnvs) {
+        String[] parts = cEnv.split("="); // split on '='
+        String value = (String) env.get(parts[0]);
+        if (value != null) {
+          // replace $env with the child's env constructed by tt's
+          // example LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp
+          value = parts[1].replace("$" + parts[0], value);
+        } else {
+          // this key is not configured by the tt for the child .. get it 
+          // from the tt's env
+          // example PATH=$PATH:/tmp
+          value = System.getenv(parts[0]); // Get from NM?
+          if (value != null) {
+            // the env key is present in the tt's env
+            value = parts[1].replace("$" + parts[0], value);
+          } else {
+            // the env key is note present anywhere .. simply set it
+            // example X=$X:/tmp or X=/tmp
+            value = parts[1].replace("$" + parts[0], "");
+          }
+        }
+        env.put(parts[0], value);
+      }
+    }
+
+    //This should not be set here (If an OS check is requied. moved to ContainerLuanch)
+    // env.put("JVM_PID", "`echo $$`");
+
+    env.put(Constants.STDOUT_LOGFILE_ENV,
+        getTaskLogFile(containerLogDir, TaskLog.LogName.STDOUT).toString());
+    env.put(Constants.STDERR_LOGFILE_ENV,
+        getTaskLogFile(containerLogDir, TaskLog.LogName.STDERR).toString());
+  }
+
+  private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) {
+    if (isMapTask) {
+      return jobConf.get(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, jobConf.get(
+          JobConf.MAPRED_TASK_JAVA_OPTS,
+          JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+    }
+    return jobConf
+        .get(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, jobConf.get(
+            JobConf.MAPRED_TASK_JAVA_OPTS,
+            JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
+  }
+
+  private static void setupLog4jProperties(Vector<String> vargs,
+      long logSize, String containerLogDir) {
+    vargs.add("-Dlog4j.configuration=container-log4j.properties");
+    vargs.add("-Dhadoop.yarn.mr.containerLogDir=" + containerLogDir);
+    vargs.add("-Dhadoop.yarn.mr.totalLogFileSize=" + logSize);
+  }
+
+  public static List<String> getVMCommand(
+      InetSocketAddress taskAttemptListenerAddr, Task task, String javaHome,
+      String workDir, String logDir, String childTmpDir, ID jvmID) {
+
+    TaskAttemptID attemptID = task.getTaskID();
+    JobConf conf = task.conf;
+
+    Vector<String> vargs = new Vector<String>(8);
+
+    vargs.add("exec");
+    vargs.add(javaHome + "/bin/java");
+
+    // Add child (task) java-vm options.
+    //
+    // The following symbols if present in mapred.{map|reduce}.child.java.opts 
+    // value are replaced:
+    // + @taskid@ is interpolated with value of TaskID.
+    // Other occurrences of @ will not be altered.
+    //
+    // Example with multiple arguments and substitutions, showing
+    // jvm GC logging, and start of a passwordless JVM JMX agent so can
+    // connect with jconsole and the likes to watch child memory, threads
+    // and get thread dumps.
+    //
+    //  <property>
+    //    <name>mapred.map.child.java.opts</name>
+    //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@taskid@.gc \
+    //           -Dcom.sun.management.jmxremote.authenticate=false \
+    //           -Dcom.sun.management.jmxremote.ssl=false \
+    //    </value>
+    //  </property>
+    //
+    //  <property>
+    //    <name>mapred.reduce.child.java.opts</name>
+    //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@taskid@.gc \
+    //           -Dcom.sun.management.jmxremote.authenticate=false \
+    //           -Dcom.sun.management.jmxremote.ssl=false \
+    //    </value>
+    //  </property>
+    //
+    String javaOpts = getChildJavaOpts(conf, task.isMapTask());
+    javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
+    String [] javaOptsSplit = javaOpts.split(" ");
+    
+    // Add java.library.path; necessary for loading native libraries.
+    //
+    // 1. We add the 'cwd' of the task to it's java.library.path to help 
+    //    users distribute native libraries via the DistributedCache.
+    // 2. The user can also specify extra paths to be added to the 
+    //    java.library.path via mapred.{map|reduce}.child.java.opts.
+    //
+    String libraryPath = workDir;
+    boolean hasUserLDPath = false;
+    for(int i=0; i<javaOptsSplit.length ;i++) { 
+      if(javaOptsSplit[i].startsWith("-Djava.library.path=")) {
+        // TODO: Does the above take care of escaped space chars
+        javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath;
+        hasUserLDPath = true;
+        break;
+      }
+    }
+    if(!hasUserLDPath) {
+      vargs.add("-Djava.library.path=" + libraryPath);
+    }
+    for (int i = 0; i < javaOptsSplit.length; i++) {
+      vargs.add(javaOptsSplit[i]);
+    }
+
+    if (childTmpDir != null) {
+      vargs.add("-Djava.io.tmpdir=" + childTmpDir);
+    }
+
+    // Setup the log4j prop
+    long logSize = TaskLog.getTaskLogLength(conf);
+    setupLog4jProperties(vargs, logSize, logDir);
+
+    if (conf.getProfileEnabled()) {
+      if (conf.getProfileTaskRange(task.isMapTask()
+                                   ).isIncluded(task.getPartition())) {
+        File prof = getTaskLogFile(logDir, TaskLog.LogName.PROFILE);
+        vargs.add(String.format(conf.getProfileParams(), prof.toString()));
+      }
+    }
+
+    // Add main class and its arguments 
+    vargs.add(YarnChild.class.getName());  // main of Child
+    // pass TaskAttemptListener's address
+    vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
+    vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
+    vargs.add(attemptID.toString());                      // pass task identifier
+
+    // Finally add the jvmID
+    vargs.add(String.valueOf(jvmID.getId()));
+    vargs.add("1>" + getTaskLogFile(logDir, TaskLog.LogName.STDERR));
+    vargs.add("2>" + getTaskLogFile(logDir, TaskLog.LogName.STDOUT));
+
+    // Final commmand
+    StringBuilder mergedCommand = new StringBuilder();
+    for (CharSequence str : vargs) {
+      mergedCommand.append(str).append(" ");
+    }
+    Vector<String> vargsFinal = new Vector<String>(1);
+    vargsFinal.add(mergedCommand.toString());
+    return vargsFinal;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapTaskAttemptImpl.java

@@ -0,0 +1,65 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+public class MapTaskAttemptImpl extends TaskAttemptImpl {
+
+  private final TaskSplitMetaInfo splitInfo;
+
+  public MapTaskAttemptImpl(TaskId taskId, int attempt, 
+      EventHandler eventHandler, Path jobFile, 
+      int partition, TaskSplitMetaInfo splitInfo, Configuration conf,
+      TaskAttemptListener taskAttemptListener, 
+      OutputCommitter committer, Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
+    super(taskId, attempt, eventHandler, 
+        taskAttemptListener, jobFile, partition, conf, splitInfo.getLocations(),
+        committer, jobToken, fsTokens, clock);
+    this.splitInfo = splitInfo;
+  }
+
+  @Override
+  public Task createRemoteTask() {
+    //job file name is set in TaskAttempt, setting it null here
+    MapTask mapTask =
+      new MapTask("", TypeConverter.fromYarn(getID()), partition,
+          splitInfo.getSplitIndex(), 1); // YARN doesn't have the concept of slots per task, set it as 1.
+    mapTask.setUser(conf.get(MRJobConfig.USER_NAME));
+    mapTask.setConf(conf);
+    return mapTask;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ReduceTaskAttemptImpl.java

@@ -0,0 +1,64 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+
+public class ReduceTaskAttemptImpl extends TaskAttemptImpl {
+
+  private final int numMapTasks;
+
+  public ReduceTaskAttemptImpl(TaskId id, int attempt,
+      EventHandler eventHandler, Path jobFile, int partition,
+      int numMapTasks, Configuration conf,
+      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
+      Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
+    super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
+        conf, new String[] {}, committer, jobToken, fsTokens, clock);
+    this.numMapTasks = numMapTasks;
+  }
+
+  @Override
+  public Task createRemoteTask() {
+  //job file name is set in TaskAttempt, setting it null here
+    ReduceTask reduceTask =
+      new ReduceTask("", TypeConverter.fromYarn(getID()), partition,
+          numMapTasks, 1); // YARN doesn't have the concept of slots per task, set it as 1.
+  reduceTask.setUser(conf.get(MRJobConfig.USER_NAME));
+  reduceTask.setConf(conf);
+    return reduceTask;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java

@@ -0,0 +1,434 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.mapred.SortedRanges.Range;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.service.CompositeService;
+
+/**
+ * This class is responsible for talking to the task umblical.
+ * It also converts all the old data structures
+ * to yarn data structures.
+ * 
+ * This class HAS to be in this package to access package private 
+ * methods/classes.
+ */
+public class TaskAttemptListenerImpl extends CompositeService 
+    implements TaskUmbilicalProtocol, TaskAttemptListener {
+
+  private static final Log LOG = LogFactory.getLog(TaskAttemptListenerImpl.class);
+
+  private AppContext context;
+  private Server server;
+  private TaskHeartbeatHandler taskHeartbeatHandler;
+  private InetSocketAddress address;
+  private Map<WrappedJvmID, org.apache.hadoop.mapred.Task> jvmIDToAttemptMap = 
+    Collections.synchronizedMap(new HashMap<WrappedJvmID, 
+        org.apache.hadoop.mapred.Task>());
+  private JobTokenSecretManager jobTokenSecretManager = null;
+  
+  public TaskAttemptListenerImpl(AppContext context,
+      JobTokenSecretManager jobTokenSecretManager) {
+    super(TaskAttemptListenerImpl.class.getName());
+    this.context = context;
+    this.jobTokenSecretManager = jobTokenSecretManager;
+  }
+
+  @Override
+  public void init(Configuration conf) {
+   registerHeartbeatHandler();
+   super.init(conf);
+  }
+
+  @Override
+  public void start() {
+    startRpcServer();
+    super.start();
+  }
+
+  protected void registerHeartbeatHandler() {
+    taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), 
+        context.getClock());
+    addService(taskHeartbeatHandler);
+  }
+
+  protected void startRpcServer() {
+    Configuration conf = getConfig();
+    try {
+      server =
+          RPC.getServer(TaskUmbilicalProtocol.class, this, "0.0.0.0", 0, 
+              conf.getInt(AMConstants.AM_TASK_LISTENER_THREADS, 
+                  AMConstants.DEFAULT_AM_TASK_LISTENER_THREADS),
+              false, conf, jobTokenSecretManager);
+      server.start();
+      InetSocketAddress listenerAddress = server.getListenerAddress();
+      this.address =
+          NetUtils.createSocketAddr(listenerAddress.getAddress()
+              .getLocalHost().getCanonicalHostName()
+              + ":" + listenerAddress.getPort());
+    } catch (IOException e) {
+      throw new YarnException(e);
+    }
+  }
+
+  @Override
+  public void stop() {
+    stopRpcServer();
+    super.stop();
+  }
+
+  protected void stopRpcServer() {
+    server.stop();
+  }
+
+  @Override
+  public InetSocketAddress getAddress() {
+    return address;
+  }
+
+  /**
+   * Child checking whether it can commit.
+   * 
+   * <br/>
+   * Commit is a two-phased protocol. First the attempt informs the
+   * ApplicationMaster that it is
+   * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
+   * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
+   * a legacy from the centralized commit protocol handling by the JobTracker.
+   */
+  @Override
+  public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException {
+    LOG.info("Commit go/no-go request from " + taskAttemptID.toString());
+    // An attempt is asking if it can commit its output. This can be decided
+    // only by the task which is managing the multiple attempts. So redirect the
+    // request there.
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+        TypeConverter.toYarn(taskAttemptID);
+
+    taskHeartbeatHandler.receivedPing(attemptID);
+
+    Job job = context.getJob(attemptID.getTaskId().getJobId());
+    Task task = job.getTask(attemptID.getTaskId());
+    return task.canCommit(attemptID);
+  }
+
+  /**
+   * TaskAttempt is reporting that it is in commit_pending and it is waiting for
+   * the commit Response
+   * 
+   * <br/>
+   * Commit it a two-phased protocol. First the attempt informs the
+   * ApplicationMaster that it is
+   * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
+   * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
+   * a legacy from the centralized commit protocol handling by the JobTracker.
+   */
+  @Override
+  public void commitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu)
+          throws IOException, InterruptedException {
+    LOG.info("Commit-pending state update from " + taskAttemptID.toString());
+    // An attempt is asking if it can commit its output. This can be decided
+    // only by the task which is managing the multiple attempts. So redirect the
+    // request there.
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+        TypeConverter.toYarn(taskAttemptID);
+
+    taskHeartbeatHandler.receivedPing(attemptID);
+    //Ignorable TaskStatus? - since a task will send a LastStatusUpdate
+    context.getEventHandler().handle(
+        new TaskAttemptEvent(attemptID, 
+            TaskAttemptEventType.TA_COMMIT_PENDING));
+  }
+
+  @Override
+  public void done(TaskAttemptID taskAttemptID) throws IOException {
+    LOG.info("Done acknowledgement from " + taskAttemptID.toString());
+
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+        TypeConverter.toYarn(taskAttemptID);
+
+    taskHeartbeatHandler.receivedPing(attemptID);
+
+    context.getEventHandler().handle(
+        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_DONE));
+  }
+
+  @Override
+  public void fatalError(TaskAttemptID taskAttemptID, String msg)
+      throws IOException {
+    // This happens only in Child and in the Task.
+    LOG.fatal("Task: " + taskAttemptID + " - exited : " + msg);
+    reportDiagnosticInfo(taskAttemptID, "Error: " + msg);
+
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+        TypeConverter.toYarn(taskAttemptID);
+    context.getEventHandler().handle(
+        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+  }
+
+  @Override
+  public void fsError(TaskAttemptID taskAttemptID, String message)
+      throws IOException {
+    // This happens only in Child.
+    LOG.fatal("Task: " + taskAttemptID + " - failed due to FSError: "
+        + message);
+    reportDiagnosticInfo(taskAttemptID, "FSError: " + message);
+
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+        TypeConverter.toYarn(taskAttemptID);
+    context.getEventHandler().handle(
+        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
+  }
+
+  @Override
+  public void shuffleError(TaskAttemptID taskAttemptID, String message) throws IOException {
+    // TODO: This isn't really used in any MR code. Ask for removal.    
+  }
+
+  @Override
+  public MapTaskCompletionEventsUpdate getMapCompletionEvents(
+      JobID jobIdentifier, int fromEventId, int maxEvents,
+      TaskAttemptID taskAttemptID) throws IOException {
+    LOG.info("MapCompletionEvents request from " + taskAttemptID.toString()
+        + ". fromEventID " + fromEventId + " maxEvents " + maxEvents);
+
+    // TODO: shouldReset is never used. See TT. Ask for Removal.
+    boolean shouldReset = false;
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+      TypeConverter.toYarn(taskAttemptID);
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events =
+        context.getJob(attemptID.getTaskId().getJobId()).getTaskAttemptCompletionEvents(
+            fromEventId, maxEvents);
+
+    taskHeartbeatHandler.receivedPing(attemptID);
+
+    // filter the events to return only map completion events in old format
+    List<TaskCompletionEvent> mapEvents = new ArrayList<TaskCompletionEvent>();
+    for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent event : events) {
+      if (TaskType.MAP.equals(event.getAttemptId().getTaskId().getTaskType())) {
+        mapEvents.add(TypeConverter.fromYarn(event));
+      }
+    }
+    
+    return new MapTaskCompletionEventsUpdate(
+        mapEvents.toArray(new TaskCompletionEvent[0]), shouldReset);
+  }
+
+  @Override
+  public boolean ping(TaskAttemptID taskAttemptID) throws IOException {
+    LOG.info("Ping from " + taskAttemptID.toString());
+    taskHeartbeatHandler.receivedPing(TypeConverter.toYarn(taskAttemptID));
+    return true;
+  }
+
+  @Override
+  public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo)
+ throws IOException {
+    LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": "
+        + diagnosticInfo);
+
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+      TypeConverter.toYarn(taskAttemptID);
+    taskHeartbeatHandler.receivedPing(attemptID);
+
+    // This is mainly used for cases where we want to propagate exception traces
+    // of tasks that fail.
+
+    // This call exists as a hadoop mapreduce legacy wherein all changes in
+    // counters/progress/phase/output-size are reported through statusUpdate()
+    // call but not diagnosticInformation.
+    context.getEventHandler().handle(
+        new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo));
+  }
+
+  @Override
+  public boolean statusUpdate(TaskAttemptID taskAttemptID,
+      TaskStatus taskStatus) throws IOException, InterruptedException {
+    LOG.info("Status update from " + taskAttemptID.toString());
+    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID =
+        TypeConverter.toYarn(taskAttemptID);
+    taskHeartbeatHandler.receivedPing(yarnAttemptID);
+    TaskAttemptStatus taskAttemptStatus =
+        new TaskAttemptStatus();
+    taskAttemptStatus.id = yarnAttemptID;
+    // Task sends the updated progress to the TT.
+    taskAttemptStatus.progress = taskStatus.getProgress();
+    LOG.info("Progress of TaskAttempt " + taskAttemptID + " is : "
+        + taskStatus.getProgress());
+    // Task sends the diagnostic information to the TT
+    taskAttemptStatus.diagnosticInfo = taskStatus.getDiagnosticInfo();
+    // Task sends the updated state-string to the TT.
+    taskAttemptStatus.stateString = taskStatus.getStateString();
+    // Set the output-size when map-task finishes. Set by the task itself.
+    taskAttemptStatus.outputSize = taskStatus.getOutputSize();
+    // Task sends the updated phase to the TT.
+    taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
+    // Counters are updated by the task.
+    taskAttemptStatus.counters =
+        TypeConverter.toYarn(taskStatus.getCounters());
+
+    // Map Finish time set by the task (map only)
+    if (taskStatus.getIsMap() && taskStatus.getMapFinishTime() != 0) {
+      taskAttemptStatus.mapFinishTime = taskStatus.getMapFinishTime();
+    }
+
+    // Shuffle Finish time set by the task (reduce only).
+    if (!taskStatus.getIsMap() && taskStatus.getShuffleFinishTime() != 0) {
+      taskAttemptStatus.shuffleFinishTime = taskStatus.getShuffleFinishTime();
+    }
+
+    // Sort finish time set by the task (reduce only).
+    if (!taskStatus.getIsMap() && taskStatus.getSortFinishTime() != 0) {
+      taskAttemptStatus.sortFinishTime = taskStatus.getSortFinishTime();
+    }
+
+    // Not Setting the task state. Used by speculation - will be set in TaskAttemptImpl
+    //taskAttemptStatus.taskState =  TypeConverter.toYarn(taskStatus.getRunState());
+    
+    //set the fetch failures
+    if (taskStatus.getFetchFailedMaps() != null 
+        && taskStatus.getFetchFailedMaps().size() > 0) {
+      taskAttemptStatus.fetchFailedMaps = 
+        new ArrayList<org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId>();
+      for (TaskAttemptID failedMapId : taskStatus.getFetchFailedMaps()) {
+        taskAttemptStatus.fetchFailedMaps.add(
+            TypeConverter.toYarn(failedMapId));
+      }
+    }
+
+ // Task sends the information about the nextRecordRange to the TT
+    
+//    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
+//    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
+//    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
+//    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
+//    // This was used by TT to do counter updates only once every minute. So this
+//    // isn't ever changed by the Task itself.
+//    taskStatus.getIncludeCounters();
+
+    context.getEventHandler().handle(
+        new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id,
+            taskAttemptStatus));
+    return true;
+  }
+
+  @Override
+  public long getProtocolVersion(String arg0, long arg1) throws IOException {
+    return TaskUmbilicalProtocol.versionID;
+  }
+
+  @Override
+  public void reportNextRecordRange(TaskAttemptID taskAttemptID, Range range)
+      throws IOException {
+    // This is used when the feature of skipping records is enabled.
+
+    // This call exists as a hadoop mapreduce legacy wherein all changes in
+    // counters/progress/phase/output-size are reported through statusUpdate()
+    // call but not the next record range information.
+    throw new IOException("Not yet implemented.");
+  }
+
+  @Override
+  public JvmTask getTask(JvmContext context) throws IOException {
+
+    // A rough imitation of code from TaskTracker.
+
+    JVMId jvmId = context.jvmId;
+    LOG.info("JVM with ID : " + jvmId + " asked for a task");
+
+    // TODO: Is it an authorised container to get a task? Otherwise return null.
+
+    // TODO: Is the request for task-launch still valid?
+
+    // TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
+    // to jobId and task-type.
+
+    WrappedJvmID wJvmID = new WrappedJvmID(jvmId.getJobId(), jvmId.isMap,
+        jvmId.getId());
+    org.apache.hadoop.mapred.Task task = jvmIDToAttemptMap.get(wJvmID);
+    if (task != null) { //there may be lag in the attempt getting added here
+      LOG.info("JVM with ID: " + jvmId + " given task: " + task.getTaskID());
+      JvmTask jvmTask = new JvmTask(task, false);
+      
+      //remove the task as it is no more needed and free up the memory
+      jvmIDToAttemptMap.remove(wJvmID);
+      
+      return jvmTask;
+    }
+    return null;
+  }
+
+  @Override
+  public void register(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
+      org.apache.hadoop.mapred.Task task, WrappedJvmID jvmID) {
+    //create the mapping so that it is easy to look up
+    //when it comes back to ask for Task.
+    jvmIDToAttemptMap.put(jvmID, task);
+    //register this attempt
+    taskHeartbeatHandler.register(attemptID);
+  }
+
+  @Override
+  public void unregister(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
+      WrappedJvmID jvmID) {
+    //remove the mapping if not already removed
+    jvmIDToAttemptMap.remove(jvmID);
+
+    //unregister this attempt
+    taskHeartbeatHandler.unregister(attemptID);
+  }
+
+  @Override
+  public ProtocolSignature getProtocolSignature(String protocol,
+      long clientVersion, int clientMethodsHash) throws IOException {
+    return ProtocolSignature.getProtocolSignature(this, 
+        protocol, clientVersion, clientMethodsHash);
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedJvmID.java

@@ -0,0 +1,30 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+/**
+ * A simple wrapper for increasing the visibility.
+ */
+public class WrappedJvmID extends JVMId {
+
+  public WrappedJvmID(JobID jobID, boolean mapTask, int nextInt) {
+    super(jobID, mapTask, nextInt);
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedPeriodicStatsAccumulator.java

@@ -0,0 +1,15 @@
+package org.apache.hadoop.mapred;
+
+//Workaround for PeriodicStateAccumulator being package access
+public class WrappedPeriodicStatsAccumulator {
+
+  private PeriodicStatsAccumulator real;
+
+  public WrappedPeriodicStatsAccumulator(PeriodicStatsAccumulator real) {
+    this.real = real;
+  }
+  
+  public void extend(double newProgress, int newValue) {
+    real.extend(newProgress, newValue);
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedProgressSplitsBlock.java

@@ -0,0 +1,52 @@
+package org.apache.hadoop.mapred;
+
+// Workaround for ProgressSplitBlock being package access
+public class WrappedProgressSplitsBlock extends ProgressSplitsBlock {
+
+  public static final int DEFAULT_NUMBER_PROGRESS_SPLITS = 12;
+
+  private WrappedPeriodicStatsAccumulator wrappedProgressWallclockTime;
+  private WrappedPeriodicStatsAccumulator wrappedProgressCPUTime;
+  private WrappedPeriodicStatsAccumulator wrappedProgressVirtualMemoryKbytes;
+  private WrappedPeriodicStatsAccumulator wrappedProgressPhysicalMemoryKbytes;
+
+  public WrappedProgressSplitsBlock(int numberSplits) {
+    super(numberSplits);
+  }
+
+  public int[][] burst() {
+    return super.burst();
+  }
+
+  public WrappedPeriodicStatsAccumulator getProgressWallclockTime() {
+    if (wrappedProgressWallclockTime == null) {
+      wrappedProgressWallclockTime = new WrappedPeriodicStatsAccumulator(
+          progressWallclockTime);
+    }
+    return wrappedProgressWallclockTime;
+  }
+
+  public WrappedPeriodicStatsAccumulator getProgressCPUTime() {
+    if (wrappedProgressCPUTime == null) {
+      wrappedProgressCPUTime = new WrappedPeriodicStatsAccumulator(
+          progressCPUTime);
+    }
+    return wrappedProgressCPUTime;
+  }
+
+  public WrappedPeriodicStatsAccumulator getProgressVirtualMemoryKbytes() {
+    if (wrappedProgressVirtualMemoryKbytes == null) {
+      wrappedProgressVirtualMemoryKbytes = new WrappedPeriodicStatsAccumulator(
+          progressVirtualMemoryKbytes);
+    }
+    return wrappedProgressVirtualMemoryKbytes;
+  }
+
+  public WrappedPeriodicStatsAccumulator getProgressPhysicalMemoryKbytes() {
+    if (wrappedProgressPhysicalMemoryKbytes == null) {
+      wrappedProgressPhysicalMemoryKbytes = new WrappedPeriodicStatsAccumulator(
+          progressPhysicalMemoryKbytes);
+    }
+    return wrappedProgressPhysicalMemoryKbytes;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java

@@ -0,0 +1,346 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSError;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.log4j.LogManager;
+
+/**
+ * The main() for MapReduce task processes.
+ */
+class YarnChild {
+
+  private static final Log LOG = LogFactory.getLog(YarnChild.class);
+
+  static volatile TaskAttemptID taskid = null;
+
+  public static void main(String[] args) throws Throwable {
+    LOG.debug("Child starting");
+
+    final JobConf defaultConf = new JobConf();
+    defaultConf.addResource(MRConstants.JOB_CONF_FILE);
+    UserGroupInformation.setConfiguration(defaultConf);
+
+    String host = args[0];
+    int port = Integer.parseInt(args[1]);
+    final InetSocketAddress address = new InetSocketAddress(host, port);
+    final TaskAttemptID firstTaskid = TaskAttemptID.forName(args[2]);
+    int jvmIdInt = Integer.parseInt(args[3]);
+    JVMId jvmId = new JVMId(firstTaskid.getJobID(),
+        firstTaskid.getTaskType() == TaskType.MAP, jvmIdInt);
+
+    // initialize metrics
+    DefaultMetricsSystem.initialize(
+        StringUtils.camelize(firstTaskid.getTaskType().name()) +"Task");
+
+    Token<JobTokenIdentifier> jt = loadCredentials(defaultConf, address);
+
+    // Create TaskUmbilicalProtocol as actual task owner.
+    UserGroupInformation taskOwner =
+      UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
+    taskOwner.addToken(jt);
+    final TaskUmbilicalProtocol umbilical =
+      taskOwner.doAs(new PrivilegedExceptionAction<TaskUmbilicalProtocol>() {
+      @Override
+      public TaskUmbilicalProtocol run() throws Exception {
+        return (TaskUmbilicalProtocol)RPC.getProxy(TaskUmbilicalProtocol.class,
+            TaskUmbilicalProtocol.versionID, address, defaultConf);
+      }
+    });
+
+    // report non-pid to application master
+    JvmContext context = new JvmContext(jvmId, "-1000");
+    LOG.debug("PID: " + System.getenv().get("JVM_PID"));
+    Task task = null;
+    UserGroupInformation childUGI = null;
+
+    try {
+      int idleLoopCount = 0;
+      JvmTask myTask = null;;
+      // poll for new task
+      for (int idle = 0; null == myTask; ++idle) {
+        long sleepTimeMilliSecs = Math.min(idle * 500, 1500);
+        LOG.info("Sleeping for " + sleepTimeMilliSecs
+            + "ms before retrying again. Got null now.");
+        MILLISECONDS.sleep(sleepTimeMilliSecs);
+        myTask = umbilical.getTask(context);
+      }
+      if (myTask.shouldDie()) {
+        return;
+      }
+
+      task = myTask.getTask();
+      YarnChild.taskid = task.getTaskID();
+
+      // Create the job-conf and set credentials
+      final JobConf job =
+        configureTask(task, defaultConf.getCredentials(), jt);
+
+      // Initiate Java VM metrics
+      JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
+      LOG.debug("Remote user: " + job.get("user.name"));
+      childUGI = UserGroupInformation.createRemoteUser(job.get("user.name"));
+      // Add tokens to new user so that it may execute its task correctly.
+      for(Token<?> token : UserGroupInformation.getCurrentUser().getTokens()) {
+        childUGI.addToken(token);
+      }
+
+      // Create a final reference to the task for the doAs block
+      final Task taskFinal = task;
+      childUGI.doAs(new PrivilegedExceptionAction<Object>() {
+        @Override
+        public Object run() throws Exception {
+          // use job-specified working directory
+          FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
+          taskFinal.run(job, umbilical); // run the task
+          return null;
+        }
+      });
+    } catch (FSError e) {
+      LOG.fatal("FSError from child", e);
+      umbilical.fsError(taskid, e.getMessage());
+    } catch (Exception exception) {
+      LOG.warn("Exception running child : "
+          + StringUtils.stringifyException(exception));
+      try {
+        if (task != null) {
+          // do cleanup for the task
+          if (childUGI == null) { // no need to job into doAs block
+            task.taskCleanup(umbilical);
+          } else {
+            final Task taskFinal = task;
+            childUGI.doAs(new PrivilegedExceptionAction<Object>() {
+              @Override
+              public Object run() throws Exception {
+                taskFinal.taskCleanup(umbilical);
+                return null;
+              }
+            });
+          }
+        }
+      } catch (Exception e) {
+        LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
+      }
+      // Report back any failures, for diagnostic purposes
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      exception.printStackTrace(new PrintStream(baos));
+      if (taskid != null) {
+        umbilical.reportDiagnosticInfo(taskid, baos.toString());
+      }
+    } catch (Throwable throwable) {
+      LOG.fatal("Error running child : "
+    	        + StringUtils.stringifyException(throwable));
+      if (taskid != null) {
+        Throwable tCause = throwable.getCause();
+        String cause = tCause == null
+                                 ? throwable.getMessage()
+                                 : StringUtils.stringifyException(tCause);
+        umbilical.fatalError(taskid, cause);
+      }
+    } finally {
+      RPC.stopProxy(umbilical);
+      DefaultMetricsSystem.shutdown();
+      // Shutting down log4j of the child-vm...
+      // This assumes that on return from Task.run()
+      // there is no more logging done.
+      LogManager.shutdown();
+    }
+  }
+
+  private static Token<JobTokenIdentifier> loadCredentials(JobConf conf,
+      InetSocketAddress address) throws IOException {
+    //load token cache storage
+    String tokenFileLocation =
+        System.getenv(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME);
+    String jobTokenFile =
+        new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf))
+            .toUri().getPath();
+    Credentials credentials =
+      TokenCache.loadTokens(jobTokenFile, conf);
+    LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
+        "; from file=" + jobTokenFile);
+    Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
+    jt.setService(new Text(address.getAddress().getHostAddress() + ":"
+        + address.getPort()));
+    UserGroupInformation current = UserGroupInformation.getCurrentUser();
+    current.addToken(jt);
+    for (Token<? extends TokenIdentifier> tok : credentials.getAllTokens()) {
+      current.addToken(tok);
+    }
+    // Set the credentials
+    conf.setCredentials(credentials);
+    return jt;
+  }
+
+  /**
+   * Configure mapred-local dirs. This config is used by the task for finding
+   * out an output directory.
+   */
+  private static void configureLocalDirs(Task task, JobConf job) {
+    String[] localSysDirs = StringUtils.getTrimmedStrings(
+        System.getenv(ApplicationConstants.LOCAL_DIR_ENV));
+    job.setStrings(MRConfig.LOCAL_DIR, localSysDirs);
+    LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR));
+  }
+
+  private static JobConf configureTask(Task task, Credentials credentials,
+      Token<JobTokenIdentifier> jt) throws IOException {
+    final JobConf job = new JobConf(MRConstants.JOB_CONF_FILE);
+    job.setCredentials(credentials);
+    // set tcp nodelay
+    job.setBoolean("ipc.client.tcpnodelay", true);
+    job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
+        YarnOutputFiles.class, MapOutputFile.class);
+    // set the jobTokenFile into task
+    task.setJobTokenSecret(
+        JobTokenSecretManager.createSecretKey(jt.getPassword()));
+
+    // setup the child's MRConfig.LOCAL_DIR.
+    configureLocalDirs(task, job);
+
+    // setup the child's attempt directories
+    // Do the task-type specific localization
+    task.localizeConfiguration(job);
+
+    // Set up the DistributedCache related configs
+    setupDistributedCacheConfig(job);
+
+    // Overwrite the localized task jobconf which is linked to in the current
+    // work-dir.
+    Path localTaskFile = new Path(Constants.JOBFILE);
+    writeLocalJobFile(localTaskFile, job);
+    task.setJobFile(localTaskFile.toString());
+    task.setConf(job);
+    return job;
+  }
+
+  /**
+   * Set up the DistributedCache related configs to make
+   * {@link DistributedCache#getLocalCacheFiles(Configuration)}
+   * and
+   * {@link DistributedCache#getLocalCacheArchives(Configuration)}
+   * working.
+   * @param job
+   * @throws IOException
+   */
+  private static void setupDistributedCacheConfig(final JobConf job)
+      throws IOException {
+
+    String localWorkDir = System.getenv("PWD");
+    //        ^ ^ all symlinks are created in the current work-dir
+
+    // Update the configuration object with localized archives.
+    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
+    if (cacheArchives != null) {
+      List<String> localArchives = new ArrayList<String>();
+      for (int i = 0; i < cacheArchives.length; ++i) {
+        URI u = cacheArchives[i];
+        Path p = new Path(u);
+        Path name =
+            new Path((null == u.getFragment()) ? p.getName()
+                : u.getFragment());
+        String linkName = name.toUri().getPath();
+        localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
+      }
+      if (!localArchives.isEmpty()) {
+        job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
+            .arrayToString(localArchives.toArray(new String[localArchives
+                .size()])));
+      }
+    }
+
+    // Update the configuration object with localized files.
+    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
+    if (cacheFiles != null) {
+      List<String> localFiles = new ArrayList<String>();
+      for (int i = 0; i < cacheFiles.length; ++i) {
+        URI u = cacheFiles[i];
+        Path p = new Path(u);
+        Path name =
+            new Path((null == u.getFragment()) ? p.getName()
+                : u.getFragment());
+        String linkName = name.toUri().getPath();
+        localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
+      }
+      if (!localFiles.isEmpty()) {
+        job.set(MRJobConfig.CACHE_LOCALFILES,
+            StringUtils.arrayToString(localFiles
+                .toArray(new String[localFiles.size()])));
+      }
+    }
+  }
+
+  private static final FsPermission urw_gr =
+    FsPermission.createImmutable((short) 0640);
+
+  /**
+   * Write the task specific job-configuration file.
+   * @throws IOException
+   */
+  private static void writeLocalJobFile(Path jobFile, JobConf conf)
+      throws IOException {
+    FileSystem localFs = FileSystem.getLocal(conf);
+    localFs.delete(jobFile);
+    OutputStream out = null;
+    try {
+      out = FileSystem.create(localFs, jobFile, urw_gr);
+      conf.writeXml(out);
+    } finally {
+      IOUtils.cleanup(LOG, out);
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java

@@ -0,0 +1,238 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRConfig;
+
+/**
+ * Manipulate the working area for the transient store for maps and reduces.
+ *
+ * This class is used by map and reduce tasks to identify the directories that
+ * they need to write to/read from for intermediate files. The callers of
+ * these methods are from child space.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class YarnOutputFiles extends MapOutputFile {
+
+  private JobConf conf;
+
+  private static final String JOB_OUTPUT_DIR = "output";
+  private static final String SPILL_FILE_PATTERN = "%s_spill_%d.out";
+  private static final String SPILL_INDEX_FILE_PATTERN = SPILL_FILE_PATTERN
+      + ".index";
+
+  public YarnOutputFiles() {
+  }
+
+  // assume configured to $localdir/usercache/$user/appcache/$appId
+  private LocalDirAllocator lDirAlloc = 
+    new LocalDirAllocator(MRConfig.LOCAL_DIR);
+
+  private Path getAttemptOutputDir() {
+    return new Path(JOB_OUTPUT_DIR, conf.get(JobContext.TASK_ATTEMPT_ID));
+  }
+  
+  /**
+   * Return the path to local map output file created earlier
+   * 
+   * @return path
+   * @throws IOException
+   */
+  public Path getOutputFile() throws IOException {
+    Path attemptOutput =
+      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING);
+    return lDirAlloc.getLocalPathToRead(attemptOutput.toString(), conf);
+  }
+
+  /**
+   * Create a local map output file name.
+   * 
+   * @param size the size of the file
+   * @return path
+   * @throws IOException
+   */
+  public Path getOutputFileForWrite(long size) throws IOException {
+    Path attemptOutput = 
+      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING);
+    return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf);
+  }
+
+  /**
+   * Create a local map output file name on the same volume.
+   */
+  public Path getOutputFileForWriteInVolume(Path existing) {
+    Path outputDir = new Path(existing.getParent(), JOB_OUTPUT_DIR);
+    Path attemptOutputDir = new Path(outputDir,
+        conf.get(JobContext.TASK_ATTEMPT_ID));
+    return new Path(attemptOutputDir, MAP_OUTPUT_FILENAME_STRING);
+  }
+
+  /**
+   * Return the path to a local map output index file created earlier
+   * 
+   * @return path
+   * @throws IOException
+   */
+  public Path getOutputIndexFile() throws IOException {
+    Path attemptIndexOutput =
+      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING +
+                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
+    return lDirAlloc.getLocalPathToRead(attemptIndexOutput.toString(), conf);
+  }
+
+  /**
+   * Create a local map output index file name.
+   * 
+   * @param size the size of the file
+   * @return path
+   * @throws IOException
+   */
+  public Path getOutputIndexFileForWrite(long size) throws IOException {
+    Path attemptIndexOutput =
+      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING +
+                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
+    return lDirAlloc.getLocalPathForWrite(attemptIndexOutput.toString(),
+        size, conf);
+  }
+
+  /**
+   * Create a local map output index file name on the same volume.
+   */
+  public Path getOutputIndexFileForWriteInVolume(Path existing) {
+    Path outputDir = new Path(existing.getParent(), JOB_OUTPUT_DIR);
+    Path attemptOutputDir = new Path(outputDir,
+        conf.get(JobContext.TASK_ATTEMPT_ID));
+    return new Path(attemptOutputDir, MAP_OUTPUT_FILENAME_STRING +
+                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
+  }
+
+  /**
+   * Return a local map spill file created earlier.
+   * 
+   * @param spillNumber the number
+   * @return path
+   * @throws IOException
+   */
+  public Path getSpillFile(int spillNumber) throws IOException {
+    return lDirAlloc.getLocalPathToRead(
+        String.format(SPILL_FILE_PATTERN,
+            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf);
+  }
+
+  /**
+   * Create a local map spill file name.
+   * 
+   * @param spillNumber the number
+   * @param size the size of the file
+   * @return path
+   * @throws IOException
+   */
+  public Path getSpillFileForWrite(int spillNumber, long size)
+      throws IOException {
+    return lDirAlloc.getLocalPathForWrite(
+        String.format(String.format(SPILL_FILE_PATTERN,
+            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber)), size, conf);
+  }
+
+  /**
+   * Return a local map spill index file created earlier
+   * 
+   * @param spillNumber the number
+   * @return path
+   * @throws IOException
+   */
+  public Path getSpillIndexFile(int spillNumber) throws IOException {
+    return lDirAlloc.getLocalPathToRead(
+        String.format(SPILL_INDEX_FILE_PATTERN,
+            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf);
+  }
+
+  /**
+   * Create a local map spill index file name.
+   * 
+   * @param spillNumber the number
+   * @param size the size of the file
+   * @return path
+   * @throws IOException
+   */
+  public Path getSpillIndexFileForWrite(int spillNumber, long size)
+      throws IOException {
+    return lDirAlloc.getLocalPathForWrite(
+        String.format(SPILL_INDEX_FILE_PATTERN,
+            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), size, conf);
+  }
+
+  /**
+   * Return a local reduce input file created earlier
+   * 
+   * @param mapId a map task id
+   * @return path
+   * @throws IOException 
+   */
+  public Path getInputFile(int mapId) throws IOException {
+    throw new UnsupportedOperationException("Incompatible with LocalRunner");
+  }
+
+  /**
+   * Create a local reduce input file name.
+   * 
+   * @param mapId a map task id
+   * @param size the size of the file
+   * @return path
+   * @throws IOException
+   */
+  public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId,
+      long size) throws IOException {
+    return lDirAlloc.getLocalPathForWrite(String.format(
+        REDUCE_INPUT_FILE_FORMAT_STRING,
+        getAttemptOutputDir().toString(), mapId.getId()),
+        size, conf);
+  }
+
+  /** Removes all of the files related to a task. */
+  public void removeAll() throws IOException {
+    throw new UnsupportedOperationException("Incompatible with LocalRunner");
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    if (conf instanceof JobConf) {
+      this.conf = (JobConf) conf;
+    } else {
+      this.conf = new JobConf(conf);
+    }
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+  
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class JobHistoryEvent extends AbstractEvent<EventType>{
+
+  private final JobId jobID;
+  private final HistoryEvent historyEvent;
+
+  public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent) {
+    super(historyEvent.getEventType());
+    this.jobID = jobID;
+    this.historyEvent = historyEvent;
+  }
+
+  public JobId getJobID() {
+    return jobID;
+  }
+
+  public HistoryEvent getHistoryEvent() {
+    return historyEvent;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -0,0 +1,675 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.v2.api.records.Counter;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+/**
+ * The job history events get routed to this class. This class writes the Job
+ * history events to the DFS directly into a staging dir and then moved to a
+ * done-dir. JobHistory implementation is in this package to access package
+ * private classes.
+ */
+public class JobHistoryEventHandler extends AbstractService
+    implements EventHandler<JobHistoryEvent> {
+
+  private final AppContext context;
+  private final int startCount;
+
+  //TODO Does the FS object need to be different ? 
+  private FileSystem stagingDirFS; // log Dir FileSystem
+  private FileSystem doneDirFS; // done Dir FileSystem
+
+  private Configuration conf;
+
+  private Path stagingDirPath = null;
+  private Path doneDirPrefixPath = null; // folder for completed jobs
+
+
+  private BlockingQueue<JobHistoryEvent> eventQueue =
+    new LinkedBlockingQueue<JobHistoryEvent>();
+  private Thread eventHandlingThread;
+  private volatile boolean stopped;
+  private final Object lock = new Object();
+
+  private static final Log LOG = LogFactory.getLog(
+      JobHistoryEventHandler.class);
+
+  private static final Map<JobId, MetaInfo> fileMap =
+    Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
+
+  public JobHistoryEventHandler(AppContext context, int startCount) {
+    super("JobHistoryEventHandler");
+    this.context = context;
+    this.startCount = startCount;
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.hadoop.yarn.service.AbstractService#init(org.apache.hadoop.conf.Configuration)
+   * Initializes the FileSystem and Path objects for the log and done directories.
+   * Creates these directories if they do not already exist.
+   */
+  @Override
+  public void init(Configuration conf) {
+
+    this.conf = conf;
+
+    String stagingDirStr = null;
+    String doneDirStr = null;
+    String userDoneDirStr = null;
+    try {
+      stagingDirStr = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf);
+      doneDirStr =
+          JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+      userDoneDirStr =
+          JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+    } catch (IOException e) {
+      LOG.error("Failed while getting the configured log directories", e);
+      throw new YarnException(e);
+    }
+
+    //Check for the existence of the history staging dir. Maybe create it. 
+    try {
+      stagingDirPath =
+          FileSystem.get(conf).makeQualified(new Path(stagingDirStr));
+      stagingDirFS = FileSystem.get(stagingDirPath.toUri(), conf);
+      mkdir(stagingDirFS, stagingDirPath, new FsPermission(
+          JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
+    } catch (IOException e) {
+      LOG.error("Failed while checking for/creating  history staging path: ["
+          + stagingDirPath + "]", e);
+      throw new YarnException(e);
+    }
+
+    //Check for the existence of intermediate done dir.
+    Path doneDirPath = null;
+    try {
+      doneDirPath = FileSystem.get(conf).makeQualified(new Path(doneDirStr));
+      doneDirFS = FileSystem.get(doneDirPath.toUri(), conf);
+      // This directory will be in a common location, or this may be a cluster
+      // meant for a single user. Creating based on the conf. Should ideally be
+      // created by the JobHistoryServer or as part of deployment.
+      if (!doneDirFS.exists(doneDirPath)) {
+      if (JobHistoryUtils.shouldCreateNonUserDirectory(conf)) {
+        LOG.info("Creating intermediate history logDir: ["
+            + doneDirPath
+            + "] + based on conf. Should ideally be created by the JobHistoryServer: "
+            + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY);
+          mkdir(
+              doneDirFS,
+              doneDirPath,
+              new FsPermission(
+            JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
+                .toShort()));
+          // TODO Temporary toShort till new FsPermission(FsPermissions)
+          // respects
+        // sticky
+      } else {
+          String message = "Not creating intermediate history logDir: ["
+                + doneDirPath
+                + "] based on conf: "
+                + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY
+                + ". Either set to true or pre-create this directory with appropriate permissions";
+        LOG.error(message);
+        throw new YarnException(message);
+      }
+      }
+    } catch (IOException e) {
+      LOG.error("Failed checking for the existance of history intermediate done directory: ["
+          + doneDirPath + "]");
+      throw new YarnException(e);
+    }
+
+    //Check/create user directory under intermediate done dir.
+    try {
+      doneDirPrefixPath =
+          FileSystem.get(conf).makeQualified(new Path(userDoneDirStr));
+      mkdir(doneDirFS, doneDirPrefixPath, new FsPermission(
+          JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS));
+    } catch (IOException e) {
+      LOG.error("Error creating user intermediate history done directory: [ "
+          + doneDirPrefixPath + "]", e);
+      throw new YarnException(e);
+    }
+
+    super.init(conf);
+  }
+
+  private void mkdir(FileSystem fs, Path path, FsPermission fsp)
+      throws IOException {
+    if (!fs.exists(path)) {
+      try {
+        fs.mkdirs(path, fsp);
+        FileStatus fsStatus = fs.getFileStatus(path);
+        LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
+            + ", Expected: " + fsp.toShort());
+        if (fsStatus.getPermission().toShort() != fsp.toShort()) {
+          LOG.info("Explicitly setting permissions to : " + fsp.toShort()
+              + ", " + fsp);
+          fs.setPermission(path, fsp);
+        }
+      } catch (FileAlreadyExistsException e) {
+        LOG.info("Directory: [" + path + "] already exists.");
+      }
+    }
+  }
+
+  @Override
+  public void start() {
+    eventHandlingThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        JobHistoryEvent event = null;
+        while (!stopped && !Thread.currentThread().isInterrupted()) {
+          try {
+            event = eventQueue.take();
+          } catch (InterruptedException e) {
+            LOG.info("EventQueue take interrupted. Returning");
+            return;
+          }
+          // If an event has been removed from the queue. Handle it.
+          // The rest of the queue is handled via stop()
+          // Clear the interrupt status if it's set before calling handleEvent
+          // and set it if it was set before calling handleEvent. 
+          // Interrupts received from other threads during handleEvent cannot be
+          // dealth with - Shell.runCommand() ignores them.
+          synchronized (lock) {
+            boolean isInterrupted = Thread.interrupted();
+            handleEvent(event);
+            if (isInterrupted) {
+              Thread.currentThread().interrupt();
+            }
+          }
+        }
+      }
+    });
+    eventHandlingThread.start();
+    super.start();
+  }
+
+  @Override
+  public void stop() {
+    LOG.info("Stopping JobHistoryEventHandler");
+    stopped = true;
+    //do not interrupt while event handling is in progress
+    synchronized(lock) {
+      eventHandlingThread.interrupt();
+    }
+
+    try {
+      eventHandlingThread.join();
+    } catch (InterruptedException ie) {
+      LOG.info("Interruped Exception while stopping", ie);
+    }
+    //write all the events remaining in queue
+    Iterator<JobHistoryEvent> it = eventQueue.iterator();
+    while(it.hasNext()) {
+      JobHistoryEvent ev = it.next();
+      LOG.info("In stop, writing event " + ev.getType());
+      handleEvent(ev);
+    }
+    
+    //close all file handles
+    for (MetaInfo mi : fileMap.values()) {
+      try {
+        mi.closeWriter();
+      } catch (IOException e) {
+        LOG.info("Exception while closing file " + e.getMessage());
+      }
+    }
+    LOG.info("Stopped JobHistoryEventHandler. super.stop()");
+    super.stop();
+  }
+
+  /**
+   * Create an event writer for the Job represented by the jobID.
+   * Writes out the job configuration to the log directory.
+   * This should be the first call to history for a job
+   * 
+   * @param jobId the jobId.
+   * @throws IOException
+   */
+  protected void setupEventWriter(JobId jobId, JobSubmittedEvent jse)
+      throws IOException {
+    if (stagingDirPath == null) {
+      LOG.error("Log Directory is null, returning");
+      throw new IOException("Missing Log Directory for History");
+    }
+
+    MetaInfo oldFi = fileMap.get(jobId);
+    Configuration conf = getConfig();
+
+    long submitTime = oldFi == null ? jse.getSubmitTime() : oldFi
+        .getJobIndexInfo().getSubmitTime();
+    
+    // TODO Ideally this should be written out to the job dir
+    // (.staging/jobid/files - RecoveryService will need to be patched)
+    Path historyFile = JobHistoryUtils.getStagingJobHistoryFile(
+        stagingDirPath, jobId, startCount);
+    String user = UserGroupInformation.getCurrentUser().getShortUserName();
+    if (user == null) {
+      throw new IOException(
+          "User is null while setting up jobhistory eventwriter");
+    }
+
+    String jobName = context.getJob(jobId).getName();
+    EventWriter writer = (oldFi == null) ? null : oldFi.writer;
+ 
+    if (writer == null) {
+      try {
+        FSDataOutputStream out = stagingDirFS.create(historyFile, true);
+        writer = new EventWriter(out);
+        LOG.info("Event Writer setup for JobId: " + jobId + ", File: "
+            + historyFile);
+      } catch (IOException ioe) {
+        LOG.info("Could not create log file: [" + historyFile + "] + for job "
+            + "[" + jobName + "]");
+        throw ioe;
+      }
+    }
+    
+    Path logDirConfPath = null;
+    if (conf != null) {
+      // TODO Ideally this should be written out to the job dir
+      // (.staging/jobid/files - RecoveryService will need to be patched)
+      logDirConfPath = JobHistoryUtils.getStagingConfFile(stagingDirPath, jobId,
+          startCount);
+      FSDataOutputStream jobFileOut = null;
+      try {
+        if (logDirConfPath != null) {
+          jobFileOut = stagingDirFS.create(logDirConfPath, true);
+          conf.writeXml(jobFileOut);
+          jobFileOut.close();
+        }
+      } catch (IOException e) {
+        LOG.info("Failed to write the job configuration file", e);
+        throw e;
+      }
+    }
+    
+    MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer, submitTime,
+        user, jobName, jobId);
+    fi.getJobSummary().setJobId(jobId);
+    fi.getJobSummary().setJobSubmitTime(submitTime);
+    fileMap.put(jobId, fi);
+  }
+
+  /** Close the event writer for this id 
+   * @throws IOException */
+  public void closeWriter(JobId id) throws IOException {
+    try {
+      final MetaInfo mi = fileMap.get(id);
+      if (mi != null) {
+        mi.closeWriter();
+      }
+      
+    } catch (IOException e) {
+      LOG.error("Error closing writer for JobID: " + id);
+      throw e;
+    }
+  }
+
+  @Override
+  public void handle(JobHistoryEvent event) {
+    try {
+      eventQueue.put(event);
+    } catch (InterruptedException e) {
+      throw new YarnException(e);
+    }
+  }
+
+  protected void handleEvent(JobHistoryEvent event) {
+    synchronized (lock) {
+
+      // If this is JobSubmitted Event, setup the writer
+      if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) {
+        try {
+          JobSubmittedEvent jobSubmittedEvent =
+              (JobSubmittedEvent) event.getHistoryEvent();
+          setupEventWriter(event.getJobID(), jobSubmittedEvent);
+        } catch (IOException ioe) {
+          LOG.error("Error JobHistoryEventHandler in handleEvent: " + event,
+              ioe);
+          throw new YarnException(ioe);
+        }
+      }
+
+      // For all events
+      // (1) Write it out
+      // (2) Process it for JobSummary
+      MetaInfo mi = fileMap.get(event.getJobID());
+      try {
+        HistoryEvent historyEvent = event.getHistoryEvent();
+        mi.writeEvent(historyEvent);
+        processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID());
+        LOG.info("In HistoryEventHandler "
+            + event.getHistoryEvent().getEventType());
+      } catch (IOException e) {
+        LOG.error("Error writing History Event: " + event.getHistoryEvent(),
+            e);
+        throw new YarnException(e);
+      }
+
+      // If this is JobFinishedEvent, close the writer and setup the job-index
+      if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
+        try {
+          JobFinishedEvent jFinishedEvent =
+              (JobFinishedEvent) event.getHistoryEvent();
+          mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime());
+          mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps());
+          mi.getJobIndexInfo().setNumReduces(
+              jFinishedEvent.getFinishedReduces());
+          mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString());
+          closeEventWriter(event.getJobID());
+        } catch (IOException e) {
+          throw new YarnException(e);
+        }
+      }
+
+      if (event.getHistoryEvent().getEventType() == EventType.JOB_FAILED
+          || event.getHistoryEvent().getEventType() == EventType.JOB_KILLED) {
+        try {
+          JobUnsuccessfulCompletionEvent jucEvent = (JobUnsuccessfulCompletionEvent) event
+              .getHistoryEvent();
+          mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime());
+          mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps());
+          mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces());
+          mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus());
+          closeEventWriter(event.getJobID());
+        } catch (IOException e) {
+          throw new YarnException(e);
+        }
+      }
+    }
+  }
+
+  private void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId) {
+    // context.getJob could be used for some of this info as well.
+    switch (event.getEventType()) {
+    case JOB_SUBMITTED:
+      JobSubmittedEvent jse = (JobSubmittedEvent) event;
+      summary.setUser(jse.getUserName());
+      summary.setQueue(jse.getJobQueueName());
+      break;
+    case JOB_INITED:
+      JobInitedEvent jie = (JobInitedEvent) event;
+      summary.setJobLaunchTime(jie.getLaunchTime());
+      break;
+    case MAP_ATTEMPT_STARTED:
+      TaskAttemptStartedEvent mtase = (TaskAttemptStartedEvent) event;
+      if (summary.getFirstMapTaskLaunchTime() == 0)
+        summary.setFirstMapTaskLaunchTime(mtase.getStartTime());
+      break;
+    case REDUCE_ATTEMPT_STARTED:
+      TaskAttemptStartedEvent rtase = (TaskAttemptStartedEvent) event;
+      if (summary.getFirstReduceTaskLaunchTime() == 0)
+        summary.setFirstReduceTaskLaunchTime(rtase.getStartTime());
+      break;
+    case JOB_FINISHED:
+      JobFinishedEvent jfe = (JobFinishedEvent) event;
+      summary.setJobFinishTime(jfe.getFinishTime());
+      summary.setNumFinishedMaps(jfe.getFinishedMaps());
+      summary.setNumFailedMaps(jfe.getFailedMaps());
+      summary.setNumFinishedReduces(jfe.getFinishedReduces());
+      summary.setNumFailedReduces(jfe.getFailedReduces());
+      if (summary.getJobStatus() == null)
+        summary
+            .setJobStatus(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED
+                .toString());
+      // TODO JOB_FINISHED does not have state. Effectively job history does not
+      // have state about the finished job.
+      setSummarySlotSeconds(summary, jobId);
+      break;
+    case JOB_FAILED:
+    case JOB_KILLED:
+      JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
+      summary.setJobStatus(juce.getStatus());
+      summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
+      summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
+      summary.setJobFinishTime(juce.getFinishTime());
+      setSummarySlotSeconds(summary, jobId);
+      break;
+    }
+  }
+
+  private void setSummarySlotSeconds(JobSummary summary, JobId jobId) {
+    Counter slotMillisMapCounter =
+        context.getJob(jobId).getCounters()
+            .getCounter(JobCounter.SLOTS_MILLIS_MAPS);
+    if (slotMillisMapCounter != null) {
+      summary.setMapSlotSeconds(slotMillisMapCounter.getValue());
+    }
+    Counter slotMillisReduceCounter =
+        context.getJob(jobId).getCounters()
+            .getCounter(JobCounter.SLOTS_MILLIS_REDUCES);
+    if (slotMillisReduceCounter != null) {
+      summary.setMapSlotSeconds(slotMillisReduceCounter.getValue());
+    }
+  }
+
+  protected void closeEventWriter(JobId jobId) throws IOException {
+
+    final MetaInfo mi = fileMap.get(jobId);
+    if (mi == null) {
+      throw new IOException("No MetaInfo found for JobId: [" + jobId + "]");
+    }
+
+    if (!mi.isWriterActive()) {
+      throw new IOException(
+          "Inactive Writer: Likely received multiple JobFinished / JobUnsuccessful events for JobId: ["
+              + jobId + "]");
+    }
+
+    // Close the Writer
+    try {
+      mi.closeWriter();
+    } catch (IOException e) {
+      LOG.error("Error closing writer for JobID: " + jobId);
+      throw e;
+    }
+     
+    if (mi.getHistoryFile() == null) {
+      LOG.warn("No file for job-history with " + jobId + " found in cache!");
+    }
+    if (mi.getConfFile() == null) {
+      LOG.warn("No file for jobconf with " + jobId + " found in cache!");
+    }
+      
+    // Writing out the summary file.
+    // TODO JH enhancement - reuse this file to store additional indexing info
+    // like ACLs, etc. JHServer can use HDFS append to build an index file
+    // with more info than is available via the filename.
+    Path qualifiedSummaryDoneFile = null;
+    FSDataOutputStream summaryFileOut = null;
+    try {
+      String doneSummaryFileName = getTempFileName(JobHistoryUtils
+          .getIntermediateSummaryFileName(jobId));
+      qualifiedSummaryDoneFile = doneDirFS.makeQualified(new Path(
+          doneDirPrefixPath, doneSummaryFileName));
+      summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true);
+      summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
+      summaryFileOut.close();
+    } catch (IOException e) {
+      LOG.info("Unable to write out JobSummaryInfo to ["
+          + qualifiedSummaryDoneFile + "]", e);
+      throw e;
+    }
+
+    try {
+
+      // Move historyFile to Done Folder.
+      Path qualifiedDoneFile = null;
+      if (mi.getHistoryFile() != null) {
+        Path historyFile = mi.getHistoryFile();
+        Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
+        String doneJobHistoryFileName =
+            getTempFileName(FileNameIndexUtils.getDoneFileName(mi
+                .getJobIndexInfo()));
+        qualifiedDoneFile =
+            doneDirFS.makeQualified(new Path(doneDirPrefixPath,
+                doneJobHistoryFileName));
+        moveToDoneNow(qualifiedLogFile, qualifiedDoneFile);
+      }
+
+      // Move confFile to Done Folder
+      Path qualifiedConfDoneFile = null;
+      if (mi.getConfFile() != null) {
+        Path confFile = mi.getConfFile();
+        Path qualifiedConfFile = stagingDirFS.makeQualified(confFile);
+        String doneConfFileName =
+            getTempFileName(JobHistoryUtils
+                .getIntermediateConfFileName(jobId));
+        qualifiedConfDoneFile =
+            doneDirFS.makeQualified(new Path(doneDirPrefixPath,
+                doneConfFileName));
+        moveToDoneNow(qualifiedConfFile, qualifiedConfDoneFile);
+      }
+      
+      moveTmpToDone(qualifiedSummaryDoneFile);
+      moveTmpToDone(qualifiedConfDoneFile);
+      moveTmpToDone(qualifiedDoneFile);
+
+    } catch (IOException e) {
+      LOG.error("Error closing writer for JobID: " + jobId);
+      throw e;
+    }
+  }
+
+  private class MetaInfo {
+    private Path historyFile;
+    private Path confFile;
+    private EventWriter writer;
+    JobIndexInfo jobIndexInfo;
+    JobSummary jobSummary;
+
+    MetaInfo(Path historyFile, Path conf, EventWriter writer, long submitTime,
+             String user, String jobName, JobId jobId) {
+      this.historyFile = historyFile;
+      this.confFile = conf;
+      this.writer = writer;
+      this.jobIndexInfo = new JobIndexInfo(submitTime, -1, user, jobName, jobId, -1, -1, null);
+      this.jobSummary = new JobSummary();
+    }
+
+    Path getHistoryFile() { return historyFile; }
+
+    Path getConfFile() {return confFile; } 
+
+    JobIndexInfo getJobIndexInfo() { return jobIndexInfo; }
+
+    JobSummary getJobSummary() { return jobSummary; }
+
+    boolean isWriterActive() {return writer != null ; }
+
+    void closeWriter() throws IOException {
+      synchronized (lock) {
+      if (writer != null) {
+        writer.close();
+      }
+      writer = null;
+    }
+    }
+
+    void writeEvent(HistoryEvent event) throws IOException {
+      synchronized (lock) {
+      if (writer != null) {
+        writer.write(event);
+        writer.flush();
+      }
+    }
+  }
+  }
+
+  private void moveTmpToDone(Path tmpPath) throws IOException {
+    if (tmpPath != null) {
+      String tmpFileName = tmpPath.getName();
+      String fileName = getFileNameFromTmpFN(tmpFileName);
+      Path path = new Path(tmpPath.getParent(), fileName);
+      doneDirFS.rename(tmpPath, path);
+      LOG.info("Moved tmp to done: " + tmpPath + " to " + path);
+    }
+  }
+  
+  // TODO If the FS objects are the same, this should be a rename instead of a
+  // copy.
+  private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
+    // check if path exists, in case of retries it may not exist
+    if (stagingDirFS.exists(fromPath)) {
+      LOG.info("Moving " + fromPath.toString() + " to " + toPath.toString());
+      // TODO temporarily removing the existing dst
+      if (doneDirFS.exists(toPath)) {
+        doneDirFS.delete(toPath, true);
+      }
+      boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
+          false, conf);
+
+      if (copied)
+        LOG.info("Copied to done location: " + toPath);
+      else 
+          LOG.info("copy failed");
+      doneDirFS.setPermission(toPath, new FsPermission(
+          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+      
+      stagingDirFS.delete(fromPath, false);
+    }
+    }
+
+  boolean pathExists(FileSystem fileSys, Path path) throws IOException {
+    return fileSys.exists(path);
+  }
+
+  private String getTempFileName(String srcFile) {
+    return srcFile + "_tmp";
+  }
+  
+  private String getFileNameFromTmpFN(String tmpFileName) {
+    //TODO. Some error checking here.
+    return tmpFileName.substring(0, tmpFileName.length()-4);
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java

@@ -0,0 +1,231 @@
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.util.StringUtils;
+
+public class JobSummary {
+  private JobId jobId;
+  private long jobSubmitTime;
+  private long jobLaunchTime;
+  private long firstMapTaskLaunchTime; // MapAttempteStarted |
+                                       // TaskAttemptStartEvent
+  private long firstReduceTaskLaunchTime; // ReduceAttemptStarted |
+                                          // TaskAttemptStartEvent
+  private long jobFinishTime;
+  private int numFinishedMaps;
+  private int numFailedMaps;
+  private int numFinishedReduces;
+  private int numFailedReduces;
+  // private int numSlotsPerMap; | Doesn't make sense with potentially different
+  // resource models
+  // private int numSlotsPerReduce; | Doesn't make sense with potentially
+  // different resource models
+  private String user;
+  private String queue;
+  private String jobStatus;
+  private long mapSlotSeconds; // TODO Not generated yet in MRV2
+  private long reduceSlotSeconds; // TODO Not generated yet MRV2
+  // private int clusterSlotCapacity;
+
+  JobSummary() {
+  }
+
+  public JobId getJobId() {
+    return jobId;
+  }
+
+  public void setJobId(JobId jobId) {
+    this.jobId = jobId;
+  }
+
+  public long getJobSubmitTime() {
+    return jobSubmitTime;
+  }
+
+  public void setJobSubmitTime(long jobSubmitTime) {
+    this.jobSubmitTime = jobSubmitTime;
+  }
+
+  public long getJobLaunchTime() {
+    return jobLaunchTime;
+  }
+
+  public void setJobLaunchTime(long jobLaunchTime) {
+    this.jobLaunchTime = jobLaunchTime;
+  }
+
+  public long getFirstMapTaskLaunchTime() {
+    return firstMapTaskLaunchTime;
+  }
+
+  public void setFirstMapTaskLaunchTime(long firstMapTaskLaunchTime) {
+    this.firstMapTaskLaunchTime = firstMapTaskLaunchTime;
+  }
+
+  public long getFirstReduceTaskLaunchTime() {
+    return firstReduceTaskLaunchTime;
+  }
+
+  public void setFirstReduceTaskLaunchTime(long firstReduceTaskLaunchTime) {
+    this.firstReduceTaskLaunchTime = firstReduceTaskLaunchTime;
+  }
+
+  public long getJobFinishTime() {
+    return jobFinishTime;
+  }
+
+  public void setJobFinishTime(long jobFinishTime) {
+    this.jobFinishTime = jobFinishTime;
+  }
+
+  public int getNumFinishedMaps() {
+    return numFinishedMaps;
+  }
+
+  public void setNumFinishedMaps(int numFinishedMaps) {
+    this.numFinishedMaps = numFinishedMaps;
+  }
+
+  public int getNumFailedMaps() {
+    return numFailedMaps;
+  }
+
+  public void setNumFailedMaps(int numFailedMaps) {
+    this.numFailedMaps = numFailedMaps;
+  }
+
+  // public int getNumSlotsPerMap() {
+  // return numSlotsPerMap;
+  // }
+  //
+  // public void setNumSlotsPerMap(int numSlotsPerMap) {
+  // this.numSlotsPerMap = numSlotsPerMap;
+  // }
+
+  public int getNumFinishedReduces() {
+    return numFinishedReduces;
+  }
+
+  public void setNumFinishedReduces(int numFinishedReduces) {
+    this.numFinishedReduces = numFinishedReduces;
+  }
+
+  public int getNumFailedReduces() {
+    return numFailedReduces;
+  }
+
+  public void setNumFailedReduces(int numFailedReduces) {
+    this.numFailedReduces = numFailedReduces;
+  }
+
+  // public int getNumSlotsPerReduce() {
+  // return numSlotsPerReduce;
+  // }
+  //
+  // public void setNumSlotsPerReduce(int numSlotsPerReduce) {
+  // this.numSlotsPerReduce = numSlotsPerReduce;
+  // }
+
+  public String getUser() {
+    return user;
+  }
+
+  public void setUser(String user) {
+    this.user = user;
+  }
+
+  public String getQueue() {
+    return queue;
+  }
+
+  public void setQueue(String queue) {
+    this.queue = queue;
+  }
+
+  public String getJobStatus() {
+    return jobStatus;
+  }
+
+  public void setJobStatus(String jobStatus) {
+    this.jobStatus = jobStatus;
+  }
+
+  public long getMapSlotSeconds() {
+    return mapSlotSeconds;
+  }
+
+  public void setMapSlotSeconds(long mapSlotSeconds) {
+    this.mapSlotSeconds = mapSlotSeconds;
+  }
+
+  public long getReduceSlotSeconds() {
+    return reduceSlotSeconds;
+  }
+
+  public void setReduceSlotSeconds(long reduceSlotSeconds) {
+    this.reduceSlotSeconds = reduceSlotSeconds;
+  }
+
+  // public int getClusterSlotCapacity() {
+  // return clusterSlotCapacity;
+  // }
+  //
+  // public void setClusterSlotCapacity(int clusterSlotCapacity) {
+  // this.clusterSlotCapacity = clusterSlotCapacity;
+  // }
+
+  public String getJobSummaryString() {
+    SummaryBuilder summary = new SummaryBuilder()
+      .add("jobId", jobId)
+      .add("submitTime", jobSubmitTime)
+      .add("launchTime", jobLaunchTime)
+      .add("firstMapTaskLaunchTime", firstMapTaskLaunchTime)
+      .add("firstReduceTaskLaunchTime", firstReduceTaskLaunchTime)
+      .add("finishTime", jobFinishTime)
+      .add("numMaps", numFinishedMaps + numFailedMaps)
+      .add("numReduces", numFinishedReduces + numFailedReduces)
+      .add("user", user)
+      .add("queue", queue)
+      .add("status", jobStatus)
+      .add("mapSlotSeconds", mapSlotSeconds)
+      .add("reduceSlotSeconds", reduceSlotSeconds);
+    return summary.toString();
+  }
+
+  static final char EQUALS = '=';
+  static final char[] charsToEscape = { StringUtils.COMMA, EQUALS,
+      StringUtils.ESCAPE_CHAR };
+  
+  static class SummaryBuilder {
+    final StringBuilder buffer = new StringBuilder();
+
+    // A little optimization for a very common case
+    SummaryBuilder add(String key, long value) {
+      return _add(key, Long.toString(value));
+    }
+
+    <T> SummaryBuilder add(String key, T value) {
+      return _add(key, StringUtils.escapeString(String.valueOf(value),
+          StringUtils.ESCAPE_CHAR, charsToEscape));
+    }
+
+    SummaryBuilder add(SummaryBuilder summary) {
+      if (buffer.length() > 0)
+        buffer.append(StringUtils.COMMA);
+      buffer.append(summary.buffer);
+      return this;
+    }
+
+    SummaryBuilder _add(String key, String value) {
+      if (buffer.length() > 0)
+        buffer.append(StringUtils.COMMA);
+      buffer.append(key).append(EQUALS).append(value);
+      return this;
+    }
+
+    @Override
+    public String toString() {
+      return buffer.toString();
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AMConstants.java

@@ -0,0 +1,74 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+
+public interface AMConstants {
+
+  public static final String CONTAINERLAUNCHER_THREADPOOL_SIZE =
+      "yarn.mapreduce.containerlauncher.threadpool-size";
+
+  public static final String AM_RM_SCHEDULE_INTERVAL =
+      "yarn.appMaster.scheduler.interval";
+
+  public static final int DEFAULT_AM_RM_SCHEDULE_INTERVAL = 2000;
+
+  public static final String AM_TASK_LISTENER_THREADS =
+      MRConstants.YARN_MR_PREFIX + "task.listener.threads";
+
+  public static final int DEFAULT_AM_TASK_LISTENER_THREADS = 10;
+
+  public static final String AM_JOB_CLIENT_THREADS =
+      MRConstants.YARN_MR_PREFIX + "job.client.threads";
+
+  public static final int DEFAULT_AM_JOB_CLIENT_THREADS = 1;
+
+  public static final String SPECULATOR_CLASS =
+      MRConstants.YARN_MR_PREFIX + "speculator.class";
+
+  public static final String TASK_RUNTIME_ESTIMATOR_CLASS =
+      MRConstants.YARN_MR_PREFIX + "task.runtime.estimator.class";
+
+  public static final String TASK_ATTEMPT_PROGRESS_RUNTIME_LINEARIZER_CLASS =
+      MRConstants.YARN_MR_PREFIX + "task.runtime.linearizer.class";
+
+  public static final String EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS =
+      MRConstants.YARN_MR_PREFIX
+          + "task.runtime.estimator.exponential.smooth.lambda";
+
+  public static final String EXPONENTIAL_SMOOTHING_SMOOTH_RATE =
+      MRConstants.YARN_MR_PREFIX
+          + "task.runtime.estimator.exponential.smooth.smoothsrate";
+
+  public static final String RECOVERY_ENABLE = MRConstants.YARN_MR_PREFIX
+      + "recovery.enable";
+  
+  public static final float DEFAULT_REDUCE_RAMP_UP_LIMIT = 0.5f;
+  public static final String REDUCE_RAMPUP_UP_LIMIT = MRConstants.YARN_MR_PREFIX
+  + "reduce.rampup.limit";
+  
+  public static final float DEFAULT_REDUCE_PREEMPTION_LIMIT = 0.5f;
+  public static final String REDUCE_PREEMPTION_LIMIT = MRConstants.YARN_MR_PREFIX
+  + "reduce.preemption.limit";
+
+  public static final String NODE_BLACKLISTING_ENABLE = MRConstants.YARN_MR_PREFIX
+  + "node.blacklisting.enable";
+  
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java

@@ -0,0 +1,55 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+
+/**
+ * Context interface for sharing information across components in YARN App.
+ */
+@InterfaceAudience.Private
+public interface AppContext {
+
+  ApplicationId getApplicationID();
+
+  ApplicationAttemptId getApplicationAttemptId();
+
+  String getApplicationName();
+
+  long getStartTime();
+
+  CharSequence getUser();
+
+  Job getJob(JobId jobID);
+
+  Map<JobId, Job> getAllJobs();
+
+  EventHandler getEventHandler();
+
+  Clock getClock();
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java

@@ -0,0 +1,576 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.LocalContainerLauncher;
+import org.apache.hadoop.mapred.TaskAttemptListenerImpl;
+import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.client.MRClientService;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl;
+import org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
+import org.apache.hadoop.mapreduce.v2.app.recover.Recovery;
+import org.apache.hadoop.mapreduce.v2.app.recover.RecoveryService;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.speculate.DefaultSpeculator;
+import org.apache.hadoop.mapreduce.v2.app.speculate.Speculator;
+import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
+import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleaner;
+import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanerImpl;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.SystemClock;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.service.CompositeService;
+import org.apache.hadoop.yarn.service.Service;
+
+/**
+ * The Map-Reduce Application Master.
+ * The state machine is encapsulated in the implementation of Job interface.
+ * All state changes happens via Job interface. Each event 
+ * results in a Finite State Transition in Job.
+ * 
+ * MR AppMaster is the composition of loosely coupled services. The services 
+ * interact with each other via events. The components resembles the 
+ * Actors model. The component acts on received event and send out the 
+ * events to other components.
+ * This keeps it highly concurrent with no or minimal synchronization needs.
+ * 
+ * The events are dispatched by a central Dispatch mechanism. All components
+ * register to the Dispatcher.
+ * 
+ * The information is shared across different components using AppContext.
+ */
+
+public class MRAppMaster extends CompositeService {
+
+  private static final Log LOG = LogFactory.getLog(MRAppMaster.class);
+
+  private Clock clock;
+  private final long startTime = System.currentTimeMillis();
+  private String appName;
+  private final int startCount;
+  private final ApplicationId appID;
+  private final ApplicationAttemptId appAttemptID;
+  protected final MRAppMetrics metrics;
+  private Set<TaskId> completedTasksFromPreviousRun;
+  private AppContext context;
+  private Dispatcher dispatcher;
+  private ClientService clientService;
+  private ContainerAllocator containerAllocator;
+  private ContainerLauncher containerLauncher;
+  private TaskCleaner taskCleaner;
+  private Speculator speculator;
+  private TaskAttemptListener taskAttemptListener;
+  private JobTokenSecretManager jobTokenSecretManager =
+      new JobTokenSecretManager();
+
+  private Job job;
+  
+  public MRAppMaster(ApplicationId applicationId, int startCount) {
+    this(applicationId, new SystemClock(), startCount);
+  }
+
+  public MRAppMaster(ApplicationId applicationId, Clock clock, int startCount) {
+    super(MRAppMaster.class.getName());
+    this.clock = clock;
+    this.appID = applicationId;
+    this.appAttemptID = RecordFactoryProvider.getRecordFactory(null)
+        .newRecordInstance(ApplicationAttemptId.class);
+    this.appAttemptID.setApplicationId(appID);
+    this.appAttemptID.setAttemptId(startCount);
+    this.startCount = startCount;
+    this.metrics = MRAppMetrics.create();
+    LOG.info("Created MRAppMaster for application " + applicationId);
+  }
+
+  @Override
+  public void init(final Configuration conf) {
+    context = new RunningAppContext();
+
+    // Job name is the same as the app name util we support DAG of jobs
+    // for an app later
+    appName = conf.get(MRJobConfig.JOB_NAME, "<missing app name>");
+
+    if (conf.getBoolean(AMConstants.RECOVERY_ENABLE, false)
+         && startCount > 1) {
+      LOG.info("Recovery is enabled. Will try to recover from previous life.");
+      Recovery recoveryServ = new RecoveryService(appID, clock, startCount);
+      addIfService(recoveryServ);
+      dispatcher = recoveryServ.getDispatcher();
+      clock = recoveryServ.getClock();
+      completedTasksFromPreviousRun = recoveryServ.getCompletedTasks();
+    } else {
+      dispatcher = new AsyncDispatcher();
+      addIfService(dispatcher);
+    }
+
+    //service to handle requests to TaskUmbilicalProtocol
+    taskAttemptListener = createTaskAttemptListener(context);
+    addIfService(taskAttemptListener);
+
+    //service to do the task cleanup
+    taskCleaner = createTaskCleaner(context);
+    addIfService(taskCleaner);
+
+    //service to handle requests from JobClient
+    clientService = createClientService(context);
+    addIfService(clientService);
+
+    //service to log job history events
+    EventHandler<JobHistoryEvent> historyService = 
+        createJobHistoryHandler(context);
+    addIfService(historyService);
+
+    JobEventDispatcher synchronousJobEventDispatcher = new JobEventDispatcher();
+
+    //register the event dispatchers
+    dispatcher.register(JobEventType.class, synchronousJobEventDispatcher);
+    dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
+    dispatcher.register(TaskAttemptEventType.class, 
+        new TaskAttemptEventDispatcher());
+    dispatcher.register(TaskCleaner.EventType.class, taskCleaner);
+    dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class,
+        historyService);
+    
+    if (conf.getBoolean(MRJobConfig.MAP_SPECULATIVE, false)
+        || conf.getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false)) {
+      //optional service to speculate on task attempts' progress
+      speculator = createSpeculator(conf, context);
+      addIfService(speculator);
+    }
+
+    dispatcher.register(Speculator.EventType.class,
+        new SpeculatorEventDispatcher());
+
+    Credentials fsTokens = new Credentials();
+    if (UserGroupInformation.isSecurityEnabled()) {
+      // Read the file-system tokens from the localized tokens-file.
+      try {
+        Path jobSubmitDir =
+            FileContext.getLocalFSFileContext().makeQualified(
+                new Path(new File(MRConstants.JOB_SUBMIT_DIR)
+                    .getAbsolutePath()));
+        Path jobTokenFile =
+            new Path(jobSubmitDir, MRConstants.APPLICATION_TOKENS_FILE);
+        fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
+        LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
+            + jobTokenFile);
+
+        UserGroupInformation currentUser =
+            UserGroupInformation.getCurrentUser();
+        for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) {
+          LOG.info(" --- DEBUG: Token of kind " + tk.getKind()
+              + "in current ugi in the AppMaster for service "
+              + tk.getService());
+          currentUser.addToken(tk); // For use by AppMaster itself.
+        }
+      } catch (IOException e) {
+        throw new YarnException(e);
+      }
+    }
+
+    super.init(conf);
+
+    //---- start of what used to be startJobs() code:
+
+    Configuration config = getConfig();
+
+    job = createJob(config, fsTokens);
+
+    /** create a job event for job intialization */
+    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
+    /** send init to the job (this does NOT trigger job execution) */
+    synchronousJobEventDispatcher.handle(initJobEvent);
+
+    // send init to speculator. This won't yest start as dispatcher isn't
+    // started yet.
+    dispatcher.getEventHandler().handle(
+        new SpeculatorEvent(job.getID(), clock.getTime()));
+
+    // JobImpl's InitTransition is done (call above is synchronous), so the
+    // "uber-decision" (MR-1220) has been made.  Query job and switch to
+    // ubermode if appropriate (by registering different container-allocator
+    // and container-launcher services/event-handlers).
+
+    if (job.isUber()) {
+      LOG.info("MRAppMaster uberizing job " + job.getID()
+               + " in local container (\"uber-AM\").");
+    } else {
+      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
+               + "job " + job.getID() + ".");
+    }
+
+    // service to allocate containers from RM (if non-uber) or to fake it (uber)
+    containerAllocator =
+        createContainerAllocator(clientService, context, job.isUber());
+    addIfService(containerAllocator);
+    dispatcher.register(ContainerAllocator.EventType.class, containerAllocator);
+    if (containerAllocator instanceof Service) {
+      ((Service) containerAllocator).init(config);
+    }
+
+    // corresponding service to launch allocated containers via NodeManager
+    containerLauncher = createContainerLauncher(context, job.isUber());
+    addIfService(containerLauncher);
+    dispatcher.register(ContainerLauncher.EventType.class, containerLauncher);
+    if (containerLauncher instanceof Service) {
+      ((Service) containerLauncher).init(config);
+    }
+
+  } // end of init()
+
+  /** Create and initialize (but don't start) a single job. 
+   * @param fsTokens */
+  protected Job createJob(Configuration conf, Credentials fsTokens) {
+
+    // create single job
+    Job newJob = new JobImpl(appID, conf, dispatcher.getEventHandler(),
+        taskAttemptListener, jobTokenSecretManager, fsTokens, clock, startCount,
+        completedTasksFromPreviousRun, metrics);
+    ((RunningAppContext) context).jobs.put(newJob.getID(), newJob);
+
+    dispatcher.register(JobFinishEvent.Type.class,
+        new EventHandler<JobFinishEvent>() {
+          @Override
+          public void handle(JobFinishEvent event) {
+            // job has finished
+            // this is the only job, so shut down the Appmaster
+            // note in a workflow scenario, this may lead to creation of a new
+            // job (FIXME?)
+
+            // TODO:currently just wait for some time so clients can know the
+            // final states. Will be removed once RM come on.
+            try {
+              Thread.sleep(5000);
+            } catch (InterruptedException e) {
+              e.printStackTrace();
+            }
+            LOG.info("Calling stop for all the services");
+            try {
+              stop();
+            } catch (Throwable t) {
+              LOG.warn("Graceful stop failed ", t);
+            }
+            //TODO: this is required because rpc server does not shut down
+            // in spite of calling server.stop().
+            //Bring the process down by force.
+            //Not needed after HADOOP-7140
+            LOG.info("Exiting MR AppMaster..GoodBye!");
+            System.exit(0);
+          }
+        });
+
+    return newJob;
+  } // end createJob()
+
+  protected void addIfService(Object object) {
+    if (object instanceof Service) {
+      addService((Service) object);
+    }
+  }
+
+  protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
+      AppContext context) {
+    JobHistoryEventHandler eventHandler = new JobHistoryEventHandler(context, 
+        getStartCount());
+    return eventHandler;
+  }
+
+  protected Speculator createSpeculator(Configuration conf, AppContext context) {
+    Class<? extends Speculator> speculatorClass;
+
+    try {
+      speculatorClass
+          // "yarn.mapreduce.job.speculator.class"
+          = conf.getClass(AMConstants.SPECULATOR_CLASS,
+                          DefaultSpeculator.class,
+                          Speculator.class);
+      Constructor<? extends Speculator> speculatorConstructor
+          = speculatorClass.getConstructor
+               (Configuration.class, AppContext.class);
+      Speculator result = speculatorConstructor.newInstance(conf, context);
+
+      return result;
+    } catch (InstantiationException ex) {
+      LOG.error("Can't make a speculator -- check "
+          + AMConstants.SPECULATOR_CLASS + " " + ex);
+      throw new YarnException(ex);
+    } catch (IllegalAccessException ex) {
+      LOG.error("Can't make a speculator -- check "
+          + AMConstants.SPECULATOR_CLASS + " " + ex);
+      throw new YarnException(ex);
+    } catch (InvocationTargetException ex) {
+      LOG.error("Can't make a speculator -- check "
+          + AMConstants.SPECULATOR_CLASS + " " + ex);
+      throw new YarnException(ex);
+    } catch (NoSuchMethodException ex) {
+      LOG.error("Can't make a speculator -- check "
+          + AMConstants.SPECULATOR_CLASS + " " + ex);
+      throw new YarnException(ex);
+    }
+  }
+
+  protected TaskAttemptListener createTaskAttemptListener(AppContext context) {
+    TaskAttemptListener lis =
+        new TaskAttemptListenerImpl(context, jobTokenSecretManager);
+    return lis;
+  }
+
+  protected TaskCleaner createTaskCleaner(AppContext context) {
+    return new TaskCleanerImpl(context);
+  }
+
+  protected ContainerAllocator createContainerAllocator(
+      ClientService clientService, AppContext context, boolean isLocal) {
+    //return new StaticContainerAllocator(context);
+    return isLocal
+        ? new LocalContainerAllocator(clientService, context)
+        : new RMContainerAllocator(clientService, context);
+  }
+
+  protected ContainerLauncher createContainerLauncher(AppContext context,
+                                                      boolean isLocal) {
+    return isLocal
+        ? new LocalContainerLauncher(context,
+            (TaskUmbilicalProtocol) taskAttemptListener)
+        : new ContainerLauncherImpl(context);
+  }
+
+  //TODO:should have an interface for MRClientService
+  protected ClientService createClientService(AppContext context) {
+    return new MRClientService(context);
+  }
+
+  public ApplicationId getAppID() {
+    return appID;
+  }
+
+  public int getStartCount() {
+    return startCount;
+  }
+
+  public AppContext getContext() {
+    return context;
+  }
+
+  public Dispatcher getDispatcher() {
+    return dispatcher;
+  }
+
+  public Set<TaskId> getCompletedTaskFromPreviousRun() {
+    return completedTasksFromPreviousRun;
+  }
+
+  public ContainerAllocator getContainerAllocator() {
+    return containerAllocator;
+  }
+  
+  public ContainerLauncher getContainerLauncher() {
+    return containerLauncher;
+  }
+
+  public TaskAttemptListener getTaskAttemptListener() {
+    return taskAttemptListener;
+  }
+
+  class RunningAppContext implements AppContext {
+
+    private Map<JobId, Job> jobs = new ConcurrentHashMap<JobId, Job>();
+
+    @Override
+    public ApplicationAttemptId getApplicationAttemptId() {
+      return appAttemptID;
+    }
+
+    @Override
+    public ApplicationId getApplicationID() {
+      return appID;
+    }
+
+    @Override
+    public String getApplicationName() {
+      return appName;
+    }
+
+    @Override
+    public long getStartTime() {
+      return startTime;
+    }
+
+    @Override
+    public Job getJob(JobId jobID) {
+      return jobs.get(jobID);
+    }
+
+    @Override
+    public Map<JobId, Job> getAllJobs() {
+      return jobs;
+    }
+
+    @Override
+    public EventHandler getEventHandler() {
+      return dispatcher.getEventHandler();
+    }
+
+    @Override
+    public CharSequence getUser() {
+      return getConfig().get(MRJobConfig.USER_NAME);
+    }
+
+    @Override
+    public Clock getClock() {
+      return clock;
+    }
+  }
+
+  @Override
+  public void start() {
+    // metrics system init is really init & start.
+    // It's more test friendly to put it here.
+    DefaultMetricsSystem.initialize("MRAppMaster");
+
+    startJobs();
+    //start all the components
+    super.start();
+  }
+
+  /**
+   * This can be overridden to instantiate multiple jobs and create a 
+   * workflow.
+   *
+   * TODO:  Rework the design to actually support this.  Currently much of the
+   * job stuff has been moved to init() above to support uberization (MR-1220).
+   * In a typical workflow, one presumably would want to uberize only a subset
+   * of the jobs (the "small" ones), which is awkward with the current design.
+   */
+  protected void startJobs() {
+    /** create a job-start event to get this ball rolling */
+    JobEvent startJobEvent = new JobEvent(job.getID(), JobEventType.JOB_START);
+    /** send the job-start event. this triggers the job execution. */
+    dispatcher.getEventHandler().handle(startJobEvent);
+  }
+
+  private class JobEventDispatcher implements EventHandler<JobEvent> {
+    @Override
+    public void handle(JobEvent event) {
+      ((EventHandler<JobEvent>)context.getJob(event.getJobId())).handle(event);
+    }
+  }
+
+  private class TaskEventDispatcher implements EventHandler<TaskEvent> {
+    @Override
+    public void handle(TaskEvent event) {
+      Task task = context.getJob(event.getTaskID().getJobId()).getTask(
+          event.getTaskID());
+      ((EventHandler<TaskEvent>)task).handle(event);
+    }
+  }
+
+  private class TaskAttemptEventDispatcher 
+          implements EventHandler<TaskAttemptEvent> {
+    @Override
+    public void handle(TaskAttemptEvent event) {
+      Job job = context.getJob(event.getTaskAttemptID().getTaskId().getJobId());
+      Task task = job.getTask(event.getTaskAttemptID().getTaskId());
+      TaskAttempt attempt = task.getAttempt(event.getTaskAttemptID());
+      ((EventHandler<TaskAttemptEvent>) attempt).handle(event);
+    }
+  }
+
+  private class SpeculatorEventDispatcher implements
+      EventHandler<SpeculatorEvent> {
+    @Override
+    public void handle(SpeculatorEvent event) {
+      if (getConfig().getBoolean(MRJobConfig.MAP_SPECULATIVE, false)
+          || getConfig().getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false)) {
+        // Speculator IS enabled, direct the event to there.
+        speculator.handle(event);
+      }
+    }
+  }
+
+  public static void main(String[] args) {
+    try {
+      //Configuration.addDefaultResource("job.xml");
+      ApplicationId applicationId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class);
+      
+      applicationId.setClusterTimestamp(Long.valueOf(args[0]));
+      applicationId.setId(Integer.valueOf(args[1]));
+      int failCount = Integer.valueOf(args[2]);
+      MRAppMaster appMaster = new MRAppMaster(applicationId, failCount);
+      YarnConfiguration conf = new YarnConfiguration(new JobConf());
+      conf.addResource(new Path(MRConstants.JOB_CONF_FILE));
+      conf.set(MRJobConfig.USER_NAME, 
+          System.getProperty("user.name")); 
+      UserGroupInformation.setConfiguration(conf);
+      appMaster.init(conf);
+      appMaster.start();
+    } catch (Throwable t) {
+      LOG.error("Caught throwable. Exiting:", t);
+      System.exit(1);
+    }
+  } 
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRClientSecurityInfo.java

@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import java.lang.annotation.Annotation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.yarn.proto.MRClientProtocol;
+import org.apache.hadoop.yarn.security.ApplicationTokenSelector;
+
+public class MRClientSecurityInfo extends SecurityInfo {
+
+  @Override
+  public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
+    return null;
+  }
+
+  @Override
+  public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
+    if (!protocol.equals(MRClientProtocol.MRClientProtocolService.BlockingInterface.class)) {
+      return null;
+    }
+    return new TokenInfo() {
+
+      @Override
+      public Class<? extends Annotation> annotationType() {
+        return null;
+      }
+
+      @Override
+      public Class<? extends TokenSelector<? extends TokenIdentifier>>
+          value() {
+        return ApplicationTokenSelector.class;
+      }
+    };
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskAttemptListener.java

@@ -0,0 +1,35 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.WrappedJvmID;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public interface TaskAttemptListener {
+
+  InetSocketAddress getAddress();
+
+  void register(TaskAttemptId attemptID, Task task, WrappedJvmID jvmID);
+
+  void unregister(TaskAttemptId attemptID, WrappedJvmID jvmID);
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java

@@ -0,0 +1,137 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+
+/**
+ * This class keeps track of tasks that have already been launched. It
+ * determines if a task is alive and running or marks a task as dead if it does
+ * not hear from it for a long time.
+ * 
+ */
+public class TaskHeartbeatHandler extends AbstractService {
+
+  private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
+
+  //thread which runs periodically to see the last time since a heartbeat is
+  //received from a task.
+  private Thread lostTaskCheckerThread;
+  private volatile boolean stopped;
+  private int taskTimeOut = 5*60*1000;//5 mins
+
+  private final EventHandler eventHandler;
+  private final Clock clock;
+
+  private Map<TaskAttemptId, Long> runningAttempts 
+    = new HashMap<TaskAttemptId, Long>();
+
+  public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock) {
+    super("TaskHeartbeatHandler");
+    this.eventHandler = eventHandler;
+    this.clock = clock;
+  }
+
+  @Override
+  public void init(Configuration conf) {
+   super.init(conf);
+   taskTimeOut = conf.getInt("mapreduce.task.timeout", 5*60*1000);
+  }
+
+  @Override
+  public void start() {
+    lostTaskCheckerThread = new Thread(new PingChecker());
+    lostTaskCheckerThread.start();
+    super.start();
+  }
+
+  @Override
+  public void stop() {
+    stopped = true;
+    lostTaskCheckerThread.interrupt();
+    super.stop();
+  }
+
+  public synchronized void receivedPing(TaskAttemptId attemptID) {
+    //only put for the registered attempts
+    if (runningAttempts.containsKey(attemptID)) {
+      runningAttempts.put(attemptID, clock.getTime());
+    }
+  }
+
+  public synchronized void register(TaskAttemptId attemptID) {
+    runningAttempts.put(attemptID, clock.getTime());
+  }
+
+  public synchronized void unregister(TaskAttemptId attemptID) {
+    runningAttempts.remove(attemptID);
+  }
+
+  private class PingChecker implements Runnable {
+
+    @Override
+    public void run() {
+      while (!stopped && !Thread.currentThread().isInterrupted()) {
+        synchronized (TaskHeartbeatHandler.this) {
+          Iterator<Map.Entry<TaskAttemptId, Long>> iterator = 
+            runningAttempts.entrySet().iterator();
+
+          //avoid calculating current time everytime in loop
+          long currentTime = clock.getTime();
+
+          while (iterator.hasNext()) {
+            Map.Entry<TaskAttemptId, Long> entry = iterator.next();
+            if (currentTime > entry.getValue() + taskTimeOut) {
+              //task is lost, remove from the list and raise lost event
+              iterator.remove();
+              eventHandler.handle(
+                  new TaskAttemptDiagnosticsUpdateEvent(entry.getKey(),
+                      "AttemptID:" + entry.getKey().toString() + 
+                      " Timed out after " + taskTimeOut/1000 + " secs"));
+              eventHandler.handle(new TaskAttemptEvent(entry
+                  .getKey(), TaskAttemptEventType.TA_TIMED_OUT));
+            }
+          }
+        }
+        try {
+          Thread.sleep(taskTimeOut);
+        } catch (InterruptedException e) {
+          LOG.info("TaskHeartbeatHandler thread interrupted");
+          break;
+        }
+      }
+    }
+    
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java

@@ -0,0 +1,28 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.client;
+
+import java.net.InetSocketAddress;
+
+public interface ClientService {
+
+  InetSocketAddress getBindAddress();
+
+  int getHttpPort();
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -0,0 +1,392 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.client;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.AccessControlException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.avro.ipc.Server;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
+import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
+import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+
+/**
+ * This module is responsible for talking to the 
+ * jobclient (user facing).
+ *
+ */
+public class MRClientService extends AbstractService 
+    implements ClientService {
+
+  static final Log LOG = LogFactory.getLog(MRClientService.class);
+  
+  private MRClientProtocol protocolHandler;
+  private Server server;
+  private WebApp webApp;
+  private InetSocketAddress bindAddress;
+  private AppContext appContext;
+
+  public MRClientService(AppContext appContext) {
+    super("MRClientService");
+    this.appContext = appContext;
+    this.protocolHandler = new MRClientProtocolHandler();
+  }
+
+  public void start() {
+    Configuration conf = new Configuration(getConfig()); // Just for not messing up sec-info class config
+    YarnRPC rpc = YarnRPC.create(conf);
+    InetSocketAddress address = NetUtils.createSocketAddr("0.0.0.0:0");
+    InetAddress hostNameResolved = null;
+    try {
+      hostNameResolved = InetAddress.getLocalHost();
+    } catch (UnknownHostException e) {
+      throw new YarnException(e);
+    }
+
+    ClientToAMSecretManager secretManager = null;
+    if (UserGroupInformation.isSecurityEnabled()) {
+      secretManager = new ClientToAMSecretManager();
+      String secretKeyStr =
+          System
+              .getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
+      byte[] bytes = Base64.decodeBase64(secretKeyStr);
+      ApplicationTokenIdentifier identifier =
+          new ApplicationTokenIdentifier(this.appContext.getApplicationID());
+      secretManager.setMasterKey(identifier, bytes);
+      conf.setClass(
+          YarnConfiguration.YARN_SECURITY_INFO,
+          SchedulerSecurityInfo.class, SecurityInfo.class); // Same for now.
+    }
+    server =
+        rpc.getServer(MRClientProtocol.class, protocolHandler, address,
+            conf, secretManager,
+            conf.getInt(AMConstants.AM_JOB_CLIENT_THREADS, 
+                AMConstants.DEFAULT_AM_JOB_CLIENT_THREADS));
+    server.start();
+    this.bindAddress =
+        NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
+            + ":" + server.getPort());
+    LOG.info("Instantiated MRClientService at " + this.bindAddress);
+    try {
+      webApp = WebApps.$for("yarn", AppContext.class, appContext).with(conf).
+          start(new AMWebApp());
+    } catch (Exception e) {
+      LOG.error("Webapps failed to start. Ignoring for now:", e);
+    }
+    super.start();
+  }
+
+  public void stop() {
+    server.close();
+    if (webApp != null) {
+      webApp.stop();
+    }
+    super.stop();
+  }
+
+  @Override
+  public InetSocketAddress getBindAddress() {
+    return bindAddress;
+  }
+
+  @Override
+  public int getHttpPort() {
+    return webApp.port();
+  }
+
+  class MRClientProtocolHandler implements MRClientProtocol {
+
+    private RecordFactory recordFactory = 
+      RecordFactoryProvider.getRecordFactory(null);
+
+    private Job verifyAndGetJob(JobId jobID, 
+        boolean modifyAccess) throws YarnRemoteException {
+      Job job = appContext.getJob(jobID);
+      if (job == null) {
+        throw RPCUtil.getRemoteException("Unknown job " + jobID);
+      }
+      //TODO fix job acls.
+      //JobACL operation = JobACL.VIEW_JOB;
+      //if (modifyAccess) {
+      //  operation = JobACL.MODIFY_JOB;
+      //}
+      //TO disable check access ofr now.
+      //checkAccess(job, operation);
+      return job;
+    }
+ 
+    private Task verifyAndGetTask(TaskId taskID, 
+        boolean modifyAccess) throws YarnRemoteException {
+      Task task = verifyAndGetJob(taskID.getJobId(), 
+          modifyAccess).getTask(taskID);
+      if (task == null) {
+        throw RPCUtil.getRemoteException("Unknown Task " + taskID);
+      }
+      return task;
+    }
+
+    private TaskAttempt verifyAndGetAttempt(TaskAttemptId attemptID, 
+        boolean modifyAccess) throws YarnRemoteException {
+      TaskAttempt attempt = verifyAndGetTask(attemptID.getTaskId(), 
+          modifyAccess).getAttempt(attemptID);
+      if (attempt == null) {
+        throw RPCUtil.getRemoteException("Unknown TaskAttempt " + attemptID);
+      }
+      return attempt;
+    }
+
+    private void checkAccess(Job job, JobACL jobOperation) 
+      throws YarnRemoteException {
+      if (!UserGroupInformation.isSecurityEnabled()) {
+        return;
+      }
+      UserGroupInformation callerUGI;
+      try {
+        callerUGI = UserGroupInformation.getCurrentUser();
+      } catch (IOException e) {
+        throw RPCUtil.getRemoteException(e);
+      }
+      if(!job.checkAccess(callerUGI, jobOperation)) {
+        throw RPCUtil.getRemoteException(new AccessControlException("User "
+            + callerUGI.getShortUserName() + " cannot perform operation "
+            + jobOperation.name() + " on " + job.getID()));
+      }
+    }
+
+    @Override
+    public GetCountersResponse getCounters(GetCountersRequest request) 
+      throws YarnRemoteException {
+      JobId jobId = request.getJobId();
+      Job job = verifyAndGetJob(jobId, false);
+      GetCountersResponse response =
+        recordFactory.newRecordInstance(GetCountersResponse.class);
+      response.setCounters(job.getCounters());
+      return response;
+    }
+    
+    @Override
+    public GetJobReportResponse getJobReport(GetJobReportRequest request) 
+      throws YarnRemoteException {
+      JobId jobId = request.getJobId();
+      Job job = verifyAndGetJob(jobId, false);
+      GetJobReportResponse response = 
+        recordFactory.newRecordInstance(GetJobReportResponse.class);
+      response.setJobReport(job.getReport());
+      return response;
+    }
+    
+    
+    @Override
+    public GetTaskAttemptReportResponse getTaskAttemptReport(
+        GetTaskAttemptReportRequest request) throws YarnRemoteException {
+      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
+      GetTaskAttemptReportResponse response =
+        recordFactory.newRecordInstance(GetTaskAttemptReportResponse.class);
+      response.setTaskAttemptReport(
+          verifyAndGetAttempt(taskAttemptId, false).getReport());
+      return response;
+    }
+
+    @Override
+    public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) 
+      throws YarnRemoteException {
+      TaskId taskId = request.getTaskId();
+      GetTaskReportResponse response = 
+        recordFactory.newRecordInstance(GetTaskReportResponse.class);
+      response.setTaskReport(verifyAndGetTask(taskId, false).getReport());
+      return response;
+    }
+
+    @Override
+    public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
+        GetTaskAttemptCompletionEventsRequest request) 
+        throws YarnRemoteException {
+      JobId jobId = request.getJobId();
+      int fromEventId = request.getFromEventId();
+      int maxEvents = request.getMaxEvents();
+      Job job = verifyAndGetJob(jobId, false);
+      
+      GetTaskAttemptCompletionEventsResponse response = 
+        recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
+      response.addAllCompletionEvents(Arrays.asList(
+          job.getTaskAttemptCompletionEvents(fromEventId, maxEvents)));
+      return response;
+    }
+    
+    @Override
+    public KillJobResponse killJob(KillJobRequest request) 
+      throws YarnRemoteException {
+      JobId jobId = request.getJobId();
+      String message = "Kill Job received from client " + jobId;
+      LOG.info(message);
+  	  verifyAndGetJob(jobId, true);
+      appContext.getEventHandler().handle(
+          new JobDiagnosticsUpdateEvent(jobId, message));
+      appContext.getEventHandler().handle(
+          new JobEvent(jobId, JobEventType.JOB_KILL));
+      KillJobResponse response = 
+        recordFactory.newRecordInstance(KillJobResponse.class);
+      return response;
+    }
+
+    @Override
+    public KillTaskResponse killTask(KillTaskRequest request) 
+      throws YarnRemoteException {
+      TaskId taskId = request.getTaskId();
+      String message = "Kill task received from client " + taskId;
+      LOG.info(message);
+      verifyAndGetTask(taskId, true);
+      appContext.getEventHandler().handle(
+          new TaskEvent(taskId, TaskEventType.T_KILL));
+      KillTaskResponse response = 
+        recordFactory.newRecordInstance(KillTaskResponse.class);
+      return response;
+    }
+    
+    @Override
+    public KillTaskAttemptResponse killTaskAttempt(
+        KillTaskAttemptRequest request) throws YarnRemoteException {
+      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
+      String message = "Kill task attempt received from client " + taskAttemptId;
+      LOG.info(message);
+      verifyAndGetAttempt(taskAttemptId, true);
+      appContext.getEventHandler().handle(
+          new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
+      appContext.getEventHandler().handle(
+          new TaskAttemptEvent(taskAttemptId, 
+              TaskAttemptEventType.TA_KILL));
+      KillTaskAttemptResponse response = 
+        recordFactory.newRecordInstance(KillTaskAttemptResponse.class);
+      return response;
+    }
+
+    @Override
+    public GetDiagnosticsResponse getDiagnostics(
+        GetDiagnosticsRequest request) throws YarnRemoteException {
+      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
+      
+      GetDiagnosticsResponse response = 
+        recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
+      response.addAllDiagnostics(
+          verifyAndGetAttempt(taskAttemptId, false).getDiagnostics());
+      return response;
+    }
+
+    @Override
+    public FailTaskAttemptResponse failTaskAttempt(
+        FailTaskAttemptRequest request) throws YarnRemoteException {
+      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
+      String message = "Fail task attempt received from client " + taskAttemptId;
+      LOG.info(message);
+      verifyAndGetAttempt(taskAttemptId, true);
+      appContext.getEventHandler().handle(
+          new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
+      appContext.getEventHandler().handle(
+          new TaskAttemptEvent(taskAttemptId, 
+              TaskAttemptEventType.TA_FAILMSG));
+      FailTaskAttemptResponse response = recordFactory.
+        newRecordInstance(FailTaskAttemptResponse.class);
+      return response;
+    }
+
+    @Override
+    public GetTaskReportsResponse getTaskReports(
+        GetTaskReportsRequest request) throws YarnRemoteException {
+      JobId jobId = request.getJobId();
+      TaskType taskType = request.getTaskType();
+      
+      GetTaskReportsResponse response = 
+        recordFactory.newRecordInstance(GetTaskReportsResponse.class);
+      
+      Job job = verifyAndGetJob(jobId, false);
+      LOG.info("Getting task report for " + taskType + "   " + jobId);
+      Collection<Task> tasks = job.getTasks(taskType).values();
+      LOG.info("Getting task report size " + tasks.size());
+      for (Task task : tasks) {
+        response.addTaskReport(task.getReport());
+	  }
+      return response;
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java

@@ -0,0 +1,59 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.security.UserGroupInformation;
+
+
+/**
+ * Main interface to interact with the job. Provides only getters. 
+ */
+public interface Job {
+
+  JobId getID();
+  String getName();
+  JobState getState();
+  JobReport getReport();
+  Counters getCounters();
+  Map<TaskId,Task> getTasks();
+  Map<TaskId,Task> getTasks(TaskType taskType);
+  Task getTask(TaskId taskID);
+  List<String> getDiagnostics();
+  int getTotalMaps();
+  int getTotalReduces();
+  int getCompletedMaps();
+  int getCompletedReduces();
+  boolean isUber();
+
+  TaskAttemptCompletionEvent[]
+      getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);
+
+  boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation);
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Task.java

@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job;
+
+import java.util.Map;
+
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+
+/**
+ * Read only view of Task.
+ */
+public interface Task {
+  TaskId getID();
+  TaskReport getReport();
+  TaskState getState();
+  Counters getCounters();
+  float getProgress();
+  TaskType getType();
+  Map<TaskAttemptId, TaskAttempt> getAttempts();
+  TaskAttempt getAttempt(TaskAttemptId attemptID);
+
+  /** Has Task reached the final state or not.
+   */
+  boolean isFinished();
+
+  /**
+   * Can the output of the taskAttempt be committed. Note that once the task
+   * gives a go for a commit, further canCommit requests from any other attempts
+   * should return false.
+   * 
+   * @param taskAttemptID
+   * @return whether the attempt's output can be committed or not.
+   */
+  boolean canCommit(TaskAttemptId taskAttemptID);
+
+  
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java

@@ -0,0 +1,66 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job;
+
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+
+/**
+ * Read only view of TaskAttempt.
+ */
+public interface TaskAttempt {
+  TaskAttemptId getID();
+  TaskAttemptReport getReport();
+  List<String> getDiagnostics();
+  Counters getCounters();
+  float getProgress();
+  TaskAttemptState getState();
+
+  /** Has attempt reached the final state or not.
+   */
+  boolean isFinished();
+
+  /**If container Assigned then return container ID, otherwise null.
+   */
+  ContainerId getAssignedContainerID();
+
+  /**If container Assigned then return container mgr address, otherwise null.
+   */
+  String getAssignedContainerMgrAddress();
+  
+  /**If container Assigned then return the node's http address, otherwise null.
+   */
+  String getNodeHttpAddress();
+
+  /** Returns time at which container is launched. If container is not launched
+   * yet, returns 0.
+   */
+  long getLaunchTime();
+
+  /** Returns attempt's finish time. If attempt is not finished
+   *  yet, returns 0.
+   */
+  long getFinishTime();
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobCounterUpdateEvent.java

@@ -0,0 +1,42 @@
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+
+public class JobCounterUpdateEvent extends JobEvent {
+
+  List<CounterIncrementalUpdate> counterUpdates = null;
+  
+  public JobCounterUpdateEvent(JobId jobId) {
+    super(jobId, JobEventType.JOB_COUNTER_UPDATE);
+    counterUpdates = new ArrayList<JobCounterUpdateEvent.CounterIncrementalUpdate>();
+  }
+
+  public void addCounterUpdate(Enum<?> key, long incrValue) {
+    counterUpdates.add(new CounterIncrementalUpdate(key, incrValue));
+  }
+  
+  public List<CounterIncrementalUpdate> getCounterUpdates() {
+    return counterUpdates;
+  }
+  
+  public static class CounterIncrementalUpdate {
+    Enum<?> key;
+    long incrValue;
+    
+    public CounterIncrementalUpdate(Enum<?> key, long incrValue) {
+      this.key = key;
+      this.incrValue = incrValue;
+    }
+    
+    public Enum<?> getCounterKey() {
+      return key;
+    }
+
+    public long getIncrementValue() {
+      return incrValue;
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobDiagnosticsUpdateEvent.java

@@ -0,0 +1,36 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+
+
+public class JobDiagnosticsUpdateEvent extends JobEvent {
+
+  private String diagnosticUpdate;
+
+  public JobDiagnosticsUpdateEvent(JobId jobID, String diagnostic) {
+    super(jobID, JobEventType.JOB_DIAGNOSTIC_UPDATE);
+    this.diagnosticUpdate = diagnostic;
+  }
+
+  public String getDiagnosticUpdate() {
+    return this.diagnosticUpdate;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEvent.java

@@ -0,0 +1,41 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+
+/**
+ * This class encapsulates job related events.
+ *
+ */
+public class JobEvent extends AbstractEvent<JobEventType> {
+
+  private JobId jobID;
+
+  public JobEvent(JobId jobID, JobEventType type) {
+    super(type);
+    this.jobID = jobID;
+  }
+
+  public JobId getJobId() {
+    return jobID;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java

@@ -0,0 +1,48 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+/**
+ * Event types handled by Job.
+ */
+public enum JobEventType {
+
+  //Producer:Client
+  JOB_KILL,
+
+  //Producer:MRAppMaster
+  JOB_INIT,
+  JOB_START,
+
+  //Producer:Task
+  JOB_TASK_COMPLETED,
+  JOB_MAP_TASK_RESCHEDULED,
+  JOB_TASK_ATTEMPT_COMPLETED,
+
+  //Producer:Job
+  JOB_COMPLETED,
+
+  //Producer:Any component
+  JOB_DIAGNOSTIC_UPDATE,
+  INTERNAL_ERROR,
+  JOB_COUNTER_UPDATE,
+  
+  //Producer:TaskAttemptListener
+  JOB_TASK_ATTEMPT_FETCH_FAILURE
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobFinishEvent.java

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class JobFinishEvent 
+          extends AbstractEvent<JobFinishEvent.Type> {
+
+  public enum Type {
+    STATE_CHANGED
+  }
+
+  private JobId jobID;
+
+  public JobFinishEvent(JobId jobID) {
+    super(Type.STATE_CHANGED);
+    this.jobID = jobID;
+  }
+
+  public JobId getJobId() {
+    return jobID;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobMapTaskRescheduledEvent.java

@@ -0,0 +1,38 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+
+
+
+public class JobMapTaskRescheduledEvent extends JobEvent {
+
+  private TaskId taskID;
+
+  public JobMapTaskRescheduledEvent(TaskId taskID) {
+    super(taskID.getJobId(), JobEventType.JOB_MAP_TASK_RESCHEDULED);
+    this.taskID = taskID;
+  }
+
+  public TaskId getTaskID() {
+    return taskID;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptCompletedEvent.java

@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
+
+
+public class JobTaskAttemptCompletedEvent extends JobEvent {
+
+  private TaskAttemptCompletionEvent completionEvent;
+
+  public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) {
+    super(completionEvent.getAttemptId().getTaskId().getJobId(), 
+        JobEventType.JOB_TASK_ATTEMPT_COMPLETED);
+    this.completionEvent = completionEvent;
+  }
+
+  public TaskAttemptCompletionEvent getCompletionEvent() {
+    return completionEvent;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java

@@ -0,0 +1,48 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+
+
+public class JobTaskAttemptFetchFailureEvent extends JobEvent {
+
+  private final TaskAttemptId reduce;
+  private final List<TaskAttemptId> maps;
+
+  public JobTaskAttemptFetchFailureEvent(TaskAttemptId reduce, 
+      List<TaskAttemptId> maps) {
+    super(reduce.getTaskId().getJobId(), 
+        JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE);
+    this.reduce = reduce;
+    this.maps = maps;
+  }
+
+  public List<TaskAttemptId> getMaps() {
+    return maps;
+  }
+
+  public TaskAttemptId getReduce() {
+    return reduce;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskEvent.java

@@ -0,0 +1,43 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+
+
+public class JobTaskEvent extends JobEvent {
+
+  private TaskId taskID;
+  private TaskState taskState;
+
+  public JobTaskEvent(TaskId taskID, TaskState taskState) {
+    super(taskID.getJobId(), JobEventType.JOB_TASK_COMPLETED);
+    this.taskID = taskID;
+    this.taskState = taskState;
+  }
+
+  public TaskId getTaskID() {
+    return taskID;
+  }
+
+  public TaskState getState() {
+    return taskState;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java

@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+
+
+
+public class TaskAttemptContainerAssignedEvent extends TaskAttemptEvent {
+
+  private final Container container;
+
+  public TaskAttemptContainerAssignedEvent(TaskAttemptId id,
+      Container container) {
+    super(id, TaskAttemptEventType.TA_ASSIGNED);
+    this.container = container;
+  }
+
+  public Container getContainer() {
+    return this.container;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptDiagnosticsUpdateEvent.java

@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+
+public class TaskAttemptDiagnosticsUpdateEvent extends TaskAttemptEvent {
+
+  private String diagnosticInfo;
+
+  public TaskAttemptDiagnosticsUpdateEvent(TaskAttemptId attemptID,
+      String diagnosticInfo) {
+    super(attemptID, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE);
+    this.diagnosticInfo = diagnosticInfo;
+  }
+
+  public String getDiagnosticInfo() {
+    return diagnosticInfo;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptEvent.java

@@ -0,0 +1,41 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+/**
+ * This class encapsulates task attempt related events.
+ *
+ */
+public class TaskAttemptEvent extends AbstractEvent<TaskAttemptEventType> {
+
+  private TaskAttemptId attemptID;
+
+  public TaskAttemptEvent(TaskAttemptId id, TaskAttemptEventType type) {
+    super(type);
+    this.attemptID = id;
+  }
+
+  public TaskAttemptId getTaskAttemptID() {
+    return attemptID;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptEventType.java

@@ -0,0 +1,55 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+/**
+ * Event types handled by TaskAttempt.
+ */
+public enum TaskAttemptEventType {
+
+  //Producer:Task
+  TA_SCHEDULE,
+  TA_RESCHEDULE,
+
+  //Producer:Client, Task
+  TA_KILL,
+
+  //Producer:ContainerAllocator
+  TA_ASSIGNED,
+  TA_CONTAINER_COMPLETED,
+
+  //Producer:ContainerLauncher
+  TA_CONTAINER_LAUNCHED,
+  TA_CONTAINER_LAUNCH_FAILED,
+  TA_CONTAINER_CLEANED,
+
+  //Producer:TaskAttemptListener
+  TA_DIAGNOSTICS_UPDATE,
+  TA_COMMIT_PENDING, 
+  TA_DONE,
+  TA_FAILMSG,
+  TA_UPDATE,
+  TA_TIMED_OUT,
+
+  //Producer:TaskCleaner
+  TA_CLEANUP_DONE,
+
+  //Producer:Job
+  TA_TOO_MANY_FETCH_FAILURE,
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java

@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.Phase;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+
+
+public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent {
+
+  private TaskAttemptStatus reportedTaskAttemptStatus;
+
+  public TaskAttemptStatusUpdateEvent(TaskAttemptId id,
+      TaskAttemptStatus taskAttemptStatus) {
+    super(id, TaskAttemptEventType.TA_UPDATE);
+    this.reportedTaskAttemptStatus = taskAttemptStatus;
+  }
+
+  public TaskAttemptStatus getReportedTaskAttemptStatus() {
+    return reportedTaskAttemptStatus;
+  }
+
+  /**
+   * The internal TaskAttemptStatus object corresponding to remote Task status.
+   * 
+   */
+  public static class TaskAttemptStatus {
+    public TaskAttemptId id;
+    public float progress;
+    public Counters counters;
+    public String diagnosticInfo;
+    public String stateString;
+    public Phase phase;
+    public long outputSize;
+    public List<TaskAttemptId> fetchFailedMaps;
+    public long mapFinishTime;
+    public long shuffleFinishTime;
+    public long sortFinishTime;
+    public TaskAttemptState taskState;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskEvent.java

@@ -0,0 +1,40 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+
+/**
+ * this class encapsulates task related events.
+ *
+ */
+public class TaskEvent extends AbstractEvent<TaskEventType> {
+
+  private TaskId taskID;
+
+  public TaskEvent(TaskId taskID, TaskEventType type) {
+    super(type);
+    this.taskID = taskID;
+  }
+
+  public TaskId getTaskID() {
+    return taskID;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskEventType.java

@@ -0,0 +1,41 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+/**
+ * Event types handled by Task.
+ */
+public enum TaskEventType {
+
+  //Producer:Client, Job
+  T_KILL,
+
+  //Producer:Job
+  T_SCHEDULE,
+
+  //Producer:Speculator
+  T_ADD_SPEC_ATTEMPT,
+
+  //Producer:TaskAttempt
+  T_ATTEMPT_LAUNCHED,
+  T_ATTEMPT_COMMIT_PENDING,
+  T_ATTEMPT_FAILED,
+  T_ATTEMPT_SUCCEEDED,
+  T_ATTEMPT_KILLED
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptEvent.java

@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+
+public class TaskTAttemptEvent extends TaskEvent {
+
+  private TaskAttemptId attemptID;
+
+  public TaskTAttemptEvent(TaskAttemptId id, TaskEventType type) {
+    super(id.getTaskId(), type);
+    this.attemptID = id;
+  }
+
+  public TaskAttemptId getTaskAttemptID() {
+    return attemptID;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

@@ -0,0 +1,1416 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobACLsManager;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent;
+import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
+import org.apache.hadoop.mapreduce.lib.chain.ChainReducer;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+import org.apache.hadoop.mapreduce.v2.api.records.Counter;
+import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+
+/** Implementation of Job interface. Maintains the state machines of Job.
+ * The read and write calls use ReadWriteLock for concurrency.
+ */
+public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, 
+  EventHandler<JobEvent> {
+
+  private static final Log LOG = LogFactory.getLog(JobImpl.class);
+
+  //The maximum fraction of fetch failures allowed for a map
+  private static final double MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5;
+
+  // Maximum no. of fetch-failure notifications after which map task is failed
+  private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
+
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
+  
+  //final fields
+  private final Clock clock;
+  private final JobACLsManager aclsManager;
+  private final String username;
+  private final Map<JobACL, AccessControlList> jobACLs;
+  private final int startCount;
+  private final Set<TaskId> completedTasksFromPreviousRun;
+  private final Lock readLock;
+  private final Lock writeLock;
+  private final JobId jobId;
+  private final String jobName;
+  private final org.apache.hadoop.mapreduce.JobID oldJobId;
+  private final TaskAttemptListener taskAttemptListener;
+  private final Object tasksSyncHandle = new Object();
+  private final Set<TaskId> mapTasks = new LinkedHashSet<TaskId>();
+  private final Set<TaskId> reduceTasks = new LinkedHashSet<TaskId>();
+  private final EventHandler eventHandler;
+  private final MRAppMetrics metrics;
+
+  private boolean lazyTasksCopyNeeded = false;
+  private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
+  private Counters jobCounters = newCounters();
+    // FIXME:  
+    //
+    // Can then replace task-level uber counters (MR-2424) with job-level ones
+    // sent from LocalContainerLauncher, and eventually including a count of
+    // of uber-AM attempts (probably sent from MRAppMaster).
+  public Configuration conf;
+
+  //fields initialized in init
+  private FileSystem fs;
+  private Path remoteJobSubmitDir;
+  public Path remoteJobConfFile;
+  private JobContext jobContext;
+  private OutputCommitter committer;
+  private int allowedMapFailuresPercent = 0;
+  private int allowedReduceFailuresPercent = 0;
+  private List<TaskAttemptCompletionEvent> taskAttemptCompletionEvents;
+  private final List<String> diagnostics = new ArrayList<String>();
+  
+  //task/attempt related datastructures
+  private final Map<TaskId, Integer> successAttemptCompletionEventNoMap = 
+    new HashMap<TaskId, Integer>();
+  private final Map<TaskAttemptId, Integer> fetchFailuresMapping = 
+    new HashMap<TaskAttemptId, Integer>();
+
+  private static final DiagnosticsUpdateTransition
+      DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition();
+  private static final InternalErrorTransition
+      INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
+  private static final TaskAttemptCompletedEventTransition
+      TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION =
+          new TaskAttemptCompletedEventTransition();
+  private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION =
+      new CounterUpdateTransition();
+
+  protected static final
+    StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent> 
+       stateMachineFactory
+     = new StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent>
+              (JobState.NEW)
+
+          // Transitions from NEW state
+          .addTransition(JobState.NEW, JobState.NEW,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.NEW, JobState.NEW,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition
+              (JobState.NEW,
+              EnumSet.of(JobState.INITED, JobState.FAILED),
+              JobEventType.JOB_INIT,
+              new InitTransition())
+          .addTransition(JobState.NEW, JobState.KILLED,
+              JobEventType.JOB_KILL,
+              new KillNewJobTransition())
+          .addTransition(JobState.NEW, JobState.ERROR,
+              JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+
+          // Transitions from INITED state
+          .addTransition(JobState.INITED, JobState.INITED,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.INITED, JobState.INITED,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(JobState.INITED, JobState.RUNNING,
+              JobEventType.JOB_START,
+              new StartTransition())
+          .addTransition(JobState.INITED, JobState.KILLED,
+              JobEventType.JOB_KILL,
+              new KillInitedJobTransition())
+          .addTransition(JobState.INITED, JobState.ERROR,
+              JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+
+          // Transitions from RUNNING state
+          .addTransition(JobState.RUNNING, JobState.RUNNING,
+              JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
+              TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
+          .addTransition
+              (JobState.RUNNING,
+              EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
+              JobEventType.JOB_TASK_COMPLETED,
+              new TaskCompletedTransition())
+          .addTransition
+              (JobState.RUNNING,
+              EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
+              JobEventType.JOB_COMPLETED,
+              new JobNoTasksCompletedTransition())
+          .addTransition(JobState.RUNNING, JobState.KILL_WAIT,
+              JobEventType.JOB_KILL, new KillTasksTransition())
+          .addTransition(JobState.RUNNING, JobState.RUNNING,
+              JobEventType.JOB_MAP_TASK_RESCHEDULED,
+              new MapTaskRescheduledTransition())
+          .addTransition(JobState.RUNNING, JobState.RUNNING,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.RUNNING, JobState.RUNNING,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(JobState.RUNNING, JobState.RUNNING,
+              JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
+              new TaskAttemptFetchFailureTransition())
+          .addTransition(
+              JobState.RUNNING,
+              JobState.ERROR, JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+
+          // Transitions from KILL_WAIT state.
+          .addTransition
+              (JobState.KILL_WAIT,
+              EnumSet.of(JobState.KILL_WAIT, JobState.KILLED),
+              JobEventType.JOB_TASK_COMPLETED,
+              new KillWaitTaskCompletedTransition())
+          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
+              JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
+              TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
+          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(
+              JobState.KILL_WAIT,
+              JobState.ERROR, JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+          // Ignore-able events
+          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
+              EnumSet.of(JobEventType.JOB_KILL,
+                         JobEventType.JOB_MAP_TASK_RESCHEDULED,
+                         JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
+
+          // Transitions from SUCCEEDED state
+          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(
+              JobState.SUCCEEDED,
+              JobState.ERROR, JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+          // Ignore-able events
+          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
+              EnumSet.of(JobEventType.JOB_KILL,
+                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
+
+          // Transitions from FAILED state
+          .addTransition(JobState.FAILED, JobState.FAILED,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.FAILED, JobState.FAILED,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(
+              JobState.FAILED,
+              JobState.ERROR, JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+          // Ignore-able events
+          .addTransition(JobState.FAILED, JobState.FAILED,
+              EnumSet.of(JobEventType.JOB_KILL,
+                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
+
+          // Transitions from KILLED state
+          .addTransition(JobState.KILLED, JobState.KILLED,
+              JobEventType.JOB_DIAGNOSTIC_UPDATE,
+              DIAGNOSTIC_UPDATE_TRANSITION)
+          .addTransition(JobState.KILLED, JobState.KILLED,
+              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
+          .addTransition(
+              JobState.KILLED,
+              JobState.ERROR, JobEventType.INTERNAL_ERROR,
+              INTERNAL_ERROR_TRANSITION)
+          // Ignore-able events
+          .addTransition(JobState.KILLED, JobState.KILLED,
+              EnumSet.of(JobEventType.JOB_KILL,
+                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
+
+          // No transitions from INTERNAL_ERROR state. Ignore all.
+          .addTransition(
+              JobState.ERROR,
+              JobState.ERROR,
+              EnumSet.of(JobEventType.JOB_INIT,
+                  JobEventType.JOB_KILL,
+                  JobEventType.JOB_TASK_COMPLETED,
+                  JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
+                  JobEventType.JOB_MAP_TASK_RESCHEDULED,
+                  JobEventType.JOB_DIAGNOSTIC_UPDATE,
+                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
+                  JobEventType.INTERNAL_ERROR))
+
+          // create the topology tables
+          .installTopology();
+ 
+  private final StateMachine<JobState, JobEventType, JobEvent> stateMachine;
+
+  //changing fields while the job is running
+  private int numMapTasks;
+  private int numReduceTasks;
+  private int completedTaskCount = 0;
+  private int succeededMapTaskCount = 0;
+  private int succeededReduceTaskCount = 0;
+  private int failedMapTaskCount = 0;
+  private int failedReduceTaskCount = 0;
+  private int killedMapTaskCount = 0;
+  private int killedReduceTaskCount = 0;
+  private long submitTime;
+  private long startTime;
+  private long finishTime;
+  private float setupProgress;
+  private float cleanupProgress;
+  private boolean isUber = false;
+
+  private Credentials fsTokens;
+  private Token<JobTokenIdentifier> jobToken;
+  private JobTokenSecretManager jobTokenSecretManager;
+
+  public JobImpl(ApplicationId appID, Configuration conf,
+      EventHandler eventHandler, TaskAttemptListener taskAttemptListener,
+      JobTokenSecretManager jobTokenSecretManager,
+      Credentials fsTokenCredentials, Clock clock, int startCount, 
+      Set<TaskId> completedTasksFromPreviousRun, MRAppMetrics metrics) {
+
+    this.jobId = recordFactory.newRecordInstance(JobId.class);
+    this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
+    this.conf = conf;
+    this.metrics = metrics;
+    this.clock = clock;
+    this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
+    this.startCount = startCount;
+    jobId.setAppId(appID);
+    jobId.setId(appID.getId());
+    oldJobId = TypeConverter.fromYarn(jobId);
+    LOG.info("Job created" +
+    		" appId=" + appID + 
+    		" jobId=" + jobId + 
+    		" oldJobId=" + oldJobId);
+    
+    this.taskAttemptListener = taskAttemptListener;
+    this.eventHandler = eventHandler;
+    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    this.readLock = readWriteLock.readLock();
+    this.writeLock = readWriteLock.writeLock();
+
+    this.fsTokens = fsTokenCredentials;
+    this.jobTokenSecretManager = jobTokenSecretManager;
+
+    this.aclsManager = new JobACLsManager(conf);
+    this.username = System.getProperty("user.name");
+    this.jobACLs = aclsManager.constructJobACLs(conf);
+    // This "this leak" is okay because the retained pointer is in an
+    //  instance variable.
+    stateMachine = stateMachineFactory.make(this);
+  }
+
+  protected StateMachine<JobState, JobEventType, JobEvent> getStateMachine() {
+    return stateMachine;
+  }
+
+  @Override
+  public JobId getID() {
+    return jobId;
+  }
+
+  // Getter methods that make unit testing easier (package-scoped)
+  OutputCommitter getCommitter() {
+    return this.committer;
+  }
+
+  EventHandler getEventHandler() {
+    return this.eventHandler;
+  }
+
+  JobContext getJobContext() {
+    return this.jobContext;
+  }
+
+  @Override
+  public boolean checkAccess(UserGroupInformation callerUGI, 
+      JobACL jobOperation) {
+    if (!UserGroupInformation.isSecurityEnabled()) {
+      return true;
+    }
+    AccessControlList jobACL = jobACLs.get(jobOperation);
+    return aclsManager.checkAccess(callerUGI, jobOperation, username, jobACL);
+  }
+
+  @Override
+  public Task getTask(TaskId taskID) {
+    readLock.lock();
+    try {
+      return tasks.get(taskID);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public int getCompletedMaps() {
+    readLock.lock();
+    try {
+      return succeededMapTaskCount + failedMapTaskCount + killedMapTaskCount;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public int getCompletedReduces() {
+    readLock.lock();
+    try {
+      return succeededReduceTaskCount + failedReduceTaskCount 
+                  + killedReduceTaskCount;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public boolean isUber() {
+    return isUber;
+  }
+
+  @Override
+  public Counters getCounters() {
+    Counters counters = newCounters();
+    readLock.lock();
+    try {
+      incrAllCounters(counters, jobCounters);
+      return incrTaskCounters(counters, tasks.values());
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private Counters getTypeCounters(Set<TaskId> taskIds) {
+    Counters counters = newCounters();
+    for (TaskId taskId : taskIds) {
+      Task task = tasks.get(taskId);
+      incrAllCounters(counters, task.getCounters());
+    }
+    return counters;
+  }
+
+  private Counters getMapCounters() {
+    readLock.lock();
+    try {
+      return getTypeCounters(mapTasks);
+    } finally {
+      readLock.unlock();
+    }
+  }
+  
+  private Counters getReduceCounters() {
+    readLock.lock();
+    try {
+      return getTypeCounters(reduceTasks);
+    } finally {
+      readLock.unlock();
+    }
+  }
+  
+  public static Counters newCounters() {
+    Counters counters = RecordFactoryProvider.getRecordFactory(null)
+        .newRecordInstance(Counters.class);
+    return counters;
+  }
+
+  public static Counters incrTaskCounters(Counters counters,
+                                          Collection<Task> tasks) {
+    for (Task task : tasks) {
+      incrAllCounters(counters, task.getCounters());
+    }
+    return counters;
+  }
+
+  public static void incrAllCounters(Counters counters, Counters other) {
+    if (other != null) {
+      for (CounterGroup otherGroup: other.getAllCounterGroups().values()) {
+        CounterGroup group = counters.getCounterGroup(otherGroup.getName());
+        if (group == null) {
+          group = RecordFactoryProvider.getRecordFactory(null)
+              .newRecordInstance(CounterGroup.class);
+          group.setName(otherGroup.getName());
+          counters.setCounterGroup(group.getName(), group);
+        }
+        group.setDisplayName(otherGroup.getDisplayName());
+        for (Counter otherCounter : otherGroup.getAllCounters().values()) {
+          Counter counter = group.getCounter(otherCounter.getName());
+          if (counter == null) {
+            counter = RecordFactoryProvider.getRecordFactory(null)
+                .newRecordInstance(Counter.class);
+            counter.setName(otherCounter.getName());
+            group.setCounter(counter.getName(), counter);
+          }
+          counter.setDisplayName(otherCounter.getDisplayName());
+          counter.setValue(counter.getValue() + otherCounter.getValue());
+        }
+      }
+    }
+  }
+
+  @Override
+  public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
+      int fromEventId, int maxEvents) {
+    TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
+    readLock.lock();
+    try {
+      if (taskAttemptCompletionEvents.size() > fromEventId) {
+        int actualMax = Math.min(maxEvents,
+            (taskAttemptCompletionEvents.size() - fromEventId));
+        events = taskAttemptCompletionEvents.subList(fromEventId,
+            actualMax + fromEventId).toArray(events);
+      }
+      return events;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public List<String> getDiagnostics() {
+    readLock.lock();
+    try {
+      return diagnostics;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public JobReport getReport() {
+    readLock.lock();
+    try {
+      JobReport report = recordFactory.newRecordInstance(JobReport.class);
+      report.setJobId(jobId);
+      report.setJobState(getState());
+      
+      // TODO - Fix to correctly setup report and to check state
+      if (report.getJobState() == JobState.NEW) {
+        return report;
+      }
+      
+      report.setStartTime(startTime);
+      report.setFinishTime(finishTime);
+      report.setSetupProgress(setupProgress);
+      report.setCleanupProgress(cleanupProgress);
+      report.setMapProgress(computeProgress(mapTasks));
+      report.setReduceProgress(computeProgress(reduceTasks));
+
+      return report;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private float computeProgress(Set<TaskId> taskIds) {
+    readLock.lock();
+    try {
+      float progress = 0;
+      for (TaskId taskId : taskIds) {
+        Task task = tasks.get(taskId);
+        progress += task.getProgress();
+      }
+      int taskIdsSize = taskIds.size();
+      if (taskIdsSize != 0) {
+        progress = progress/taskIdsSize;
+      }
+      return progress;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public Map<TaskId, Task> getTasks() {
+    synchronized (tasksSyncHandle) {
+      lazyTasksCopyNeeded = true;
+      return Collections.unmodifiableMap(tasks);
+    }
+  }
+
+  @Override
+  public Map<TaskId,Task> getTasks(TaskType taskType) {
+    Map<TaskId, Task> localTasksCopy = tasks;
+    Map<TaskId, Task> result = new HashMap<TaskId, Task>();
+    Set<TaskId> tasksOfGivenType = null;
+    readLock.lock();
+    try {
+      if (TaskType.MAP == taskType) {
+        tasksOfGivenType = mapTasks;
+      } else {
+        tasksOfGivenType = reduceTasks;
+      }
+      for (TaskId taskID : tasksOfGivenType)
+      result.put(taskID, localTasksCopy.get(taskID));
+      return result;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public JobState getState() {
+    readLock.lock();
+    try {
+     return getStateMachine().getCurrentState();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  protected void scheduleTasks(Set<TaskId> taskIDs) {
+    for (TaskId taskID : taskIDs) {
+      eventHandler.handle(new TaskEvent(taskID, 
+          TaskEventType.T_SCHEDULE));
+    }
+  }
+
+  @Override
+  /**
+   * The only entry point to change the Job.
+   */
+  public void handle(JobEvent event) {
+    LOG.info("Processing " + event.getJobId() + " of type " + event.getType());
+    try {
+      writeLock.lock();
+      JobState oldState = getState();
+      try {
+         getStateMachine().doTransition(event.getType(), event);
+      } catch (InvalidStateTransitonException e) {
+        LOG.error("Can't handle this event at current state", e);
+        addDiagnostic("Invalid event " + event.getType() + 
+            " on Job " + this.jobId);
+        eventHandler.handle(new JobEvent(this.jobId,
+            JobEventType.INTERNAL_ERROR));
+      }
+      //notify the eventhandler of state change
+      if (oldState != getState()) {
+        LOG.info(jobId + "Job Transitioned from " + oldState + " to "
+                 + getState());
+      }
+    }
+    
+    finally {
+      writeLock.unlock();
+    }
+  }
+
+  //helpful in testing
+  protected void addTask(Task task) {
+    synchronized (tasksSyncHandle) {
+      if (lazyTasksCopyNeeded) {
+        Map<TaskId, Task> newTasks = new LinkedHashMap<TaskId, Task>();
+        newTasks.putAll(tasks);
+        tasks = newTasks;
+        lazyTasksCopyNeeded = false;
+      }
+    }
+    tasks.put(task.getID(), task);
+    if (task.getType() == TaskType.MAP) {
+      mapTasks.add(task.getID());
+    } else if (task.getType() == TaskType.REDUCE) {
+      reduceTasks.add(task.getID());
+    }
+    metrics.waitingTask(task);
+  }
+
+  void setFinishTime() {
+    finishTime = clock.getTime();
+  }
+
+  void logJobHistoryFinishedEvent() {
+    this.setFinishTime();
+    JobFinishedEvent jfe = createJobFinishedEvent(this);
+    LOG.info("Calling handler for JobFinishedEvent ");
+    this.getEventHandler().handle(new JobHistoryEvent(this.jobId, jfe));    
+  }
+  
+  static JobState checkJobCompleteSuccess(JobImpl job) {
+    // check for Job success
+    if (job.completedTaskCount == job.getTasks().size()) {
+      try {
+        // Commit job & do cleanup
+        job.getCommitter().commitJob(job.getJobContext());
+      } catch (IOException e) {
+        LOG.warn("Could not do commit for Job", e);
+      }
+      
+      job.logJobHistoryFinishedEvent();
+      return job.finished(JobState.SUCCEEDED);
+    }
+    return null;
+  }
+
+  JobState finished(JobState finalState) {
+    if (getState() == JobState.RUNNING) {
+      metrics.endRunningJob(this);
+    }
+    if (finishTime == 0) setFinishTime();
+    eventHandler.handle(new JobFinishEvent(jobId));
+
+    switch (finalState) {
+      case KILLED:
+        metrics.killedJob(this);
+        break;
+      case FAILED:
+        metrics.failedJob(this);
+        break;
+      case SUCCEEDED:
+        metrics.completedJob(this);
+    }
+    return finalState;
+  }
+
+  @Override
+  public String getName() {
+    return jobName;
+  }
+
+  @Override
+  public int getTotalMaps() {
+    return mapTasks.size();  //FIXME: why indirection? return numMapTasks...
+                             // unless race?  how soon can this get called?
+  }
+
+  @Override
+  public int getTotalReduces() {
+    return reduceTasks.size();  //FIXME: why indirection? return numReduceTasks
+  }
+
+  public static class InitTransition 
+      implements MultipleArcTransition<JobImpl, JobEvent, JobState> {
+
+    /**
+     * Note that this transition method is called directly (and synchronously)
+     * by MRAppMaster's init() method (i.e., no RPC, no thread-switching;
+     * just plain sequential call within AM context), so we can trigger
+     * modifications in AM state from here (at least, if AM is written that
+     * way; MR version is).
+     */
+    @Override
+    public JobState transition(JobImpl job, JobEvent event) {
+      job.submitTime = job.clock.getTime();
+      job.metrics.submittedJob(job);
+      job.metrics.preparingJob(job);
+      try {
+        setup(job);
+        job.fs = FileSystem.get(job.conf);
+
+        //log to job history
+        JobSubmittedEvent jse = new JobSubmittedEvent(job.oldJobId,
+              job.conf.get(MRJobConfig.JOB_NAME, "test"), 
+            job.conf.get(MRJobConfig.USER_NAME, "mapred"),
+            job.submitTime,
+            job.remoteJobConfFile.toString(),
+            job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "default"));
+        job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
+        //TODO JH Verify jobACLs, UserName via UGI?
+
+        TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
+        job.numMapTasks = taskSplitMetaInfo.length;
+        job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0);
+
+        if (job.numMapTasks == 0 && job.numReduceTasks == 0) {
+          job.addDiagnostic("No of maps and reduces are 0 " + job.jobId);
+        }
+
+        checkTaskLimits();
+
+        
+        boolean newApiCommitter = false;
+        if ((job.numReduceTasks > 0 && 
+            job.conf.getBoolean("mapred.reducer.new-api", false)) ||
+              (job.numReduceTasks == 0 && 
+               job.conf.getBoolean("mapred.mapper.new-api", false)))  {
+          newApiCommitter = true;
+          LOG.info("Using mapred newApiCommitter.");
+        }
+        
+        LOG.info("OutputCommitter set in config " + job.conf.get("mapred.output.committer.class"));
+        
+        if (newApiCommitter) {
+          job.jobContext = new JobContextImpl(job.conf,
+              job.oldJobId);
+          org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = RecordFactoryProvider
+              .getRecordFactory(null)
+              .newRecordInstance(
+                  org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId.class);
+          attemptID.setTaskId(RecordFactoryProvider.getRecordFactory(null)
+              .newRecordInstance(TaskId.class));
+          attemptID.getTaskId().setJobId(job.jobId);
+          attemptID.getTaskId().setTaskType(TaskType.MAP);
+          TaskAttemptContext taskContext = new TaskAttemptContextImpl(job.conf,
+              TypeConverter.fromYarn(attemptID));
+          try {
+            OutputFormat outputFormat = ReflectionUtils.newInstance(
+                taskContext.getOutputFormatClass(), job.conf);
+            job.committer = outputFormat.getOutputCommitter(taskContext);
+          } catch(Exception e) {
+            throw new IOException("Failed to assign outputcommitter", e);
+          }
+        } else {
+          job.jobContext = new org.apache.hadoop.mapred.JobContextImpl(
+              new JobConf(job.conf), job.oldJobId);
+          job.committer = ReflectionUtils.newInstance(
+              job.conf.getClass("mapred.output.committer.class", FileOutputCommitter.class,
+              org.apache.hadoop.mapred.OutputCommitter.class), job.conf);
+        }
+        LOG.info("OutputCommitter is " + job.committer.getClass().getName());
+        
+        long inputLength = 0;
+        for (int i = 0; i < job.numMapTasks; ++i) {
+          inputLength += taskSplitMetaInfo[i].getInputDataLength();
+        }
+
+//FIXME:  need new memory criterion for uber-decision (oops, too late here; until AM-resizing supported, must depend on job client to pass fat-slot needs)
+        // these are no longer "system" settings, necessarily; user may override
+        int sysMaxMaps = job.conf.getInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 9);
+        int sysMaxReduces =
+            job.conf.getInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 1);
+        long sysMaxBytes = job.conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
+            job.conf.getLong("dfs.block.size", 64*1024*1024));  //FIXME: this is wrong; get FS from [File?]InputFormat and default block size from that
+        //long sysMemSizeForUberSlot = JobTracker.getMemSizeForReduceSlot(); // FIXME [could use default AM-container memory size...]
+
+        boolean uberEnabled =
+            job.conf.getBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
+        boolean smallNumMapTasks = (job.numMapTasks <= sysMaxMaps);
+        boolean smallNumReduceTasks = (job.numReduceTasks <= sysMaxReduces);
+        boolean smallInput = (inputLength <= sysMaxBytes);
+        boolean smallMemory = true;  //FIXME (see above)
+            // ignoring overhead due to UberTask and statics as negligible here:
+//  FIXME   && (Math.max(memoryPerMap, memoryPerReduce) <= sysMemSizeForUberSlot
+//              || sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT)
+        boolean notChainJob = !isChainJob(job.conf);
+
+        // User has overall veto power over uberization, or user can modify
+        // limits (overriding system settings and potentially shooting
+        // themselves in the head).  Note that ChainMapper/Reducer are
+        // fundamentally incompatible with MR-1220; they employ a blocking
+
+        // User has overall veto power over uberization, or user can modify
+        // limits (overriding system settings and potentially shooting
+        // themselves in the head).  Note that ChainMapper/Reducer are
+        // fundamentally incompatible with MR-1220; they employ a blocking
+        // queue between the maps/reduces and thus require parallel execution,
+        // while "uber-AM" (MR AM + LocalContainerLauncher) loops over tasks
+        // and thus requires sequential execution.
+        job.isUber = uberEnabled && smallNumMapTasks && smallNumReduceTasks
+            && smallInput && smallMemory && notChainJob;
+
+        if (job.isUber) {
+          LOG.info("Uberizing job " + job.jobId + ": " + job.numMapTasks + "m+"
+              + job.numReduceTasks + "r tasks (" + inputLength
+              + " input bytes) will run sequentially on single node.");
+              //TODO: also note which node?
+
+          // make sure reduces are scheduled only after all map are completed
+          job.conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART,
+                            1.0f);
+          // uber-subtask attempts all get launched on same node; if one fails,
+          // probably should retry elsewhere, i.e., move entire uber-AM:  ergo,
+          // limit attempts to 1 (or at most 2?  probably not...)
+          job.conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
+          job.conf.setInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 1);
+
+          // disable speculation:  makes no sense to speculate an entire job
+//        canSpeculateMaps = canSpeculateReduces = false; // [TODO: in old version, ultimately was from conf.getMapSpeculativeExecution(), conf.getReduceSpeculativeExecution()]
+        } else {
+          StringBuilder msg = new StringBuilder();
+          msg.append("Not uberizing ").append(job.jobId).append(" because:");
+          if (!uberEnabled)
+            msg.append(" not enabled;");
+          if (!smallNumMapTasks)
+            msg.append(" too many maps;");
+          if (!smallNumReduceTasks)
+            msg.append(" too many reduces;");
+          if (!smallInput)
+            msg.append(" too much input;");
+          if (!smallMemory)
+            msg.append(" too much RAM;");
+          if (!notChainJob)
+            msg.append(" chainjob");
+          LOG.info(msg.toString());
+        }
+
+        job.taskAttemptCompletionEvents =
+            new ArrayList<TaskAttemptCompletionEvent>(
+                job.numMapTasks + job.numReduceTasks + 10);
+
+        job.allowedMapFailuresPercent =
+            job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0);
+        job.allowedReduceFailuresPercent =
+            job.conf.getInt(MRJobConfig.REDUCE_FAILURES_MAXPERCENT, 0);
+
+        // do the setup
+        job.committer.setupJob(job.jobContext);
+        job.setupProgress = 1.0f;
+
+        // create the Tasks but don't start them yet
+        createMapTasks(job, inputLength, taskSplitMetaInfo);
+        createReduceTasks(job);
+
+        job.metrics.endPreparingJob(job);
+        return JobState.INITED;
+        //TODO XXX Should JobInitedEvent be generated here (instead of in StartTransition)
+
+      } catch (IOException e) {
+        LOG.warn("Job init failed", e);
+        job.addDiagnostic("Job init failed : "
+            + StringUtils.stringifyException(e));
+        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
+        job.metrics.endPreparingJob(job);
+        return job.finished(JobState.FAILED);
+      }
+    }
+
+    protected void setup(JobImpl job) throws IOException {
+
+      String oldJobIDString = job.oldJobId.toString();
+      String user = 
+        UserGroupInformation.getCurrentUser().getShortUserName();
+      Path path = MRApps.getStagingAreaDir(job.conf, user);
+      LOG.info("DEBUG --- startJobs:"
+          + " parent="
+          + path + " child="
+          + oldJobIDString);
+
+      job.remoteJobSubmitDir =
+          FileSystem.get(job.conf).makeQualified(
+              new Path(path, oldJobIDString));
+      job.remoteJobConfFile =
+          new Path(job.remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
+
+      // Prepare the TaskAttemptListener server for authentication of Containers
+      // TaskAttemptListener gets the information via jobTokenSecretManager.
+      JobTokenIdentifier identifier =
+          new JobTokenIdentifier(new Text(oldJobIDString));
+      job.jobToken =
+          new Token<JobTokenIdentifier>(identifier, job.jobTokenSecretManager);
+      job.jobToken.setService(identifier.getJobId());
+      // Add it to the jobTokenSecretManager so that TaskAttemptListener server
+      // can authenticate containers(tasks)
+      job.jobTokenSecretManager.addTokenForJob(oldJobIDString, job.jobToken);
+      LOG.info("Adding job token for " + oldJobIDString
+          + " to jobTokenSecretManager");
+
+      // Upload the jobTokens onto the remote FS so that ContainerManager can
+      // localize it to be used by the Containers(tasks)
+      Credentials tokenStorage = new Credentials();
+      TokenCache.setJobToken(job.jobToken, tokenStorage);
+
+      if (UserGroupInformation.isSecurityEnabled()) {
+        tokenStorage.addAll(job.fsTokens);
+      }
+
+      Path remoteJobTokenFile =
+          new Path(job.remoteJobSubmitDir,
+              MRConstants.APPLICATION_TOKENS_FILE);
+      tokenStorage.writeTokenStorageFile(remoteJobTokenFile, job.conf);
+      LOG.info("Writing back the job-token file on the remote file system:"
+          + remoteJobTokenFile.toString());
+    }
+
+    /**
+     * ChainMapper and ChainReducer must execute in parallel, so they're not
+     * compatible with uberization/LocalContainerLauncher (100% sequential).
+     */
+    boolean isChainJob(Configuration conf) {
+      boolean isChainJob = false;
+      try {
+        String mapClassName = conf.get(MRJobConfig.MAP_CLASS_ATTR);
+        if (mapClassName != null) {
+          Class<?> mapClass = Class.forName(mapClassName);
+          if (ChainMapper.class.isAssignableFrom(mapClass))
+            isChainJob = true;
+        }
+      } catch (ClassNotFoundException cnfe) {
+        // don't care; assume it's not derived from ChainMapper
+      }
+      try {
+        String reduceClassName = conf.get(MRJobConfig.REDUCE_CLASS_ATTR);
+        if (reduceClassName != null) {
+          Class<?> reduceClass = Class.forName(reduceClassName);
+          if (ChainReducer.class.isAssignableFrom(reduceClass))
+            isChainJob = true;
+        }
+      } catch (ClassNotFoundException cnfe) {
+        // don't care; assume it's not derived from ChainReducer
+      }
+      return isChainJob;
+    }
+
+    private void createMapTasks(JobImpl job, long inputLength,
+                                TaskSplitMetaInfo[] splits) {
+      for (int i=0; i < job.numMapTasks; ++i) {
+        TaskImpl task =
+            new MapTaskImpl(job.jobId, i,
+                job.eventHandler, 
+                job.remoteJobConfFile, 
+                job.conf, splits[i], 
+                job.taskAttemptListener, 
+                job.committer, job.jobToken, job.fsTokens.getAllTokens(), 
+                job.clock, job.completedTasksFromPreviousRun, job.startCount,
+                job.metrics);
+        job.addTask(task);
+      }
+      LOG.info("Input size for job " + job.jobId + " = " + inputLength
+          + ". Number of splits = " + splits.length);
+    }
+
+    private void createReduceTasks(JobImpl job) {
+      for (int i = 0; i < job.numReduceTasks; i++) {
+        TaskImpl task =
+            new ReduceTaskImpl(job.jobId, i,
+                job.eventHandler, 
+                job.remoteJobConfFile, 
+                job.conf, job.numMapTasks, 
+                job.taskAttemptListener, job.committer, job.jobToken,
+                job.fsTokens.getAllTokens(), job.clock, 
+                job.completedTasksFromPreviousRun, job.startCount, job.metrics);
+        job.addTask(task);
+      }
+      LOG.info("Number of reduces for job " + job.jobId + " = "
+          + job.numReduceTasks);
+    }
+
+    protected TaskSplitMetaInfo[] createSplits(JobImpl job, JobId jobId) {
+      TaskSplitMetaInfo[] allTaskSplitMetaInfo;
+      try {
+        allTaskSplitMetaInfo = SplitMetaInfoReader.readSplitMetaInfo(
+            job.oldJobId, job.fs, 
+            job.conf, 
+            job.remoteJobSubmitDir);
+      } catch (IOException e) {
+        throw new YarnException(e);
+      }
+      return allTaskSplitMetaInfo;
+    }
+
+    /**
+     * If the number of tasks are greater than the configured value
+     * throw an exception that will fail job initialization
+     */
+    private void checkTaskLimits() {
+      // no code, for now
+    }
+  } // end of InitTransition
+
+  public static class StartTransition
+  implements SingleArcTransition<JobImpl, JobEvent> {
+    /**
+     * This transition executes in the event-dispatcher thread, though it's
+     * triggered in MRAppMaster's startJobs() method.
+     */
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      job.startTime = job.clock.getTime();
+      job.scheduleTasks(job.mapTasks);  // schedule (i.e., start) the maps
+      job.scheduleTasks(job.reduceTasks);
+      JobInitedEvent jie =
+        new JobInitedEvent(job.oldJobId,
+             job.startTime,
+             job.numMapTasks, job.numReduceTasks,
+             job.getState().toString()); //Will transition to state running. Currently in INITED
+      job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
+      JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,
+          job.submitTime, job.startTime);
+      job.eventHandler.handle(new JobHistoryEvent(job.jobId, jice));
+      job.metrics.runningJob(job);
+
+			// If we have no tasks, just transition to job completed
+      if (job.numReduceTasks == 0 && job.numMapTasks == 0) {
+        job.eventHandler.handle(new JobEvent(job.jobId, JobEventType.JOB_COMPLETED));
+      }
+    }
+  }
+
+  private void abortJob(
+      org.apache.hadoop.mapreduce.JobStatus.State finalState) {
+    try {
+      committer.abortJob(jobContext, finalState);
+    } catch (IOException e) {
+      LOG.warn("Could not abortJob", e);
+    }
+    if (finishTime == 0) setFinishTime();
+    cleanupProgress = 1.0f;
+    JobUnsuccessfulCompletionEvent unsuccessfulJobEvent =
+      new JobUnsuccessfulCompletionEvent(oldJobId,
+          finishTime,
+          succeededMapTaskCount,
+          succeededReduceTaskCount,
+          finalState.toString());
+    eventHandler.handle(new JobHistoryEvent(jobId, unsuccessfulJobEvent));
+  }
+    
+  // JobFinishedEvent triggers the move of the history file out of the staging
+  // area. May need to create a new event type for this if JobFinished should 
+  // not be generated for KilledJobs, etc.
+  private static JobFinishedEvent createJobFinishedEvent(JobImpl job) {
+    JobFinishedEvent jfe = new JobFinishedEvent(
+        job.oldJobId, job.finishTime,
+        job.succeededMapTaskCount, job.succeededReduceTaskCount,
+        job.failedMapTaskCount, job.failedReduceTaskCount,
+        TypeConverter.fromYarn(job.getMapCounters()),
+        TypeConverter.fromYarn(job.getReduceCounters()),
+        TypeConverter.fromYarn(job.getCounters()));
+    return jfe;
+  }
+
+  // Task-start has been moved out of InitTransition, so this arc simply
+  // hardcodes 0 for both map and reduce finished tasks.
+  private static class KillNewJobTransition
+  implements SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      job.setFinishTime();
+      JobUnsuccessfulCompletionEvent failedEvent =
+          new JobUnsuccessfulCompletionEvent(job.oldJobId,
+              job.finishTime, 0, 0,
+              JobState.KILLED.toString());
+      job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
+      job.finished(JobState.KILLED);
+    }
+  }
+
+  private static class KillInitedJobTransition
+  implements SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
+      job.addDiagnostic("Job received Kill in INITED state.");
+      job.finished(JobState.KILLED);
+    }
+  }
+
+  private static class KillTasksTransition
+      implements SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      job.addDiagnostic("Job received Kill while in RUNNING state.");
+      for (Task task : job.tasks.values()) {
+        job.eventHandler.handle(
+            new TaskEvent(task.getID(), TaskEventType.T_KILL));
+      }
+      job.metrics.endRunningJob(job);
+    }
+  }
+
+  private static class TaskAttemptCompletedEventTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      TaskAttemptCompletionEvent tce = 
+        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
+      // Add the TaskAttemptCompletionEvent
+      //eventId is equal to index in the arraylist
+      tce.setEventId(job.taskAttemptCompletionEvents.size());
+      job.taskAttemptCompletionEvents.add(tce);
+      
+      //make the previous completion event as obsolete if it exists
+      Object successEventNo = 
+        job.successAttemptCompletionEventNoMap.remove(tce.getAttemptId().getTaskId());
+      if (successEventNo != null) {
+        TaskAttemptCompletionEvent successEvent = 
+          job.taskAttemptCompletionEvents.get((Integer) successEventNo);
+        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
+      }
+
+      if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) {
+        job.successAttemptCompletionEventNoMap.put(tce.getAttemptId().getTaskId(), 
+            tce.getEventId());
+      }
+    }
+  }
+
+  private static class TaskAttemptFetchFailureTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      JobTaskAttemptFetchFailureEvent fetchfailureEvent = 
+        (JobTaskAttemptFetchFailureEvent) event;
+      for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId mapId : 
+            fetchfailureEvent.getMaps()) {
+        Integer fetchFailures = job.fetchFailuresMapping.get(mapId);
+        fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1);
+        job.fetchFailuresMapping.put(mapId, fetchFailures);
+        
+        //get number of running reduces
+        int runningReduceTasks = 0;
+        for (TaskId taskId : job.reduceTasks) {
+          if (TaskState.RUNNING.equals(job.tasks.get(taskId).getState())) {
+            runningReduceTasks++;
+          }
+        }
+        
+        float failureRate = (float) fetchFailures / runningReduceTasks;
+        // declare faulty if fetch-failures >= max-allowed-failures
+        boolean isMapFaulty =
+            (failureRate >= MAX_ALLOWED_FETCH_FAILURES_FRACTION);
+        if (fetchFailures >= MAX_FETCH_FAILURES_NOTIFICATIONS && isMapFaulty) {
+          LOG.info("Too many fetch-failures for output of task attempt: " + 
+              mapId + " ... raising fetch failure to map");
+          job.eventHandler.handle(new TaskAttemptEvent(mapId, 
+              TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+          job.fetchFailuresMapping.remove(mapId);
+        }
+      }
+    }
+  }
+
+  private static class TaskCompletedTransition implements
+      MultipleArcTransition<JobImpl, JobEvent, JobState> {
+
+    @Override
+    public JobState transition(JobImpl job, JobEvent event) {
+      job.completedTaskCount++;
+      LOG.info("Num completed Tasks: " + job.completedTaskCount);
+      JobTaskEvent taskEvent = (JobTaskEvent) event;
+      Task task = job.tasks.get(taskEvent.getTaskID());
+      if (taskEvent.getState() == TaskState.SUCCEEDED) {
+        taskSucceeded(job, task);
+      } else if (taskEvent.getState() == TaskState.FAILED) {
+        taskFailed(job, task);
+      } else if (taskEvent.getState() == TaskState.KILLED) {
+        taskKilled(job, task);
+      }
+
+      return checkJobForCompletion(job);
+    }
+
+    protected JobState checkJobForCompletion(JobImpl job) {
+      //check for Job failure
+      if (job.failedMapTaskCount*100 > 
+        job.allowedMapFailuresPercent*job.numMapTasks ||
+        job.failedReduceTaskCount*100 > 
+        job.allowedReduceFailuresPercent*job.numReduceTasks) {
+        job.setFinishTime();
+
+        String diagnosticMsg = "Job failed as tasks failed. " +
+            "failedMaps:" + job.failedMapTaskCount + 
+            " failedReduces:" + job.failedReduceTaskCount;
+        LOG.info(diagnosticMsg);
+        job.addDiagnostic(diagnosticMsg);
+        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
+        return job.finished(JobState.FAILED);
+      }
+      
+      JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
+      if (jobCompleteSuccess != null) {
+        return jobCompleteSuccess;
+      }
+      
+      //return the current state, Job not finished yet
+      return job.getState();
+    }
+
+    private void taskSucceeded(JobImpl job, Task task) {
+      if (task.getType() == TaskType.MAP) {
+        job.succeededMapTaskCount++;
+      } else {
+        job.succeededReduceTaskCount++;
+      }
+      job.metrics.completedTask(task);
+    }
+  
+    private void taskFailed(JobImpl job, Task task) {
+      if (task.getType() == TaskType.MAP) {
+        job.failedMapTaskCount++;
+      } else if (task.getType() == TaskType.REDUCE) {
+        job.failedReduceTaskCount++;
+      }
+      job.addDiagnostic("Task failed " + task.getID());
+      job.metrics.failedTask(task);
+    }
+
+    private void taskKilled(JobImpl job, Task task) {
+      if (task.getType() == TaskType.MAP) {
+        job.killedMapTaskCount++;
+      } else if (task.getType() == TaskType.REDUCE) {
+        job.killedReduceTaskCount++;
+      }
+      job.metrics.killedTask(task);
+    }
+  }
+
+  // Transition class for handling jobs with no tasks
+  static class JobNoTasksCompletedTransition implements
+  MultipleArcTransition<JobImpl, JobEvent, JobState> {
+
+    @Override
+    public JobState transition(JobImpl job, JobEvent event) {
+      JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
+      if (jobCompleteSuccess != null) {
+        return jobCompleteSuccess;
+      }
+      
+      // Return the current state, Job not finished yet
+      return job.getState();
+    }
+  }
+
+  private static class MapTaskRescheduledTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      //succeeded map task is restarted back
+      job.completedTaskCount--;
+      job.succeededMapTaskCount--;
+    }
+  }
+
+  private static class KillWaitTaskCompletedTransition extends  
+      TaskCompletedTransition {
+    @Override
+    protected JobState checkJobForCompletion(JobImpl job) {
+      if (job.completedTaskCount == job.tasks.size()) {
+        job.setFinishTime();
+        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
+        return job.finished(JobState.KILLED);
+      }
+      //return the current state, Job not finished yet
+      return job.getState();
+    }
+  }
+
+  private void addDiagnostic(String diag) {
+    diagnostics.add(diag);
+  }
+  
+  private static class DiagnosticsUpdateTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      job.addDiagnostic(((JobDiagnosticsUpdateEvent) event)
+          .getDiagnosticUpdate());
+    }
+  }
+  
+  private static class CounterUpdateTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      JobCounterUpdateEvent jce = (JobCounterUpdateEvent) event;
+      for (JobCounterUpdateEvent.CounterIncrementalUpdate ci : jce
+          .getCounterUpdates()) {
+        job.jobCounters.incrCounter(ci.getCounterKey(), ci.getIncrementValue());
+      }
+    }
+  }
+
+  private static class InternalErrorTransition implements
+      SingleArcTransition<JobImpl, JobEvent> {
+    @Override
+    public void transition(JobImpl job, JobEvent event) {
+      //TODO Is this JH event required.
+      job.setFinishTime();
+      JobUnsuccessfulCompletionEvent failedEvent =
+          new JobUnsuccessfulCompletionEvent(job.oldJobId,
+              job.finishTime, 0, 0,
+              JobState.ERROR.toString());
+      job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
+      job.finished(JobState.ERROR);
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/MapTaskImpl.java

@@ -0,0 +1,97 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.impl;
+
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.MapTaskAttemptImpl;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+
+public class MapTaskImpl extends TaskImpl {
+
+  private final TaskSplitMetaInfo taskSplitMetaInfo;
+
+  public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
+      Path remoteJobConfFile, Configuration conf,
+      TaskSplitMetaInfo taskSplitMetaInfo,
+      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
+      Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock, 
+      Set<TaskId> completedTasksFromPreviousRun, int startCount,
+      MRAppMetrics metrics) {
+    super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
+        conf, taskAttemptListener, committer, jobToken, fsTokens, clock, 
+        completedTasksFromPreviousRun, startCount, metrics);
+    this.taskSplitMetaInfo = taskSplitMetaInfo;
+  }
+
+  @Override
+  protected int getMaxAttempts() {
+    return conf.getInt(MRJobConfig.MAP_MAX_ATTEMPTS, 4);
+  }
+
+  @Override
+  protected TaskAttemptImpl createAttempt() {
+    return new MapTaskAttemptImpl(getID(), nextAttemptNumber,
+        eventHandler, jobFile,
+        partition, taskSplitMetaInfo, conf, taskAttemptListener,
+        committer, jobToken, fsTokens, clock);
+  }
+
+  @Override
+  public TaskType getType() {
+    return TaskType.MAP;
+  }
+
+  protected TaskSplitMetaInfo getTaskSplitMetaInfo() {
+    return this.taskSplitMetaInfo;
+  }
+
+  /**
+   * @return a String formatted as a comma-separated list of splits.
+   */
+  @Override
+  protected String getSplitsAsString() {
+    String[] splits = getTaskSplitMetaInfo().getLocations();
+    if (splits == null || splits.length == 0)
+    return "";
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < splits.length; i++) {
+      if (i != 0) sb.append(",");
+      sb.append(splits[i]);
+    }
+    return sb.toString();
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ReduceTaskImpl.java

@@ -0,0 +1,75 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.impl;
+
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.ReduceTaskAttemptImpl;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+public class ReduceTaskImpl extends TaskImpl {
+  
+  private final int numMapTasks;
+
+  public ReduceTaskImpl(JobId jobId, int partition,
+      EventHandler eventHandler, Path jobFile, Configuration conf,
+      int numMapTasks, TaskAttemptListener taskAttemptListener,
+      OutputCommitter committer, Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock, 
+      Set<TaskId> completedTasksFromPreviousRun, int startCount,
+      MRAppMetrics metrics) {
+    super(jobId, TaskType.REDUCE, partition, eventHandler, jobFile, conf,
+        taskAttemptListener, committer, jobToken, fsTokens, clock,
+        completedTasksFromPreviousRun, startCount, metrics);
+    this.numMapTasks = numMapTasks;
+  }
+
+  @Override
+  protected int getMaxAttempts() {
+    return conf.getInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 4);
+  }
+
+  @Override
+  protected TaskAttemptImpl createAttempt() {
+    return new ReduceTaskAttemptImpl(getID(), nextAttemptNumber,
+        eventHandler, jobFile,
+        partition, numMapTasks, conf, taskAttemptListener,
+        committer, jobToken, fsTokens, clock);
+  }
+
+  @Override
+  public TaskType getType() {
+    return TaskType.REDUCE;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -0,0 +1,1442 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceChildJVM;
+import org.apache.hadoop.mapred.ProgressSplitsBlock;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.TaskAttemptContextImpl;
+import org.apache.hadoop.mapred.WrappedJvmID;
+import org.apache.hadoop.mapred.WrappedProgressSplitsBlock;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStartedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+import org.apache.hadoop.mapreduce.v2.api.records.Counter;
+import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.Phase;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
+import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
+import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.RackResolver;
+
+/**
+ * Implementation of TaskAttempt interface.
+ */
+@SuppressWarnings("all")
+public abstract class TaskAttemptImpl implements
+    org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
+      EventHandler<TaskAttemptEvent> {
+
+  private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
+  private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
+  private static final int MAP_MEMORY_MB_DEFAULT = 1024;
+  private static final int REDUCE_MEMORY_MB_DEFAULT = 1024;
+  private final static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+
+  protected final Configuration conf;
+  protected final Path jobFile;
+  protected final int partition;
+  protected final EventHandler eventHandler;
+  private final TaskAttemptId attemptId;
+  private final Clock clock;
+  private final org.apache.hadoop.mapred.JobID oldJobId;
+  private final TaskAttemptListener taskAttemptListener;
+  private final OutputCommitter committer;
+  private final Resource resourceCapability;
+  private final String[] dataLocalHosts;
+  private final List<String> diagnostics = new ArrayList<String>();
+  private final Lock readLock;
+  private final Lock writeLock;
+  private Collection<Token<? extends TokenIdentifier>> fsTokens;
+  private Token<JobTokenIdentifier> jobToken;
+  private static AtomicBoolean initialClasspathFlag = new AtomicBoolean();
+  private static String initialClasspath = null;
+  private final Object classpathLock = new Object();
+  private long launchTime;
+  private long finishTime;
+  private WrappedProgressSplitsBlock progressSplitBlock;
+
+  private static final CleanupContainerTransition CLEANUP_CONTAINER_TRANSITION =
+    new CleanupContainerTransition();
+
+  private static final DiagnosticInformationUpdater 
+    DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION 
+      = new DiagnosticInformationUpdater();
+
+  private static final StateMachineFactory
+        <TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+        stateMachineFactory
+    = new StateMachineFactory
+             <TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+           (TaskAttemptState.NEW)
+
+     // Transitions from the NEW state.
+     .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
+         TaskAttemptEventType.TA_SCHEDULE, new RequestContainerTransition(false))
+     .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
+         TaskAttemptEventType.TA_RESCHEDULE, new RequestContainerTransition(true))
+     .addTransition(TaskAttemptState.NEW, TaskAttemptState.KILLED,
+         TaskAttemptEventType.TA_KILL, new KilledTransition())
+     .addTransition(TaskAttemptState.NEW, TaskAttemptState.FAILED,
+         TaskAttemptEventType.TA_FAILMSG, new FailedTransition())
+
+     // Transitions from the UNASSIGNED state.
+     .addTransition(TaskAttemptState.UNASSIGNED,
+         TaskAttemptState.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED,
+         new ContainerAssignedTransition())
+     .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.KILLED,
+         TaskAttemptEventType.TA_KILL, new DeallocateContainerTransition(
+             TaskAttemptState.KILLED, true))
+     .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.FAILED,
+         TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition(
+             TaskAttemptState.FAILED, true))
+
+     // Transitions from the ASSIGNED state.
+     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.RUNNING,
+         TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
+         new LaunchedContainerTransition())
+     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.ASSIGNED,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED,
+         TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED,
+         new DeallocateContainerTransition(TaskAttemptState.FAILED, false))
+     .addTransition(TaskAttemptState.ASSIGNED, 
+         TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
+     .addTransition(TaskAttemptState.ASSIGNED, 
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
+
+     // Transitions from RUNNING state.
+     .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
+         TaskAttemptEventType.TA_UPDATE, new StatusUpdater())
+     .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // If no commit is required, task directly goes to success
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
+     // If commit is required, task goes through commit pending state.
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptEventType.TA_COMMIT_PENDING, new CommitPendingTransition())
+     // Failure handling while RUNNING
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
+      //for handling container exit without sending the done or fail msg
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+         CLEANUP_CONTAINER_TRANSITION)
+     // Timeout handling while RUNNING
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
+     // Kill handling
+     .addTransition(TaskAttemptState.RUNNING,
+         TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
+         CLEANUP_CONTAINER_TRANSITION)
+
+     // Transitions from COMMIT_PENDING state
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE,
+         new StatusUpdater())
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
+         CLEANUP_CONTAINER_TRANSITION)
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+         CLEANUP_CONTAINER_TRANSITION)
+     .addTransition(TaskAttemptState.COMMIT_PENDING,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
+
+     // Transitions from SUCCESS_CONTAINER_CLEANUP state
+     // kill and cleanup the container
+     .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+         TaskAttemptState.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED,
+         new SucceededTransition())
+     .addTransition(
+          TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+          TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+          TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+          DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+      // Ignore-able events
+     .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_FAILMSG,
+             TaskAttemptEventType.TA_TIMED_OUT,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED))
+
+     // Transitions from FAIL_CONTAINER_CLEANUP state.
+     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptState.FAIL_TASK_CLEANUP,
+         TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
+     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+      // Ignore-able events
+     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG,
+             TaskAttemptEventType.TA_TIMED_OUT))
+
+      // Transitions from KILL_CONTAINER_CLEANUP
+     .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         TaskAttemptState.KILL_TASK_CLEANUP,
+         TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
+     .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // Ignore-able events
+     .addTransition(
+         TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         TaskAttemptState.KILL_CONTAINER_CLEANUP,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG,
+             TaskAttemptEventType.TA_TIMED_OUT))
+
+     // Transitions from FAIL_TASK_CLEANUP
+     // run the task cleanup
+     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
+         TaskAttemptState.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE,
+         new FailedTransition())
+     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
+         TaskAttemptState.FAIL_TASK_CLEANUP,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+      // Ignore-able events
+     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
+         TaskAttemptState.FAIL_TASK_CLEANUP,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG))
+
+     // Transitions from KILL_TASK_CLEANUP
+     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
+         TaskAttemptState.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE,
+         new KilledTransition())
+     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
+         TaskAttemptState.KILL_TASK_CLEANUP,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // Ignore-able events
+     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
+         TaskAttemptState.KILL_TASK_CLEANUP,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG))
+
+      // Transitions from SUCCEEDED
+     .addTransition(TaskAttemptState.SUCCEEDED, //only possible for map attempts
+         TaskAttemptState.FAILED,
+         TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+         new TooManyFetchFailureTransition())
+     .addTransition(
+         TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // Ignore-able events for SUCCEEDED state
+     .addTransition(TaskAttemptState.SUCCEEDED,
+         TaskAttemptState.SUCCEEDED,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_FAILMSG,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED))
+
+     // Transitions from FAILED state
+     .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
+       TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+       DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // Ignore-able events for FAILED state
+     .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_ASSIGNED,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG))
+
+     // Transitions from KILLED state
+     .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
+         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+     // Ignore-able events for KILLED state
+     .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
+         EnumSet.of(TaskAttemptEventType.TA_KILL,
+             TaskAttemptEventType.TA_ASSIGNED,
+             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+             TaskAttemptEventType.TA_UPDATE,
+             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
+             TaskAttemptEventType.TA_COMMIT_PENDING,
+             TaskAttemptEventType.TA_DONE,
+             TaskAttemptEventType.TA_FAILMSG))
+
+     // create the topology tables
+     .installTopology();
+
+  private final StateMachine
+         <TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+    stateMachine;
+
+  private ContainerId containerID;
+  private String nodeHostName;
+  private String containerMgrAddress;
+  private String nodeHttpAddress;
+  private WrappedJvmID jvmID;
+  private ContainerToken containerToken;
+  private Resource assignedCapability;
+  
+  //this takes good amount of memory ~ 30KB. Instantiate it lazily
+  //and make it null once task is launched.
+  private org.apache.hadoop.mapred.Task remoteTask;
+  
+  //this is the last status reported by the REMOTE running attempt
+  private TaskAttemptStatus reportedStatus;
+
+  public TaskAttemptImpl(TaskId taskId, int i, EventHandler eventHandler,
+      TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
+      Configuration conf, String[] dataLocalHosts, OutputCommitter committer,
+      Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
+    oldJobId = TypeConverter.fromYarn(taskId.getJobId());
+    this.conf = conf;
+    this.clock = clock;
+    attemptId = recordFactory.newRecordInstance(TaskAttemptId.class);
+    attemptId.setTaskId(taskId);
+    attemptId.setId(i);
+    this.taskAttemptListener = taskAttemptListener;
+
+    // Initialize reportedStatus
+    reportedStatus = new TaskAttemptStatus();
+    initTaskAttemptStatus(reportedStatus);
+
+    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    readLock = readWriteLock.readLock();
+    writeLock = readWriteLock.writeLock();
+
+    this.fsTokens = fsTokens;
+    this.jobToken = jobToken;
+    this.eventHandler = eventHandler;
+    this.committer = committer;
+    this.jobFile = jobFile;
+    this.partition = partition;
+
+    //TODO:create the resource reqt for this Task attempt
+    this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
+    this.resourceCapability.setMemory(getMemoryRequired(conf, taskId.getTaskType()));
+    this.dataLocalHosts = dataLocalHosts;
+    RackResolver.init(conf);
+
+    // This "this leak" is okay because the retained pointer is in an
+    //  instance variable.
+    stateMachine = stateMachineFactory.make(this);
+  }
+
+  private int getMemoryRequired(Configuration conf, TaskType taskType) {
+    int memory = 1024;
+    if (taskType == TaskType.MAP)  {
+      memory = conf.getInt(MRJobConfig.MAP_MEMORY_MB, MAP_MEMORY_MB_DEFAULT);
+    } else if (taskType == TaskType.REDUCE) {
+      memory = conf.getInt(MRJobConfig.REDUCE_MEMORY_MB, REDUCE_MEMORY_MB_DEFAULT);
+    }
+    
+    return memory;
+  }
+
+  /**
+   * Create a {@link LocalResource} record with all the given parameters.
+   * TODO: This should pave way for Builder pattern.
+   */
+  private static LocalResource createLocalResource(FileSystem fc,
+      RecordFactory recordFactory, Path file, LocalResourceType type,
+      LocalResourceVisibility visibility) throws IOException {
+    FileStatus fstat = fc.getFileStatus(file);
+    LocalResource resource =
+        recordFactory.newRecordInstance(LocalResource.class);
+    resource.setResource(ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
+        .getPath())));
+    resource.setType(type);
+    resource.setVisibility(visibility);
+    resource.setSize(fstat.getLen());
+    resource.setTimestamp(fstat.getModificationTime());
+    return resource;
+  }
+
+  /**
+   * Lock this on initialClasspath so that there is only one fork in the AM for
+   * getting the initial class-path. TODO: This should go away once we construct
+   * a parent CLC and use it for all the containers.
+   */
+  private String getInitialClasspath() throws IOException {
+    synchronized (classpathLock) {
+      if (initialClasspathFlag.get()) {
+        return initialClasspath;
+      }
+      Map<String, String> env = new HashMap<String, String>();
+      MRApps.setInitialClasspath(env);
+      initialClasspath = env.get(MRApps.CLASSPATH);
+      initialClasspathFlag.set(true);
+      return initialClasspath;
+    }
+  }
+
+
+  /**
+   * Create the {@link ContainerLaunchContext} for this attempt.
+   */
+  private ContainerLaunchContext createContainerLaunchContext() {
+
+    ContainerLaunchContext container =
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
+
+    try {
+      FileSystem remoteFS = FileSystem.get(conf);
+
+      // //////////// Set up JobJar to be localized properly on the remote NM.
+      if (conf.get(MRJobConfig.JAR) != null) {
+        Path remoteJobJar = (new Path(remoteTask.getConf().get(
+              MRJobConfig.JAR))).makeQualified(remoteFS.getUri(), 
+                                               remoteFS.getWorkingDirectory());
+        container.setLocalResource(
+            MRConstants.JOB_JAR,
+            createLocalResource(remoteFS, recordFactory, remoteJobJar,
+                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
+        LOG.info("The job-jar file on the remote FS is "
+            + remoteJobJar.toUri().toASCIIString());
+      } else {
+        // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
+        // mapreduce jar itself which is already on the classpath.
+        LOG.info("Job jar is not present. "
+            + "Not adding any jar to the list of resources.");
+      }
+      // //////////// End of JobJar setup
+
+      // //////////// Set up JobConf to be localized properly on the remote NM.
+      Path path =
+          MRApps.getStagingAreaDir(conf, UserGroupInformation
+              .getCurrentUser().getShortUserName());
+      Path remoteJobSubmitDir =
+          new Path(path, oldJobId.toString());
+      Path remoteJobConfPath = 
+          new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
+      container.setLocalResource(
+          MRConstants.JOB_CONF_FILE,
+          createLocalResource(remoteFS, recordFactory, remoteJobConfPath,
+              LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
+      LOG.info("The job-conf file on the remote FS is "
+          + remoteJobConfPath.toUri().toASCIIString());
+      // //////////// End of JobConf setup
+
+      // Setup DistributedCache
+      setupDistributedCache(remoteFS, conf, container);
+
+      // Setup up tokens
+      Credentials taskCredentials = new Credentials();
+
+      if (UserGroupInformation.isSecurityEnabled()) {
+        // Add file-system tokens
+        for (Token<? extends TokenIdentifier> token : fsTokens) {
+          LOG.info("Putting fs-token for NM use for launching container : "
+              + token.toString());
+          taskCredentials.addToken(token.getService(), token);
+        }
+      }
+
+      // LocalStorageToken is needed irrespective of whether security is enabled
+      // or not.
+      TokenCache.setJobToken(jobToken, taskCredentials);
+
+      DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+      LOG.info("Size of containertokens_dob is "
+          + taskCredentials.numberOfTokens());
+      taskCredentials.writeTokenStorageToStream(containerTokens_dob);
+      container.setContainerTokens(
+          ByteBuffer.wrap(containerTokens_dob.getData(), 0,
+              containerTokens_dob.getLength()));
+
+      // Add shuffle token
+      LOG.info("Putting shuffle token in serviceData");
+      DataOutputBuffer jobToken_dob = new DataOutputBuffer();
+      jobToken.write(jobToken_dob);
+      container
+          .setServiceData(
+              ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+              ByteBuffer.wrap(jobToken_dob.getData(), 0,
+                  jobToken_dob.getLength()));
+
+      MRApps.addToClassPath(container.getAllEnv(), getInitialClasspath());
+    } catch (IOException e) {
+      throw new YarnException(e);
+    }
+    
+    container.setContainerId(containerID);
+    container.setUser(conf.get(MRJobConfig.USER_NAME)); // TODO: Fix
+
+    File workDir = new File("$PWD"); // Will be expanded by the shell.
+    String containerLogDir =
+        new File(ApplicationConstants.LOG_DIR_EXPANSION_VAR).toString();
+    String childTmpDir = new File(workDir, "tmp").toString();
+    String javaHome = "${JAVA_HOME}"; // Will be expanded by the shell.
+    String nmLdLibraryPath = "{LD_LIBRARY_PATH}"; // Expanded by the shell?
+    List<String> classPaths = new ArrayList<String>();
+
+    String localizedApplicationTokensFile =
+        new File(workDir, MRConstants.APPLICATION_TOKENS_FILE).toString();
+    classPaths.add(MRConstants.JOB_JAR);
+    classPaths.add(MRConstants.YARN_MAPREDUCE_APP_JAR_PATH);
+    classPaths.add(workDir.toString()); // TODO
+
+    // Construct the actual Container
+    container.addAllCommands(MapReduceChildJVM.getVMCommand(
+        taskAttemptListener.getAddress(), remoteTask, javaHome,
+        workDir.toString(), containerLogDir, childTmpDir, jvmID));
+
+    MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths,
+        workDir.toString(), containerLogDir, nmLdLibraryPath, remoteTask,
+        localizedApplicationTokensFile);
+
+    // Construct the actual Container
+    container.setContainerId(containerID);
+    container.setUser(conf.get(MRJobConfig.USER_NAME));
+    container.setResource(assignedCapability);
+    return container;
+  }
+
+  private static long[] parseTimeStamps(String[] strs) {
+    if (null == strs) {
+      return null;
+    }
+    long[] result = new long[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Long.parseLong(strs[i]);
+    }
+    return result;
+  }
+
+  private void setupDistributedCache(FileSystem remoteFS, Configuration conf, 
+      ContainerLaunchContext container) throws IOException {
+    
+    // Cache archives
+    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE, 
+        DistributedCache.getCacheArchives(conf), 
+        parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), 
+        getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
+        DistributedCache.getArchiveVisibilities(conf), 
+        DistributedCache.getArchiveClassPaths(conf));
+    
+    // Cache files
+    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE, 
+        DistributedCache.getCacheFiles(conf),
+        parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
+        getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
+        DistributedCache.getFileVisibilities(conf),
+        DistributedCache.getFileClassPaths(conf));
+  }
+
+  // TODO - Move this to MR!
+  // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], 
+  // long[], boolean[], Path[], FileType)
+  private void parseDistributedCacheArtifacts(
+      FileSystem remoteFS, ContainerLaunchContext container, LocalResourceType type,
+      URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], 
+      Path[] pathsToPutOnClasspath) throws IOException {
+
+    if (uris != null) {
+      // Sanity check
+      if ((uris.length != timestamps.length) || (uris.length != sizes.length) ||
+          (uris.length != visibilities.length)) {
+        throw new IllegalArgumentException("Invalid specification for " +
+        		"distributed-cache artifacts of type " + type + " :" +
+        		" #uris=" + uris.length +
+        		" #timestamps=" + timestamps.length +
+        		" #visibilities=" + visibilities.length
+        		);
+      }
+      
+      Map<String, Path> classPaths = new HashMap<String, Path>();
+      if (pathsToPutOnClasspath != null) {
+        for (Path p : pathsToPutOnClasspath) {
+          p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
+              remoteFS.getWorkingDirectory()));
+          classPaths.put(p.toUri().getPath().toString(), p);
+        }
+      }
+      for (int i = 0; i < uris.length; ++i) {
+        URI u = uris[i];
+        Path p = new Path(u);
+        p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
+            remoteFS.getWorkingDirectory()));
+        // Add URI fragment or just the filename
+        Path name = new Path((null == u.getFragment())
+          ? p.getName()
+          : u.getFragment());
+        if (name.isAbsolute()) {
+          throw new IllegalArgumentException("Resource name must be relative");
+        }
+        String linkName = name.toUri().getPath();
+        container.setLocalResource(
+            linkName,
+            BuilderUtils.newLocalResource(recordFactory,
+                p.toUri(), type, 
+                visibilities[i]
+                  ? LocalResourceVisibility.PUBLIC
+                  : LocalResourceVisibility.PRIVATE,
+                sizes[i], timestamps[i])
+        );
+        if (classPaths.containsKey(u.getPath())) {
+          Map<String, String> environment = container.getAllEnv();
+          MRApps.addToClassPath(environment, linkName);
+        }
+      }
+    }
+  }
+  
+  // TODO - Move this to MR!
+  private static long[] getFileSizes(Configuration conf, String key) {
+    String[] strs = conf.getStrings(key);
+    if (strs == null) {
+      return null;
+    }
+    long[] result = new long[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Long.parseLong(strs[i]);
+    }
+    return result;
+  }
+  
+  @Override
+  public ContainerId getAssignedContainerID() {
+    readLock.lock();
+    try {
+      return containerID;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public String getAssignedContainerMgrAddress() {
+    readLock.lock();
+    try {
+      return containerMgrAddress;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public long getLaunchTime() {
+    readLock.lock();
+    try {
+      return launchTime;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public long getFinishTime() {
+    readLock.lock();
+    try {
+      return finishTime;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  /**If container Assigned then return the node's address, otherwise null.
+   */
+  @Override
+  public String getNodeHttpAddress() {
+    readLock.lock();
+    try {
+      return nodeHttpAddress;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  protected abstract org.apache.hadoop.mapred.Task createRemoteTask();
+
+  @Override
+  public TaskAttemptId getID() {
+    return attemptId;
+  }
+
+  @Override
+  public boolean isFinished() {
+    readLock.lock();
+    try {
+      // TODO: Use stateMachine level method?
+      return (getState() == TaskAttemptState.SUCCEEDED || 
+          getState() == TaskAttemptState.FAILED ||
+          getState() == TaskAttemptState.KILLED);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public TaskAttemptReport getReport() {
+    TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class);
+    readLock.lock();
+    try {
+      result.setTaskAttemptId(attemptId);
+      //take the LOCAL state of attempt
+      //DO NOT take from reportedStatus
+      
+      result.setTaskAttemptState(getState());
+      result.setProgress(reportedStatus.progress);
+      result.setStartTime(launchTime);
+      result.setFinishTime(finishTime);
+      result.setDiagnosticInfo(reportedStatus.diagnosticInfo);
+      result.setPhase(reportedStatus.phase);
+      result.setStateString(reportedStatus.stateString);
+      result.setCounters(getCounters());
+      return result;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public List<String> getDiagnostics() {
+    List<String> result = new ArrayList<String>();
+    readLock.lock();
+    try {
+      result.addAll(diagnostics);
+      return result;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public Counters getCounters() {
+    readLock.lock();
+    try {
+      Counters counters = reportedStatus.counters;
+      if (counters == null) {
+        counters = recordFactory.newRecordInstance(Counters.class);
+//        counters.groups = new HashMap<String, CounterGroup>();
+      }
+      return counters;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public float getProgress() {
+    readLock.lock();
+    try {
+      return reportedStatus.progress;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public TaskAttemptState getState() {
+    readLock.lock();
+    try {
+      return stateMachine.getCurrentState();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public void handle(TaskAttemptEvent event) {
+    LOG.info("Processing " + event.getTaskAttemptID() +
+        " of type " + event.getType());
+    writeLock.lock();
+    try {
+      final TaskAttemptState oldState = getState();
+      try {
+        stateMachine.doTransition(event.getType(), event);
+      } catch (InvalidStateTransitonException e) {
+        LOG.error("Can't handle this event at current state", e);
+        eventHandler.handle(new JobDiagnosticsUpdateEvent(
+            this.attemptId.getTaskId().getJobId(), "Invalid event " + event.getType() + 
+            " on TaskAttempt " + this.attemptId));
+        eventHandler.handle(new JobEvent(this.attemptId.getTaskId().getJobId(),
+            JobEventType.INTERNAL_ERROR));
+      }
+      if (oldState != getState()) {
+          LOG.info(attemptId + " TaskAttempt Transitioned from " 
+           + oldState + " to "
+           + getState());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  //always called in write lock
+  private void setFinishTime() {
+    //set the finish time only if launch time is set
+    if (launchTime != 0) {
+      finishTime = clock.getTime();
+    }
+  }
+  
+  private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
+    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
+    int slotMemoryReq =
+        taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
+    int simSlotsRequired =
+        slotMemoryReq
+            / (taskType == TaskType.MAP ? MAP_MEMORY_MB_DEFAULT
+                : REDUCE_MEMORY_MB_DEFAULT);
+    // Simulating MRv1 slots for counters by assuming *_MEMORY_MB_DEFAULT
+    // corresponds to a MrV1 slot.
+    // Fallow slot millis is not applicable in MRv2 - since a container is
+    // either assigned with the required memory or is not. No partial
+    // reserveations
+    long slotMillisIncrement =
+        simSlotsRequired
+            * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
+    return slotMillisIncrement;
+  }
+  
+  private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
+      TaskAttemptImpl taskAttempt) {
+    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
+    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
+    
+    long slotMillisIncrement = computeSlotMillis(taskAttempt);
+    
+    if (taskType == TaskType.MAP) {
+      jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
+      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
+    } else {
+      jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
+      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
+    }
+    return jce;
+  }
+
+  private static TaskAttemptUnsuccessfulCompletionEvent createTaskAttemptUnsuccessfulCompletionEvent(
+      TaskAttemptImpl taskAttempt, TaskAttemptState attemptState) {
+    TaskAttemptUnsuccessfulCompletionEvent tauce = new TaskAttemptUnsuccessfulCompletionEvent(
+        TypeConverter.fromYarn(taskAttempt.attemptId),
+        TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()),
+        attemptState.toString(), taskAttempt.finishTime,
+        taskAttempt.nodeHostName == null ? "UNKNOWN" : taskAttempt.nodeHostName,
+        taskAttempt.reportedStatus.diagnosticInfo.toString(),
+        taskAttempt.getProgressSplitBlock().burst());
+    return tauce;
+  }
+
+  private WrappedProgressSplitsBlock getProgressSplitBlock() {
+    readLock.lock();
+    try {
+      if (progressSplitBlock == null) {
+        progressSplitBlock = new WrappedProgressSplitsBlock(conf.getInt(
+            JHConfig.JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS_KEY,
+            WrappedProgressSplitsBlock.DEFAULT_NUMBER_PROGRESS_SPLITS));
+      }
+      return progressSplitBlock;
+    } finally {
+      readLock.unlock();
+    }
+  }
+  
+  private void updateProgressSplits() {
+    double newProgress = reportedStatus.progress;
+    Counters counters = reportedStatus.counters;
+    if (counters == null)
+      return;
+
+    WrappedProgressSplitsBlock splitsBlock = getProgressSplitBlock();
+    if (splitsBlock != null) {
+      long now = clock.getTime();
+      long start = getLaunchTime(); // TODO Ensure not 0
+
+      if (start != 0 && now - start <= Integer.MAX_VALUE) {
+        splitsBlock.getProgressWallclockTime().extend(newProgress,
+            (int) (now - start));
+      }
+
+      Counter cpuCounter = counters.getCounter(
+          TaskCounter.CPU_MILLISECONDS);
+      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
+        splitsBlock.getProgressCPUTime().extend(newProgress,
+            (int) cpuCounter.getValue());
+      }
+
+      Counter virtualBytes = counters.getCounter(
+          TaskCounter.VIRTUAL_MEMORY_BYTES);
+      if (virtualBytes != null) {
+        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
+            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
+      }
+
+      Counter physicalBytes = counters.getCounter(
+          TaskCounter.PHYSICAL_MEMORY_BYTES);
+      if (physicalBytes != null) {
+        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
+            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
+      }
+    }
+  }
+
+  private static class RequestContainerTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    private final boolean rescheduled;
+    public RequestContainerTransition(boolean rescheduled) {
+      this.rescheduled = rescheduled;
+    }
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      // Tell any speculator that we're requesting a container
+      taskAttempt.eventHandler.handle
+          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1));
+      //request for container
+      if (rescheduled) {
+        taskAttempt.eventHandler.handle(
+            ContainerRequestEvent.createContainerRequestEventForFailedContainer(
+                taskAttempt.attemptId, 
+                taskAttempt.resourceCapability));
+      } else {
+        int i = 0;
+        String[] racks = new String[taskAttempt.dataLocalHosts.length];
+        for (String host : taskAttempt.dataLocalHosts) {
+          racks[i++] = RackResolver.resolve(host).getNetworkLocation();
+        }
+        taskAttempt.eventHandler.handle(
+            new ContainerRequestEvent(taskAttempt.attemptId, 
+                taskAttempt.resourceCapability, 
+                taskAttempt.dataLocalHosts, racks));
+      }
+    }
+  }
+
+  private static class ContainerAssignedTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(final TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      TaskAttemptContainerAssignedEvent cEvent = 
+        (TaskAttemptContainerAssignedEvent) event;
+      taskAttempt.containerID = cEvent.getContainer().getId();
+      taskAttempt.nodeHostName = cEvent.getContainer().getNodeId().getHost();
+      taskAttempt.containerMgrAddress = cEvent.getContainer().getNodeId()
+          .toString();
+      taskAttempt.nodeHttpAddress = cEvent.getContainer().getNodeHttpAddress();
+      taskAttempt.containerToken = cEvent.getContainer().getContainerToken();
+      taskAttempt.assignedCapability = cEvent.getContainer().getResource();
+      // this is a _real_ Task (classic Hadoop mapred flavor):
+      taskAttempt.remoteTask = taskAttempt.createRemoteTask();
+      taskAttempt.jvmID = new WrappedJvmID(
+          taskAttempt.remoteTask.getTaskID().getJobID(), 
+          taskAttempt.remoteTask.isMapTask(), taskAttempt.containerID.getId());
+      
+      //launch the container
+      //create the container object to be launched for a given Task attempt
+      taskAttempt.eventHandler.handle(
+          new ContainerRemoteLaunchEvent(taskAttempt.attemptId, 
+              taskAttempt.containerID, 
+              taskAttempt.containerMgrAddress, taskAttempt.containerToken) {
+        @Override
+        public ContainerLaunchContext getContainer() {
+          return taskAttempt.createContainerLaunchContext();
+        }
+        @Override
+        public Task getRemoteTask() {  // classic mapred Task, not YARN version
+          return taskAttempt.remoteTask;
+        }
+      });
+
+      // send event to speculator that our container needs are satisfied
+      taskAttempt.eventHandler.handle
+          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
+    }
+  }
+
+  private static class DeallocateContainerTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    private final TaskAttemptState finalState;
+    private final boolean withdrawsContainerRequest;
+    DeallocateContainerTransition
+        (TaskAttemptState finalState, boolean withdrawsContainerRequest) {
+      this.finalState = finalState;
+      this.withdrawsContainerRequest = withdrawsContainerRequest;
+    }
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      //set the finish time
+      taskAttempt.setFinishTime();
+      //send the deallocate event to ContainerAllocator
+      taskAttempt.eventHandler.handle(
+          new ContainerAllocatorEvent(taskAttempt.attemptId,
+          ContainerAllocator.EventType.CONTAINER_DEALLOCATE));
+
+      // send event to speculator that we withdraw our container needs, if
+      //  we're transitioning out of UNASSIGNED
+      if (withdrawsContainerRequest) {
+        taskAttempt.eventHandler.handle
+            (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
+      }
+
+      switch(finalState) {
+        case FAILED:
+          taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+              taskAttempt.attemptId,
+              TaskEventType.T_ATTEMPT_FAILED));
+          break;
+        case KILLED:
+          taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+              taskAttempt.attemptId,
+              TaskEventType.T_ATTEMPT_KILLED));
+          break;
+      }
+      if (taskAttempt.getLaunchTime() != 0) {
+        TaskAttemptUnsuccessfulCompletionEvent tauce =
+            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
+                finalState);
+        taskAttempt.eventHandler
+            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+        taskAttempt.eventHandler.handle(new JobHistoryEvent(
+            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
+      } else {
+        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
+            + taskAttempt.getID());
+      }
+    }
+  }
+
+  private static class LaunchedContainerTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      //set the launch time
+      taskAttempt.launchTime = taskAttempt.clock.getTime();
+      // register it to TaskAttemptListener so that it start listening
+      // for it
+      taskAttempt.taskAttemptListener.register(
+          taskAttempt.attemptId, taskAttempt.remoteTask, taskAttempt.jvmID);
+      //TODO Resolve to host / IP in case of a local address.
+      InetSocketAddress nodeHttpInetAddr =
+          NetUtils.createSocketAddr(taskAttempt.nodeHttpAddress); // TODO:
+                                                                  // Costly?
+      JobCounterUpdateEvent jce =
+          new JobCounterUpdateEvent(taskAttempt.attemptId.getTaskId()
+              .getJobId());
+      jce.addCounterUpdate(
+          taskAttempt.attemptId.getTaskId().getTaskType() == TaskType.MAP ? 
+              JobCounter.TOTAL_LAUNCHED_MAPS: JobCounter.TOTAL_LAUNCHED_REDUCES
+              , 1);
+      taskAttempt.eventHandler.handle(jce);
+      
+      TaskAttemptStartedEvent tase =
+        new TaskAttemptStartedEvent(TypeConverter.fromYarn(taskAttempt.attemptId),
+            TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()),
+            taskAttempt.launchTime,
+            nodeHttpInetAddr.getHostName(), nodeHttpInetAddr.getPort());
+      taskAttempt.eventHandler.handle
+          (new JobHistoryEvent(taskAttempt.attemptId.getTaskId().getJobId(), tase));
+      taskAttempt.eventHandler.handle
+          (new SpeculatorEvent
+              (taskAttempt.attemptId, true, taskAttempt.clock.getTime()));
+      //make remoteTask reference as null as it is no more needed
+      //and free up the memory
+      taskAttempt.remoteTask = null;
+      
+      //tell the Task that attempt has started
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId, 
+         TaskEventType.T_ATTEMPT_LAUNCHED));
+    }
+  }
+   
+  private static class CommitPendingTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId, 
+         TaskEventType.T_ATTEMPT_COMMIT_PENDING));
+    }
+  }
+
+  private static class TaskCleanupTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      TaskAttemptContext taskContext =
+        new TaskAttemptContextImpl(new JobConf(taskAttempt.conf),
+            TypeConverter.fromYarn(taskAttempt.attemptId));
+      taskAttempt.eventHandler.handle(new TaskCleanupEvent(
+          taskAttempt.attemptId,
+          taskAttempt.committer,
+          taskContext));
+    }
+  }
+
+  private static class SucceededTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      //set the finish time
+      taskAttempt.setFinishTime();
+      String taskType = 
+          TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()).toString();
+      LOG.info("In TaskAttemptImpl taskType: " + taskType);
+      long slotMillis = computeSlotMillis(taskAttempt);
+      JobCounterUpdateEvent jce =
+          new JobCounterUpdateEvent(taskAttempt.attemptId.getTaskId()
+              .getJobId());
+      jce.addCounterUpdate(
+        taskAttempt.attemptId.getTaskId().getTaskType() == TaskType.MAP ? 
+          JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
+          slotMillis);
+      taskAttempt.eventHandler.handle(jce);
+      taskAttempt.logAttemptFinishedEvent(TaskAttemptState.SUCCEEDED);
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId,
+          TaskEventType.T_ATTEMPT_SUCCEEDED));
+      taskAttempt.eventHandler.handle
+      (new SpeculatorEvent
+          (taskAttempt.reportedStatus, taskAttempt.clock.getTime()));
+   }
+  }
+
+  private static class FailedTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+      // set the finish time
+      taskAttempt.setFinishTime();
+      
+      if (taskAttempt.getLaunchTime() != 0) {
+        taskAttempt.eventHandler
+            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+        TaskAttemptUnsuccessfulCompletionEvent tauce =
+            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
+                TaskAttemptState.FAILED);
+        taskAttempt.eventHandler.handle(new JobHistoryEvent(
+            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
+        // taskAttempt.logAttemptFinishedEvent(TaskAttemptState.FAILED); Not
+        // handling failed map/reduce events.
+      }else {
+        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
+            + taskAttempt.getID());
+      }
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
+    }
+  }
+
+  private void logAttemptFinishedEvent(TaskAttemptState state) {
+    //Log finished events only if an attempt started.
+    if (getLaunchTime() == 0) return; 
+    if (attemptId.getTaskId().getTaskType() == TaskType.MAP) {
+      MapAttemptFinishedEvent mfe =
+         new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
+         TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
+         state.toString(),
+         this.reportedStatus.mapFinishTime,
+         finishTime, this.nodeHostName == null ? "UNKNOWN" : this.nodeHostName,
+         this.reportedStatus.stateString,
+         TypeConverter.fromYarn(getCounters()),
+         getProgressSplitBlock().burst());
+         eventHandler.handle(
+           new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe));
+    } else {
+       ReduceAttemptFinishedEvent rfe =
+         new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
+         TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
+         state.toString(),
+         this.reportedStatus.shuffleFinishTime,
+         this.reportedStatus.sortFinishTime,
+         finishTime, this.containerMgrAddress == null ? "UNKNOWN" : this.containerMgrAddress,
+         this.reportedStatus.stateString,
+         TypeConverter.fromYarn(getCounters()),
+         getProgressSplitBlock().burst());
+         eventHandler.handle(
+           new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe));
+    }
+  }
+
+  private static class TooManyFetchFailureTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+      //add to diagnostic
+      taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
+      //set the finish time
+      taskAttempt.setFinishTime();
+      
+      if (taskAttempt.getLaunchTime() != 0) {
+        taskAttempt.eventHandler
+            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+        TaskAttemptUnsuccessfulCompletionEvent tauce =
+            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
+                TaskAttemptState.FAILED);
+        taskAttempt.eventHandler.handle(new JobHistoryEvent(
+            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
+      }else {
+        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
+            + taskAttempt.getID());
+      }
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
+    }
+  }
+
+  private static class KilledTransition implements
+      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt,
+        TaskAttemptEvent event) {
+      //set the finish time
+      taskAttempt.setFinishTime();
+      if (taskAttempt.getLaunchTime() != 0) {
+        taskAttempt.eventHandler
+            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+        TaskAttemptUnsuccessfulCompletionEvent tauce =
+            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
+                TaskAttemptState.KILLED);
+        taskAttempt.eventHandler.handle(new JobHistoryEvent(
+            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
+      }else {
+        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
+            + taskAttempt.getID());
+      }
+//      taskAttempt.logAttemptFinishedEvent(TaskAttemptState.KILLED); Not logging Map/Reduce attempts in case of failure.
+      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
+          taskAttempt.attemptId,
+          TaskEventType.T_ATTEMPT_KILLED));
+    }
+  }
+
+  private static class CleanupContainerTransition implements
+       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      // unregister it to TaskAttemptListener so that it stops listening
+      // for it
+      taskAttempt.taskAttemptListener.unregister(
+          taskAttempt.attemptId, taskAttempt.jvmID);
+      //send the cleanup event to containerLauncher
+      taskAttempt.eventHandler.handle(new ContainerLauncherEvent(
+          taskAttempt.attemptId, 
+          taskAttempt.containerID, taskAttempt.containerMgrAddress,
+          taskAttempt.containerToken,
+          ContainerLauncher.EventType.CONTAINER_REMOTE_CLEANUP));
+    }
+  }
+
+  private void addDiagnosticInfo(String diag) {
+    if (diag != null && !diag.equals("")) {
+      diagnostics.add(diag);
+    }
+  }
+
+  private static class StatusUpdater 
+       implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      // Status update calls don't really change the state of the attempt.
+      TaskAttemptStatus newReportedStatus =
+          ((TaskAttemptStatusUpdateEvent) event)
+              .getReportedTaskAttemptStatus();
+      // Now switch the information in the reportedStatus
+      taskAttempt.reportedStatus = newReportedStatus;
+      taskAttempt.reportedStatus.taskState = taskAttempt.getState();
+
+      // send event to speculator about the reported status
+      taskAttempt.eventHandler.handle
+          (new SpeculatorEvent
+              (taskAttempt.reportedStatus, taskAttempt.clock.getTime()));
+      
+      //add to diagnostic
+      taskAttempt.addDiagnosticInfo(newReportedStatus.diagnosticInfo);
+      taskAttempt.updateProgressSplits();
+      
+      //if fetch failures are present, send the fetch failure event to job
+      //this only will happen in reduce attempt type
+      if (taskAttempt.reportedStatus.fetchFailedMaps != null && 
+          taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) {
+        taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent(
+            taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps));
+      }
+    }
+  }
+
+  private static class DiagnosticInformationUpdater 
+        implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @Override
+    public void transition(TaskAttemptImpl taskAttempt, 
+        TaskAttemptEvent event) {
+      TaskAttemptDiagnosticsUpdateEvent diagEvent =
+          (TaskAttemptDiagnosticsUpdateEvent) event;
+      LOG.info("Diagnostics report from " + taskAttempt.attemptId + ": "
+          + diagEvent.getDiagnosticInfo());
+      taskAttempt.addDiagnosticInfo(diagEvent.getDiagnosticInfo());
+    }
+  }
+
+  private void initTaskAttemptStatus(TaskAttemptStatus result) {
+    result.progress = 0.0f;
+    result.diagnosticInfo = "";
+    result.phase = Phase.STARTING;
+    result.stateString = "NEW";
+    result.taskState = TaskAttemptState.NEW;
+    Counters counters = recordFactory.newRecordInstance(Counters.class);
+//    counters.groups = new HashMap<String, CounterGroup>();
+    result.counters = counters;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java

@@ -0,0 +1,887 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.impl;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
+import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
+import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobMapTaskRescheduledEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+
+/**
+ * Implementation of Task interface.
+ */
+public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
+
+  private static final Log LOG = LogFactory.getLog(TaskImpl.class);
+
+  protected final Configuration conf;
+  protected final Path jobFile;
+  protected final OutputCommitter committer;
+  protected final int partition;
+  protected final TaskAttemptListener taskAttemptListener;
+  protected final EventHandler eventHandler;
+  private final TaskId taskId;
+  private Map<TaskAttemptId, TaskAttempt> attempts;
+  private final int maxAttempts;
+  protected final Clock clock;
+  private final Lock readLock;
+  private final Lock writeLock;
+  private final MRAppMetrics metrics;
+  private long scheduledTime;
+  
+  private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+  
+  protected Collection<Token<? extends TokenIdentifier>> fsTokens;
+  protected Token<JobTokenIdentifier> jobToken;
+  
+  // counts the number of attempts that are either running or in a state where
+  //  they will come to be running when they get a Container
+  private int numberUncompletedAttempts = 0;
+
+  private boolean historyTaskStartGenerated = false;
+  
+  private static final SingleArcTransition<TaskImpl, TaskEvent> 
+     ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
+  private static final SingleArcTransition<TaskImpl, TaskEvent> 
+     KILL_TRANSITION = new KillTransition();
+
+  private static final StateMachineFactory
+               <TaskImpl, TaskState, TaskEventType, TaskEvent> 
+            stateMachineFactory 
+           = new StateMachineFactory<TaskImpl, TaskState, TaskEventType, TaskEvent>
+               (TaskState.NEW)
+
+    // define the state machine of Task
+
+    // Transitions from NEW state
+    .addTransition(TaskState.NEW, TaskState.SCHEDULED, 
+        TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
+    .addTransition(TaskState.NEW, TaskState.KILLED, 
+        TaskEventType.T_KILL, new KillNewTransition())
+
+    // Transitions from SCHEDULED state
+      //when the first attempt is launched, the task state is set to RUNNING
+     .addTransition(TaskState.SCHEDULED, TaskState.RUNNING, 
+         TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition())
+     .addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT, 
+         TaskEventType.T_KILL, KILL_TRANSITION)
+     .addTransition(TaskState.SCHEDULED, TaskState.SCHEDULED, 
+         TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION)
+     .addTransition(TaskState.SCHEDULED, 
+        EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED), 
+        TaskEventType.T_ATTEMPT_FAILED, 
+        new AttemptFailedTransition())
+ 
+    // Transitions from RUNNING state
+    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
+        TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
+    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
+        TaskEventType.T_ATTEMPT_COMMIT_PENDING,
+        new AttemptCommitPendingTransition())
+    .addTransition(TaskState.RUNNING, TaskState.RUNNING,
+        TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
+    .addTransition(TaskState.RUNNING, TaskState.SUCCEEDED, 
+        TaskEventType.T_ATTEMPT_SUCCEEDED,
+        new AttemptSucceededTransition())
+    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
+        TaskEventType.T_ATTEMPT_KILLED,
+        ATTEMPT_KILLED_TRANSITION)
+    .addTransition(TaskState.RUNNING, 
+        EnumSet.of(TaskState.RUNNING, TaskState.FAILED), 
+        TaskEventType.T_ATTEMPT_FAILED,
+        new AttemptFailedTransition())
+    .addTransition(TaskState.RUNNING, TaskState.KILL_WAIT, 
+        TaskEventType.T_KILL, KILL_TRANSITION)
+
+    // Transitions from KILL_WAIT state
+    .addTransition(TaskState.KILL_WAIT,
+        EnumSet.of(TaskState.KILL_WAIT, TaskState.KILLED),
+        TaskEventType.T_ATTEMPT_KILLED,
+        new KillWaitAttemptKilledTransition())
+    // Ignore-able transitions.
+    .addTransition(
+        TaskState.KILL_WAIT,
+        TaskState.KILL_WAIT,
+        EnumSet.of(TaskEventType.T_KILL,
+            TaskEventType.T_ATTEMPT_LAUNCHED,
+            TaskEventType.T_ATTEMPT_COMMIT_PENDING,
+            TaskEventType.T_ATTEMPT_FAILED,
+            TaskEventType.T_ATTEMPT_SUCCEEDED,
+            TaskEventType.T_ADD_SPEC_ATTEMPT))
+
+    // Transitions from SUCCEEDED state
+    .addTransition(TaskState.SUCCEEDED, //only possible for map tasks
+        EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED),
+        TaskEventType.T_ATTEMPT_FAILED, new MapRetroactiveFailureTransition())
+    // Ignore-able transitions.
+    .addTransition(
+        TaskState.SUCCEEDED, TaskState.SUCCEEDED,
+        EnumSet.of(TaskEventType.T_KILL,
+            TaskEventType.T_ADD_SPEC_ATTEMPT,
+            TaskEventType.T_ATTEMPT_LAUNCHED,
+            TaskEventType.T_ATTEMPT_KILLED))
+
+    // Transitions from FAILED state        
+    .addTransition(TaskState.FAILED, TaskState.FAILED,
+        EnumSet.of(TaskEventType.T_KILL,
+                   TaskEventType.T_ADD_SPEC_ATTEMPT))
+
+    // Transitions from KILLED state
+    .addTransition(TaskState.KILLED, TaskState.KILLED,
+        EnumSet.of(TaskEventType.T_KILL,
+                   TaskEventType.T_ADD_SPEC_ATTEMPT))
+
+    // create the topology tables
+    .installTopology();
+
+  private final StateMachine<TaskState, TaskEventType, TaskEvent>
+    stateMachine;
+  
+  protected int nextAttemptNumber;
+
+  //should be set to one which comes first
+  //saying COMMIT_PENDING
+  private TaskAttemptId commitAttempt;
+
+  private TaskAttemptId successfulAttempt;
+
+  private int failedAttempts;
+  private int finishedAttempts;//finish are total of success, failed and killed
+
+  @Override
+  public TaskState getState() {
+    return stateMachine.getCurrentState();
+  }
+
+  public TaskImpl(JobId jobId, TaskType taskType, int partition,
+      EventHandler eventHandler, Path remoteJobConfFile, Configuration conf,
+      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
+      Token<JobTokenIdentifier> jobToken,
+      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock,
+      Set<TaskId> completedTasksFromPreviousRun, int startCount,
+      MRAppMetrics metrics) {
+    this.conf = conf;
+    this.clock = clock;
+    this.jobFile = remoteJobConfFile;
+    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+    readLock = readWriteLock.readLock();
+    writeLock = readWriteLock.writeLock();
+    this.attempts = Collections.emptyMap();
+    // This overridable method call is okay in a constructor because we
+    //  have a convention that none of the overrides depends on any
+    //  fields that need initialization.
+    maxAttempts = getMaxAttempts();
+    taskId = recordFactory.newRecordInstance(TaskId.class);
+    taskId.setJobId(jobId);
+    taskId.setId(partition);
+    taskId.setTaskType(taskType);
+    this.partition = partition;
+    this.taskAttemptListener = taskAttemptListener;
+    this.eventHandler = eventHandler;
+    this.committer = committer;
+    this.fsTokens = fsTokens;
+    this.jobToken = jobToken;
+    this.metrics = metrics;
+
+    if (completedTasksFromPreviousRun != null
+        && completedTasksFromPreviousRun.contains(taskId)) {
+      LOG.info("Task is from previous run " + taskId);
+      startCount = startCount - 1;
+    }
+
+    //attempt ids are generated based on MR app startCount so that attempts
+    //from previous lives don't overstep the current one.
+    //this assumes that a task won't have more than 1000 attempts in its single 
+    //life
+    nextAttemptNumber = (startCount - 1) * 1000;
+
+    // This "this leak" is okay because the retained pointer is in an
+    //  instance variable.
+    stateMachine = stateMachineFactory.make(this);
+  }
+
+  @Override
+  public Map<TaskAttemptId, TaskAttempt> getAttempts() {
+    readLock.lock();
+
+    try {
+      if (attempts.size() <= 1) {
+        return attempts;
+      }
+      
+      Map<TaskAttemptId, TaskAttempt> result
+          = new LinkedHashMap<TaskAttemptId, TaskAttempt>();
+      result.putAll(attempts);
+
+      return result;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public TaskAttempt getAttempt(TaskAttemptId attemptID) {
+    readLock.lock();
+    try {
+      return attempts.get(attemptID);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public TaskId getID() {
+    return taskId;
+  }
+
+  @Override
+  public boolean isFinished() {
+    readLock.lock();
+    try {
+     // TODO: Use stateMachine level method?
+      return (getState() == TaskState.SUCCEEDED ||
+          getState() == TaskState.FAILED ||
+          getState() == TaskState.KILLED);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public TaskReport getReport() {
+    TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
+    readLock.lock();
+    try {
+      report.setTaskId(taskId);
+      report.setStartTime(getLaunchTime());
+      report.setFinishTime(getFinishTime());
+      report.setTaskState(getState());
+      report.setProgress(getProgress());
+      report.setCounters(getCounters());
+
+      for (TaskAttempt attempt : attempts.values()) {
+        if (TaskAttemptState.RUNNING.equals(attempt.getState())) {
+          report.addRunningAttempt(attempt.getID());
+        }
+      }
+
+      report.setSuccessfulAttempt(successfulAttempt);
+      
+      for (TaskAttempt att : attempts.values()) {
+        String prefix = "AttemptID:" + att.getID() + " Info:";
+        for (CharSequence cs : att.getDiagnostics()) {
+          report.addDiagnostics(prefix + cs);
+          
+        }
+      }
+      return report;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public Counters getCounters() {
+    Counters counters = null;
+    readLock.lock();
+    try {
+      TaskAttempt bestAttempt = selectBestAttempt();
+      if (bestAttempt != null) {
+        counters = bestAttempt.getCounters();
+      } else {
+        counters = recordFactory.newRecordInstance(Counters.class);
+//        counters.groups = new HashMap<CharSequence, CounterGroup>();
+      }
+      return counters;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public float getProgress() {
+    readLock.lock();
+    try {
+      TaskAttempt bestAttempt = selectBestAttempt();
+      if (bestAttempt == null) {
+        return 0;
+      }
+      return bestAttempt.getProgress();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  //this is always called in read/write lock
+  private long getLaunchTime() {
+    long launchTime = 0;
+    for (TaskAttempt at : attempts.values()) {
+      //select the least launch time of all attempts
+      if (launchTime == 0  || launchTime > at.getLaunchTime()) {
+        launchTime = at.getLaunchTime();
+      }
+    }
+    if (launchTime == 0) {
+      return this.scheduledTime;
+    }
+    return launchTime;
+  }
+
+  //this is always called in read/write lock
+  //TODO Verify behaviour is Task is killed (no finished attempt)
+  private long getFinishTime() {
+    if (!isFinished()) {
+      return 0;
+    }
+    long finishTime = 0;
+    for (TaskAttempt at : attempts.values()) {
+      //select the max finish time of all attempts
+      if (finishTime < at.getFinishTime()) {
+        finishTime = at.getFinishTime();
+      }
+    }
+    return finishTime;
+  }
+
+  private long getFinishTime(TaskAttemptId taId) {
+    if (taId == null) {
+      return clock.getTime();
+    }
+    long finishTime = 0;
+    for (TaskAttempt at : attempts.values()) {
+      //select the max finish time of all attempts
+      if (at.getID().equals(taId)) {
+        return at.getFinishTime();
+      }
+    }
+    return finishTime;
+  }
+  
+  private TaskState finished(TaskState finalState) {
+    if (getState() == TaskState.RUNNING) {
+      metrics.endRunningTask(this);
+    }
+    return finalState;
+  }
+
+  //select the nextAttemptNumber with best progress
+  // always called inside the Read Lock
+  private TaskAttempt selectBestAttempt() {
+    float progress = 0f;
+    TaskAttempt result = null;
+    for (TaskAttempt at : attempts.values()) {
+      if (result == null) {
+        result = at; //The first time around
+      }
+      //TODO: consider the nextAttemptNumber only if it is not failed/killed ?
+      // calculate the best progress
+      if (at.getProgress() > progress) {
+        result = at;
+        progress = at.getProgress();
+      }
+    }
+    return result;
+  }
+
+  @Override
+  public boolean canCommit(TaskAttemptId taskAttemptID) {
+    readLock.lock();
+    boolean canCommit = false;
+    try {
+      if (commitAttempt != null) {
+        canCommit = taskAttemptID.equals(commitAttempt);
+        LOG.info("Result of canCommit for " + taskAttemptID + ":" + canCommit);
+      }
+    } finally {
+      readLock.unlock();
+    }
+    return canCommit;
+  }
+
+  protected abstract TaskAttemptImpl createAttempt();
+
+  // No override of this method may require that the subclass be initialized.
+  protected abstract int getMaxAttempts();
+
+  protected TaskAttempt getSuccessfulAttempt() {
+    readLock.lock();
+    try {
+      if (null == successfulAttempt) {
+        return null;
+      }
+      return attempts.get(successfulAttempt);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  // This is always called in the Write Lock
+  private void addAndScheduleAttempt() {
+    TaskAttempt attempt = createAttempt();
+    LOG.info("Created attempt " + attempt.getID());
+    switch (attempts.size()) {
+      case 0:
+        attempts = Collections.singletonMap(attempt.getID(), attempt);
+        break;
+        
+      case 1:
+        Map newAttempts
+            = new LinkedHashMap<TaskAttemptId, TaskAttempt>(maxAttempts);
+        newAttempts.putAll(attempts);
+        attempts = newAttempts;
+        attempts.put(attempt.getID(), attempt);
+        break;
+
+      default:
+        attempts.put(attempt.getID(), attempt);
+        break;
+    }
+    ++nextAttemptNumber;
+    ++numberUncompletedAttempts;
+    //schedule the nextAttemptNumber
+    if (failedAttempts > 0) {
+      eventHandler.handle(new TaskAttemptEvent(attempt.getID(),
+        TaskAttemptEventType.TA_RESCHEDULE));
+    } else {
+      eventHandler.handle(new TaskAttemptEvent(attempt.getID(),
+          TaskAttemptEventType.TA_SCHEDULE));
+    }
+  }
+
+  @Override
+  public void handle(TaskEvent event) {
+    LOG.info("Processing " + event.getTaskID() + " of type " + event.getType());
+    try {
+      writeLock.lock();
+      TaskState oldState = getState();
+      try {
+        stateMachine.doTransition(event.getType(), event);
+      } catch (InvalidStateTransitonException e) {
+        LOG.error("Can't handle this event at current state", e);
+        internalError(event.getType());
+      }
+      if (oldState != getState()) {
+        LOG.info(taskId + " Task Transitioned from " + oldState + " to "
+            + getState());
+      }
+
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void internalError(TaskEventType type) {
+    eventHandler.handle(new JobDiagnosticsUpdateEvent(
+        this.taskId.getJobId(), "Invalid event " + type + 
+        " on Task " + this.taskId));
+    eventHandler.handle(new JobEvent(this.taskId.getJobId(),
+        JobEventType.INTERNAL_ERROR));
+  }
+
+  // always called inside a transition, in turn inside the Write Lock
+  private void handleTaskAttemptCompletion(TaskAttemptId attemptId,
+      TaskAttemptCompletionEventStatus status) {
+    finishedAttempts++;
+    TaskAttempt attempt = attempts.get(attemptId);
+    //raise the completion event only if the container is assigned
+    // to nextAttemptNumber
+    if (attempt.getNodeHttpAddress() != null) {
+      TaskAttemptCompletionEvent tce = recordFactory.newRecordInstance(TaskAttemptCompletionEvent.class);
+      tce.setEventId(-1);
+      //TODO: XXXXXX  hardcoded port
+      tce.setMapOutputServerAddress("http://" + attempt.getNodeHttpAddress().split(":")[0] + ":8080");
+      tce.setStatus(status);
+      tce.setAttemptId(attempt.getID());
+      int runTime = 0;
+      if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
+        runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
+      tce.setAttemptRunTime(runTime);
+      
+      //raise the event to job so that it adds the completion event to its
+      //data structures
+      eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
+    }
+  }
+
+  private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskState taskState) {
+    TaskFinishedEvent tfe =
+      new TaskFinishedEvent(TypeConverter.fromYarn(task.taskId),
+        task.getFinishTime(task.successfulAttempt),
+        TypeConverter.fromYarn(task.taskId.getTaskType()),
+        taskState.toString(),
+        TypeConverter.fromYarn(task.getCounters()));
+    return tfe;
+  }
+  
+  private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskState taskState, TaskAttemptId taId) {
+    StringBuilder errorSb = new StringBuilder();
+    if (diag != null) {
+      for (String d : diag) {
+        errorSb.append(", ").append(d);
+      }
+    }
+    TaskFailedEvent taskFailedEvent = new TaskFailedEvent(
+        TypeConverter.fromYarn(task.taskId),
+     // Hack since getFinishTime needs isFinished to be true and that doesn't happen till after the transition.
+        task.getFinishTime(taId),
+        TypeConverter.fromYarn(task.getType()),
+        errorSb.toString(),
+        taskState.toString(),
+        taId == null ? null : TypeConverter.fromYarn(taId));
+    return taskFailedEvent;
+  }
+
+  /**
+  * @return a String representation of the splits.
+  *
+  * Subclasses can override this method to provide their own representations
+  * of splits (if any).
+  *
+  */
+  protected String getSplitsAsString(){
+	  return "";
+  }
+
+  private static class InitialScheduleTransition
+    implements SingleArcTransition<TaskImpl, TaskEvent> {
+
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      task.addAndScheduleAttempt();
+      task.scheduledTime = task.clock.getTime();
+      TaskStartedEvent tse = new TaskStartedEvent(
+          TypeConverter.fromYarn(task.taskId), task.getLaunchTime(),
+          TypeConverter.fromYarn(task.taskId.getTaskType()),
+          task.getSplitsAsString());
+      task.eventHandler
+          .handle(new JobHistoryEvent(task.taskId.getJobId(), tse));
+      task.historyTaskStartGenerated = true;
+    }
+  }
+
+  // Used when creating a new attempt while one is already running.
+  //  Currently we do this for speculation.  In the future we may do this
+  //  for tasks that failed in a way that might indicate application code
+  //  problems, so we can take later failures in parallel and flush the
+  //  job quickly when this happens.
+  private static class RedundantScheduleTransition
+    implements SingleArcTransition<TaskImpl, TaskEvent> {
+
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      LOG.info("Scheduling a redundant attempt for task " + task.taskId);
+      task.addAndScheduleAttempt();
+    }
+  }
+
+  private static class AttemptCommitPendingTransition 
+          implements SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      TaskTAttemptEvent ev = (TaskTAttemptEvent) event;
+      // The nextAttemptNumber is commit pending, decide on set the commitAttempt
+      TaskAttemptId attemptID = ev.getTaskAttemptID();
+      if (task.commitAttempt == null) {
+        // TODO: validate attemptID
+        task.commitAttempt = attemptID;
+        LOG.info(attemptID + " given a go for committing the task output.");
+      } else {
+        // Don't think this can be a pluggable decision, so simply raise an
+        // event for the TaskAttempt to delete its output.
+        LOG.info(task.commitAttempt
+            + " already given a go for committing the task output, so killing "
+            + attemptID);
+        task.eventHandler.handle(new TaskAttemptEvent(
+            attemptID, TaskAttemptEventType.TA_KILL));
+      }
+    }
+  }
+
+  private static class AttemptSucceededTransition 
+      implements SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      task.handleTaskAttemptCompletion(
+          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
+          TaskAttemptCompletionEventStatus.SUCCEEDED);
+      --task.numberUncompletedAttempts;
+      task.successfulAttempt = ((TaskTAttemptEvent) event).getTaskAttemptID();
+      task.eventHandler.handle(new JobTaskEvent(
+          task.taskId, TaskState.SUCCEEDED));
+      LOG.info("Task succeeded with attempt " + task.successfulAttempt);
+      // issue kill to all other attempts
+      if (task.historyTaskStartGenerated) {
+        TaskFinishedEvent tfe = createTaskFinishedEvent(task,
+            TaskState.SUCCEEDED);
+        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
+            tfe));
+      }
+      for (TaskAttempt attempt : task.attempts.values()) {
+        if (attempt.getID() != task.successfulAttempt &&
+            // This is okay because it can only talk us out of sending a
+            //  TA_KILL message to an attempt that doesn't need one for
+            //  other reasons.
+            !attempt.isFinished()) {
+          LOG.info("Issuing kill to other attempt " + attempt.getID());
+          task.eventHandler.handle(
+              new TaskAttemptEvent(attempt.getID(), 
+                  TaskAttemptEventType.TA_KILL));
+        }
+      }
+      task.finished(TaskState.SUCCEEDED);
+    }
+  }
+
+  private static class AttemptKilledTransition implements
+      SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      task.handleTaskAttemptCompletion(
+          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
+          TaskAttemptCompletionEventStatus.KILLED);
+      --task.numberUncompletedAttempts;
+      if (task.successfulAttempt == null) {
+        task.addAndScheduleAttempt();
+      }
+    }
+  }
+
+
+  private static class KillWaitAttemptKilledTransition implements
+      MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
+
+    protected TaskState finalState = TaskState.KILLED;
+
+    @Override
+    public TaskState transition(TaskImpl task, TaskEvent event) {
+      task.handleTaskAttemptCompletion(
+          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
+          TaskAttemptCompletionEventStatus.KILLED);
+      // check whether all attempts are finished
+      if (task.finishedAttempts == task.attempts.size()) {
+        if (task.historyTaskStartGenerated) {
+        TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
+              finalState, null); // TODO JH verify failedAttempt null
+        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
+            taskFailedEvent)); 
+        } else {
+          LOG.debug("Not generating HistoryFinish event since start event not" +
+          		" generated for task: " + task.getID());
+        }
+
+        task.eventHandler.handle(
+            new JobTaskEvent(task.taskId, finalState));
+        return finalState;
+      }
+      return task.getState();
+    }
+  }
+
+  private static class AttemptFailedTransition implements
+    MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
+
+    @Override
+    public TaskState transition(TaskImpl task, TaskEvent event) {
+      task.failedAttempts++;
+      TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
+      TaskAttempt attempt = task.attempts.get(castEvent.getTaskAttemptID());
+      if (attempt.getAssignedContainerMgrAddress() != null) {
+        //container was assigned
+        task.eventHandler.handle(new ContainerFailedEvent(attempt.getID(), 
+            attempt.getAssignedContainerMgrAddress()));
+      }
+      
+      if (task.failedAttempts < task.maxAttempts) {
+        task.handleTaskAttemptCompletion(
+            ((TaskTAttemptEvent) event).getTaskAttemptID(), 
+            TaskAttemptCompletionEventStatus.FAILED);
+        // we don't need a new event if we already have a spare
+        if (--task.numberUncompletedAttempts == 0
+            && task.successfulAttempt == null) {
+          task.addAndScheduleAttempt();
+        }
+      } else {
+        task.handleTaskAttemptCompletion(
+            ((TaskTAttemptEvent) event).getTaskAttemptID(), 
+            TaskAttemptCompletionEventStatus.TIPFAILED);
+        TaskTAttemptEvent ev = (TaskTAttemptEvent) event;
+        TaskAttemptId taId = ev.getTaskAttemptID();
+        
+        if (task.historyTaskStartGenerated) {
+        TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, attempt.getDiagnostics(),
+            TaskState.FAILED, taId);
+        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
+            taskFailedEvent));
+        } else {
+          LOG.debug("Not generating HistoryFinish event since start event not" +
+          		" generated for task: " + task.getID());
+        }
+        task.eventHandler.handle(
+            new JobTaskEvent(task.taskId, TaskState.FAILED));
+        return task.finished(TaskState.FAILED);
+      }
+      return getDefaultState(task);
+    }
+
+    protected TaskState getDefaultState(Task task) {
+      return task.getState();
+    }
+
+    protected void unSucceed(TaskImpl task) {
+      ++task.numberUncompletedAttempts;
+      task.successfulAttempt = null;
+    }
+  }
+
+  private static class MapRetroactiveFailureTransition
+      extends AttemptFailedTransition {
+
+    @Override
+    public TaskState transition(TaskImpl task, TaskEvent event) {
+      //verify that this occurs only for map task
+      //TODO: consider moving it to MapTaskImpl
+      if (!TaskType.MAP.equals(task.getType())) {
+        LOG.error("Unexpected event for REDUCE task " + event.getType());
+        task.internalError(event.getType());
+      }
+      
+      // tell the job about the rescheduling
+      task.eventHandler.handle(
+          new JobMapTaskRescheduledEvent(task.taskId));
+      // super.transition is mostly coded for the case where an
+      //  UNcompleted task failed.  When a COMPLETED task retroactively
+      //  fails, we have to let AttemptFailedTransition.transition
+      //  believe that there's no redundancy.
+      unSucceed(task);
+      return super.transition(task, event);
+    }
+
+    @Override
+    protected TaskState getDefaultState(Task task) {
+      return TaskState.SCHEDULED;
+    }
+  }
+
+  private static class KillNewTransition 
+    implements SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      
+      if (task.historyTaskStartGenerated) {
+      TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
+            TaskState.KILLED, null); // TODO Verify failedAttemptId is null
+      task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
+          taskFailedEvent));
+      }else {
+        LOG.debug("Not generating HistoryFinish event since start event not" +
+        		" generated for task: " + task.getID());
+      }
+
+      task.eventHandler.handle(
+          new JobTaskEvent(task.taskId, TaskState.KILLED));
+      task.metrics.endWaitingTask(task);
+    }
+  }
+
+  private void killUnfinishedAttempt(TaskAttempt attempt, String logMsg) {
+    if (attempt != null && !attempt.isFinished()) {
+      eventHandler.handle(
+          new TaskAttemptEvent(attempt.getID(),
+              TaskAttemptEventType.TA_KILL));
+    }
+  }
+
+  private static class KillTransition 
+    implements SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      // issue kill to all non finished attempts
+      for (TaskAttempt attempt : task.attempts.values()) {
+        task.killUnfinishedAttempt
+            (attempt, "Task KILL is received. Killing attempt!");
+      }
+
+      task.numberUncompletedAttempts = 0;
+    }
+  }
+
+  static class LaunchTransition
+      implements SingleArcTransition<TaskImpl, TaskEvent> {
+    @Override
+    public void transition(TaskImpl task, TaskEvent event) {
+      task.metrics.launchedTask(task);
+      task.metrics.runningTask(task);
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java

@@ -0,0 +1,31 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.launcher;
+
+
+import org.apache.hadoop.yarn.event.EventHandler;
+
+public interface ContainerLauncher 
+    extends EventHandler<ContainerLauncherEvent> {
+
+  enum EventType {
+    CONTAINER_REMOTE_LAUNCH,
+    CONTAINER_REMOTE_CLEANUP
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java

@@ -0,0 +1,114 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.launcher;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public class ContainerLauncherEvent 
+    extends AbstractEvent<ContainerLauncher.EventType> {
+
+  private TaskAttemptId taskAttemptID;
+  private ContainerId containerID;
+  private String containerMgrAddress;
+  private ContainerToken containerToken;
+
+  public ContainerLauncherEvent(TaskAttemptId taskAttemptID, 
+      ContainerId containerID,
+      String containerMgrAddress,
+      ContainerToken containerToken,
+      ContainerLauncher.EventType type) {
+    super(type);
+    this.taskAttemptID = taskAttemptID;
+    this.containerID = containerID;
+    this.containerMgrAddress = containerMgrAddress;
+    this.containerToken = containerToken;
+  }
+
+  public TaskAttemptId getTaskAttemptID() {
+    return this.taskAttemptID;
+  }
+
+  public ContainerId getContainerID() {
+    return containerID;
+  }
+
+  public String getContainerMgrAddress() {
+    return containerMgrAddress;
+  }
+
+  public ContainerToken getContainerToken() {
+    return containerToken;
+  }
+
+  @Override
+  public String toString() {
+    return super.toString() + " for taskAttempt " + taskAttemptID;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + ((containerID == null) ? 0 : containerID.hashCode());
+    result = prime * result
+        + ((containerMgrAddress == null) ? 0 : containerMgrAddress.hashCode());
+    result = prime * result
+        + ((containerToken == null) ? 0 : containerToken.hashCode());
+    result = prime * result
+        + ((taskAttemptID == null) ? 0 : taskAttemptID.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    ContainerLauncherEvent other = (ContainerLauncherEvent) obj;
+    if (containerID == null) {
+      if (other.containerID != null)
+        return false;
+    } else if (!containerID.equals(other.containerID))
+      return false;
+    if (containerMgrAddress == null) {
+      if (other.containerMgrAddress != null)
+        return false;
+    } else if (!containerMgrAddress.equals(other.containerMgrAddress))
+      return false;
+    if (containerToken == null) {
+      if (other.containerToken != null)
+        return false;
+    } else if (!containerToken.equals(other.containerToken))
+      return false;
+    if (taskAttemptID == null) {
+      if (other.taskAttemptID != null)
+        return false;
+    } else if (!taskAttemptID.equals(other.taskAttemptID))
+      return false;
+    return true;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java

@@ -0,0 +1,279 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.launcher;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.ContainerManager;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+/**
+ * This class is responsible for launching of containers.
+ */
+public class ContainerLauncherImpl extends AbstractService implements
+    ContainerLauncher {
+
+  private static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
+
+  private AppContext context;
+  private ThreadPoolExecutor launcherPool;
+  private Thread eventHandlingThread;
+  private BlockingQueue<ContainerLauncherEvent> eventQueue =
+      new LinkedBlockingQueue<ContainerLauncherEvent>();
+  private RecordFactory recordFactory;
+  //have a cache/map of UGIs so as to avoid creating too many RPC
+  //client connection objects to the same NodeManager
+  private Map<String, UserGroupInformation> ugiMap = 
+    new HashMap<String, UserGroupInformation>();
+
+  public ContainerLauncherImpl(AppContext context) {
+    super(ContainerLauncherImpl.class.getName());
+    this.context = context;
+  }
+
+  @Override
+  public synchronized void init(Configuration conf) {
+    // Clone configuration for this component so that the SecurityInfo setting
+    // doesn't affect the original configuration
+    Configuration myLocalConfig = new Configuration(conf);
+    myLocalConfig.setClass(
+        YarnConfiguration.YARN_SECURITY_INFO,
+        ContainerManagerSecurityInfo.class, SecurityInfo.class);
+    this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
+    super.init(myLocalConfig);
+  }
+
+  public void start() {
+    launcherPool =
+        new ThreadPoolExecutor(getConfig().getInt(
+            AMConstants.CONTAINERLAUNCHER_THREADPOOL_SIZE, 10),
+            Integer.MAX_VALUE, 1, TimeUnit.HOURS,
+            new LinkedBlockingQueue<Runnable>());
+    launcherPool.prestartAllCoreThreads(); // Wait for work.
+    eventHandlingThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        ContainerLauncherEvent event = null;
+        while (!Thread.currentThread().isInterrupted()) {
+          try {
+            event = eventQueue.take();
+          } catch (InterruptedException e) {
+            LOG.error("Returning, interrupted : " + e);
+            return;
+          }
+          // the events from the queue are handled in parallel
+          // using a thread pool
+          launcherPool.execute(new EventProcessor(event));
+
+          // TODO: Group launching of multiple containers to a single
+          // NodeManager into a single connection
+        }
+      }
+    });
+    eventHandlingThread.start();
+    super.start();
+  }
+
+  public void stop() {
+    eventHandlingThread.interrupt();
+    launcherPool.shutdown();
+    super.stop();
+  }
+
+  protected ContainerManager getCMProxy(ContainerId containerID,
+      final String containerManagerBindAddr, ContainerToken containerToken)
+      throws IOException {
+
+    UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+    // TODO: Synchronization problems!!
+    if (UserGroupInformation.isSecurityEnabled()) {
+      if(!ugiMap.containsKey(containerManagerBindAddr)) {
+        Token<ContainerTokenIdentifier> token =
+          new Token<ContainerTokenIdentifier>(
+              containerToken.getIdentifier().array(),
+              containerToken.getPassword().array(), new Text(
+                  containerToken.getKind()), new Text(
+                      containerToken.getService()));
+        //the user in createRemoteUser in this context is not important
+        user = UserGroupInformation.createRemoteUser(containerManagerBindAddr);
+        user.addToken(token);
+        ugiMap.put(containerManagerBindAddr, user);
+      } else {
+        user = ugiMap.get(containerManagerBindAddr);    
+      }
+    }
+    ContainerManager proxy =
+        user.doAs(new PrivilegedAction<ContainerManager>() {
+          @Override
+          public ContainerManager run() {
+            YarnRPC rpc = YarnRPC.create(getConfig());
+            return (ContainerManager) rpc.getProxy(ContainerManager.class,
+                NetUtils.createSocketAddr(containerManagerBindAddr),
+                getConfig());
+          }
+        });
+    return proxy;
+  }
+
+  /**
+   * Setup and start the container on remote nodemanager.
+   */
+  private class EventProcessor implements Runnable {
+    private ContainerLauncherEvent event;
+
+    EventProcessor(ContainerLauncherEvent event) {
+      this.event = event;
+    }
+
+    @Override
+    public void run() {
+      LOG.info("Processing the event " + event.toString());
+
+      // Load ContainerManager tokens before creating a connection.
+      // TODO: Do it only once per NodeManager.
+      final String containerManagerBindAddr = event.getContainerMgrAddress();
+      ContainerId containerID = event.getContainerID();
+      ContainerToken containerToken = event.getContainerToken();
+
+      switch(event.getType()) {
+
+      case CONTAINER_REMOTE_LAUNCH:
+        ContainerRemoteLaunchEvent launchEv = (ContainerRemoteLaunchEvent) event;
+
+        TaskAttemptId taskAttemptID = launchEv.getTaskAttemptID();
+        try {
+          
+          ContainerManager proxy = 
+            getCMProxy(containerID, containerManagerBindAddr, containerToken);
+          
+          // Construct the actual Container
+          ContainerLaunchContext containerLaunchContext =
+              launchEv.getContainer();
+
+          // Now launch the actual container
+          StartContainerRequest startRequest = recordFactory
+              .newRecordInstance(StartContainerRequest.class);
+          startRequest.setContainerLaunchContext(containerLaunchContext);
+          proxy.startContainer(startRequest);
+
+          LOG.info("Returning from container-launch for " + taskAttemptID);
+
+          // after launching, send launched event to task attempt to move
+          // it from ASSIGNED to RUNNING state
+          context.getEventHandler().handle(
+              new TaskAttemptEvent(taskAttemptID,
+                  TaskAttemptEventType.TA_CONTAINER_LAUNCHED));
+        } catch (Throwable t) {
+          String message = "Container launch failed for " + containerID
+              + " : " + StringUtils.stringifyException(t);
+          LOG.error(message);
+          context.getEventHandler().handle(
+              new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID, message));
+          context.getEventHandler().handle(
+              new TaskAttemptEvent(taskAttemptID,
+                  TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
+        }
+
+        break;
+
+      case CONTAINER_REMOTE_CLEANUP:
+        // We will have to remove the launch (meant "cleanup"? FIXME) event if it is still in eventQueue
+        // and not yet processed
+        if (eventQueue.contains(event)) {
+          eventQueue.remove(event); // TODO: Any synchro needed?
+          //deallocate the container
+          context.getEventHandler().handle(
+              new ContainerAllocatorEvent(event.getTaskAttemptID(),
+              ContainerAllocator.EventType.CONTAINER_DEALLOCATE));
+        } else {
+          try {
+            ContainerManager proxy = 
+              getCMProxy(containerID, containerManagerBindAddr, containerToken);
+            // TODO:check whether container is launched
+
+            // kill the remote container if already launched
+            StopContainerRequest stopRequest = recordFactory
+                .newRecordInstance(StopContainerRequest.class);
+            stopRequest.setContainerId(event.getContainerID());
+            proxy.stopContainer(stopRequest);
+
+          } catch (Throwable t) {
+            //ignore the cleanup failure
+            LOG.warn("cleanup failed for container " + event.getContainerID() ,
+                t);
+          }
+
+          // after killing, send killed event to taskattempt
+          context.getEventHandler().handle(
+              new TaskAttemptEvent(event.getTaskAttemptID(),
+                  TaskAttemptEventType.TA_CONTAINER_CLEANED));
+        }
+        break;
+      }
+    }
+    
+  }
+
+  @Override
+  public void handle(ContainerLauncherEvent event) {
+    try {
+      eventQueue.put(event);
+    } catch (InterruptedException e) {
+      throw new YarnException(e);
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java

@@ -0,0 +1,40 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.launcher;
+
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+
+public abstract class ContainerRemoteLaunchEvent extends ContainerLauncherEvent {
+
+  public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID,
+      ContainerId containerID, String containerMgrAddress,
+      ContainerToken containerToken) {
+    super(taskAttemptID, containerID, containerMgrAddress,
+        containerToken,
+        ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH);
+  }
+  public abstract ContainerLaunchContext getContainer();
+
+  public abstract Task getRemoteTask();
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java

@@ -0,0 +1,100 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.local;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Allocates containers locally. Doesn't allocate a real container;
+ * instead sends an allocated event for all requests.
+ */
+public class LocalContainerAllocator extends RMCommunicator
+    implements ContainerAllocator {
+
+  private static final Log LOG =
+      LogFactory.getLog(LocalContainerAllocator.class);
+
+  private final EventHandler eventHandler;
+  private final ApplicationId appID;
+  private AtomicInteger containerCount = new AtomicInteger();
+
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
+
+  public LocalContainerAllocator(ClientService clientService,
+                                 AppContext context) {
+    super(clientService, context);
+    this.eventHandler = context.getEventHandler();
+    this.appID = context.getApplicationID();
+  }
+
+  @Override
+  public void handle(ContainerAllocatorEvent event) {
+    if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
+      LOG.info("Processing the event " + event.toString());
+      ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
+      cID.setAppId(appID);
+      // use negative ids to denote that these are local. Need a better way ??
+      cID.setId((-1) * containerCount.getAndIncrement());
+      
+      Container container = recordFactory.newRecordInstance(Container.class);
+      container.setId(cID);
+      NodeId nodeId = Records.newRecord(NodeId.class);
+      nodeId.setHost("localhost");
+      nodeId.setPort(1234);
+      container.setNodeId(nodeId);
+      container.setContainerToken(null);
+      container.setNodeHttpAddress("localhost:9999");
+      // send the container-assigned event to task attempt
+
+      if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
+        JobCounterUpdateEvent jce =
+            new JobCounterUpdateEvent(event.getAttemptID().getTaskId()
+                .getJobId());
+        // TODO Setting OTHER_LOCAL_MAP for now.
+        jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
+        eventHandler.handle(jce);
+      }
+      eventHandler.handle(new TaskAttemptContainerAssignedEvent(
+          event.getAttemptID(), container));
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java

@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.v2.app.metrics;
+
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterInt;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+
+@Metrics(about="MR App Metrics", context="mapred")
+public class MRAppMetrics {
+  @Metric MutableCounterInt jobsSubmitted;
+  @Metric MutableCounterInt jobsCompleted;
+  @Metric MutableCounterInt jobsFailed;
+  @Metric MutableCounterInt jobsKilled;
+  @Metric MutableGaugeInt jobsPreparing;
+  @Metric MutableGaugeInt jobsRunning;
+
+  @Metric MutableCounterInt mapsLaunched;
+  @Metric MutableCounterInt mapsCompleted;
+  @Metric MutableCounterInt mapsFailed;
+  @Metric MutableCounterInt mapsKilled;
+  @Metric MutableGaugeInt mapsRunning;
+  @Metric MutableGaugeInt mapsWaiting;
+
+  @Metric MutableCounterInt reducesLaunched;
+  @Metric MutableCounterInt reducesCompleted;
+  @Metric MutableCounterInt reducesFailed;
+  @Metric MutableCounterInt reducesKilled;
+  @Metric MutableGaugeInt reducesRunning;
+  @Metric MutableGaugeInt reducesWaiting;
+  
+  public static MRAppMetrics create() {
+    return create(DefaultMetricsSystem.instance());
+  }
+
+  public static MRAppMetrics create(MetricsSystem ms) {
+    JvmMetrics.create("MRAppMaster", null, ms);
+    return ms.register(new MRAppMetrics());
+  }
+
+  // potential instrumentation interface methods
+
+  public void submittedJob(Job job) {
+    jobsSubmitted.incr();
+  }
+
+  public void completedJob(Job job) {
+    jobsCompleted.incr();
+  }
+
+  public void failedJob(Job job) {
+    jobsFailed.incr();
+  }
+
+  public void killedJob(Job job) {
+    jobsKilled.incr();
+  }
+
+  public void preparingJob(Job job) {
+    jobsPreparing.incr();
+  }
+
+  public void endPreparingJob(Job job) {
+    jobsPreparing.decr();
+  }
+
+  public void runningJob(Job job) {
+    jobsRunning.incr();
+  }
+
+  public void endRunningJob(Job job) {
+    jobsRunning.decr();
+  }
+
+  public void launchedTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsLaunched.incr();
+        break;
+      case REDUCE:
+        reducesLaunched.incr();
+        break;
+    }
+    endWaitingTask(task);
+  }
+
+  public void completedTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsCompleted.incr();
+        break;
+      case REDUCE:
+        reducesCompleted.incr();
+        break;
+    }
+  }
+
+  public void failedTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsFailed.incr();
+        break;
+      case REDUCE:
+        reducesFailed.incr();
+        break;
+    }
+  }
+
+  public void killedTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsKilled.incr();
+        break;
+      case REDUCE:
+        reducesKilled.incr();
+        break;
+    }
+  }
+
+  public void runningTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsRunning.incr();
+        break;
+      case REDUCE:
+        reducesRunning.incr();
+        break;
+    }
+  }
+
+  public void endRunningTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsRunning.decr();
+        break;
+      case REDUCE:
+        reducesRunning.decr();
+        break;
+    }
+  }
+
+  public void waitingTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsWaiting.incr();
+        break;
+      case REDUCE:
+        reducesWaiting.incr();
+    }
+  }
+
+  public void endWaitingTask(Task task) {
+    switch (task.getType()) {
+      case MAP:
+        mapsWaiting.decr();
+        break;
+      case REDUCE:
+        reducesWaiting.decr();
+        break;
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/ControlledClock.java

@@ -0,0 +1,43 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.mapreduce.v2.app.recover;
+
+import org.apache.hadoop.yarn.Clock;
+
+class ControlledClock implements Clock {
+  private long time = -1;
+  private final Clock actualClock;
+  ControlledClock(Clock actualClock) {
+    this.actualClock = actualClock;
+  }
+  synchronized void setTime(long time) {
+    this.time = time;
+  }
+  synchronized void reset() {
+    time = -1;
+  }
+
+  @Override
+  public synchronized long getTime() {
+    if (time != -1) {
+      return time;
+    }
+    return actualClock.getTime();
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/Recovery.java

@@ -0,0 +1,34 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.recover;
+
+import java.util.Set;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.event.Dispatcher;
+
+public interface Recovery {
+
+  Dispatcher getDispatcher();
+
+  Clock getClock();
+  
+  Set<TaskId> getCompletedTasks();
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java

@@ -0,0 +1,368 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.recover;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.Phase;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
+import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
+import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleaner;
+import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.service.CompositeService;
+import org.apache.hadoop.yarn.service.Service;
+
+/*
+ * Recovers the completed tasks from the previous life of Application Master.
+ * The completed tasks are deciphered from the history file of the previous life.
+ * Recovery service intercepts and replay the events for completed tasks.
+ * While recovery is in progress, the scheduling of new tasks are delayed by 
+ * buffering the task schedule events.
+ * The recovery service controls the clock while recovery is in progress.
+ */
+
+//TODO:
+//task cleanup for all non completed tasks
+//change job output committer to have 
+//    - atomic job output promotion
+//    - recover output of completed tasks
+
+public class RecoveryService extends CompositeService implements Recovery {
+
+  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+
+  private static final Log LOG = LogFactory.getLog(RecoveryService.class);
+
+  private final ApplicationId appID;
+  private final Dispatcher dispatcher;
+  private final ControlledClock clock;
+  private final int startCount;
+
+  private JobInfo jobInfo = null;
+  private final Map<TaskId, TaskInfo> completedTasks =
+    new HashMap<TaskId, TaskInfo>();
+
+  private final List<TaskEvent> pendingTaskScheduleEvents =
+    new ArrayList<TaskEvent>();
+
+  private volatile boolean recoveryMode = false;
+
+  public RecoveryService(ApplicationId appID, Clock clock, int startCount) {
+    super("RecoveringDispatcher");
+    this.appID = appID;
+    this.startCount = startCount;
+    this.dispatcher = new RecoveryDispatcher();
+    this.clock = new ControlledClock(clock);
+      addService((Service) dispatcher);
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    super.init(conf);
+    // parse the history file
+    try {
+      parse();
+      if (completedTasks.size() > 0) {
+        recoveryMode = true;
+        LOG.info("SETTING THE RECOVERY MODE TO TRUE. NO OF COMPLETED TASKS " + 
+            "TO RECOVER " + completedTasks.size());
+        LOG.info("Job launch time " + jobInfo.getLaunchTime());
+        clock.setTime(jobInfo.getLaunchTime());
+      }
+    } catch (IOException e) {
+      LOG.warn(e);
+      LOG.warn("Could not parse the old history file. Aborting recovery. "
+          + "Starting afresh.");
+    }
+  }
+
+  @Override
+  public Dispatcher getDispatcher() {
+    return dispatcher;
+  }
+
+  @Override
+  public Clock getClock() {
+    return clock;
+  }
+
+  @Override
+  public Set<TaskId> getCompletedTasks() {
+    return completedTasks.keySet();
+  }
+
+  private void parse() throws IOException {
+    // TODO: parse history file based on startCount
+    String jobName = TypeConverter.fromYarn(appID).toString();
+    String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(getConfig());
+    FSDataInputStream in = null;
+    Path historyFile = null;
+    Path histDirPath = FileContext.getFileContext(getConfig()).makeQualified(
+        new Path(jobhistoryDir));
+    FileContext fc = FileContext.getFileContext(histDirPath.toUri(),
+        getConfig());
+    historyFile = fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(
+        histDirPath, jobName, startCount - 1));          //read the previous history file
+    in = fc.open(historyFile);
+    JobHistoryParser parser = new JobHistoryParser(in);
+    jobInfo = parser.parse();
+    Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
+        .getAllTasks();
+    for (TaskInfo taskInfo : taskInfos.values()) {
+      if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
+        completedTasks
+            .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo);
+        LOG.info("Read from history task "
+            + TypeConverter.toYarn(taskInfo.getTaskId()));
+      }
+    }
+    LOG.info("Read completed tasks from history "
+        + completedTasks.size());
+  }
+
+  class RecoveryDispatcher extends AsyncDispatcher {
+    private final EventHandler actualHandler;
+    private final EventHandler handler;
+
+    RecoveryDispatcher() {
+      actualHandler = super.getEventHandler();
+      handler = new InterceptingEventHandler(actualHandler);
+    }
+
+    @Override
+    public void dispatch(Event event) {
+      if (recoveryMode) {
+        if (event.getType() == TaskAttemptEventType.TA_CONTAINER_LAUNCHED) {
+          TaskAttemptInfo attInfo = getTaskAttemptInfo(((TaskAttemptEvent) event)
+              .getTaskAttemptID());
+          LOG.info("Attempt start time " + attInfo.getStartTime());
+          clock.setTime(attInfo.getStartTime());
+
+        } else if (event.getType() == TaskAttemptEventType.TA_DONE
+            || event.getType() == TaskAttemptEventType.TA_FAILMSG
+            || event.getType() == TaskAttemptEventType.TA_KILL) {
+          TaskAttemptInfo attInfo = getTaskAttemptInfo(((TaskAttemptEvent) event)
+              .getTaskAttemptID());
+          LOG.info("Attempt finish time " + attInfo.getFinishTime());
+          clock.setTime(attInfo.getFinishTime());
+        }
+
+        else if (event.getType() == TaskEventType.T_ATTEMPT_FAILED
+            || event.getType() == TaskEventType.T_ATTEMPT_KILLED
+            || event.getType() == TaskEventType.T_ATTEMPT_SUCCEEDED) {
+          TaskTAttemptEvent tEvent = (TaskTAttemptEvent) event;
+          LOG.info("Recovered Task attempt " + tEvent.getTaskAttemptID());
+          TaskInfo taskInfo = completedTasks.get(tEvent.getTaskAttemptID()
+              .getTaskId());
+          taskInfo.getAllTaskAttempts().remove(
+              TypeConverter.fromYarn(tEvent.getTaskAttemptID()));
+          // remove the task info from completed tasks if all attempts are
+          // recovered
+          if (taskInfo.getAllTaskAttempts().size() == 0) {
+            completedTasks.remove(tEvent.getTaskAttemptID().getTaskId());
+            // checkForRecoveryComplete
+            LOG.info("CompletedTasks() " + completedTasks.size());
+            if (completedTasks.size() == 0) {
+              recoveryMode = false;
+              clock.reset();
+              LOG.info("Setting the recovery mode to false. " +
+                 "Recovery is complete!");
+
+              // send all pending tasks schedule events
+              for (TaskEvent tEv : pendingTaskScheduleEvents) {
+                actualHandler.handle(tEv);
+              }
+
+            }
+          }
+        }
+      }
+      super.dispatch(event);
+    }
+
+    @Override
+    public EventHandler getEventHandler() {
+      return handler;
+    }
+  }
+
+  private TaskAttemptInfo getTaskAttemptInfo(TaskAttemptId id) {
+    TaskInfo taskInfo = completedTasks.get(id.getTaskId());
+    return taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn(id));
+  }
+
+  private class InterceptingEventHandler implements EventHandler {
+    EventHandler actualHandler;
+
+    InterceptingEventHandler(EventHandler actualHandler) {
+      this.actualHandler = actualHandler;
+    }
+
+    @Override
+    public void handle(Event event) {
+      if (!recoveryMode) {
+        // delegate to the dispatcher one
+        actualHandler.handle(event);
+        return;
+      }
+
+      else if (event.getType() == TaskEventType.T_SCHEDULE) {
+        TaskEvent taskEvent = (TaskEvent) event;
+        // delay the scheduling of new tasks till previous ones are recovered
+        if (completedTasks.get(taskEvent.getTaskID()) == null) {
+          LOG.debug("Adding to pending task events "
+              + taskEvent.getTaskID());
+          pendingTaskScheduleEvents.add(taskEvent);
+          return;
+        }
+      }
+
+      else if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
+        TaskAttemptId aId = ((ContainerAllocatorEvent) event).getAttemptID();
+        TaskAttemptInfo attInfo = getTaskAttemptInfo(aId);
+        LOG.debug("CONTAINER_REQ " + aId);
+        sendAssignedEvent(aId, attInfo);
+        return;
+      }
+
+      else if (event.getType() == TaskCleaner.EventType.TASK_CLEAN) {
+        TaskAttemptId aId = ((TaskCleanupEvent) event).getAttemptID();
+        LOG.debug("TASK_CLEAN");
+        actualHandler.handle(new TaskAttemptEvent(aId,
+            TaskAttemptEventType.TA_CLEANUP_DONE));
+        return;
+      }
+
+      else if (event.getType() == ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH) {
+        TaskAttemptId aId = ((ContainerRemoteLaunchEvent) event)
+            .getTaskAttemptID();
+        TaskAttemptInfo attInfo = getTaskAttemptInfo(aId);
+        actualHandler.handle(new TaskAttemptEvent(aId,
+            TaskAttemptEventType.TA_CONTAINER_LAUNCHED));
+        // send the status update event
+        sendStatusUpdateEvent(aId, attInfo);
+
+        TaskAttemptState state = TaskAttemptState.valueOf(attInfo.getTaskStatus());
+        switch (state) {
+        case SUCCEEDED:
+          // send the done event
+          LOG.info("Sending done event to " + aId);
+          actualHandler.handle(new TaskAttemptEvent(aId,
+              TaskAttemptEventType.TA_DONE));
+          break;
+        case KILLED:
+          LOG.info("Sending kill event to " + aId);
+          actualHandler.handle(new TaskAttemptEvent(aId,
+              TaskAttemptEventType.TA_KILL));
+          break;
+        default:
+          LOG.info("Sending fail event to " + aId);
+          actualHandler.handle(new TaskAttemptEvent(aId,
+              TaskAttemptEventType.TA_FAILMSG));
+          break;
+        }
+        return;
+      }
+
+      // delegate to the actual handler
+      actualHandler.handle(event);
+    }
+
+    private void sendStatusUpdateEvent(TaskAttemptId yarnAttemptID,
+        TaskAttemptInfo attemptInfo) {
+      LOG.info("Sending status update event to " + yarnAttemptID);
+      TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
+      taskAttemptStatus.id = yarnAttemptID;
+      taskAttemptStatus.progress = 1.0f;
+      taskAttemptStatus.diagnosticInfo = "";
+      taskAttemptStatus.stateString = attemptInfo.getTaskStatus(); 
+      // taskAttemptStatus.outputSize = attemptInfo.getOutputSize();
+      taskAttemptStatus.phase = Phase.CLEANUP;
+      org.apache.hadoop.mapreduce.Counters cntrs = attemptInfo.getCounters();
+      if (cntrs == null) {
+        taskAttemptStatus.counters = null;
+      } else {
+        taskAttemptStatus.counters = TypeConverter.toYarn(attemptInfo
+            .getCounters());
+      }
+      actualHandler.handle(new TaskAttemptStatusUpdateEvent(
+          taskAttemptStatus.id, taskAttemptStatus));
+    }
+
+    private void sendAssignedEvent(TaskAttemptId yarnAttemptID,
+        TaskAttemptInfo attemptInfo) {
+      LOG.info("Sending assigned event to " + yarnAttemptID);
+      ContainerId cId = recordFactory
+          .newRecordInstance(ContainerId.class);
+      Container container = recordFactory
+          .newRecordInstance(Container.class);
+      container.setId(cId);
+      container.setNodeId(recordFactory
+          .newRecordInstance(NodeId.class));
+      container.setContainerToken(null);
+      container.setNodeHttpAddress(attemptInfo.getHostname() + ":" + 
+          attemptInfo.getHttpPort());
+      actualHandler.handle(new TaskAttemptContainerAssignedEvent(yarnAttemptID,
+          container));
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerAllocator.java

@@ -0,0 +1,32 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import org.apache.hadoop.yarn.event.EventHandler;
+
+public interface ContainerAllocator extends EventHandler<ContainerAllocatorEvent>{
+
+  enum EventType {
+
+    CONTAINER_REQ,
+    CONTAINER_DEALLOCATE,
+    CONTAINER_FAILED
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerAllocatorEvent.java

@@ -0,0 +1,38 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public class ContainerAllocatorEvent extends 
+    AbstractEvent<ContainerAllocator.EventType> {
+  
+  private TaskAttemptId attemptID;
+
+  public ContainerAllocatorEvent(TaskAttemptId attemptID,
+      ContainerAllocator.EventType type) {
+    super(type);
+    this.attemptID = attemptID;
+  }
+
+  public TaskAttemptId getAttemptID() {
+    return attemptID;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerFailedEvent.java

@@ -0,0 +1,18 @@
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+public class ContainerFailedEvent extends ContainerAllocatorEvent {
+
+  private final String contMgrAddress;
+  
+  public ContainerFailedEvent(TaskAttemptId attemptID, String contMgrAddr) {
+    super(attemptID, ContainerAllocator.EventType.CONTAINER_FAILED);
+    this.contMgrAddress = contMgrAddr;
+  }
+
+  public String getContMgrAddress() {
+    return contMgrAddress;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerRequestEvent.java

@@ -0,0 +1,68 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+
+public class ContainerRequestEvent extends ContainerAllocatorEvent {
+  
+  private final Resource capability;
+  private final String[] hosts;
+  private final String[] racks;
+  private boolean earlierAttemptFailed = false;
+
+  public ContainerRequestEvent(TaskAttemptId attemptID, 
+      Resource capability,
+      String[] hosts, String[] racks) {
+    super(attemptID, ContainerAllocator.EventType.CONTAINER_REQ);
+    this.capability = capability;
+    this.hosts = hosts;
+    this.racks = racks;
+  }
+  
+  ContainerRequestEvent(TaskAttemptId attemptID, Resource capability) {
+    this(attemptID, capability, new String[0], new String[0]);
+    this.earlierAttemptFailed = true;
+  }
+  
+  public static ContainerRequestEvent createContainerRequestEventForFailedContainer(
+      TaskAttemptId attemptID, 
+      Resource capability) {
+    //ContainerRequest for failed events does not consider rack / node locality?
+    return new ContainerRequestEvent(attemptID, capability);
+  }
+
+  public Resource getCapability() {
+    return capability;
+  }
+
+  public String[] getHosts() {
+    return hosts;
+  }
+  
+  public String[] getRacks() {
+    return racks;
+  }
+  
+  public boolean getEarlierAttemptFailed() {
+    return earlierAttemptFailed;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java

@@ -0,0 +1,280 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+/**
+ * Registers/unregisters to RM and sends heartbeats to RM.
+ */
+public class RMCommunicator extends AbstractService  {
+  private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
+  private int rmPollInterval;//millis
+  protected ApplicationId applicationId;
+  protected ApplicationAttemptId applicationAttemptId;
+  private volatile boolean stopped;
+  protected Thread allocatorThread;
+  protected EventHandler eventHandler;
+  protected AMRMProtocol scheduler;
+  private final ClientService clientService;
+  private int lastResponseID;
+  private Resource minContainerCapability;
+  private Resource maxContainerCapability;
+
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
+  
+  private final AppContext context;
+  private Job job;
+
+  public RMCommunicator(ClientService clientService, AppContext context) {
+    super("RMCommunicator");
+    this.clientService = clientService;
+    this.context = context;
+    this.eventHandler = context.getEventHandler();
+    this.applicationId = context.getApplicationID();
+    this.applicationAttemptId = context.getApplicationAttemptId();
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    super.init(conf);
+    rmPollInterval =
+        conf.getInt(AMConstants.AM_RM_SCHEDULE_INTERVAL,
+            AMConstants.DEFAULT_AM_RM_SCHEDULE_INTERVAL);
+  }
+
+  @Override
+  public void start() {
+    scheduler= createSchedulerProxy();
+    //LOG.info("Scheduler is " + scheduler);
+    register();
+    startAllocatorThread();
+    JobID id = TypeConverter.fromYarn(context.getApplicationID());
+    JobId jobId = TypeConverter.toYarn(id);
+    job = context.getJob(jobId);
+    super.start();
+  }
+
+  protected AppContext getContext() {
+    return context;
+  }
+
+  protected Job getJob() {
+    return job;
+  }
+
+  protected void register() {
+    //Register
+    String host = 
+      clientService.getBindAddress().getAddress().getHostAddress();
+    try {
+      RegisterApplicationMasterRequest request =
+        recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
+      request.setApplicationAttemptId(applicationAttemptId);
+      request.setHost(host);
+      request.setRpcPort(clientService.getBindAddress().getPort());
+      request.setTrackingUrl(host + ":" + clientService.getHttpPort());
+      RegisterApplicationMasterResponse response = 
+        scheduler.registerApplicationMaster(request);
+      minContainerCapability = response.getMinimumResourceCapability();
+      maxContainerCapability = response.getMaximumResourceCapability();
+      LOG.info("minContainerCapability: " + minContainerCapability.getMemory());
+      LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
+    } catch (Exception are) {
+      LOG.info("Exception while registering", are);
+      throw new YarnException(are);
+    }
+  }
+
+  protected void unregister() {
+    try {
+      String finalState = "RUNNING";
+      if (job.getState() == JobState.SUCCEEDED) {
+        finalState = "SUCCEEDED";
+      } else if (job.getState() == JobState.KILLED) {
+        finalState = "KILLED";
+      } else if (job.getState() == JobState.FAILED
+          || job.getState() == JobState.ERROR) {
+        finalState = "FAILED";
+      }
+      StringBuffer sb = new StringBuffer();
+      for (String s : job.getDiagnostics()) {
+        sb.append(s).append("\n");
+      }
+      LOG.info("Setting job diagnostics to " + sb.toString());
+      
+      String historyUrl = JobHistoryUtils.getHistoryUrl(getConfig(), 
+          context.getApplicationID());
+      LOG.info("History url is " + historyUrl);
+
+      FinishApplicationMasterRequest request =
+          recordFactory.newRecordInstance(FinishApplicationMasterRequest.class);
+      request.setAppAttemptId(this.applicationAttemptId);
+      request.setFinalState(finalState.toString());
+      request.setDiagnostics(sb.toString());
+      request.setTrackingUrl(historyUrl);
+      scheduler.finishApplicationMaster(request);
+    } catch(Exception are) {
+      LOG.info("Exception while unregistering ", are);
+    }
+  }
+
+  protected Resource getMinContainerCapability() {
+    return minContainerCapability;
+  }
+  
+  protected Resource getMaxContainerCapability() {
+    return maxContainerCapability;
+  }
+
+  @Override
+  public void stop() {
+    stopped = true;
+    allocatorThread.interrupt();
+    try {
+      allocatorThread.join();
+    } catch (InterruptedException ie) {
+      LOG.info("InterruptedException while stopping", ie);
+    }
+    unregister();
+    super.stop();
+  }
+
+  protected void startAllocatorThread() {
+    allocatorThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        while (!stopped && !Thread.currentThread().isInterrupted()) {
+          try {
+            Thread.sleep(rmPollInterval);
+            try {
+              heartbeat();
+            } catch (Exception e) {
+              LOG.error("ERROR IN CONTACTING RM. ", e);
+              // TODO: for other exceptions
+            }
+          } catch (InterruptedException e) {
+            LOG.info("Allocated thread interrupted. Returning.");
+            return;
+          }
+        }
+      }
+    });
+    allocatorThread.start();
+  }
+
+  protected AMRMProtocol createSchedulerProxy() {
+    final YarnRPC rpc = YarnRPC.create(getConfig());
+    final Configuration conf = new Configuration(getConfig());
+    final String serviceAddr = conf.get(
+        YarnConfiguration.SCHEDULER_ADDRESS,
+        YarnConfiguration.DEFAULT_SCHEDULER_BIND_ADDRESS);
+
+    UserGroupInformation currentUser;
+    try {
+      currentUser = UserGroupInformation.getCurrentUser();
+    } catch (IOException e) {
+      throw new YarnException(e);
+    }
+
+    if (UserGroupInformation.isSecurityEnabled()) {
+      conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
+          SchedulerSecurityInfo.class, SecurityInfo.class);
+
+      String tokenURLEncodedStr = System.getenv().get(
+          ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
+      LOG.debug("AppMasterToken is " + tokenURLEncodedStr);
+      Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
+
+      try {
+        token.decodeFromUrlString(tokenURLEncodedStr);
+      } catch (IOException e) {
+        throw new YarnException(e);
+      }
+
+      currentUser.addToken(token);
+    }
+
+    return currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
+      @Override
+      public AMRMProtocol run() {
+        return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
+            NetUtils.createSocketAddr(serviceAddr), conf);
+      }
+    });
+  }
+
+  protected synchronized void heartbeat() throws Exception {
+    AllocateRequest allocateRequest =
+        recordFactory.newRecordInstance(AllocateRequest.class);
+    allocateRequest.setApplicationAttemptId(applicationAttemptId);
+    allocateRequest.setResponseId(lastResponseID);
+    allocateRequest.addAllAsks(new ArrayList<ResourceRequest>());
+    allocateRequest.addAllReleases(new ArrayList<ContainerId>());
+    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
+    AMResponse response = allocateResponse.getAMResponse();
+    if (response.getReboot()) {
+      LOG.info("Event from RM: shutting down Application Master");
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

@@ -0,0 +1,784 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.RackResolver;
+
+/**
+ * Allocates the container from the ResourceManager scheduler.
+ */
+public class RMContainerAllocator extends RMContainerRequestor
+    implements ContainerAllocator {
+
+  private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
+  
+  public static final 
+  float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f;
+  
+  private static final Priority PRIORITY_FAST_FAIL_MAP;
+  private static final Priority PRIORITY_REDUCE;
+  private static final Priority PRIORITY_MAP;
+  
+  static {
+    PRIORITY_FAST_FAIL_MAP = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
+    PRIORITY_FAST_FAIL_MAP.setPriority(5);
+    PRIORITY_REDUCE = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
+    PRIORITY_REDUCE.setPriority(10);
+    PRIORITY_MAP = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
+    PRIORITY_MAP.setPriority(20);
+  }
+  
+  /*
+  Vocabulory Used: 
+  pending -> requests which are NOT yet sent to RM
+  scheduled -> requests which are sent to RM but not yet assigned
+  assigned -> requests which are assigned to a container
+  completed -> request corresponding to which container has completed
+  
+  Lifecycle of map
+  scheduled->assigned->completed
+  
+  Lifecycle of reduce
+  pending->scheduled->assigned->completed
+  
+  Maps are scheduled as soon as their requests are received. Reduces are 
+  added to the pending and are ramped up (added to scheduled) based 
+  on completed maps and current availability in the cluster.
+  */
+  
+  //reduces which are not yet scheduled
+  private final LinkedList<ContainerRequest> pendingReduces = 
+    new LinkedList<ContainerRequest>();
+
+  //holds information about the assigned containers to task attempts
+  private final AssignedRequests assignedRequests = new AssignedRequests();
+  
+  //holds scheduled requests to be fulfilled by RM
+  private final ScheduledRequests scheduledRequests = new ScheduledRequests();
+  
+  private int containersAllocated = 0;
+  private int containersReleased = 0;
+  private int hostLocalAssigned = 0;
+  private int rackLocalAssigned = 0;
+  
+  private boolean recalculateReduceSchedule = false;
+  private int mapResourceReqt;//memory
+  private int reduceResourceReqt;//memory
+  private int completedMaps = 0;
+  private int completedReduces = 0;
+  
+  private boolean reduceStarted = false;
+  private float maxReduceRampupLimit = 0;
+  private float maxReducePreemptionLimit = 0;
+  private float reduceSlowStart = 0;
+
+  public RMContainerAllocator(ClientService clientService, AppContext context) {
+    super(clientService, context);
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    super.init(conf);
+    reduceSlowStart = conf.getFloat(
+        MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 
+        DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART);
+    maxReduceRampupLimit = conf.getFloat(
+        AMConstants.REDUCE_RAMPUP_UP_LIMIT, 
+        AMConstants.DEFAULT_REDUCE_RAMP_UP_LIMIT);
+    maxReducePreemptionLimit = conf.getFloat(
+        AMConstants.REDUCE_PREEMPTION_LIMIT,
+        AMConstants.DEFAULT_REDUCE_PREEMPTION_LIMIT);
+    RackResolver.init(conf);
+  }
+
+  @Override
+  protected synchronized void heartbeat() throws Exception {
+    LOG.info("Before Scheduling: " + getStat());
+    List<Container> allocatedContainers = getResources();
+    LOG.info("After Scheduling: " + getStat());
+    if (allocatedContainers.size() > 0) {
+      LOG.info("Before Assign: " + getStat());
+      scheduledRequests.assign(allocatedContainers);
+      LOG.info("After Assign: " + getStat());
+    }
+    
+    if (recalculateReduceSchedule) {
+      preemptReducesIfNeeded();
+      scheduleReduces();
+      recalculateReduceSchedule = false;
+    }
+  }
+
+  @Override
+  public void stop() {
+    super.stop();
+    LOG.info("Final Stats: " + getStat());
+  }
+
+  @Override
+  public synchronized void handle(ContainerAllocatorEvent event) {
+    LOG.info("Processing the event " + event.toString());
+    recalculateReduceSchedule = true;
+    if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
+      ContainerRequestEvent reqEvent = (ContainerRequestEvent) event;
+      if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
+        if (mapResourceReqt == 0) {
+          mapResourceReqt = reqEvent.getCapability().getMemory();
+          int minSlotMemSize = getMinContainerCapability().getMemory();
+          mapResourceReqt = (int) Math.ceil((float) mapResourceReqt/minSlotMemSize) * minSlotMemSize;
+          LOG.info("mapResourceReqt:"+mapResourceReqt);
+          if (mapResourceReqt > getMaxContainerCapability().getMemory()) {
+            String diagMsg = "MAP capability required is more than the supported " +
+            "max container capability in the cluster. Killing the Job. mapResourceReqt: " + 
+            mapResourceReqt + " maxContainerCapability:" + getMaxContainerCapability().getMemory();
+            LOG.info(diagMsg);
+            eventHandler.handle(new JobDiagnosticsUpdateEvent(
+                getJob().getID(), diagMsg));
+            eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.JOB_KILL));
+          }
+        }
+        //set the rounded off memory
+        reqEvent.getCapability().setMemory(mapResourceReqt);
+        scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
+      } else {
+        if (reduceResourceReqt == 0) {
+          reduceResourceReqt = reqEvent.getCapability().getMemory();
+          int minSlotMemSize = getMinContainerCapability().getMemory();
+          //round off on slotsize
+          reduceResourceReqt = (int) Math.ceil((float) reduceResourceReqt/minSlotMemSize) * minSlotMemSize;
+          LOG.info("reduceResourceReqt:"+reduceResourceReqt);
+          if (reduceResourceReqt > getMaxContainerCapability().getMemory()) {
+            String diagMsg = "REDUCE capability required is more than the supported " +
+            "max container capability in the cluster. Killing the Job. reduceResourceReqt: " + 
+            reduceResourceReqt + " maxContainerCapability:" + getMaxContainerCapability().getMemory();
+            LOG.info(diagMsg);
+            eventHandler.handle(new JobDiagnosticsUpdateEvent(
+                getJob().getID(), diagMsg));
+            eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.JOB_KILL));
+          }
+        }
+        //set the rounded off memory
+        reqEvent.getCapability().setMemory(reduceResourceReqt);
+        if (reqEvent.getEarlierAttemptFailed()) {
+          //add to the front of queue for fail fast
+          pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE));
+        } else {
+          pendingReduces.add(new ContainerRequest(reqEvent, PRIORITY_REDUCE));//reduces are added to pending and are slowly ramped up
+        }
+      }
+      
+    } else if (
+        event.getType() == ContainerAllocator.EventType.CONTAINER_DEALLOCATE) {
+      TaskAttemptId aId = event.getAttemptID();
+      
+      boolean removed = scheduledRequests.remove(aId);
+      if (!removed) {
+        ContainerId containerId = assignedRequests.get(aId);
+        if (containerId != null) {
+          removed = true;
+          assignedRequests.remove(aId);
+          containersReleased++;
+          release(containerId);
+        }
+      }
+      if (!removed) {
+        LOG.error("Could not deallocate container for task attemptId " + 
+            aId);
+      }
+    } else if (
+        event.getType() == ContainerAllocator.EventType.CONTAINER_FAILED) {
+      ContainerFailedEvent fEv = (ContainerFailedEvent) event;
+      String host = getHost(fEv.getContMgrAddress());
+      containerFailedOnHost(host);
+    }
+  }
+
+  private static String getHost(String contMgrAddress) {
+    String host = contMgrAddress;
+    String[] hostport = host.split(":");
+    if (hostport.length == 2) {
+      host = hostport[0];
+    }
+    return host;
+  }
+
+  private void preemptReducesIfNeeded() {
+    if (reduceResourceReqt == 0) {
+      return; //no reduces
+    }
+    //check if reduces have taken over the whole cluster and there are 
+    //unassigned maps
+    if (scheduledRequests.maps.size() > 0) {
+      int memLimit = getMemLimit();
+      int availableMemForMap = memLimit - ((assignedRequests.reduces.size() -
+          assignedRequests.preemptionWaitingReduces.size()) * reduceResourceReqt);
+      //availableMemForMap must be sufficient to run atleast 1 map
+      if (availableMemForMap < mapResourceReqt) {
+        //to make sure new containers are given to maps and not reduces
+        //ramp down all scheduled reduces if any
+        //(since reduces are scheduled at higher priority than maps)
+        LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size());
+        for (ContainerRequest req : scheduledRequests.reduces.values()) {
+          pendingReduces.add(req);
+        }
+        scheduledRequests.reduces.clear();
+        
+        //preempt for making space for atleast one map
+        int premeptionLimit = Math.max(mapResourceReqt, 
+            (int) (maxReducePreemptionLimit * memLimit));
+        
+        int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt, 
+            premeptionLimit);
+        
+        int toPreempt = (int) Math.ceil((float) preemptMem/reduceResourceReqt);
+        toPreempt = Math.min(toPreempt, assignedRequests.reduces.size());
+        
+        LOG.info("Going to preempt " + toPreempt);
+        assignedRequests.preemptReduce(toPreempt);
+      }
+    }
+  }
+
+  private void scheduleReduces() {
+    
+    if (pendingReduces.size() == 0) {
+      return;
+    }
+    
+    LOG.info("Recalculating schedule...");
+    
+    //if all maps are assigned, then ramp up all reduces irrespective of the 
+    //headroom
+    if (scheduledRequests.maps.size() == 0 && pendingReduces.size() > 0) {
+      LOG.info("All maps assigned. Ramping up all remaining reduces:" + pendingReduces.size());
+      for (ContainerRequest req : pendingReduces) {
+        scheduledRequests.addReduce(req);
+      }
+      pendingReduces.clear();
+      return;
+    }
+    
+    
+    int totalMaps = assignedRequests.maps.size() + completedMaps + scheduledRequests.maps.size();
+    
+    //check for slow start
+    if (!reduceStarted) {//not set yet
+      int completedMapsForReduceSlowstart = (int)Math.ceil(reduceSlowStart * 
+                      totalMaps);
+      if(completedMaps < completedMapsForReduceSlowstart) {
+        LOG.info("Reduce slow start threshold not met. " +
+              "completedMapsForReduceSlowstart " + completedMapsForReduceSlowstart);
+        return;
+      } else {
+        LOG.info("Reduce slow start threshold reached. Scheduling reduces.");
+        reduceStarted = true;
+      }
+    }
+    
+    float completedMapPercent = 0f;
+    if (totalMaps != 0) {//support for 0 maps
+      completedMapPercent = (float)completedMaps/totalMaps;
+    } else {
+      completedMapPercent = 1;
+    }
+    
+    int netScheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt
+        + assignedRequests.maps.size() * mapResourceReqt;
+
+    int netScheduledReduceMem = scheduledRequests.reduces.size()
+        * reduceResourceReqt + assignedRequests.reduces.size()
+        * reduceResourceReqt;
+
+    int finalMapMemLimit = 0;
+    int finalReduceMemLimit = 0;
+    
+    // ramp up the reduces based on completed map percentage
+    int totalMemLimit = getMemLimit();
+    int idealReduceMemLimit = Math.min((int)(completedMapPercent * totalMemLimit),
+        (int) (maxReduceRampupLimit * totalMemLimit));
+    int idealMapMemLimit = totalMemLimit - idealReduceMemLimit;
+
+    // check if there aren't enough maps scheduled, give the free map capacity
+    // to reduce
+    if (idealMapMemLimit > netScheduledMapMem) {
+      int unusedMapMemLimit = idealMapMemLimit - netScheduledMapMem;
+      finalReduceMemLimit = idealReduceMemLimit + unusedMapMemLimit;
+      finalMapMemLimit = totalMemLimit - finalReduceMemLimit;
+    } else {
+      finalMapMemLimit = idealMapMemLimit;
+      finalReduceMemLimit = idealReduceMemLimit;
+    }
+    
+    LOG.info("completedMapPercent " + completedMapPercent +
+        " totalMemLimit:" + totalMemLimit +
+        " finalMapMemLimit:" + finalMapMemLimit +
+        " finalReduceMemLimit:" + finalReduceMemLimit + 
+        " netScheduledMapMem:" + netScheduledMapMem +
+        " netScheduledReduceMem:" + netScheduledReduceMem);
+    
+    int rampUp = (finalReduceMemLimit - netScheduledReduceMem)
+        / reduceResourceReqt;
+    
+    if (rampUp > 0) {
+      rampUp = Math.min(rampUp, pendingReduces.size());
+      LOG.info("Ramping up " + rampUp);
+      //more reduce to be scheduled
+      for (int i = 0; i < rampUp; i++) {
+        ContainerRequest request = pendingReduces.removeFirst();
+        scheduledRequests.addReduce(request);
+      }
+    } else if (rampUp < 0){
+      int rampDown = -1 * rampUp;
+      rampDown = Math.min(rampDown, scheduledRequests.reduces.size());
+      LOG.info("Ramping down " + rampDown);
+      //remove from the scheduled and move back to pending
+      for (int i = 0; i < rampDown; i++) {
+        ContainerRequest request = scheduledRequests.removeReduce();
+        pendingReduces.add(request);
+      }
+    }
+  }
+
+  /**
+   * Synchronized to avoid findbugs warnings
+   */
+  private synchronized String getStat() {
+    return "PendingReduces:" + pendingReduces.size() +
+        " ScheduledMaps:" + scheduledRequests.maps.size() +
+        " ScheduledReduces:" + scheduledRequests.reduces.size() +
+        " AssignedMaps:" + assignedRequests.maps.size() + 
+        " AssignedReduces:" + assignedRequests.reduces.size() +
+        " completedMaps:" + completedMaps +
+        " completedReduces:" + completedReduces +
+        " containersAllocated:" + containersAllocated +
+        " containersReleased:" + containersReleased +
+        " hostLocalAssigned:" + hostLocalAssigned + 
+        " rackLocalAssigned:" + rackLocalAssigned +
+        " availableResources(headroom):" + getAvailableResources();
+  }
+  
+  private List<Container> getResources() throws Exception {
+    int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
+    AMResponse response = makeRemoteRequest();
+    int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
+    List<Container> newContainers = response.getNewContainerList();
+    List<Container> finishedContainers = response.getFinishedContainerList();
+    if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
+      //something changed
+      recalculateReduceSchedule = true;
+    }
+    
+    List<Container> allocatedContainers = new ArrayList<Container>();
+    for (Container cont : newContainers) {
+        allocatedContainers.add(cont);
+        LOG.debug("Received new Container :" + cont);
+    }
+    for (Container cont : finishedContainers) {
+      LOG.info("Received completed container " + cont);
+      TaskAttemptId attemptID = assignedRequests.get(cont.getId());
+      if (attemptID == null) {
+        LOG.error("Container complete event for unknown container id "
+            + cont.getId());
+      } else {
+        assignedRequests.remove(attemptID);
+        if (attemptID.getTaskId().getTaskType().equals(TaskType.MAP)) {
+          completedMaps++;
+        } else {
+          completedReduces++;
+        }
+        // send the container completed event to Task attempt
+        eventHandler.handle(new TaskAttemptEvent(attemptID,
+            TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+        // Send the diagnostics
+        String diagnostics = cont.getContainerStatus().getDiagnostics();
+        eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
+            diagnostics));
+      }
+    }
+    return newContainers;
+  }
+
+  private int getMemLimit() {
+    int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
+    return headRoom + assignedRequests.maps.size() * mapResourceReqt + 
+       assignedRequests.reduces.size() * reduceResourceReqt;
+  }
+  
+  private class ScheduledRequests {
+    
+    private final LinkedList<TaskAttemptId> earlierFailedMaps = 
+      new LinkedList<TaskAttemptId>();
+    
+    /** Maps from a host to a list of Map tasks with data on the host */
+    private final Map<String, LinkedList<TaskAttemptId>> mapsHostMapping = 
+      new HashMap<String, LinkedList<TaskAttemptId>>();
+    private final Map<String, LinkedList<TaskAttemptId>> mapsRackMapping = 
+      new HashMap<String, LinkedList<TaskAttemptId>>();
+    private final Map<TaskAttemptId, ContainerRequest> maps = 
+      new LinkedHashMap<TaskAttemptId, ContainerRequest>();
+    
+    private final LinkedHashMap<TaskAttemptId, ContainerRequest> reduces = 
+      new LinkedHashMap<TaskAttemptId, ContainerRequest>();
+    
+    boolean remove(TaskAttemptId tId) {
+      ContainerRequest req = null;
+      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
+        req = maps.remove(tId);
+      } else {
+        req = reduces.remove(tId);
+      }
+      
+      if (req == null) {
+        return false;
+      } else {
+        decContainerReq(req);
+        return true;
+      }
+    }
+    
+    ContainerRequest removeReduce() {
+      Iterator<Entry<TaskAttemptId, ContainerRequest>> it = reduces.entrySet().iterator();
+      if (it.hasNext()) {
+        Entry<TaskAttemptId, ContainerRequest> entry = it.next();
+        it.remove();
+        decContainerReq(entry.getValue());
+        return entry.getValue();
+      }
+      return null;
+    }
+    
+    void addMap(ContainerRequestEvent event) {
+      ContainerRequest request = null;
+      
+      if (event.getEarlierAttemptFailed()) {
+        earlierFailedMaps.add(event.getAttemptID());
+        request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP);
+      } else {
+        for (String host : event.getHosts()) {
+          //host comes from data splitLocations which are hostnames. Containers
+          // use IP addresses.
+          //TODO Temporary fix for locality. Use resolvers from h-common. 
+          // Cache to make this more efficient ?
+          InetAddress addr = null;
+          try {
+            addr = InetAddress.getByName(host);
+          } catch (UnknownHostException e) {
+            LOG.warn("Unable to resolve host to IP for host [: " + host + "]");
+          }
+          if (addr != null) //Fallback to host if resolve fails.
+            host = addr.getHostAddress();
+          LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
+          if (list == null) {
+            list = new LinkedList<TaskAttemptId>();
+            mapsHostMapping.put(host, list);
+          }
+          list.add(event.getAttemptID());
+          LOG.info("Added attempt req to host " + host);
+       }
+       for (String rack: event.getRacks()) {
+         LinkedList<TaskAttemptId> list = mapsRackMapping.get(rack);
+         if (list == null) {
+           list = new LinkedList<TaskAttemptId>();
+           mapsRackMapping.put(rack, list);
+         }
+         list.add(event.getAttemptID());
+         LOG.info("Added attempt req to rack " + rack);
+       }
+       request = new ContainerRequest(event, PRIORITY_MAP);
+      }
+      maps.put(event.getAttemptID(), request);
+      addContainerReq(request);
+    }
+    
+    
+    void addReduce(ContainerRequest req) {
+      reduces.put(req.attemptID, req);
+      addContainerReq(req);
+    }
+    
+    private void assign(List<Container> allocatedContainers) {
+      Iterator<Container> it = allocatedContainers.iterator();
+      LOG.info("Got allocated containers " + allocatedContainers.size());
+      containersAllocated += allocatedContainers.size();
+      while (it.hasNext()) {
+        Container allocated = it.next();
+        LOG.info("Assigning container " + allocated);
+        ContainerRequest assigned = assign(allocated);
+          
+        if (assigned != null) {
+          // Update resource requests
+          decContainerReq(assigned);
+
+          // send the container-assigned event to task attempt
+          eventHandler.handle(new TaskAttemptContainerAssignedEvent(
+              assigned.attemptID, allocated));
+
+          assignedRequests.add(allocated.getId(), assigned.attemptID);
+          
+          LOG.info("Assigned container (" + allocated + ") " +
+              " to task " + assigned.attemptID +
+              " on node " + allocated.getNodeId().toString());
+        } else {
+          //not assigned to any request, release the container
+          LOG.info("Releasing unassigned and invalid container " + allocated
+              + ". RM has gone crazy, someone go look!"
+              + " Hey RM, if you are so rich, go donate to non-profits!");
+          containersReleased++;
+          release(allocated.getId());
+        }
+      }
+    }
+    
+    private ContainerRequest assign(Container allocated) {
+      ContainerRequest assigned = null;
+      
+      if (mapResourceReqt != reduceResourceReqt) {
+        //assign based on size
+        LOG.info("Assigning based on container size");
+        if (allocated.getResource().getMemory() == mapResourceReqt) {
+          assigned = assignToFailedMap(allocated);
+          if (assigned == null) {
+            assigned = assignToMap(allocated);
+          }
+        } else if (allocated.getResource().getMemory() == reduceResourceReqt) {
+          assigned = assignToReduce(allocated);
+        }
+        
+        return assigned;
+      }
+      
+      //container can be given to either map or reduce
+      //assign based on priority
+      
+      //try to assign to earlierFailedMaps if present
+      assigned = assignToFailedMap(allocated);
+      
+      //Assign to reduces before assigning to maps ?
+      if (assigned == null) {
+        assigned = assignToReduce(allocated);
+      }
+      
+      //try to assign to maps if present
+      if (assigned == null) {
+        assigned = assignToMap(allocated);
+      }
+      
+      return assigned;
+    }
+    
+    
+    private ContainerRequest assignToFailedMap(Container allocated) {
+      //try to assign to earlierFailedMaps if present
+      ContainerRequest assigned = null;
+      while (assigned == null && earlierFailedMaps.size() > 0 && 
+          allocated.getResource().getMemory() >= mapResourceReqt) {
+        TaskAttemptId tId = earlierFailedMaps.removeFirst();
+        if (maps.containsKey(tId)) {
+          assigned = maps.remove(tId);
+          JobCounterUpdateEvent jce =
+            new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
+          jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
+          eventHandler.handle(jce);
+          LOG.info("Assigned from earlierFailedMaps");
+          break;
+        }
+      }
+      return assigned;
+    }
+    
+    private ContainerRequest assignToReduce(Container allocated) {
+      ContainerRequest assigned = null;
+      //try to assign to reduces if present
+      if (assigned == null && reduces.size() > 0
+          && allocated.getResource().getMemory() >= reduceResourceReqt) {
+        TaskAttemptId tId = reduces.keySet().iterator().next();
+        assigned = reduces.remove(tId);
+        LOG.info("Assigned to reduce");
+      }
+      return assigned;
+    }
+    
+    private ContainerRequest assignToMap(Container allocated) {
+    //try to assign to maps if present 
+      //first by host, then by rack, followed by *
+      ContainerRequest assigned = null;
+      while (assigned == null && maps.size() > 0
+          && allocated.getResource().getMemory() >= mapResourceReqt) {
+        String host = getHost(allocated.getNodeId().toString());
+        LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
+        while (list != null && list.size() > 0) {
+          LOG.info("Host matched to the request list " + host);
+          TaskAttemptId tId = list.removeFirst();
+          if (maps.containsKey(tId)) {
+            assigned = maps.remove(tId);
+            JobCounterUpdateEvent jce =
+              new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
+            jce.addCounterUpdate(JobCounter.DATA_LOCAL_MAPS, 1);
+            eventHandler.handle(jce);
+            hostLocalAssigned++;
+            LOG.info("Assigned based on host match " + host);
+            break;
+          }
+        }
+        if (assigned == null) {
+          String rack = RackResolver.resolve(host).getNetworkLocation();
+          list = mapsRackMapping.get(rack);
+          while (list != null && list.size() > 0) {
+            TaskAttemptId tId = list.removeFirst();
+            if (maps.containsKey(tId)) {
+              assigned = maps.remove(tId);
+              JobCounterUpdateEvent jce =
+                new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
+              jce.addCounterUpdate(JobCounter.RACK_LOCAL_MAPS, 1);
+              eventHandler.handle(jce);
+              rackLocalAssigned++;
+              LOG.info("Assigned based on rack match " + rack);
+              break;
+            }
+          }
+          if (assigned == null && maps.size() > 0) {
+            TaskAttemptId tId = maps.keySet().iterator().next();
+            assigned = maps.remove(tId);
+            JobCounterUpdateEvent jce =
+              new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
+            jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
+            eventHandler.handle(jce);
+            LOG.info("Assigned based on * match");
+            break;
+          }
+        }
+      }
+      return assigned;
+    }
+  }
+
+  private class AssignedRequests {
+    private final Map<ContainerId, TaskAttemptId> containerToAttemptMap =
+      new HashMap<ContainerId, TaskAttemptId>();
+    private final LinkedHashMap<TaskAttemptId, ContainerId> maps = 
+      new LinkedHashMap<TaskAttemptId, ContainerId>();
+    private final LinkedHashMap<TaskAttemptId, ContainerId> reduces = 
+      new LinkedHashMap<TaskAttemptId, ContainerId>();
+    private final Set<TaskAttemptId> preemptionWaitingReduces = 
+      new HashSet<TaskAttemptId>();
+    
+    void add(ContainerId containerId, TaskAttemptId tId) {
+      LOG.info("Assigned container " + containerId.toString()
+          + " to " + tId);
+      containerToAttemptMap.put(containerId, tId);
+      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
+        maps.put(tId, containerId);
+      } else {
+        reduces.put(tId, containerId);
+      }
+    }
+
+    void preemptReduce(int toPreempt) {
+      List<TaskAttemptId> reduceList = new ArrayList(reduces.keySet());
+      //sort reduces on progress
+      Collections.sort(reduceList,
+          new Comparator<TaskAttemptId>() {
+        @Override
+        public int compare(TaskAttemptId o1, TaskAttemptId o2) {
+          float p = getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress() -
+              getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress();
+          return p >= 0 ? 1 : -1;
+        }
+      });
+      
+      for (int i = 0; i < toPreempt && reduceList.size() > 0; i++) {
+        TaskAttemptId id = reduceList.remove(0);//remove the one on top
+        LOG.info("Preempting " + id);
+        preemptionWaitingReduces.add(id);
+        eventHandler.handle(new TaskAttemptEvent(id, TaskAttemptEventType.TA_KILL));
+      }
+    }
+    
+    boolean remove(TaskAttemptId tId) {
+      ContainerId containerId = null;
+      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
+        containerId = maps.remove(tId);
+      } else {
+        containerId = reduces.remove(tId);
+        if (containerId != null) {
+          boolean preempted = preemptionWaitingReduces.remove(tId);
+          if (preempted) {
+            LOG.info("Reduce preemption successful " + tId);
+          }
+        }
+      }
+      
+      if (containerId != null) {
+        containerToAttemptMap.remove(containerId);
+        return true;
+      }
+      return false;
+    }
+    
+    TaskAttemptId get(ContainerId cId) {
+      return containerToAttemptMap.get(cId);
+    }
+
+    ContainerId get(TaskAttemptId tId) {
+      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
+        return maps.get(tId);
+      } else {
+        return reduces.get(tId);
+      }
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java

@@ -0,0 +1,274 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.rm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.AMResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+
+/**
+ * Keeps the data structures to send container requests to RM.
+ */
+public abstract class RMContainerRequestor extends RMCommunicator {
+  
+  private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class);
+  static final String ANY = "*";
+
+  private int lastResponseID;
+  private Resource availableResources;
+
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
+  //Key -> Priority
+  //Value -> Map
+  //Key->ResourceName (e.g., hostname, rackname, *)
+  //Value->Map
+  //Key->Resource Capability
+  //Value->ResourceReqeust
+  private final Map<Priority, Map<String, Map<Resource, ResourceRequest>>>
+  remoteRequestsTable =
+      new TreeMap<Priority, Map<String, Map<Resource, ResourceRequest>>>();
+
+  private final Set<ResourceRequest> ask = new TreeSet<ResourceRequest>();
+  private final Set<ContainerId> release = new TreeSet<ContainerId>(); 
+
+  private boolean nodeBlacklistingEnabled;
+  private int maxTaskFailuresPerNode;
+  private final Map<String, Integer> nodeFailures = new HashMap<String, Integer>();
+  private final Set<String> blacklistedNodes = new HashSet<String>();
+
+  public RMContainerRequestor(ClientService clientService, AppContext context) {
+    super(clientService, context);
+  }
+
+  static class ContainerRequest {
+    final TaskAttemptId attemptID;
+    final Resource capability;
+    final String[] hosts;
+    final String[] racks;
+    //final boolean earlierAttemptFailed;
+    final Priority priority;
+    public ContainerRequest(ContainerRequestEvent event, Priority priority) {
+      this.attemptID = event.getAttemptID();
+      this.capability = event.getCapability();
+      this.hosts = event.getHosts();
+      this.racks = event.getRacks();
+      //this.earlierAttemptFailed = event.getEarlierAttemptFailed();
+      this.priority = priority;
+    }
+  }
+
+  @Override
+  public void init(Configuration conf) {
+    super.init(conf);
+    nodeBlacklistingEnabled = 
+      conf.getBoolean(AMConstants.NODE_BLACKLISTING_ENABLE, true);
+    LOG.info("nodeBlacklistingEnabled:" + nodeBlacklistingEnabled);
+    maxTaskFailuresPerNode = 
+      conf.getInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 3);
+    LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
+  }
+
+  protected abstract void heartbeat() throws Exception;
+
+  protected AMResponse makeRemoteRequest() throws YarnRemoteException {
+    AllocateRequest allocateRequest = recordFactory
+        .newRecordInstance(AllocateRequest.class);
+    allocateRequest.setApplicationAttemptId(applicationAttemptId);
+    allocateRequest.setResponseId(lastResponseID);
+    allocateRequest.addAllAsks(new ArrayList<ResourceRequest>(ask));
+    allocateRequest.addAllReleases(new ArrayList<ContainerId>(release));
+    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
+    AMResponse response = allocateResponse.getAMResponse();
+    lastResponseID = response.getResponseId();
+    availableResources = response.getAvailableResources();
+
+    LOG.info("getResources() for " + applicationId + ":" + " ask="
+        + ask.size() + " release= " + release.size() + " newContainers="
+        + response.getNewContainerCount() + " finishedContainers="
+        + response.getFinishedContainerCount()
+        + " resourcelimit=" + availableResources);
+
+    ask.clear();
+    release.clear();
+    return response;
+  }
+
+  protected void containerFailedOnHost(String hostName) {
+    if (!nodeBlacklistingEnabled) {
+      return;
+    }
+    if (blacklistedNodes.contains(hostName)) {
+      LOG.info("Host " + hostName + " is already blacklisted.");
+      return; //already blacklisted
+    }
+    Integer failures = nodeFailures.remove(hostName);
+    failures = failures == null ? 0 : failures;
+    failures++;
+    LOG.info(failures + " failures on node " + hostName);
+    if (failures >= maxTaskFailuresPerNode) {
+      blacklistedNodes.add(hostName);
+      LOG.info("Blacklisted host " + hostName);
+      
+      //remove all the requests corresponding to this hostname
+      for (Map<String, Map<Resource, ResourceRequest>> remoteRequests 
+          : remoteRequestsTable.values()){
+        //remove from host
+        Map<Resource, ResourceRequest> reqMap = remoteRequests.remove(hostName);
+        if (reqMap != null) {
+          for (ResourceRequest req : reqMap.values()) {
+            ask.remove(req);
+          }
+        }
+        //TODO: remove from rack
+      }
+    } else {
+      nodeFailures.put(hostName, failures);
+    }
+  }
+
+  protected Resource getAvailableResources() {
+    return availableResources;
+  }
+  
+  protected void addContainerReq(ContainerRequest req) {
+    // Create resource requests
+    for (String host : req.hosts) {
+      // Data-local
+      addResourceRequest(req.priority, host, req.capability);
+    }
+
+    // Nothing Rack-local for now
+    for (String rack : req.racks) {
+      addResourceRequest(req.priority, rack, req.capability);
+    }
+
+    // Off-switch
+    addResourceRequest(req.priority, ANY, req.capability); 
+  }
+
+  protected void decContainerReq(ContainerRequest req) {
+    // Update resource requests
+    for (String hostName : req.hosts) {
+      decResourceRequest(req.priority, hostName, req.capability);
+    }
+    
+    for (String rack : req.racks) {
+      decResourceRequest(req.priority, rack, req.capability);
+    }
+   
+    decResourceRequest(req.priority, ANY, req.capability);
+  }
+
+  private void addResourceRequest(Priority priority, String resourceName,
+      Resource capability) {
+    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
+      this.remoteRequestsTable.get(priority);
+    if (remoteRequests == null) {
+      remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
+      this.remoteRequestsTable.put(priority, remoteRequests);
+      LOG.info("Added priority=" + priority);
+    }
+    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
+    if (reqMap == null) {
+      reqMap = new HashMap<Resource, ResourceRequest>();
+      remoteRequests.put(resourceName, reqMap);
+    }
+    ResourceRequest remoteRequest = reqMap.get(capability);
+    if (remoteRequest == null) {
+      remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
+      remoteRequest.setPriority(priority);
+      remoteRequest.setHostName(resourceName);
+      remoteRequest.setCapability(capability);
+      remoteRequest.setNumContainers(0);
+      reqMap.put(capability, remoteRequest);
+    }
+    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
+
+    // Note this down for next interaction with ResourceManager
+    ask.add(remoteRequest);
+    LOG.info("addResourceRequest:" + " applicationId=" + applicationId.getId()
+        + " priority=" + priority.getPriority() + " resourceName=" + resourceName
+        + " numContainers=" + remoteRequest.getNumContainers() + " #asks="
+        + ask.size());
+  }
+
+  private void decResourceRequest(Priority priority, String resourceName,
+      Resource capability) {
+    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
+      this.remoteRequestsTable.get(priority);
+    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
+    ResourceRequest remoteRequest = reqMap.get(capability);
+
+    LOG.info("BEFORE decResourceRequest:" + " applicationId=" + applicationId.getId()
+        + " priority=" + priority.getPriority() + " resourceName=" + resourceName
+        + " numContainers=" + remoteRequest.getNumContainers() + " #asks="
+        + ask.size());
+
+    remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
+    if (remoteRequest.getNumContainers() == 0) {
+      reqMap.remove(capability);
+      if (reqMap.size() == 0) {
+        remoteRequests.remove(resourceName);
+      }
+      if (remoteRequests.size() == 0) {
+        remoteRequestsTable.remove(priority);
+      }
+      //remove from ask if it may have
+      ask.remove(remoteRequest);
+    } else {
+      ask.add(remoteRequest);//this will override the request if ask doesn't
+      //already have it.
+    }
+
+    LOG.info("AFTER decResourceRequest:" + " applicationId="
+             + applicationId.getId() + " priority=" + priority.getPriority()
+             + " resourceName=" + resourceName + " numContainers="
+             + remoteRequest.getNumContainers() + " #asks=" + ask.size());
+  }
+
+  protected void release(ContainerId containerId) {
+    release.add(containerId);
+  }
+  
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DataStatistics.java

@@ -0,0 +1,78 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+public class DataStatistics {
+  private int count = 0;
+  private double sum = 0;
+  private double sumSquares = 0;
+
+  public DataStatistics() {
+  }
+
+  public DataStatistics(double initNum) {
+    this.count = 1;
+    this.sum = initNum;
+    this.sumSquares = initNum * initNum;
+  }
+
+  public synchronized void add(double newNum) {
+    this.count++;
+    this.sum += newNum;
+    this.sumSquares += newNum * newNum;
+  }
+
+  public synchronized void updateStatistics(double old, double update) {
+	this.sum += update - old;
+	this.sumSquares += (update * update) - (old * old);
+  }
+
+  public synchronized double mean() {
+    return count == 0 ? 0.0 : sum/count;
+  }
+
+  public synchronized double var() {
+    // E(X^2) - E(X)^2
+    if (count <= 1) {
+      return 0.0;
+    }
+    double mean = mean();
+    return Math.max((sumSquares/count) - mean * mean, 0.0d);
+  }
+
+  public synchronized double std() {
+    return Math.sqrt(this.var());
+  }
+
+  public synchronized double outlier(float sigma) {
+    if (count != 0.0) {
+      return mean() + std() * sigma;
+    }
+
+    return 0.0;
+  }
+
+  public synchronized double count() {
+    return count;
+  }
+
+  public String toString() {
+    return "DataStatistics: count is " + count + ", sum is " + sum +
+    ", sumSquares is " + sumSquares + " mean is " + mean() + " std() is " + std();
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java

@@ -0,0 +1,512 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.MRConstants;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+
+public class DefaultSpeculator extends AbstractService implements
+    Speculator {
+
+  private static final long ON_SCHEDULE = Long.MIN_VALUE;
+  private static final long ALREADY_SPECULATING = Long.MIN_VALUE + 1;
+  private static final long TOO_NEW = Long.MIN_VALUE + 2;
+  private static final long PROGRESS_IS_GOOD = Long.MIN_VALUE + 3;
+  private static final long NOT_RUNNING = Long.MIN_VALUE + 4;
+  private static final long TOO_LATE_TO_SPECULATE = Long.MIN_VALUE + 5;
+
+  private static final long SOONEST_RETRY_AFTER_NO_SPECULATE = 1000L * 1L;
+  private static final long SOONEST_RETRY_AFTER_SPECULATE = 1000L * 15L;
+
+  private static final double PROPORTION_RUNNING_TASKS_SPECULATABLE = 0.1;
+  private static final double PROPORTION_TOTAL_TASKS_SPECULATABLE = 0.01;
+  private static final int  MINIMUM_ALLOWED_SPECULATIVE_TASKS = 10;
+
+  private static final Log LOG = LogFactory.getLog(DefaultSpeculator.class);
+
+  private final ConcurrentMap<TaskId, Boolean> runningTasks
+      = new ConcurrentHashMap<TaskId, Boolean>();
+
+  private final Map<Task, AtomicBoolean> pendingSpeculations
+      = new ConcurrentHashMap<Task, AtomicBoolean>();
+
+  // These are the current needs, not the initial needs.  For each job, these
+  //  record the number of attempts that exist and that are actively
+  //  waiting for a container [as opposed to running or finished]
+  private final ConcurrentMap<JobId, AtomicInteger> mapContainerNeeds
+      = new ConcurrentHashMap<JobId, AtomicInteger>();
+  private final ConcurrentMap<JobId, AtomicInteger> reduceContainerNeeds
+      = new ConcurrentHashMap<JobId, AtomicInteger>();
+
+  private final Set<TaskId> mayHaveSpeculated = new HashSet();
+
+  private final Configuration conf;
+  private AppContext context;
+  private Thread speculationBackgroundThread = null;
+  private BlockingQueue<SpeculatorEvent> eventQueue
+      = new LinkedBlockingQueue<SpeculatorEvent>();
+  private TaskRuntimeEstimator estimator;
+
+  private BlockingQueue<Object> scanControl = new LinkedBlockingQueue<Object>();
+
+  private final Clock clock;
+
+  private final EventHandler<TaskEvent> eventHandler;
+
+  public DefaultSpeculator(Configuration conf, AppContext context) {
+    this(conf, context, context.getClock());
+  }
+
+  public DefaultSpeculator(Configuration conf, AppContext context, Clock clock) {
+    this(conf, context, getEstimator(conf, context), clock);
+  }
+  
+  static private TaskRuntimeEstimator getEstimator
+      (Configuration conf, AppContext context) {
+    TaskRuntimeEstimator estimator;
+    
+    try {
+      // "yarn.mapreduce.job.task.runtime.estimator.class"
+      Class<? extends TaskRuntimeEstimator> estimatorClass
+          = conf.getClass(AMConstants.TASK_RUNTIME_ESTIMATOR_CLASS,
+                          LegacyTaskRuntimeEstimator.class,
+                          TaskRuntimeEstimator.class);
+
+      Constructor<? extends TaskRuntimeEstimator> estimatorConstructor
+          = estimatorClass.getConstructor();
+
+      estimator = estimatorConstructor.newInstance();
+
+      estimator.contextualize(conf, context);
+    } catch (InstantiationException ex) {
+      LOG.error("Can't make a speculation runtime extimator" + ex);
+      throw new YarnException(ex);
+    } catch (IllegalAccessException ex) {
+      LOG.error("Can't make a speculation runtime extimator" + ex);
+      throw new YarnException(ex);
+    } catch (InvocationTargetException ex) {
+      LOG.error("Can't make a speculation runtime extimator" + ex);
+      throw new YarnException(ex);
+    } catch (NoSuchMethodException ex) {
+      LOG.error("Can't make a speculation runtime extimator" + ex);
+      throw new YarnException(ex);
+    }
+    
+  return estimator;
+  }
+
+  // This constructor is designed to be called by other constructors.
+  //  However, it's public because we do use it in the test cases.
+  // Normally we figure out our own estimator.
+  public DefaultSpeculator
+      (Configuration conf, AppContext context,
+       TaskRuntimeEstimator estimator, Clock clock) {
+    super(DefaultSpeculator.class.getName());
+
+    this.conf = conf;
+    this.context = context;
+    this.estimator = estimator;
+    this.clock = clock;
+    this.eventHandler = context.getEventHandler();
+  }
+
+/*   *************************************************************    */
+
+  // This is the task-mongering that creates the two new threads -- one for
+  //  processing events from the event queue and one for periodically
+  //  looking for speculation opportunities
+
+  @Override
+  public void start() {
+    Runnable speculationBackgroundCore
+        = new Runnable() {
+            @Override
+            public void run() {
+              while (!Thread.currentThread().isInterrupted()) {
+                long backgroundRunStartTime = clock.getTime();
+                try {
+                  int speculations = computeSpeculations();
+                  long mininumRecomp
+                      = speculations > 0 ? SOONEST_RETRY_AFTER_SPECULATE
+                                         : SOONEST_RETRY_AFTER_NO_SPECULATE;
+
+                  long wait = Math.max(mininumRecomp,
+                        clock.getTime() - backgroundRunStartTime);
+
+                  if (speculations > 0) {
+                    LOG.info("We launched " + speculations
+                        + " speculations.  Sleeping " + wait + " milliseconds.");
+                  }
+
+                  Object pollResult
+                      = scanControl.poll(wait, TimeUnit.MILLISECONDS);
+                } catch (InterruptedException e) {
+                  LOG.error("Background thread returning, interrupted : " + e);
+                  e.printStackTrace(System.out);
+                  return;
+                }
+              }
+            }
+          };
+    speculationBackgroundThread = new Thread
+        (speculationBackgroundCore, "DefaultSpeculator background processing");
+    speculationBackgroundThread.start();
+
+    super.start();
+  }
+
+  @Override
+  public void stop() {
+    // this could be called before background thread is established
+    if (speculationBackgroundThread != null) {
+      speculationBackgroundThread.interrupt();
+    }
+    super.stop();
+  }
+
+  @Override
+  public void handleAttempt(TaskAttemptStatus status) {
+    long timestamp = clock.getTime();
+    statusUpdate(status, timestamp);
+  }
+
+  // This section is not part of the Speculator interface; it's used only for
+  //  testing
+  public boolean eventQueueEmpty() {
+    return eventQueue.isEmpty();
+  }
+
+  // This interface is intended to be used only for test cases.
+  public void scanForSpeculations() {
+    LOG.info("We got asked to run a debug speculation scan.");
+    // debug
+    System.out.println("We got asked to run a debug speculation scan.");
+    System.out.println("There are " + scanControl.size()
+        + " events stacked already.");
+    scanControl.add(new Object());
+    Thread.yield();
+  }
+
+
+/*   *************************************************************    */
+
+  // This section contains the code that gets run for a SpeculatorEvent
+
+  private AtomicInteger containerNeed(TaskId taskID) {
+    JobId jobID = taskID.getJobId();
+    TaskType taskType = taskID.getTaskType();
+
+    ConcurrentMap<JobId, AtomicInteger> relevantMap
+        = taskType == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;
+
+    AtomicInteger result = relevantMap.get(jobID);
+
+    if (result == null) {
+      relevantMap.putIfAbsent(jobID, new AtomicInteger(0));
+      result = relevantMap.get(jobID);
+    }
+
+    return result;
+  }
+
+  private synchronized void processSpeculatorEvent(SpeculatorEvent event) {
+    switch (event.getType()) {
+      case ATTEMPT_STATUS_UPDATE:
+        statusUpdate(event.getReportedStatus(), event.getTimestamp());
+        break;
+
+      case TASK_CONTAINER_NEED_UPDATE:
+      {
+        AtomicInteger need = containerNeed(event.getTaskID());
+        need.addAndGet(event.containersNeededChange());
+        break;
+      }
+
+      case ATTEMPT_START:
+      {
+        LOG.info("ATTEMPT_START " + event.getTaskID());
+        estimator.enrollAttempt
+            (event.getReportedStatus(), event.getTimestamp());
+        break;
+      }
+      
+      case JOB_CREATE:
+      {
+        LOG.info("JOB_CREATE " + event.getJobID());
+        estimator.contextualize(getConfig(), context);
+        break;
+      }
+    }
+  }
+
+  /**
+   * Absorbs one TaskAttemptStatus
+   *
+   * @param reportedStatus the status report that we got from a task attempt
+   *        that we want to fold into the speculation data for this job
+   * @param timestamp the time this status corresponds to.  This matters
+   *        because statuses contain progress.
+   */
+  protected void statusUpdate(TaskAttemptStatus reportedStatus, long timestamp) {
+
+    String stateString = reportedStatus.taskState.toString();
+
+    TaskAttemptId attemptID = reportedStatus.id;
+    TaskId taskID = attemptID.getTaskId();
+    Job job = context.getJob(taskID.getJobId());
+
+    if (job == null) {
+      return;
+    }
+
+    Task task = job.getTask(taskID);
+
+    if (task == null) {
+      return;
+    }
+
+    estimator.updateAttempt(reportedStatus, timestamp);
+
+    // If the task is already known to be speculation-bait, don't do anything
+    if (pendingSpeculations.get(task) != null) {
+      if (pendingSpeculations.get(task).get()) {
+        return;
+      }
+    }
+
+    if (stateString.equals(TaskAttemptState.RUNNING.name())) {
+      runningTasks.putIfAbsent(taskID, Boolean.TRUE);
+    } else {
+      runningTasks.remove(taskID, Boolean.TRUE);
+    }
+  }
+
+/*   *************************************************************    */
+
+// This is the code section that runs periodically and adds speculations for
+//  those jobs that need them.
+
+
+  // This can return a few magic values for tasks that shouldn't speculate:
+  //  returns ON_SCHEDULE if thresholdRuntime(taskID) says that we should not
+  //     considering speculating this task
+  //  returns ALREADY_SPECULATING if that is true.  This has priority.
+  //  returns TOO_NEW if our companion task hasn't gotten any information
+  //  returns PROGRESS_IS_GOOD if the task is sailing through
+  //  returns NOT_RUNNING if the task is not running
+  //
+  // All of these values are negative.  Any value that should be allowed to
+  //  speculate is 0 or positive.
+  private long speculationValue(TaskId taskID, long now) {
+    Job job = context.getJob(taskID.getJobId());
+    Task task = job.getTask(taskID);
+    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
+    long acceptableRuntime = Long.MIN_VALUE;
+    long result = Long.MIN_VALUE;
+
+    if (!mayHaveSpeculated.contains(taskID)) {
+      acceptableRuntime = estimator.thresholdRuntime(taskID);
+      if (acceptableRuntime == Long.MAX_VALUE) {
+        return ON_SCHEDULE;
+      }
+    }
+
+    TaskAttemptId runningTaskAttemptID = null;
+
+    int numberRunningAttempts = 0;
+
+    for (TaskAttempt taskAttempt : attempts.values()) {
+      if (taskAttempt.getState() == TaskAttemptState.RUNNING
+          || taskAttempt.getState() == TaskAttemptState.ASSIGNED) {
+        if (++numberRunningAttempts > 1) {
+          return ALREADY_SPECULATING;
+        }
+        runningTaskAttemptID = taskAttempt.getID();
+
+        long estimatedRunTime = estimator.estimatedRuntime(runningTaskAttemptID);
+
+        long taskAttemptStartTime
+            = estimator.attemptEnrolledTime(runningTaskAttemptID);
+        if (taskAttemptStartTime > now) {
+          // This background process ran before we could process the task
+          //  attempt status change that chronicles the attempt start
+          return TOO_NEW;
+        }
+
+        long estimatedEndTime = estimatedRunTime + taskAttemptStartTime;
+
+        long estimatedReplacementEndTime
+            = now + estimator.estimatedNewAttemptRuntime(taskID);
+
+        if (estimatedEndTime < now) {
+          return PROGRESS_IS_GOOD;
+        }
+
+        if (estimatedReplacementEndTime >= estimatedEndTime) {
+          return TOO_LATE_TO_SPECULATE;
+        }
+
+        result = estimatedEndTime - estimatedReplacementEndTime;
+      }
+    }
+
+    // If we are here, there's at most one task attempt.
+    if (numberRunningAttempts == 0) {
+      return NOT_RUNNING;
+    }
+
+
+
+    if (acceptableRuntime == Long.MIN_VALUE) {
+      acceptableRuntime = estimator.thresholdRuntime(taskID);
+      if (acceptableRuntime == Long.MAX_VALUE) {
+        return ON_SCHEDULE;
+      }
+    }
+
+    return result;
+  }
+
+  //Add attempt to a given Task.
+  protected void addSpeculativeAttempt(TaskId taskID) {
+    LOG.info
+        ("DefaultSpeculator.addSpeculativeAttempt -- we are speculating " + taskID);
+    eventHandler.handle(new TaskEvent(taskID, TaskEventType.T_ADD_SPEC_ATTEMPT));
+    mayHaveSpeculated.add(taskID);
+  }
+
+  @Override
+  public void handle(SpeculatorEvent event) {
+    processSpeculatorEvent(event);
+  }
+
+
+  private int maybeScheduleAMapSpeculation() {
+    return maybeScheduleASpeculation(TaskType.MAP);
+  }
+
+  private int maybeScheduleAReduceSpeculation() {
+    return maybeScheduleASpeculation(TaskType.REDUCE);
+  }
+
+  private int maybeScheduleASpeculation(TaskType type) {
+    int successes = 0;
+
+    long now = clock.getTime();
+
+    ConcurrentMap<JobId, AtomicInteger> containerNeeds
+        = type == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;
+
+    for (ConcurrentMap.Entry<JobId, AtomicInteger> jobEntry : containerNeeds.entrySet()) {
+      // This race conditon is okay.  If we skip a speculation attempt we
+      //  should have tried because the event that lowers the number of
+      //  containers needed to zero hasn't come through, it will next time.
+      // Also, if we miss the fact that the number of containers needed was
+      //  zero but increased due to a failure it's not too bad to launch one
+      //  container prematurely.
+      if (jobEntry.getValue().get() > 0) {
+        continue;
+      }
+
+      int numberSpeculationsAlready = 0;
+      int numberRunningTasks = 0;
+
+      // loop through the tasks of the kind
+      Job job = context.getJob(jobEntry.getKey());
+
+      Map<TaskId, Task> tasks = job.getTasks(type);
+
+      int numberAllowedSpeculativeTasks
+          = (int) Math.max(MINIMUM_ALLOWED_SPECULATIVE_TASKS,
+                           PROPORTION_TOTAL_TASKS_SPECULATABLE * tasks.size());
+
+      TaskId bestTaskID = null;
+      long bestSpeculationValue = -1L;
+
+      // this loop is potentially pricey.
+      // TODO track the tasks that are potentially worth looking at
+      for (Map.Entry<TaskId, Task> taskEntry : tasks.entrySet()) {
+        long mySpeculationValue = speculationValue(taskEntry.getKey(), now);
+
+        if (mySpeculationValue == ALREADY_SPECULATING) {
+          ++numberSpeculationsAlready;
+        }
+
+        if (mySpeculationValue != NOT_RUNNING) {
+          ++numberRunningTasks;
+        }
+
+        if (mySpeculationValue > bestSpeculationValue) {
+          bestTaskID = taskEntry.getKey();
+          bestSpeculationValue = mySpeculationValue;
+        }
+      }
+      numberAllowedSpeculativeTasks
+          = (int) Math.max(numberAllowedSpeculativeTasks,
+                           PROPORTION_RUNNING_TASKS_SPECULATABLE * numberRunningTasks);
+
+      // If we found a speculation target, fire it off
+      if (bestTaskID != null
+          && numberAllowedSpeculativeTasks > numberSpeculationsAlready) {
+        addSpeculativeAttempt(bestTaskID);
+        ++successes;
+      }
+    }
+
+    return successes;
+  }
+
+  private int computeSpeculations() {
+    // We'll try to issue one map and one reduce speculation per job per run
+    return maybeScheduleAMapSpeculation() + maybeScheduleAReduceSpeculation();
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java

@@ -0,0 +1,195 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.app.AMConstants;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+
+/**
+ * This estimator exponentially smooths the rate of progress versus wallclock
+ * time.  Conceivably we could write an estimator that smooths time per
+ * unit progress, and get different results.
+ */
+public class ExponentiallySmoothedTaskRuntimeEstimator extends StartEndTimesBase {
+
+  private final ConcurrentMap<TaskAttemptId, AtomicReference<EstimateVector>> estimates
+      = new ConcurrentHashMap<TaskAttemptId, AtomicReference<EstimateVector>>();
+
+  private SmoothedValue smoothedValue;
+
+  private long lambda;
+
+  public enum SmoothedValue {
+    RATE, TIME_PER_UNIT_PROGRESS
+  }
+
+  ExponentiallySmoothedTaskRuntimeEstimator
+      (long lambda, SmoothedValue smoothedValue) {
+    super();
+    this.smoothedValue = smoothedValue;
+    this.lambda = lambda;
+  }
+
+  public ExponentiallySmoothedTaskRuntimeEstimator() {
+    super();
+  }
+
+  // immutable
+  private class EstimateVector {
+    final double value;
+    final float basedOnProgress;
+    final long atTime;
+
+    EstimateVector(double value, float basedOnProgress, long atTime) {
+      this.value = value;
+      this.basedOnProgress = basedOnProgress;
+      this.atTime = atTime;
+    }
+
+    EstimateVector incorporate(float newProgress, long newAtTime) {
+      if (newAtTime <= atTime || newProgress < basedOnProgress) {
+        return this;
+      }
+
+      double oldWeighting
+          = value < 0.0
+              ? 0.0 : Math.exp(((double) (newAtTime - atTime)) / lambda);
+
+      double newRead = (newProgress - basedOnProgress) / (newAtTime - atTime);
+
+      if (smoothedValue == SmoothedValue.TIME_PER_UNIT_PROGRESS) {
+        newRead = 1.0 / newRead;
+      }
+
+      return new EstimateVector
+          (value * oldWeighting + newRead * (1.0 - oldWeighting),
+           newProgress, newAtTime);
+    }
+  }
+
+  private void incorporateReading
+      (TaskAttemptId attemptID, float newProgress, long newTime) {
+    //TODO: Refactor this method, it seems more complicated than necessary.
+    AtomicReference<EstimateVector> vectorRef = estimates.get(attemptID);
+
+    if (vectorRef == null) {
+      estimates.putIfAbsent(attemptID, new AtomicReference<EstimateVector>(null));
+      incorporateReading(attemptID, newProgress, newTime);
+      return;
+    }
+
+    EstimateVector oldVector = vectorRef.get();
+
+    if (oldVector == null) {
+      if (vectorRef.compareAndSet(null,
+             new EstimateVector(-1.0, 0.0F, Long.MIN_VALUE))) {
+        return;
+      }
+
+      incorporateReading(attemptID, newProgress, newTime);
+      return;
+    }
+
+    while (!vectorRef.compareAndSet
+            (oldVector, oldVector.incorporate(newProgress, newTime))) {
+      oldVector = vectorRef.get();
+    }
+  }
+
+  private EstimateVector getEstimateVector(TaskAttemptId attemptID) {
+    AtomicReference<EstimateVector> vectorRef = estimates.get(attemptID);
+
+    if (vectorRef == null) {
+      return null;
+    }
+
+    return vectorRef.get();
+  }
+
+  private static final long DEFAULT_EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS
+      = 1000L * 60;
+
+  @Override
+  public void contextualize(Configuration conf, AppContext context) {
+    super.contextualize(conf, context);
+
+    lambda
+        = conf.getLong(AMConstants.EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS,
+            DEFAULT_EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS);
+    smoothedValue
+        = conf.getBoolean(AMConstants.EXPONENTIAL_SMOOTHING_SMOOTH_RATE, true)
+            ? SmoothedValue.RATE : SmoothedValue.TIME_PER_UNIT_PROGRESS;
+  }
+
+  @Override
+  public long estimatedRuntime(TaskAttemptId id) {
+    Long startTime = startTimes.get(id);
+
+    if (startTime == null) {
+      return -1L;
+    }
+
+    EstimateVector vector = getEstimateVector(id);
+
+    if (vector == null) {
+      return -1L;
+    }
+
+    long sunkTime = vector.atTime - startTime;
+
+    double value = vector.value;
+    float progress = vector.basedOnProgress;
+
+    if (value == 0) {
+      return -1L;
+    }
+
+    double rate = smoothedValue == SmoothedValue.RATE ? value : 1.0 / value;
+
+    if (rate == 0.0) {
+      return -1L;
+    }
+
+    double remainingTime = (1.0 - progress) / rate;
+
+    return sunkTime + (long)remainingTime;
+  }
+
+  @Override
+  public long runtimeEstimateVariance(TaskAttemptId id) {
+    return -1L;
+  }
+
+  @Override
+  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
+    super.updateAttempt(status, timestamp);
+    TaskAttemptId attemptID = status.id;
+
+    float progress = status.progress;
+
+    incorporateReading(attemptID, progress, timestamp);
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/LegacyTaskRuntimeEstimator.java

@@ -0,0 +1,150 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+
+
+
+
+public class LegacyTaskRuntimeEstimator extends StartEndTimesBase {
+
+  private final Map<TaskAttempt, AtomicLong> attemptRuntimeEstimates
+      = new ConcurrentHashMap<TaskAttempt, AtomicLong>();
+  private final ConcurrentHashMap<TaskAttempt, AtomicLong> attemptRuntimeEstimateVariances
+      = new ConcurrentHashMap<TaskAttempt, AtomicLong>();
+
+  @Override
+  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
+    super.updateAttempt(status, timestamp);
+    
+
+    TaskAttemptId attemptID = status.id;
+    TaskId taskID = attemptID.getTaskId();
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+
+    if (job == null) {
+      return;
+    }
+
+    Task task = job.getTask(taskID);
+
+    if (task == null) {
+      return;
+    }
+
+    TaskAttempt taskAttempt = task.getAttempt(attemptID);
+
+    if (taskAttempt == null) {
+      return;
+    }
+
+    Long boxedStart = startTimes.get(attemptID);
+    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
+
+    // We need to do two things.
+    //  1: If this is a completion, we accumulate statistics in the superclass
+    //  2: If this is not a completion, we learn more about it.
+
+    // This is not a completion, but we're cooking.
+    //
+    if (taskAttempt.getState() == TaskAttemptState.RUNNING) {
+      // See if this task is already in the registry
+      AtomicLong estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
+      AtomicLong estimateVarianceContainer
+          = attemptRuntimeEstimateVariances.get(taskAttempt);
+
+      if (estimateContainer == null) {
+        if (attemptRuntimeEstimates.get(taskAttempt) == null) {
+          attemptRuntimeEstimates.put(taskAttempt, new AtomicLong());
+
+          estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
+        }
+      }
+
+      if (estimateVarianceContainer == null) {
+        attemptRuntimeEstimateVariances.putIfAbsent(taskAttempt, new AtomicLong());
+        estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
+      }
+
+
+      long estimate = -1;
+      long varianceEstimate = -1;
+
+      // This code assumes that we'll never consider starting a third
+      //  speculative task attempt if two are already running for this task
+      if (start > 0 && timestamp > start) {
+        estimate = (long) ((timestamp - start) / Math.max(0.0001, status.progress));
+        varianceEstimate = (long) (estimate * status.progress / 10);
+      }
+      if (estimateContainer != null) {
+        estimateContainer.set(estimate);
+      }
+      if (estimateVarianceContainer != null) {
+        estimateVarianceContainer.set(varianceEstimate);
+      }
+    }
+  }
+
+  private long storedPerAttemptValue
+       (Map<TaskAttempt, AtomicLong> data, TaskAttemptId attemptID) {
+    TaskId taskID = attemptID.getTaskId();
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+
+    Task task = job.getTask(taskID);
+
+    if (task == null) {
+      return -1L;
+    }
+
+    TaskAttempt taskAttempt = task.getAttempt(attemptID);
+
+    if (taskAttempt == null) {
+      return -1L;
+    }
+
+    AtomicLong estimate = data.get(taskAttempt);
+
+    return estimate == null ? -1L : estimate.get();
+
+  }
+
+  @Override
+  public long estimatedRuntime(TaskAttemptId attemptID) {
+    return storedPerAttemptValue(attemptRuntimeEstimates, attemptID);
+  }
+
+  @Override
+  public long runtimeEstimateVariance(TaskAttemptId attemptID) {
+    return storedPerAttemptValue(attemptRuntimeEstimateVariances, attemptID);
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/NullTaskRuntimesEngine.java

@@ -0,0 +1,72 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+
+
+/*
+ * This class is provided solely as an exemplae of the values that mean
+ *  that nothing needs to be computed.  It's not currently used.
+ */
+public class NullTaskRuntimesEngine implements TaskRuntimeEstimator {
+  @Override
+  public void enrollAttempt(TaskAttemptStatus status, long timestamp) {
+    // no code
+  }
+
+  @Override
+  public long attemptEnrolledTime(TaskAttemptId attemptID) {
+    return Long.MAX_VALUE;
+  }
+
+  @Override
+  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
+    // no code
+  }
+
+  @Override
+  public void contextualize(Configuration conf, AppContext context) {
+    // no code
+  }
+
+  @Override
+  public long thresholdRuntime(TaskId id) {
+    return Long.MAX_VALUE;
+  }
+
+  @Override
+  public long estimatedRuntime(TaskAttemptId id) {
+    return -1L;
+  }
+  @Override
+  public long estimatedNewAttemptRuntime(TaskId id) {
+    return -1L;
+  }
+
+  @Override
+  public long runtimeEstimateVariance(TaskAttemptId id) {
+    return -1L;
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/Speculator.java

@@ -0,0 +1,45 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.yarn.event.EventHandler;
+
+/**
+ * Speculator component. Task Attempts' status updates are sent to this
+ * component. Concrete implementation runs the speculative algorithm and
+ * sends the TaskEventType.T_ADD_ATTEMPT.
+ *
+ * An implementation also has to arrange for the jobs to be scanned from
+ * time to time, to launch the speculations.
+ */
+public interface Speculator
+              extends EventHandler<SpeculatorEvent> {
+
+  enum EventType {
+    ATTEMPT_STATUS_UPDATE,
+    ATTEMPT_START,
+    TASK_CONTAINER_NEED_UPDATE,
+    JOB_CREATE
+  }
+
+  // This will be implemented if we go to a model where the events are
+  //  processed within the TaskAttempts' state transitions' code.
+  public void handleAttempt(TaskAttemptStatus status);
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/SpeculatorEvent.java

@@ -0,0 +1,86 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+
+public class SpeculatorEvent extends AbstractEvent<Speculator.EventType> {
+
+  // valid for ATTEMPT_STATUS_UPDATE
+  private TaskAttemptStatus reportedStatus;
+
+  // valid for TASK_CONTAINER_NEED_UPDATE
+  private TaskId taskID;
+  private int containersNeededChange;
+  
+  // valid for CREATE_JOB
+  private JobId jobID;
+
+  public SpeculatorEvent(JobId jobID, long timestamp) {
+    super(Speculator.EventType.JOB_CREATE, timestamp);
+    this.jobID = jobID;
+  }
+
+  public SpeculatorEvent(TaskAttemptStatus reportedStatus, long timestamp) {
+    super(Speculator.EventType.ATTEMPT_STATUS_UPDATE, timestamp);
+    this.reportedStatus = reportedStatus;
+  }
+
+  public SpeculatorEvent(TaskAttemptId attemptID, boolean flag, long timestamp) {
+    super(Speculator.EventType.ATTEMPT_START, timestamp);
+    this.reportedStatus = new TaskAttemptStatus();
+    this.reportedStatus.id = attemptID;
+    this.taskID = attemptID.getTaskId();
+  }
+
+  /*
+   * This c'tor creates a TASK_CONTAINER_NEED_UPDATE event .
+   * We send a +1 event when a task enters a state where it wants a container,
+   *  and a -1 event when it either gets one or withdraws the request.
+   * The per job sum of all these events is the number of containers requested
+   *  but not granted.  The intent is that we only do speculations when the
+   *  speculation wouldn't compete for containers with tasks which need
+   *  to be run.
+   */
+  public SpeculatorEvent(TaskId taskID, int containersNeededChange) {
+    super(Speculator.EventType.TASK_CONTAINER_NEED_UPDATE);
+    this.taskID = taskID;
+    this.containersNeededChange = containersNeededChange;
+  }
+
+  public TaskAttemptStatus getReportedStatus() {
+    return reportedStatus;
+  }
+
+  public int containersNeededChange() {
+    return containersNeededChange;
+  }
+
+  public TaskId getTaskID() {
+    return taskID;
+  }
+  
+  public JobId getJobID() {
+    return jobID;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java

@@ -0,0 +1,213 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
+
+abstract class StartEndTimesBase implements TaskRuntimeEstimator {
+  static final float MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE
+      = 0.05F;
+  static final int MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
+      = 1;
+
+  protected Configuration conf = null;
+  protected AppContext context = null;
+
+  protected final Map<TaskAttemptId, Long> startTimes
+      = new ConcurrentHashMap<TaskAttemptId, Long>();
+
+  // XXXX This class design assumes that the contents of AppContext.getAllJobs
+  //   never changes.  Is that right?
+  //
+  // This assumption comes in in several places, mostly in data structure that
+  //   can grow without limit if a AppContext gets new Job's when the old ones
+  //   run out.  Also, these mapper statistics blocks won't cover the Job's
+  //   we don't know about.
+  protected final Map<Job, DataStatistics> mapperStatistics
+      = new HashMap<Job, DataStatistics>();
+  protected final Map<Job, DataStatistics> reducerStatistics
+      = new HashMap<Job, DataStatistics>();
+
+
+  private final Map<Job, Float> slowTaskRelativeTresholds
+      = new HashMap<Job, Float>();
+
+  protected final Set<Task> doneTasks = new HashSet<Task>();
+
+  @Override
+  public void enrollAttempt(TaskAttemptStatus status, long timestamp) {
+    startTimes.put(status.id,timestamp);
+  }
+
+  @Override
+  public long attemptEnrolledTime(TaskAttemptId attemptID) {
+    Long result = startTimes.get(attemptID);
+
+    return result == null ? Long.MAX_VALUE : result;
+  }
+
+
+  @Override
+  public void contextualize(Configuration conf, AppContext context) {
+    this.conf = conf;
+    this.context = context;
+
+    Map<JobId, Job> allJobs = context.getAllJobs();
+
+    for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) {
+      final Job job = entry.getValue();
+      mapperStatistics.put(job, new DataStatistics());
+      reducerStatistics.put(job, new DataStatistics());
+      slowTaskRelativeTresholds.put
+          (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f));
+    }
+  }
+
+  protected DataStatistics dataStatisticsForTask(TaskId taskID) {
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+
+    if (job == null) {
+      return null;
+    }
+
+    Task task = job.getTask(taskID);
+
+    if (task == null) {
+      return null;
+    }
+
+    return task.getType() == TaskType.MAP
+            ? mapperStatistics.get(job)
+            : task.getType() == TaskType.REDUCE
+                ? reducerStatistics.get(job)
+                : null;
+  }
+
+  @Override
+  public long thresholdRuntime(TaskId taskID) {
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+
+    TaskType type = taskID.getTaskType();
+
+    DataStatistics statistics
+        = dataStatisticsForTask(taskID);
+
+    int completedTasksOfType
+        = type == TaskType.MAP
+            ? job.getCompletedMaps() : job.getCompletedReduces();
+
+    int totalTasksOfType
+        = type == TaskType.MAP
+            ? job.getTotalMaps() : job.getTotalReduces();
+
+    if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
+        || (((float)completedTasksOfType) / totalTasksOfType)
+              < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) {
+      return Long.MAX_VALUE;
+    }
+
+    long result =  statistics == null
+        ? Long.MAX_VALUE
+        : (long)statistics.outlier(slowTaskRelativeTresholds.get(job));
+    return result;
+  }
+
+  @Override
+  public long estimatedNewAttemptRuntime(TaskId id) {
+    DataStatistics statistics = dataStatisticsForTask(id);
+
+    if (statistics == null) {
+      return -1L;
+    }
+
+    return (long)statistics.mean();
+  }
+
+  @Override
+  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
+
+    TaskAttemptId attemptID = status.id;
+    TaskId taskID = attemptID.getTaskId();
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+
+    if (job == null) {
+      return;
+    }
+
+    Task task = job.getTask(taskID);
+
+    if (task == null) {
+      return;
+    }
+
+    Long boxedStart = startTimes.get(attemptID);
+    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
+    
+    TaskAttempt taskAttempt = task.getAttempt(attemptID);
+
+    if (taskAttempt.getState() == TaskAttemptState.SUCCEEDED) {
+      boolean isNew = false;
+      // is this  a new success?
+      synchronized (doneTasks) {
+        if (!doneTasks.contains(task)) {
+          doneTasks.add(task);
+          isNew = true;
+        }
+      }
+
+      // It's a new completion
+      // Note that if a task completes twice [because of a previous speculation
+      //  and a race, or a success followed by loss of the machine with the
+      //  local data] we only count the first one.
+      if (isNew) {
+        long finish = timestamp;
+        if (start > 1L && finish > 1L && start <= finish) {
+          long duration = finish - start;
+
+          DataStatistics statistics
+          = dataStatisticsForTask(taskID);
+
+          if (statistics != null) {
+            statistics.add(duration);
+          }
+        }
+      }
+    }
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/TaskRuntimeEstimator.java

@@ -0,0 +1,90 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+
+
+
+public interface TaskRuntimeEstimator {
+  public void enrollAttempt(TaskAttemptStatus reportedStatus, long timestamp);
+
+  public long attemptEnrolledTime(TaskAttemptId attemptID);
+
+  public void updateAttempt(TaskAttemptStatus reportedStatus, long timestamp);
+
+  public void contextualize(Configuration conf, AppContext context);
+
+  /**
+   *
+   * Find a maximum reasonable execution wallclock time.  Includes the time
+   * already elapsed.
+   *
+   * Find a maximum reasonable execution time.  Includes the time
+   * already elapsed.  If the projected total execution time for this task
+   * ever exceeds its reasonable execution time, we may speculate it.
+   *
+   * @param id the {@link TaskID} of the task we are asking about
+   * @return the task's maximum reasonable runtime, or MAX_VALUE if
+   *         we don't have enough information to rule out any runtime,
+   *         however long.
+   *
+   */
+  public long thresholdRuntime(TaskId id);
+
+  /**
+   *
+   * Estimate a task attempt's total runtime.  Includes the time already
+   * elapsed.
+   *
+   * @param id the {@link TaskAttemptID} of the attempt we are asking about
+   * @return our best estimate of the attempt's runtime, or {@code -1} if
+   *         we don't have enough information yet to produce an estimate.
+   *
+   */
+  public long estimatedRuntime(TaskAttemptId id);
+
+  /**
+   *
+   * Estimates how long a new attempt on this task will take if we start
+   *  one now
+   *
+   * @param id the {@link TaskID} of the task we are asking about
+   * @return our best estimate of a new attempt's runtime, or {@code -1} if
+   *         we don't have enough information yet to produce an estimate.
+   *
+   */
+  public long estimatedNewAttemptRuntime(TaskId id);
+
+  /**
+   *
+   * Computes the width of the error band of our estimate of the task
+   *  runtime as returned by {@link estimatedRuntime}
+   *
+   * @param id the {@link TaskAttemptID} of the attempt we are asking about
+   * @return our best estimate of the attempt's runtime, or {@code -1} if
+   *         we don't have enough information yet to produce an estimate.
+   *
+   */
+  public long runtimeEstimateVariance(TaskAttemptId id);
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/TaskSpeculationPredicate.java

@@ -0,0 +1,39 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.speculate;
+
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.Task;
+
+
+public class TaskSpeculationPredicate {
+  boolean canSpeculate(AppContext context, TaskId taskID) {
+    // This class rejects speculating any task that already has speculations,
+    //  or isn't running.
+    //  Subclasses should call TaskSpeculationPredicate.canSpeculate(...) , but
+    //  can be even more restrictive.
+    JobId jobID = taskID.getJobId();
+    Job job = context.getJob(jobID);
+    Task task = job.getTask(taskID);
+    return task.getAttempts().size() == 1;
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleaner.java

@@ -0,0 +1,28 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.taskclean;
+
+import org.apache.hadoop.yarn.event.EventHandler;
+
+public interface TaskCleaner extends EventHandler<TaskCleanupEvent> {
+
+  enum EventType {
+    TASK_CLEAN
+  }
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java

@@ -0,0 +1,108 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.taskclean;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+public class TaskCleanerImpl extends AbstractService implements TaskCleaner {
+
+  private static final Log LOG = LogFactory.getLog(TaskCleanerImpl.class);
+
+  private final AppContext context;
+  private ThreadPoolExecutor launcherPool;
+  private Thread eventHandlingThread;
+  private BlockingQueue<TaskCleanupEvent> eventQueue =
+      new LinkedBlockingQueue<TaskCleanupEvent>();
+
+  public TaskCleanerImpl(AppContext context) {
+    super("TaskCleaner");
+    this.context = context;
+  }
+
+  public void start() {
+    launcherPool = new ThreadPoolExecutor(1, 5, 1, 
+        TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
+    eventHandlingThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        TaskCleanupEvent event = null;
+        while (!Thread.currentThread().isInterrupted()) {
+          try {
+            event = eventQueue.take();
+          } catch (InterruptedException e) {
+            LOG.error("Returning, interrupted : " + e);
+            return;
+          }
+          // the events from the queue are handled in parallel
+          // using a thread pool
+          launcherPool.execute(new EventProcessor(event));        }
+      }
+    });
+    eventHandlingThread.start();
+    super.start();
+  }
+
+  public void stop() {
+    eventHandlingThread.interrupt();
+    launcherPool.shutdown();
+    super.stop();
+  }
+
+  private class EventProcessor implements Runnable {
+    private TaskCleanupEvent event;
+
+    EventProcessor(TaskCleanupEvent event) {
+      this.event = event;
+    }
+
+    @Override
+    public void run() {
+      LOG.info("Processing the event " + event.toString());
+      try {
+        event.getCommitter().abortTask(event.getAttemptContext());
+      } catch (Exception e) {
+        LOG.warn("Task cleanup failed for attempt " + event.getAttemptID(), e);
+      }
+      context.getEventHandler().handle(
+          new TaskAttemptEvent(event.getAttemptID(), 
+              TaskAttemptEventType.TA_CLEANUP_DONE));
+    }
+  }
+
+  @Override
+  public void handle(TaskCleanupEvent event) {
+    try {
+      eventQueue.put(event);
+    } catch (InterruptedException e) {
+      throw new YarnException(e);
+    }
+  }
+
+}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanupEvent.java

@@ -0,0 +1,56 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.taskclean;
+
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+/**
+ * This class encapsulates task cleanup event.
+ *
+ */
+public class TaskCleanupEvent extends AbstractEvent<TaskCleaner.EventType> {
+
+  private final TaskAttemptId attemptID;
+  private final OutputCommitter committer;
+  private final TaskAttemptContext attemptContext;
+
+  public TaskCleanupEvent(TaskAttemptId attemptID, OutputCommitter committer, 
+      TaskAttemptContext attemptContext) {
+    super(TaskCleaner.EventType.TASK_CLEAN);
+    this.attemptID = attemptID;
+    this.committer = committer;
+    this.attemptContext = attemptContext;
+  }
+
+  public TaskAttemptId getAttemptID() {
+    return attemptID;
+  }
+
+  public OutputCommitter getCommitter() {
+    return committer;
+  }
+
+  public TaskAttemptContext getAttemptContext() {
+    return attemptContext;
+  }
+
+}