Throw-away branch for testing MAPREDUCE-279 merge to trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-279-merge@1158944 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce/.eclipse.templates/.launches/AllMapredTests.launch

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
-<stringAttribute key="bad_container_name" value="/@PROJECT@/.l"/>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
-<listEntry value="/@PROJECT@/src/test/mapred"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
-<listEntry value="2"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
-<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
-<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
-</listAttribute>
-<stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value="=@PROJECT@/src\/test\/mapred"/>
-<booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
-<stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
-<stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
-<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-</listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
-<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value=""/>
-<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms256m -Xmx512m -Dtest.build.data=${workspace_loc:@PROJECT@}/build/test -Dtest.cache.data=${workspace_loc:@PROJECT@}/build/test/cache -Dtest.debug.data=${workspace_loc:@PROJECT@}/build/test/debug -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/build/test/log -Dtest.src.dir=${workspace_loc:@PROJECT@}/build/test/src/mapred -Dtest.build.extraconf=${workspace_loc:@PROJECT@}/build/test/extraconf -Dhadoop.policy.file=hadoop-policy.xml"/>
-</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/JobTracker.launch

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
-<listEntry value="/@PROJECT@/src/mapred/org/apache/hadoop/mapred/JobTracker.java"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
-<listEntry value="1"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
-<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
-<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
-</listAttribute>
-<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/conf}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-</listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
-<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.mapred.JobTracker"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx1000m -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=${workspace_loc:@PROJECT@} -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/logs"/>
-</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/SpecificTestTemplate.launch

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
-<stringAttribute key="bad_container_name" value="/@PROJECT@/.l"/>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
-<listEntry value="/@PROJECT@"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
-<listEntry value="4"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
-<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
-<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
-</listAttribute>
-<stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value=""/>
-<booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
-<stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
-<stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>
-<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-</listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
-<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.TestNameHere"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms256m -Xmx512m -Dtest.build.data=${workspace_loc:@PROJECT@}/build/test -Dtest.cache.data=${workspace_loc:@PROJECT@}/build/test/cache -Dtest.debug.data=${workspace_loc:@PROJECT@}/build/test/debug -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/build/test/log -Dtest.src.dir=${workspace_loc:@PROJECT@}/build/test/src -Dtest.build.extraconf=${workspace_loc:@PROJECT@}/build/test/extraconf -Dhadoop.policy.file=hadoop-policy.xml"/>
-</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/.launches/TaskTracker.launch

@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
-<listEntry value="/@PROJECT@/src/mapred/org/apache/hadoop/mapred/TaskTracker.java"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
-<listEntry value="1"/>
-</listAttribute>
-<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
-<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>
-<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
-</listAttribute>
-<listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry containerPath=&quot;org.eclipse.jdt.launching.JRE_CONTAINER&quot; javaProject=&quot;@PROJECT@&quot; path=&quot;1&quot; type=&quot;4&quot;/&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/conf}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build/classes}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.variableClasspathEntry&quot;&gt;&#10;&lt;memento path=&quot;3&quot; variableString=&quot;${workspace_loc:@PROJECT@/build}&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-<listEntry value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&#10;&lt;runtimeClasspathEntry id=&quot;org.eclipse.jdt.launching.classpathentry.defaultClasspath&quot;&gt;&#10;&lt;memento exportedEntriesOnly=&quot;false&quot; project=&quot;@PROJECT@&quot;/&gt;&#10;&lt;/runtimeClasspathEntry&gt;&#10;"/>
-</listAttribute>
-<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
-<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.mapred.TaskTracker"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx1000m -Dhadoop.root.logger=INFO,console -Dhadoop.policy.file=hadoop-policy.xml -Dhadoop.log.file=hadoop.log -Dhadoop.home.dir=${workspace_loc:@PROJECT@} -Dhadoop.log.dir=${workspace_loc:@PROJECT@}/logs"/>
-</launchConfiguration>

hadoop-mapreduce/.eclipse.templates/README.txt

@@ -1,6 +0,0 @@
-This directory contains templates for generating Eclipse files to configure
-Eclipse for Hadoop development.
-
-For further information please consult
-
-http://wiki.apache.org/hadoop/EclipseEnvironment 

hadoop-mapreduce/.gitignore

@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*~
-.classpath
-.project
-.launches
-.settings
-*.iml
-*.ipr
-*.iws
-.idea
-.svn
-build/
-build-fi/
-build.properties
-conf/masters
-conf/slaves
-conf/core-site.xml
-conf/hdfs-site.xml
-conf/hadoop-env.sh
-conf/hadoop-site.xml
-conf/mapred-site.xml
-conf/hadoop-policy.xml
-conf/capacity-scheduler.xml
-conf/fair-scheduler.xml
-conf/mapred-queue-acls.xml
-conf/mapred-queue-acls.xml.template
-conf/mapred-queues.xml
-docs/api/
-logs/
-src/contrib/capacity-scheduler/src/java/mapred-queues.xml
-src/contrib/index/conf/index-config.xml
-src/docs/build
-src/docs/cn/build
-src/docs/cn/src/documentation/sitemap.xmap
-src/docs/cn/uming.conf

hadoop-mapreduce/CHANGES.txt

@@ -1,2526 +0,0 @@
-Hadoop MapReduce Change Log
-
-Trunk (unreleased changes)
-
-  INCOMPATIBLE CHANGES
-
-    MAPREDUCE-2455. Remove deprecated JobTracker.State in favour of
-    JobTrackerStatus. (tomwhite)
-
-    MAPREDUCE-2430. Remove mrunit contrib. (nigel via eli)
-
-    MAPREDUCE-2606. Remove IsolationRunner. (Alejandro Abdelnur via eli)
-
-  NEW FEATURES
-
-    MAPREDUCE-2682. Add "mapred classpath" command to print classpath
-    for MR applications. (vinodkv via acmurthy) 
-
-    MAPREDUCE-2107. [Gridmix] Total heap usage emulation in Gridmix.
-    (Amar Kamat and Ravi Gummadi via amarrk)
-
-    MAPREDUCE-2106. [Gridmix] Cumulative CPU usage emulation in Gridmix. 
-    (amarrk)
-
-    MAPREDUCE-2543. [Gridmix] High-Ram feature emulation in Gridmix. (amarrk)
-
-    MAPREDUCE-2408. [Gridmix] Compression emulation in Gridmix. (amarrk)
-
-    MAPREDUCE-2473. Add "mapred groups" command to query the server-side groups
-    resolved for a user. (Aaron T. Myers via todd)
-
-    MAPREDUCE-461. Enable ServicePlugins for the JobTracker.
-    (Fredrik Hedberg via tomwhite)
-
-    MAPREDUCE-2521. Create RPM and Debian packages for MapReduce. Changes 
-    deployment layout to be consistent across the binary tgz, rpm, and deb.
-    (Eric Yang via omalley)
-
-    MAPREDUCE-2323. Add metrics to the fair scheduler. (todd)
-
-    MAPREDUCE-2037. Capture intermediate progress, CPU and memory usage for
-    tasks. (Dick King via acmurthy) 
-
-  IMPROVEMENTS
-
-    MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
-    acmurthy) 
-
-    MAPREDUCE-2365. Add counters to track bytes (read,written) via 
-    File(Input,Output)Format. (Siddharth Seth via acmurthy)
- 
-    MAPREDUCE-2680. Display queue name in job client CLI. (acmurthy) 
- 
-    MAPREDUCE-2679. Minor changes to sync trunk with MR-279 branch. (acmurthy) 
- 
-    MAPREDUCE-2400. Remove Cluster's dependency on JobTracker via a 
-    ServiceProvider for the actual implementation. (tomwhite via acmurthy) 
- 
-    MAPREDUCE-2596. [Gridmix] Summarize Gridmix runs. (amarrk)
-
-    MAPREDUCE-2563. [Gridmix] Add High-Ram emulation system tests to 
-    Gridmix. (Vinay Kumar Thota via amarrk)
-
-    MAPREDUCE-2104. [Rumen] Add Cpu, Memory and Heap usages to 
-    TraceBuilder's output. (amarrk)
-
-    MAPREDUCE-2554. [Gridmix]  Add distributed cache emulation system tests 
-    to Gridmix. (Vinay Kumar Thota via amarrk)
- 
-    MAPREDUCE-2543. [Gridmix] High-Ram feature emulation testcase. (amarrk)
-
-    MAPREDUCE-2469. Task counters should also report the total heap usage of 
-    the task. (Ravi Gummadi and Amar Ramesh Kamat via amarrk)
-
-    MAPREDUCE-2544. [Gridmix] Add compression emulation system tests to 
-    Gridmix. (Vinay Kumar Thota via amarrk)
-
-    MAPREDUCE-2517. [Gridmix] Add system tests to Gridmix. 
-    (Vinay Kumar Thota via amarrk)
-
-    MAPREDUCE-2492. The new MapReduce API should make available task's
-    progress to the task. (amarrk)
-
-    MAPREDUCE-2153. Bring in more job configuration properties in to the trace 
-    file. (Rajesh Balamohan via amarrk)
-
-    MAPREDUCE-1461. Feature to instruct rumen-folder utility to skip jobs worth 
-    of specific duration. (Rajesh Balamohan via amarrk)
-
-    MAPREDUCE-2172. Added test-patch.properties required by test-patch.sh 
-    (nigel)
-
-    MAPREDUCE-2156. Raid-aware FSCK. (Patrick Kling via dhruba)
-
-    MAPREDUCE-2215. A more elegant FileSystem#listCorruptFileBlocks API
-    (RAID changes) (Patrick Kling via hairong)
-
-    MAPREDUCE-1831. BlockPlacement policy for HDFS-RAID.
-    (Scott Chen via dhruba)
-
-    MAPREDUCE-1906. Lower minimum heartbeat interval for TaskTracker
-    (Scott Carey and Todd Lipcon via todd)
-
-    MAPREDUCE-1382. MRAsyncDiscService should tolerate missing local.dir.
-    (Zheng Shao and tomwhite via tomwhite)
-
-    MAPREDUCE-2263. MapReduce side of HADOOP-6904: RPC compatibility.
-    (hairong)
-
-    MAPREDUCE-1706. Log RAID recoveries on HDFS. (schen)
-
-    MAPREDUCE-2334. Update BlockPlacementPolicyRaid for the new method
-    in BlockPlacementPolicy.  (szetszwo)
-
-    MAPREDUCE-2254. Allow setting of end-of-record delimiter for
-    TextInputFormat (Ahmed Radwan via todd)
-
-    MAPREDUCE-1927. Unit test for HADOOP-6835 (concatenated gzip support).
-    (Greg Roelofs via tomwhite)
-
-    MAPREDUCE-2206. The task-cleanup tasks should be optional. (schen)
-
-    MAPREDUCE-2225. MultipleOutputs should not require the use of 'Writable'.
-    (Harsh J Chouraria via tomwhite)
-
-    MAPREDUCE-1811. Job.monitorAndPrintJob() should print status of the job
-    at completion. (Harsh J Chouraria via tomwhite)
-
-    MAPREDUCE-993. bin/hadoop job -events <jobid> <from-event-#> <#-of-events>
-    help message is confusing. (Harsh J Chouraria via tomwhite)
-
-    MAPREDUCE-2302. Add static factory methods in GaloisField. (schen)
-
-    MAPREDUCE-2351. mapred.job.tracker.history.completed.location should
-    support an arbitrary filesystem URI. (tomwhite)
-
-    MAPREDUCE-2239. BlockPlacementPolicyRaid should call getBlockLocations
-    only when necessary. (schen)
-
-    MAPREDUCE-2331. Add coverage of task graph servlet to fair scheduler system
-    test. (todd)
-
-    MAPREDUCE-2367. Allow using a file to exclude certain tests from build.
-    (todd)
-
-    MAPREDUCE-2202. Generalize CLITest structure and interfaces to faciliate
-    upstream adoption (e.g. for web or system testing). (cos)
-
-    MAPREDUCE-2420. JobTracker should be able to renew delegation token over 
-    HTTP (Boris Shkolnik via jitendra)
-
-    MAPREDUCE-2474. Add docs to the new API Partitioner on how to access the
-    Job Configuration. (Harsh J Chouraria via todd)
-    
-    MAPREDUCE-2475. Disable IPV6 for junit tests. (suresh srinivas via mahadev)
-
-    MAPREDUCE-2422. Removed unused internal methods from DistributedCache.
-    (tomwhite)
-
-    MAPREDUCE-2456. Log the reduce taskID and associated TaskTrackers with
-    failed fetch notifications in the JobTracker log.
-    (Jeffrey Naisbitt via cdouglas)
-
-    MAPREDUCE-869. Documentation for config to set map/reduce task environment
-    (Alejandro Abdelnur via todd)
-
-    MAPREDUCE-2410. Add entry to streaming FAQ about how streaming reducers
-    receive keys. (Harsh J Chouraria via todd)
-
-    MAPREDUCE-2499. MR part of HADOOP-7291. (eli)
-
-    MAPREDUCE-2497. Missing spaces in error messages. (eli)
-
-    MAPREDUCE-2502. JobSubmitter should use mapreduce.job.maps instead of
-    its deprecated equivalent. (eli via todd)
-
-    MAPREDUCE-2381. JobTracker instrumentation not consistent about error
-    handling. (Philip Zeyliger via tomwhite)
-
-    MAPREDUCE-2449. Allow for command line arguments when performing
-    "Run on Hadoop" action in Eclipse plugin. (Jeff Zemerick via todd)
-
-    MAPREDUCE-2483. Remove duplication of jars between Hadoop subprojects
-    from build artifacts. (Eric Yang via omalley)
-
-    MAPREDUCE-2372. TaskLogAppender mechanism shouldn't be set up in
-    log4j.properties (todd)
-
-    MAPREDUCE-2516. Rename webinterface.private.actions to
-    mapreduce.jobtracker.webinterface.trusted (Ari Rabkin via todd)
-
-    MAPREDUCE-2459. Cache HAR filesystem metadata. (Mac Yang via mahadev)
-
-    HADOOP-7259. Contrib modules should include the build.properties from
-    the enclosing hadoop directory. (omalley)
-
-    MAPREDUCE-2494. Order distributed cache deletions by LRU. (Robert Joseph
-    Evans via cdouglas)
-
-    MAPREDUCE-2452. Makes the cancellation of delegation tokens happen in a 
-    separate thread. (ddas)
-
-    HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects.
-    (todd, nigel)
-
-    MAPREDUCE-2249. Check the reflexive property of Counters objects when
-    comparing equality. (Devaraj K via todd)
-
-    MAPREDUCE-2623. Update ClusterMapReduceTestCase to use
-    MiniDFSCluster.Builder (Harsh J Chouraria via eli)
-
-    MAPREDUCE-2602. Allow setting of end-of-record delimiter for
-    TextInputFormat for the old API. (Ahmed Radwan via todd)
-
-    MAPREDUCE-2705. Permits parallel multiple task launches. 
-    (Thomas Graves via ddas)
-
-    MAPREDUCE-2489. Jobsplits with random hostnames can make the queue 
-    unusable (jeffrey naisbit via mahadev)
-
-  OPTIMIZATIONS
-
-    MAPREDUCE-2026. Make JobTracker.getJobCounters() and
-    JobInProgress.getCounters() aquire locks in a shorter time period.
-    (Joydeep Sen Sarma via schen)
-
-    MAPREDUCE-2740. MultipleOutputs in new API creates needless
-    TaskAttemptContexts. (todd)
-
-    MAPREDUCE-901. Efficient framework counters. (llu via acmurthy)
-
-  BUG FIXES
-
-    MAPREDUCE-2603. Disable High-Ram emulation in system tests. 
-    (Vinay Kumar Thota via amarrk)
-
-    MAPREDUCE-2539. Fixed NPE in getMapTaskReports in JobClient. (Robert Evans via
-    acmurthy) 
-
-    MAPREDUCE-2531. Fixed jobcontrol to downgrade JobID. (Robert Evans via
-    acmurthy) 
-
-    MAPREDUCE-1978. Rumen TraceBuilder should provide recursive
-    input folder scanning.
-
-    MAPREDUCE-2416. Remove the restriction of specifying group names in
-    users-list file for Gridmix in RoundRobinUserResolver mode.
-
-    MAPREDUCE-2417. Fix Gridmix in RoundRobinUserResolver mode to
-    map testing/proxy users to unique users in a trace.
-
-    MAPREDUCE-2307. Exception thrown in Jobtracker logs, when the Scheduler
-    configured is FairScheduler. (Devaraj K via matei)
-
-    MAPREDUCE-2199. build is broken 0.22 branch creation. (cos)
-
-    MAPREDUCE-1752. Implement getFileBlockLocations in HarFilesystem.
-    (Patrick Kling via dhruba)
-
-    MAPREDUCE-2155. RaidNode should optionally use the mapreduce jobs to 
-    fix missing blocks.  (Patrick Kling via dhruba)
-
-    MAPREDUCE-1334. Fix TestIndexUpdater by ignoring _SUCCESS file in HDFS.
-    (Kay Kay via yhemanth)
-
-    MAPREDUCE-2232. Add missing methods to TestMapredGroupMappingServiceRefresh.
-    (Todd Lipcon via eli)
-
-    MAPREDUCE-2271. Fix TestSetupTaskScheduling failure on trunk.
-    (Liyin Liang via todd)
-
-    MAPREDUCE-2290. Fix compilation error in TestTaskCommit. (eli)
-
-    MAPREDUCE-2294. Fix compilation error in mumak. (todd)
-
-    MAPREDUCE-2300. Fix TestUmbilicalProtocolWithJobToken on trunk after
-    HADOOP-6904. (todd)
-
-    MAPREDUCE-2296. Fix references to misspelled method name
-    getProtocolSigature (todd)
-
-    MAPREDUCE-2311. Fix TestFairScheduler failure (schen)
-
-    MAPREDUCE-1996. API: Reducer.reduce() method detail misstatement.
-    (Harsh J Chouraria via tomwhite)
-
-    MAPREDUCE-2203. Wrong javadoc for TaskRunner's appendJobJarClasspaths
-    method. (Jingguo Yao via tomwhite)
-
-    MAPREDUCE-2074. Task should fail when symlink creation fails.
-    (Priyo Mustafi via tomwhite)
-
-    MAPREDUCE-1242. Chain APIs error misleading.
-    (Harsh J Chouraria via tomwhite)
-
-    MAPREDUCE-2379. Adds missing DistributedCache configurations in 
-    mapred-default.xml (Todd Lipcon via amareshwari)
-
-    MAPREDUCE-2348. Disable mumak tests on trunk since they currently time out
-    (todd)
-
-    MAPREDUCE-2395. TestBlockFixer timing out on trunk. (Ramkumar Vadali via
-    todd)
-
-    MAPREDUCE-2426. Make TestFairSchedulerSystem fail with more verbose output
-    (todd)
-
-    MAPREDUCE-2448. NoSuchMethodError:
-    org.apache.hadoop.hdfs.TestDatanodeBlockScanner.corruptReplica(..) (eli)
-
-    MAPREDUCE-2460. Fix flaky test TestFairSchedulerSystem. (todd)
-
-    MAPREDUCE-2451. Log the details from health check script at the
-    JobTracker. (Thomas Graves via cdouglas)
-
-    MAPREDUCE-2467. HDFS-1052 changes break the raid contrib module in 
-    MapReduce. (suresh srinivas via mahadev)
-
-    MAPREDUCE-2258. IFile reader closes stream and compressor in wrong order.
-    (todd via tomwhite)
-
-    MAPREDUCE-2518. The t flag is missing in distcp help message.  (Wei Yongjun
-    via szetszwo)
-
-    MAPREDUCE-2514. Fix typo in TaskTracker ReinitTrackerAction log message.
-    (Jonathan Eagles via cdouglas)
-
-    MAPREDUCE-2490. Add logging to graylist and blacklist activity to aid
-    diagnosis of related issues. (Jonathan Eagles via cdouglas)
-
-    MAPREDUCE-2495. exit() the TaskTracker when the distributed cache cleanup
-    thread dies. (Robert Joseph Evans via cdouglas)
-
-    MAPREDUCE-2470. Fix NPE in RunningJobs::getCounters. (Robert Joseph Evans
-    via cdouglas)
-
-    MAPREDUCE-2536. Update FsShell -mv command usage in TestMRCLI.  (Daryn
-    Sharp via szetszwo)
-
-    MAPREDUCE-2529. Add support for regex-based shuffle metric counting
-    exceptions. (Thomas Graves via cdouglas)
-
-    MAPREDUCE-2559. ant binary fails due to missing c++ lib dir. (eli)
-
-    MAPREDUCE-2573. Fix new findbugs warning introduced by MAPREDUCE-2494.
-    (Robert Joseph Evans via todd)
-
-    MAPREDUCE-2581. Spelling errors in log messages. (Tim Sell via eli)
-
-    MAPREDUCE-2588. Change raid to the new DataTransferProtocol API.  (szetszwo)
-
-    MAPREDUCE-2576. Typo in comment in SimulatorLaunchTaskAction.java.
-    (Tim Sell via jghoman)
-
-    MAPREDUCE-2550. Fix bin/mapred to work properly from within a source
-    checkout (Eric Yang via todd)
-
-    MAPREDUCE-2620. Update RAID for HDFS-2087.  (szetszwo)
-
-    MAPREDUCE-2624. Update RAID for HDFS-2107.  (szetszwo)
-
-    MAPREDUCE-2670. Fixing spelling mistake in FairSchedulerServlet.java. (eli)
-
-    MAPREDUCE-2710. Update JobSubmitter.printTokens(..) for HDFS-2161.
-    (szetszwo)
-
-    MAPREDUCE-2409. DistributedCache maps files and archives to the same path,
-    despite semantic incompatibility. (Siddharth Seth via cdouglas)
-
-    MAPREDUCE-2575. TestMiniMRDFSCaching fails if test.build.dir is set 
-    to something other than build/test (Thomas Graves via mahadev)
-
-    MAPREDUCE-2622. Remove the last remaining reference to the deprecated
-    configuration "io.sort.mb". (Harsh J Chouraria via todd)
-
-    MAPREDUCE-2732. Remove directly accessing FSNamesystem.LOG from
-    TestCopyFiles and TestDistCh.  (szetszwo)
-
-    MAPREDUCE-2463. Job history files are not moved to done folder when job
-    history location is hdfs.  (Devaraj K via szetszwo)
-
-    MAPREDUCE-2243. Close streams propely in a finally-block to avoid leakage
-    in CompletedJobStatusStore, TaskLog, EventWriter and TotalOrderPartitioner.
-    (Devaraj K via szetszwo)
-
-    MAPREDUCE-2741. Make ant build system work with hadoop-common JAR
-    generated by Maven. (Alejandro Abdelnur via tomwhite)
-
-    MAPREDUCE-2760. mapreduce.jobtracker.split.metainfo.maxsize typoed
-    in mapred-default.xml. (todd via eli)
-
-    MAPREDUCE-2797. Update mapreduce tests and RAID for HDFS-2239.  (szetszwo)
-
-    MAPREDUCE-2805. Update RAID for HDFS-2241.  (szetszwo)
-    
-    MAPREDUCE-2837. Ported bug fixes from y-merge to prepare for MAPREDUCE-279
-    merge. (acmurthy) 
-
-    MAPREDUCE-2541. Fixed a race condition in IndexCache.removeMap. (Binglin
-    Chang via acmurthy) 
-
-    MAPREDUCE-2839. Fixed TokenCache to get delegation tokens using both new
-    and old apis. (Siddharth Seth via acmurthy)
-
-    MAPREDUCE-2727. Fix divide-by-zero error in SleepJob for sleepCount equals
-    0. (Jeffrey Naisbitt via acmurthy)
-
-
-Release 0.22.0 - Unreleased
-
-  INCOMPATIBLE CHANGES
-
-    MAPREDUCE-1866. Removes deprecated class
-    org.apache.hadoop.streaming.UTF8ByteArrayUtils. (amareshwari)
-
-    MAPREDUCE-1664. Changes the behaviour of the combination of job-acls
-    when they function together with queue-acls. (Ravi Gummadi via vinodkv)
-
-  NEW FEATURES
-
-    MAPREDUCE-1804. Stress-test tool for HDFS introduced in HDFS-708.
-    (Joshua Harlow via shv)
-
-    MAPREDUCE-220. Collect cpu and memory statistics per task. (Scott Chen via
-    acmurthy)
-
-    MAPREDUCE-1970.  Reed-Solomon code implementation for HDFS RAID.
-    (Scott Chen via dhruba)
-
-    MAPREDUCE-2169. Integrated Reed-Solomon code with RaidNode. (Ramkumar
-    Vadali via schen)
-
-  IMPROVEMENTS
-
-    MAPREDUCE-2141. Add an "extra data" field to Task for use by Mesos. (matei)
-
-    MAPREDUCE-2140. Regenerate fair scheduler design doc PDF. (matei)
-
-    MAPREDUCE-1546. Redirect all job pages to corresponding history page
-    if job is not in JT memory. (Scott Chen via sharad)
-
-    MAPREDUCE-1092. Enable assertions for unit tests. (Eli Collins via
-    cdouglas)
-
-    MAPREDUCE-1680. Add a metric recording JobTracker heartbeats processed.
-    (Dick King via cdouglas)
-
-    MAPREDUCE-1761. FairScheduler allows separate configuration of node
-    and rack locality wait time (Scott Chen via dhruba)
-
-    MAPREDUCE-1539. authorization checks for inter-server protocol
-    (based on HADOOP-6600) (Boris Shkolnik via shv)
-
-    MAPREDUCE-1798. Names the configuration keys for the Kerberos
-    principals better. (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1773. streaming doesn't support jobclient.output.filter.
-    (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1785. Add streaming config option for not emitting the key.
-    (Eli Collins via sharad)
-
-    MAPREDUCE-572. If #link is missing from uri format of -cacheArchive
-    then streaming does not throw error. (Amareshwari Sriramadasu via
-    vinodkv)
-
-    MAPREDUCE-1545. Add timestamps for first task type launched in job summary.
-    (Luke Lu via cdouglas)
-
-    MAPREDUCE-1543. Add an audit log for authentication events. (Amar Kamat and
-    Luke Lu via cdouglas)
-
-    MAPREDUCE-1762. Add ability to set values of task counters. (Scott Chen via
-    cdouglas)
-
-    MAPREDUCE-1533. Reduce overhead of logging and string manipulation during
-    heartbeat processing. (Amar Kamat and Dick King via cdouglas)
-
-    MAPREDUCE-1516. JobTracker issues delegation tokens only if the user's
-    authentication is Kerberos. (Jitendra Pandey via ddas)
-
-    MAPREDUCE-647. Update distcp forrest documentation to reflect the changes
-    of HADOOP-5472, MAPREDUCE-642 and HADOOP-5620.  (Rodrigo Schmidt via
-    szetszwo)
-
-    MAPREDUCE-1851. Documents configuration parameters in streaming.
-    (amareshwari)
-    
-    MAPREDUCE-1868. Add a read and connection timeout to JobClient while
-    pulling tasklogs. (Krishna Ramachandran via acmurthy)
-
-    MAPREDUCE-1778. Ensure failure to setup CompletedJobStatusStore is not
-    silently ignored by the JobTracker. (Krishna Ramachandran via acmurthy)
-
-    MAPREDUCE-1850. Includes job submit host information (name and ip) in
-    jobconf and jobdetails display (Krishna Ramachandran via amareshwari)
-
-    MAPREDUCE-1893. Slive with multiple reducers. (shv)
-
-    MAPREDUCE-1248. Fixes redudant memory copying in StreamKeyValUtil.
-    (Ruibang He via amareshwari)
-
-    MAPREDUCE-1840. Enhancements to Gridmix benchmark simulating user
-    diversity, queue replay, and task duration for JobTracker load testing.
-    Also includes compatibility with security enhancements, and scalability
-    improvements. (Amar Kamat, Rahul Singh, Hong Tang, and cdouglas)
-
-    MAPREDUCE-1848. Put number of speculative, data local, rack local 
-    tasks in JobTracker metrics. (Scott Chen via dhruba)
-
-    MAPREDUCE-1935. Makes the Distcp to work in a secure environment.
-    (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1945. The MapReduce component for HADOOP-6632.
-    (Kan Zhang & Jitendra Pandey via ddas)
-
-    MAPREDUCE-1936. Modify Gridmix3 to support more tunable parameters for
-    stress submission and sleep jobs. (Hong Tang via cdouglas)
-
-    MAPREDUCE-1733. Makes pipes applications secure. (Jitendra Pandey via ddas)
-
-    MAPREDUCE-1566. Adds a configuration attribute using which job clients can
-    specify a credentials file. The tokens from there will be passed to the job.
-    (Jitendra Pandey and Owen O'Malley via ddas)
-
-    MAPREDUCE-1624. Documents the job credentials and associated details to do 
-    with delegation tokens (on the client side). 
-    (Jitendra Pandey and Devaraj Das via ddas)
-
-    MAPREDUCE-1834. TestSimulatorDeterministicReplay timesout on trunk.
-    (Hong Tang via mahadev)
-
-    MAPREDUCE-1993. Fixes test failure
-    TestTrackerDistributedCacheManagerWithLinuxTaskController. (Devaraj Das
-    via amareshwari)
-
-    MAPREDUCE-1523. Making Mumak work with Capacity-Scheduler (Anirban Das
-    via mahadev)
-
-    MAPREDUCE-1920. Enables completed jobstatus store by default. (Tom White
-    via amareshwari)
-
-    MAPREDUCE-1881. Improve TaskTrackerInstrumentation to enable collection of
-    advanced metrics. (Matei Zaharia via acmurthy)
-
-    MAPREDUCE-1548. Hadoop archives preserve times and other properties from 
-    original files. (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1517. Supports streaming job to run in the background. (Bochun Bai
-    via amareshwari)
-
-    MAPREDUCE-1819. RaidNode is now smarter in submitting Raid jobs. (Ramkumar
-    Vadali via schen)
-
-    MAPREDUCE-2132. A command line option in RaidShell to fix blocks using raid
-
-    MAPREDUCE-2147. Remove redundant lines in JobInProgress ctor.
-    (Harsh J Chouraria via cutting)
-   
-    HADOOP-7007. Update the hudson-test-patch ant target to work with the
-    latest test-patch.sh script (gkesavan)
-
-    MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar
-    Vadali via schen)
-
-    MAPREDUCE-2051. Contribute a fair scheduler preemption system test.
-    (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-1892. RaidNode can allow layered policies more efficiently.
-    (Ramkumar Vadali via schen)
-
-    MAPREDUCE-1592. Generate Eclipse's .classpath file from Ivy config. 
-    (tomwhite via nigel)
-
-    MAPREDUCE-2073. TestTrackerDistributedCacheManager should be up-front
-    about requirements on build environment. (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-2093. Herriot JT and TT clients should vend statistics. (cos)
-
-    MAPREDUCE-2167. Faster directory traversal for raid node. (Ramkumar Vadali
-    via schen)
-
-    MAPREDUCE-1931. Gridmix forrest documentation . (Ranjit Mathew via vinodkv).
-
-    MAPREDUCE-2184. Port DistRaid.java to new mapreduce API. (Ramkumar Vadali
-    via schen)
-
-    MAPREDUCE-1878. Add MRUnit documentation. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test (todd)
-
-    MAPREDUCE-2250. Fix logging in raid code. (Ramkumar Vadali via schen)
-
-    MAPREDUCE-2260. Remove auto-generated native build files. (rvs via eli)
-
-    MAPREDUCE-2314. configure files that are generated as part of the released
-    tarball need to have executable bit set (rvs via cos)
-
-    MAPREDUCE-1159.  Limit Job name in web UI to be 80 char long.  (Harsh J
-    Chouraria via szetszwo)
-
-    MAPREDUCE-2337. Remove dependence of public MapReduce API on classes in
-    server package. (tomwhite)
-
-    MAPREDUCE-2383. Improve documentation of DistributedCache methods (Harsh J
-    Chouraria via todd)
-
-    MAPREDUCE-2222. Ivy resolve force mode should be turned off by default.
-    (Luke Lu via tomwhite)
-
-    MAPREDUCE-2103. task-controller shouldn't require o-r permissions.
-    (todd via eli)
-
-    MAPREDUCE-2505. Explain how to use ACLs in the fair scheduler.
-    (matei via eli)
-
-  OPTIMIZATIONS
-
-    MAPREDUCE-1354. Enhancements to JobTracker for better performance and
-    scalability. (Arun C. Murthy & Richard King via acmurthy) 
-
-    MAPREDUCE-1829. JobInProgress.findSpeculativeTask should use min() to
-    find the candidate instead of sort(). (Scott Chen via vinodkv)
-
-  BUG FIXES
-
-    MAPREDUCE-1845. FairScheduler.tasksToPreempt() can return negative number.
-    (Scott Chen via matei)
-
-    MAPREDUCE-1707. TaskRunner can get NPE in getting ugi from TaskTracker.
-    (Vinod Kumar Vavilapalli)
-    
-    MAPREDUCE-1532. Ensures that delegation tokens is obtained as the
-    actual user when the proxy-user is used for submitting jobs. Also
-    refactors the DelegationTokenToRenew class. (ddas)
-
-    MAPREDUCE-1558. Fixes MRAdmin to look up the principal of the 
-    JobTracker and use that in the RefreshUserToGroupsMapping protocol and
-    RefreshAuthorizationPolicyProtocol. (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1662. Remove unused methods from TaskRunner. (Amareshwari
-    Sriramadasu via cdouglas)
-
-    MAPREDUCE-1617. Use IPv4 stack for unit tests. (Amar Kamat and Luke Lu via
-    cdouglas)
-
-    MAPREDUCE-1599. Fixes MRBench so that it reuses tokens across jobs
-    correctly. (Jitendra Nath Pandey via ddas)
-
-    MAPREDUCE-1836. Refresh for proxy superuser config (mr part for HDFS-1096).
-    (Boris Shkolnik via shv)
-
-    MAPREDUCE-1505. Create RPC client on job submission, not in cstr of Job
-    instance. (Dick King via cdouglas)
-
-    MAPREDUCE-1813. NPE in PipeMapred.MRErrorThread. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1225. Fixes DistributedCache to check if the file is fresh or not,
-    for the first localization also. (Zhong Wang via amareshwari)
-
-    MAPREDUCE-1559. Fixes the token renewer to use the JobTracker's 
-    credentials for talking to the NameNode. (ddas)
-
-    MAPREDUCE-1492. Delete  obsolete har files used on the parity files
-    of hdfs raid. (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1857. Removes unused configuration parameters in streaming.
-    (amareshwari)
-
-    MAPREDUCE-1887. MRAsyncDiskService now properly absolutizes volume root
-    paths. (Aaron Kimball via zshao)
-
-    MAPREDUCE-1863. Fix NPE in Rumen when processing null CDF for failed task
-    attempts. (Amar Kamat via cdouglas)
-
-    MAPREDUCE-1864. Removes uninitialized/unused variables in
-    org.apache.hadoop.streaming.PipeMapRed. (amareshwari)
-
-    MAPREDUCE-1888. Fixes Streaming to override output key and value types, 
-    only if mapper/reducer is a command. (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-577. Fixes duplicate records in StreamXmlRecordReader.
-    (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-1894. Fixed a bug in DistributedRaidFileSystem.readFully() 
-    that was causing it to loop infinitely. (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-1838. Reduce the time needed for raiding a bunch of files
-    by randomly assigning files to map tasks. (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-1820. Fix InputSampler to clone sampled keys. (Alex Kozlov via
-    cdouglas)
-
-    MAPREDUCE-1528. Incorporates the changes to the credentials API done in
-    HADOOP-6845. Also, introduces Credentials in JobConf, and in JobContext.
-    (Jitendra Pandey and Arun Murthy via ddas)
-
-    MAPREDUCE-1865. Rumen should also support jobhistory files generated using
-    trunk. (Amar Kamat via amareshwari)
-
-    MAPREDUCE-1621. Fixes NPE in TextOutputReader.getLastOutput if it has never
-    read any output. (amareshwari)
-
-    MAPREDUCE-1911. Fixes errors in -info message in streaming. (amareshwari) 
-
-    MAPREDUCE-1772. Corrects errors in streaming documentation in forrest.
-    (amareshwari)
-
-    MAPREDUCE-1925. Fix failing TestRumenJobTraces. (Ravi Gummadi via cdouglas)
-
-    MAPREDUCE-1718. Fixes a bug in the construction of jobconf key for the
-    mapping that the tasks use at runtime for looking up delegation tokens.
-    (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1701. Fixes a problem to do with exception handling in 
-    delegation-token renewals. (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1686. Fixes StreamUtil.goodClassOrNull to find classes without
-    package names. (Paul Burkhardt via amareshwari)
-
-    MAPREDUCE-1288. Fixes TrackerDistributedCacheManager to take into account
-    the owner of the localized file in the mapping from cache URIs to
-    CacheStatus objects. (ddas)
-
-    MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either 
-    of configuration properties "mapreduce.job.name" and "mapred.job.name".
-    (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-1958. The MapReduce part corresponding to the HADOOP-6873.
-    (Boris Shkolnik & Owen O'Malley via ddas)
-
-    MAPREDUCE-1900. TaskTracker and JobTracker closes FileSystems, opened on
-    behalf of users that it no longer requires. (Kan Zhang and ddas via ddas)
-
-    MAPREDUCE-1992. Fixes a problem to do with bringing up the JobTracker in
-    unsecure mode. (Kan Zhang via ddas)
-
-    MAPREDUCE-1999. Fixes ClientProtocol to use the correct 
-    DelegationTokenSelector. (Jitendra Pandey via ddas)
-
-    MAPREDUCE-1780. AccessControlList.toString() is used for serialization of
-    ACL in JobStatus.java. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1961. Fix ConcurrentModificationException in Gridmix during
-    shutdown. (Hong Tang via cdouglas)
-
-    MAPREDUCE-2000. Fix parsing of JobHistory lines in Rumen when quotes are
-    escaped. (Hong Tang via cdouglas)
-
-    MAPREDUCE-2022. Fixes compilation errors in TestSubmitJob. (amareshwari)
-
-    MAPREDUCE-1670. RAID policies should not scan their own destination path.
-    (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-1668. RaidNode Hars a directory only if all its parity files 
-    have been created. (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-2021. Fixes duplicate hostnames in CombineFileInputFormat's
-    split locations. (amareshwari)
-
-    MAPREDUCE-1375. Fixes flaky test TestFileArgs. (Todd Lipcon via
-    amareshwari)
-
-    MAPREDUCE-2023. TestDFSIO should not stop reading if curSize != bufferSize.
-    (Hong Tang via szetszwo)
-
-    MAPREDUCE-2031. Fixes test failures TestTaskLauncher and
-    TestTaskTrackerLocalization. (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-2046. Fixes CombineFileInputFormat to allow splits with size
-    less than DFS block size. (dhruba borthakur via amareshwari)
-
-    MAPREDUCE-1975. Fixes unnecessary InterruptedException log in gridmix.
-    (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-1597. Fixes CombineFileInputFormat to work with non-splittable
-    files. (amareshwari)
-
-    MAPREDUCE-2032. Fixes TestJobCleanup to cleanup test directory in
-    tearDown. (Dick King via amareshwari)
-
-    MAPREDUCE-1979. Fixes "Output directory already exists" error in gridmix
-    when gridmix.output.directory is not defined. (Ravi Gummadi via
-    amareshwari)
-
-    MAPREDUCE-1918. Adds documentation to Rumen. (Amar Kamat via amareshwari)
-
-    MAPREDUCE-2078. Fixes TraceBuilder to generate traces when a globbed job
-    history path is given. (Amar Kamat via amareshwari)
-
-    MAPREDUCE-1989. Fixes error message in gridmix when user resolver is set
-    and no user list is given. (Ravi Gummadi via amareshwari)
-
-    MAPREDUCE-2067.  Distinct minicluster services (e.g. NN and JT) overwrite
-    each other's service policies.  (Aaron T. Myers via tomwhite)
-
-    MAPREDUCE-2029. DistributedRaidFileSystem removes itself from FileSystem
-    cache when it is closed. (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable 
-    partfile size. (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-2082. Fixes Pipes to create the jobtoken file in the right
-    place. (Jitendra Pandey via ddas)
-
-    MAPREDUCE-2095. Fixes Gridmix to run from compressed traces. (Ranjit
-    Mathew via amareshwari)
-
-    MAPREDUCE-1908. DistributedRaidFileSystem now handles ChecksumException
-    correctly. (Ramkumar Vadali via schen)
-
-    MAPREDUCE-2126. JobQueueJobInProgressListener's javadoc is inconsistent
-    with source code. (Jingguo Yao via tomwhite)
-
-    MAPREDUCE-2143.  HarFileSystem is able to handle spaces in pathnames.
-    (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-1867.  Remove unused methods in
-    org.apache.hadoop.streaming.StreamUtil.  (amareshwari via tomwhite)
-
-    MAPREDUCE-2146.  Raid does not affect access time of a source file.
-    (Ramkumar Vadali via dhruba)
-
-    MAPREDUCE-2150.  RaidNode periodically fixes corrupt blocks. (Ramkumar Vadali via
-    schen)
-
-    MAPREDUCE-2099.  RaidNode recreates outdated parity HARs. (Ramkumar Vadali
-    via schen)
-
-    MAPREDUCE-2173.  Fix race condition in TestBlockFixer that was
-    causing  intermittent failure (Patrick Kling via dhruba)
-
-    MAPREDUCE-2142.  Refactor RaidNode so that the map-reduce component is
-    clearly separated out. (Patrick Kling via dhruba)
-
-    MAPREDUCE-2179. Fix RaidBlockSender compilation failure. (Ramkumar Vadali
-    via schen)
-
-    MAPREDUCE-2034. TestSubmitJob triggers NPE instead of permissions error.
-    (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-2195. New property for local conf directory in
-    system-test-mapreduce.xml file. (cos)
-
-    MAPREDUCE-1783. FairScheduler initializes tasks only when the job can be
-    run. (Ramkumar Vadali via schen)
-
-    MAPREDUCE-2224. Fix synchronization bugs in JvmManager. (todd)
-
-    MAPREDUCE-714. JobConf.findContainingJar unescapes unnecessarily on linux (todd)
-
-    MAPREDUCE-2096. Secure local filesystem IO from symlink vulnerabilities (todd)
-
-    MAPREDUCE-2234. If Localizer can't create task log directory, it should fail
-    on the spot. (todd)
-
-    MAPREDUCE-2219. JobTracker should not try to remove mapred.system.dir
-    during startup. (todd)
-
-    MAPREDUCE-2207. Task-cleanup task should not be scheduled on the node that
-    the task just failed. (Liyin Liang via schen)
-
-    MAPREDUCE-2084. Remove deprecate annotation for package file.  The package
-    classes themselves are already deprecated. This removes an Eclipse error.
-    (tomwhite via nigel)
-
-    MAPREDUCE-2248. DistributedRaidFileSystem should unraid only the corrupt
-    block (Ramkumar Vadali via schen)
-
-    MAPREDUCE-1085. For tasks, "ulimit -v -1" is being run when user doesn't
-    specify a ulimit (todd)
-
-    MAPREDUCE-2282. Fix TestMRServerPorts for the changes in
-    TestHDFSServerPorts.  (shv via szetszwo)
-
-    MAPREDUCE-2238. Fix permissions handling to avoid leaving undeletable
-    directories in local dirs. (todd)
-
-    MAPREDUCE-2277. TestCapacitySchedulerWithJobTracker needs to wait for jobs
-    to complete before testing status. (todd)
-
-    MAPREDUCE-2253. Servlets should specify content type (todd)
-
-    MAPREDUCE-2283. Add timeout for Raid Tests (Ramkumar Vadali via schen)
-
-    MAPREDUCE-1754. Replace mapred.persmissions.supergroup with an 
-    acl : mapreduce.cluster.administrators (Amareshwari Sriramadasu via shv)
-
-    MAPREDUCE-2256. FairScheduler fairshare preemption from multiple pools may
-    preempt all tasks from one pool causing that pool to go below fairshare.
-    (Priyo Mustafi via shv)
-
-    MAPREDUCE-2281. MR part of HADOOP-6642. (Chris Douglas, Po Cheung via shv)
-
-    MAPREDUCE-2200. TestUmbilicalProtocolWithJobToken is failing without Krb
-    evironment: needs to be conditional. (cos)
-
-    MAPREDUCE-2077. Resolve name clash in the deprecated 
-    o.a.h.util.MemoryCalculatorPlugin (Luke Lu via shv)
-
-    MAPREDUCE-2188. The new API MultithreadedMapper doesn't initialize
-    RecordReader. (Owen O'Malley via shv)
-
-    MAPREDUCE-1915. Fix IndexOutOfBoundsException in IndexCache.
-    (Priyo Mustafi via shv)
-
-    MAPREDUCE-1974. Fix multiple preemtions of the same task in FairScheduler.
-    (Scott Chen via shv)
-
-    MAPREDUCE-2304. Fix TestMRCLI to allow hostname with a hyphen (-).
-    (Priyo Mustafi via shv)
-
-    MAPREDUCE-1825. jobqueue_details.jsp and FairSchedulerServelet should not
-    call finishedMaps and finishedReduces when job is not initialized.
-    (Scott Chen via shv)
-
-    MAPREDUCE-2285. MiniMRCluster does not start after ant test-patch
-    (todd)
-
-    MAPREDUCE-2315. javadoc is failing in nightly build (todd)
-
-    MAPREDUCE-2054. Hierarchical queue implementation broke dynamic queue
-    addition in Dynamic Scheduler. (Thomas Sandholm via tomwhite)
-
-    MAPREDUCE-2272. Job ACL file should not be executable
-    (Harsh J Chouraria via todd)
-
-    MAPREDUCE-2241. ClusterWithLinuxTaskController should accept relative path
-    on the command line. (todd)
-
-    MAPREDUCE-2251. Remove unused mapreduce.job.userhistorylocation config.
-    (Harsh J Chouraria via todd)
-
-    MAPREDUCE-2284. TestLocalRunner.testMultiMaps times out (todd)
-
-    MAPREDUCE-2336. Tool-related packages should be in the Tool javadoc group.
-    (tomwhite)
-    
-    MAPREDUCE-2394. JUnit output format doesn't propagate into raid contrib
-    build. (todd)
-
-    MAPREDUCE-2392. TaskTracker shutdown in the tests sometimes take 60s.
-    (tomwhite)
-
-    MAPREDUCE-2437. SLive uses only part* files to generating the final report.
-    (shv)
-
-    MAPREDUCE-2428. start-mapred.sh script fails if HADOOP_HOME is not set.
-    (tomwhite via eli)
-
-    MAPREDUCE-2445. Fix TestMiniMRWithDFSWithDistinctUsers to be a valid test.
-    (todd)
-
-    MAPREDUCE-2457. Job submission should inject group.name on the JobTracker
-    (Alejandro Abdelnur via todd)
-
-    MAPREDUCE-2472. Extra whitespace in mapred.child.java.opts breaks JVM
-    initialization. (Aaron T. Myers via todd)
-
-    MAPREDUCE-2222. Ivy resolve force mode should be turned off by default.
-    (Luke Lu via tomwhite)
-
-    MAPREDUCE-2486. Incorrect snapshot dependency published in .pom files
-    (todd)
-
-    MAPREDUCE-2327. MapTask doesn't need to put username information in
-    SpillRecord. (todd via tomwhite)
-
-    MAPREDUCE-2515. MapReduce code references some deprecated options
-    (Ari Rabkin via todd)
-
-    MAPREDUCE-2487. ChainReducer uses MAPPER_BY_VALUE instead of
-    REDUCER_BY_VALUE. (Devaraj K via todd)
-
-    MAPREDUCE-2185. Fix infinite loop at creating splits using
-    CombineFileInputFormat. (Ramkumar Vadali via schen)
-
-    MAPREDUCE-2571. CombineFileInputFormat.getSplits throws a
-    java.lang.ArrayStoreException. (Bochun Bai via todd)
-
-Release 0.21.1 - Unreleased
-
-  NEW FEATURES
-
-    MAPREDUCE-2040. Forrest Documentation for Dynamic Priority Scheduler.
-    (Thomas Sandholm via tomwhite)
-
-  BUG FIXES
-
-    MAPREDUCE-1897. trunk build broken on compile-mapred-test (cos)
-
-    MAPREDUCE-1280. Update Eclipse plugin to the new eclipse.jdt API.
-    (Alex Kozlov via szetszwo)
-
-    MAPREDUCE-1984. herriot TestCluster fails because exclusion is not there
-    (Balaji Rajagopalan via cos)
-
-    MAPREDUCE-2090. Clover build doesn't generate per-test coverage. (cos)
-
-    MAPREDUCE-2134. ant binary-system is broken in mapreduce project. (cos)
-
-    MAPREDUCE-1905. Fixes Context.setStatus() and progress() apis.
-    (amareshwari)
-
-    MAPREDUCE-1809. Ant build changes for Streaming system tests in contrib
-    projects. (Vinay Kumar Thota via amareshwari)
-
-    MAPREDUCE-2223. TestMRCLI might fail on Ubuntu with default /etc/hosts
-    (cos)
-
-    MAPREDUCE-2228. Remove java5 dependencies from build. (cos)
-
-    MAPREDUCE-1929. Allow artifacts to be published to the staging Apache Nexus
-    Maven Repository. (tomwhite)
-
-    MAPREDUCE-2317. Fix a NPE in HadoopArchives.  (Devaraj K via szetszwo)
-
-    MAPREDUCE-2127. mapreduce trunk builds are filing on hudson. 
-    (Bruno Mahé via eli)
-
-Release 0.21.0 - 2010-08-13
-
-  INCOMPATIBLE CHANGES
-
-    MAPREDUCE-516. Fix the starvation problem in the Capacity Scheduler 
-    when running High RAM Jobs. (Arun Murthy via yhemanth)
-
-    MAPREDUCE-358. Change org.apache.hadoop.examples. AggregateWordCount 
-    and org.apache.hadoop.examples.AggregateWordHistogram to use new 
-    mapreduce api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-245. Change Job and jobcontrol classes to use the List interface
-    rather than ArrayList in APIs. (Tom White via cdouglas)
-
-    MAPREDUCE-766. Enhanced list-blacklisted-trackers to display reasons
-    for blacklisting a node. (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-817. Add a cache for retired jobs with minimal job info and 
-    provide a way to access history file url. (sharad)
-
-    MAPREDUCE-711. Moved Distributed Cache from Common to Map/Reduce
-    project. (Vinod Kumar Vavilapalli via yhemanth)
-
-    MAPREDUCE-895. Per the contract elucidated in HADOOP-6201, throw
-    FileNotFoundException from FileSystem::listStatus rather than returning
-    null. (Jakob Homan via cdouglas)
-
-    MAPREDUCE-479. Provide full task id to map output servlet rather than the
-    reduce id, only. (Jiaqi Tan via cdouglas)
-
-    MAPREDUCE-873. Simplify job recovery. Incomplete jobs are resubmitted on 
-    jobtracker restart. Removes a public constructor in JobInProgress. (sharad)
-
-    HADOOP-6230. Moved process tree and memory calculator related classes from
-    Common to Map/Reduce. (Vinod Kumar Vavilapalli via yhemanth)
-
-    MAPREDUCE-157. Refactor job history APIs and change the history format to 
-    JSON. (Jothi Padmanabhan via sharad)
-
-    MAPREDUCE-849. Rename configuration properties. (Amareshwari Sriramadasu 
-    via sharad)
-
-    MAPREDUCE-1287. Only call the partitioner with more than one reducer.
-    (cdouglas)
-
-    MAPREDUCE-1385. Use the new UserGroupInformation from HADOOP-6299.
-    (ddas via omalley)
-
-    MAPREDUCE-1493. Authorization for job-history pages. (vinodkv)
-
-    MAPREDUCE-1607. Task controller may not set permissions for a
-    task cleanup attempt's log directory (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1683. Remove JNI calls from ClusterStatus cstr. (Arun Murthy and
-    Luke Lu via cdouglas)
-
-    MAPREDUCE-1855. Makes the refresh methods (for groups and proxy users) 
-    independent of the client side configuration. (Boris Shkolnik via ddas)
-
-  NEW FEATURES
-
-    MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth
-    Ramakrishnan, Konstantin Boudnik, at all via cos)
-
-    MAPREDUCE-706. Support for FIFO pools in the fair scheduler.
-    (Matei Zaharia)
-
-    MAPREDUCE-546. Provide sample fair scheduler config file in conf/ and use
-    it by default if no other config file is specified. (Matei Zaharia)
-
-    MAPREDUCE-551. Preemption support in the Fair Scheduler. (Matei Zaharia)
-
-    MAPREDUCE-567. Add a new example MR that always fails. (Philip Zeyliger
-    via tomwhite)
-
-    MAPREDUCE-211. Provides ability to run a health check script on the
-    tasktracker nodes and blacklist nodes if they are unhealthy.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-637. Add an example, distbbp, which able to compute the n th bit
-    of Pi for some large n.  (szetszwo)
-
-    MAPREDUCE-532. Provide a way to limit the number of used slots 
-    per queue in the capacity scheduler.
-    (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-467. Provide ability to collect statistics about total tasks 
-    and succeeded tasks in different time windows. (sharad)
-
-    MAPREDUCE-740. Log a job-summary at the end of a job, while allowing it
-    to be configured to use a custom appender if desired. (acmurthy)
-
-    MAPREDUCE-814. Provide a way to configure completed job history files 
-    to be on HDFS. (sharad)
-
-    MAPREDUCE-800. MRUnit should support the new API. (Aaron Kimball via
-    tomwhite)
-
-    MAPREDUCE-798. MRUnit should be able to test a succession of MapReduce
-    passes. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-768. Provide an option to dump jobtracker configuration in JSON
-    format to standard output. (V.V.Chaitanya Krishna via yhemanth)
-
-    MAPREDUCE-824. Add support for a hierarchy of queues in the capacity 
-    scheduler. (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-751. Add Rumen, a tool for extracting statistics from job tracker
-    logs and generating job traces for simulation and analysis. (Dick King via
-    cdouglas)
-
-    MAPREDUCE-830. Add support for splittable compression to TextInputFormats.
-    (Abdul Qadeer via cdouglas)
-
-    MAPREDUCE-861. Add support for hierarchical queues in the Map/Reduce
-    framework. (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-776. Add Gridmix, a benchmark processing Rumen traces to simulate
-    a measured mix of jobs on a cluster. (cdouglas)
-
-    MAPREDUCE-862. Enhance JobTracker UI to display hierarchical queues.
-    (V.V.Chaitanya Krishna via yhemanth)
-
-    MAPREDUCE-777. Brand new apis to track and query jobs as a
-    replacement for JobClient. (Amareshwari Sriramadasu via acmurthy)
-
-    MAPREDUCE-775. Add native and streaming support for Vertica as an input
-    or output format taking advantage of parallel read and write properties of 
-    the DBMS. (Omer Trajman via ddas)
-
-    MAPREDUCE-679. XML-based metrics as JSP servlet for JobTracker.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-980. Modify JobHistory to use Avro for serialization. (cutting)
-
-    MAPREDUCE-728. Add Mumak, a Hadoop map/reduce simulator. (Arun C Murthy,
-    Tamas Sarlos, Anirban Dasgupta, Guanying Wang, and Hong Tang via cdouglas)
-
-    MAPREDUCE-1383. Automates fetching of delegation tokens in File*Formats
-    Distributed Cache and Distcp. Also, provides a config 
-    mapreduce.job.hdfs-servers that the jobs can populate with a comma
-    separated list of namenodes. The job client automatically fetches
-    delegation tokens from those namenodes. (Boris Shkolnik via ddas)    
-
-    MAPREDUCE-698. Per-pool task limits for the fair scheduler.
-    (Kevin Peterson via matei)
-
-    MAPREDUCE-1026. Does mutual authentication of the shuffle
-    transfers using a shared JobTracker generated key.
-    (Boris Shkolnik via ddas)
-
-    MAPREDUCE-744. Introduces the notion of a public distributed cache.
-    (Devaraj Das)
-
-    MAPREDUCE-1338. Introduces the notion of token cache using which
-    tokens and secrets can be sent by the Job client to the JobTracker.
-    (Boris Shkolnik via ddas)
-
-    HDFS-503. This patch implements an optional layer over HDFS that 
-    implements offline erasure-coding.  It can be used to reduce the 
-    total storage requirements of HDFS.  (dhruba)
-
-    MAPREDUCE-1432. Adds hooks in the jobtracker and tasktracker 
-    for loading the tokens in the user's ugi. This is required
-    for the copying of files from the hdfs. (ddas)
-
-    MAPREDUCE-1335. Adds SASL Kerberos/Digest authentication in MapReduce.
-    (Kan Zhang via ddas)
-
-    MAPREDUCE-1464. Makes a compatible change in JobTokenIdentifier to
-    account for HADOOP-6510. (Jitendra Nath Pandey via ddas)
-
-    MAPREDUCE-1433. Add a delegation token for MapReduce. (omalley)
-
-    MAPREDUCE-1307. Introduces the Job level ACLs feature. 
-    (Vinod Kumar Vavilapalli via ddas)
-
-    MAPREDUCE-1430. JobTracker automatically renews delegation tokens for jobs.
-    (Boris Shkolnik via ddas)
-
-    MAPREDUCE-1455. Introduces job-level authorization for mapreduce servlets.
-    (Ravi Gummadi via vinodkv)
-
-  IMPROVEMENTS
-
-    MAPREDUCE-463. Makes job setup and cleanup tasks as optional.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-502. Allow jobtracker to be configured with zero completed jobs
-    in memory. (Amar Kamat via sharad)
-
-    MAPREDUCE-416. Moves the history file to a "done" folder whenever a job 
-    completes. (Amar Kamat via ddas)
-
-    MAPREDUCE-646. Increase srcfilelist replication number in dictcp job.
-    (Ravi Gummadi via szetszwo)
-   
-    HADOOP-6106. Updated hadoop-core and test jars from hudson trunk 
-    build #12. (Giridharan Kesavan)
-
-    MAPREDUCE-642. A option to distcp that allows preserving the full
-    source path of a file in the specified destination directory.
-    (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-686. Move TestSpeculativeExecution.Fake* into a separate class
-    so that it can be used by other tests. (Jothi Padmanabhan via sharad)
-
-    MAPREDUCE-625. Modify TestTaskLimits to improve execution time.
-    (Jothi Padmanabhan via sharad)
-
-    MAPREDUCE-465. Deprecate o.a.h.mapred.lib.MultithreadedMapRunner and add
-    test for o.a.h.mapreduce.lib.MultithreadedMapper.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-701. Improves the runtime of the TestRackAwareTaskPlacement
-    by making it a unit test.  (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-371. Change KeyFieldBasedComparator and KeyFieldBasedPartitioner
-    to use new api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-623. Resolve javac warnings in mapreduce. (Jothi Padmanabhan
-    via sharad)
-
-    MAPREDUCE-655. Change KeyValueLineRecordReader and KeyValueTextInputFormat
-    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-632. Merge TestCustomOutputCommitter with 
-    TestCommandLineJobSubmission. (Jothi Padmanabhan via sharad)
-
-    MAPREDUCE-627. Improves execution time of TestTrackerBlacklistAcrossJobs.
-    (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-630. Improves execution time of TestKillCompletedJob.
-    (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-626. Improves the execution time of TestLostTracker.
-    (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-353. Makes the shuffle read and connection timeouts
-    configurable. (Ravi Gummadi via ddas)
-
-    MAPREDUCE-739. Allow relative paths to be created in archives. (Mahadev
-    Konar via cdouglas)
-
-    MAPREDUCE-772. Merge HADOOP-4010 changes to LineRecordReader into mapreduce
-    package. (Abdul Qadeer via cdouglas)
-
-    MAPREDUCE-785. Separate sub-test of TestReduceFetch to be included in
-    MR-670. (Jothi Padmanabhan via cdouglas)
-
-    MAPREDUCE-784. Modify TestUserDefinedCounters to use LocalJobRunner 
-    instead of MiniMR. (Jothi Padmanabhan via sharad)
-
-    HADOOP-6160. Fix releaseaudit target to run on specific directories. 
-    (gkesavan)
-
-    MAPREDUCE-782. Use PureJavaCrc32 in SpillRecord.  (Todd Lipcon via
-    szetszwo)
-
-    MAPREDUCE-369. Change org.apache.hadoop.mapred.lib.MultipleInputs to 
-    use new api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-373. Change org.apache.hadoop.mapred.lib.FieldSelectionMapReduce
-    to use new api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-628. Improves the execution time of TestJobInProgress.
-    (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-793. Creates a new test that consolidates a few tests to
-    include in the commit-test list. (Jothi Padmanabhan via ddas)
-    
-    MAPREDUCE-797. Adds combiner support to MRUnit MapReduceDriver.
-    (Aaron Kimball via johan)    
-
-    MAPREDUCE-656. Change org.apache.hadoop.mapred.SequenceFile* classes
-    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-670. Creates ant target for 10 mins patch test build.
-    (Jothi Padmanabhan via gkesavan)
-
-    MAPREDUCE-375. Change org.apache.hadoop.mapred.lib.NLineInputFormat 
-    and org.apache.hadoop.mapred.MapFileOutputFormat to use new api.
-    (Amareshwari Sriramadasu via ddas)
-
-    MAPREDUCE-779. Added node health failure counts into 
-    JobTrackerStatistics. (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-842. Setup secure permissions for localized job files,
-    intermediate outputs and log files on tasktrackers.
-    (Vinod Kumar Vavilapalli via yhemanth)
-    
-    MAPREDUCE-478. Allow map and reduce jvm parameters, environment variables
-    and ulimit to be set separately.
-    Configuration changes:
-      add mapred.map.child.java.opts
-      add mapred.reduce.child.java.opts
-      add mapred.map.child.env
-      add mapred.reduce.child.ulimit
-      add mapred.map.child.env
-      add mapred.reduce.child.ulimit
-      deprecated mapred.child.java.opts
-      deprecated mapred.child.env
-      deprecated mapred.child.ulimit
-    (acmurthy)
-
-    MAPREDUCE-767. Remove the dependence on the CLI 2.0 snapshot.
-    (Amar Kamat via omalley)
-
-    MAPREDUCE-712. Minor efficiency tweaks to RandomTextWriter. (cdouglas)
-
-    MAPREDUCE-870. Remove the job retire thread and the associated 
-    config parameters. (sharad)
-
-    MAPREDUCE-874. Rename the PiEstimator example to QuasiMonteCarlo.
-    (szetszwo)
-
-    MAPREDUCE-336. Allow logging level of map/reduce tasks to be configurable. 
-    Configuration changes:
-      add mapred.map.child.log.level 
-      add mapred.reduce.child.log.level 
-    (acmurthy)
-
-    MAPREDUCE-355. Update mapred.join package to use the new API. (Amareshwari
-    Sriramadasu via cdouglas)
-
-    HADOOP-6184. Updated hadoop common and test jars to get the new API
-    in Configuration for dumping in JSON format from Hudson trunk build #68.
-    (yhemanth)
-
-    MAPREDUCE-476. Extend DistributedCache to work locally (LocalJobRunner).
-    (Philip Zeyliger via tomwhite)
-
-    MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests
-    in local mode. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-910. Support counters in MRUnit. (Aaron Kimball via cdouglas)
-
-    MAPREDUCE-788. Update gridmix2 to use the new API (Amareshwari Sriramadasu
-    via cdouglas)
-
-    MAPREDUCE-875. Make DBRecordReader execute queries lazily. (Aaron Kimball 
-    via enis)
-
-    MAPREDUCE-318. Modularizes the shuffle code. (Jothi Padmanabhan and 
-    Arun Murthy via ddas)
-
-    MAPREDUCE-936. Allow a load difference for fairshare scheduler.
-    (Zheng Shao via dhruba)
-
-    MAPREDUCE-370. Update MultipleOutputs to use the API, merge funcitonality
-    of MultipleOutputFormat. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-898. Changes DistributedCache to use the new API.
-    (Amareshwari Sriramadasu via ddas)
-
-    MAPREDUCE-144. Includes dump of the process tree in task diagnostics when 
-    a task is killed due to exceeding memory limits.
-    (Vinod Kumar Vavilapalli via yhemanth)
-
-    MAPREDUCE-945. Modifies MRBench and TestMapRed to use ToolRunner so that
-    options such as queue name can be passed via command line.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-963. Deprecate o.a.h.mapred.FileAlreadyExistsException and
-    replace it with o.a.h.fs.FileAlreadyExistsException.  (Boris Shkolnik
-    via szetszwo)
-
-    MAPREDUCE-960. Remove an unnecessary intermediate copy and obsolete API
-    from KeyValueLineRecordReader. (cdouglas)
-
-    MAPREDUCE-930. Modify Rumen to resolve paths in the canonical way, rather
-    than defaulting to the local filesystem. (cdouglas)
-
-    MAPREDUCE-944. Extend the LoadManager API of the fair-share scheduler
-    to support regulating tasks for a job based on resources currently in use
-    by that job. (dhruba)
-
-    MAPREDUCE-973. Move FailJob and SleepJob from examples to test. (cdouglas 
-    via omalley)
-
-    MAPREDUCE-966. Modify Rumen to clean up interfaces and simplify integration
-    with other tools. (Hong Tang via cdouglas)
-
-    MAPREDUCE-856. Setup secure permissions for distributed cache files.
-    (Vinod Kumar Vavilapalli via yhemanth)
-
-    MAPREDUCE-885. More efficient SQL queries for DBInputFormat. (Aaron Kimball 
-    via enis)
-
-    MAPREDUCE-284. Enables ipc.client.tcpnodelay in Tasktracker's Child.
-    (Ravi Gummadi via sharad)
-
-    MAPREDUCE-916. Split the documentation to match the project split.
-    (Corinne Chandel via omalley)
-
-    MAPREDUCE-649. Validate a copy by comparing the source and destination
-    checksums in distcp. Also adds an intra-task retry mechanism for errors
-    detected during the copy. (Ravi Gummadi via cdouglas)
-
-    MAPREDUCE-654. Add a -dryrun option to distcp printing a summary of the
-    file data to be copied, without actually performing the copy. (Ravi Gummadi
-    via cdouglas)
-
-    MAPREDUCE-664. Display the number of files deleted by distcp when the
-    -delete option is specified. (Ravi Gummadi via cdouglas)
-
-    MAPREDUCE-781. Let the name of distcp jobs be configurable. (Venkatesh S
-    via cdouglas)
-
-    MAPREDUCE-975. Add an API in job client to get the history file url for 
-    a given job id. (sharad)
-
-    MAPREDUCE-905. Add Eclipse launch tasks for MapReduce. (Philip Zeyliger
-    via tomwhite)
-
-    MAPREDUCE-277. Makes job history counters available on the job history
-    viewers. (Jothi Padmanabhan via ddas)
-
-    MAPREDUCE-893. Provides an ability to refresh queue configuration
-    without restarting the JobTracker.
-    (Vinod Kumar Vavilapalli and Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-1011. Add build.properties to svn and git ignore. (omalley)
-
-    MAPREDUCE-954. Change Map-Reduce context objects to be interfaces.
-    (acmurthy) 
-
-    MAPREDUCE-639. Change Terasort example to reflect the 2009 updates. 
-    (omalley)
-
-    MAPREDUCE-1063. Document gridmix benchmark. (cdouglas)
-
-    MAPREDUCE-931. Use built-in interpolation classes for making up task
-    runtimes in Rumen. (Dick King via cdouglas)
-
-    MAPREDUCE-1012. Mark Context interfaces as public evolving. (Tom White via
-    cdouglas)
-
-    MAPREDUCE-971. Document use of distcp when copying to s3, managing timeouts
-    in particular. (Aaron Kimball via cdouglas)
-
-    HDFS-663. DFSIO for append. (shv)
-
-    HDFS-641. Move all of the components that depend on map/reduce to 
-    map/reduce. (omalley)
-
-    HADOOP-5107. Use Maven ant tasks to publish artifacts. (Giridharan Kesavan
-    via omalley)
-
-    MAPREDUCE-1229. Allow customization of job submission policy in Mumak.
-    (Hong Tang via cdouglas)
-
-    MAPREDUCE-1317. Reduce the memory footprint of Rumen objects by interning
-    host Strings. (Hong Tang via cdouglas)
-
-    MAPREDUCE-1097. Add support for Vertica 3.5 to its contrib module. (Omer
-    Trajman via cdouglas)
-
-    MAPREDUCE-1627. HadoopArchives should not uses a method in DistCp.
-    (szetszwo)
-
-    MAPREDUCE-1198. Alternatively schedule different types of tasks in
-    fair share scheduler. (Scott Chen via matei)
-
-    MAPREDUCE-707. Provide a jobconf property for explicitly assigning a job to 
-    a pool in the Fair Scheduler. (Alan Heirich via matei)
-
-    MAPREDUCE-947. Added commitJob and abortJob apis to OutputCommitter.
-    Enhanced FileOutputCommitter to create a _SUCCESS file for successful
-    jobs. (Amar Kamat & Jothi Padmanabhan via acmurthy) 
-
-    MAPREDUCE-1103. Added more metrics to Jobtracker. (sharad) 
-
-    MAPREDUCE-1048. Add occupied/reserved slot usage summary on jobtracker UI.
-    (Amareshwari Sriramadasu and Hemanth Yamijala via sharad)
-
-    MAPREDUCE-1090. Modified log statement in TaskMemoryManagerThread to
-    include task attempt id. (yhemanth)
-
-    MAPREDUCE-1189. Reduce ivy console output to ovservable level (cos)
-
-    MAPREDUCE-1167. ProcfsBasedProcessTree collects rss memory information.
-    (Scott Chen via dhruba)
-
-    MAPREDUCE-1231. Added a new DistCp option, -skipcrccheck, so that the CRC
-    check during setup can be skipped.  (Jothi Padmanabhan via szetszwo)
-
-    MAPREDUCE-1190. Add package documentation for BBP example.
-    (Tsz Wo (Nicholas) Sze via cdouglas)
-
-    MAPREDUCE-1119. When tasks fail to report status, show tasks's stack dump
-    before killing. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1185. Redirect running job url to history url if job is already 
-    retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
-
-    MAPREDUCE-1050. Introduce a mock object testing framework. (tomwhite)
-
-    MAPREDUCE-1084. Implementing aspects development and fault injeciton
-    framework for MapReduce. (Sreekanth Ramakrishnan via cos)
-
-    MAPREDUCE-1209. Move common specific part of the test TestReflectionUtils
-    out of mapred into common. (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-967. TaskTracker does not need to fully unjar job jars.
-    (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-1083. Changes in MapReduce so that group information of users 
-    can be refreshed in the JobTracker via command line. 
-    (Boris Shkolnik via ddas)
-
-    MAPREDUCE-181. Changes the job submission process to be secure.
-    (Devaraj Das)
-
-    MAPREDUCE-1250. Refactors the JobToken to use Common's Token interface.
-    (Kan Zhang via ddas)
-
-    MAPREDUCE-896. Enhance tasktracker to cleanup files that might have
-    been created by user tasks with non-writable permissions.
-    (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-372. Change org.apache.hadoop.mapred.lib.ChainMapper/Reducer 
-    to use new mapreduce api. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-1295. Add a tool in Rumen for folding and manipulating job
-    traces. (Dick King via cdouglas)
-
-    MAPREDUCE-1302. TrackerDistributedCacheManager deletes file
-    asynchronously, thus reducing task initialization delays.
-    (Zheng Shao via dhruba)
-
-    MAPREDUCE-1218. TaskTrackers send cpu and memory usage of
-    node to JobTracker. (Scott Chen via dhruba)
-
-    MAPREDUCE-847. Fix Releaseaudit warning count to zero
-    (Giridharan Kesavan)
-
-    MAPREDUCE-1337. Use generics in StreamJob to improve readability of that
-    class. (Kay Kay via cdouglas)
-
-    MAPREDUCE-361. Port terasort example to the new mapreduce API. (Amareshwari
-    Sriramadasu via cdouglas)
-
-    MAPREDUCE-1367. LocalJobRunner should support parallel mapper execution.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-64. Eliminate io.sort.record.percent from MapTask configuration.
-    (cdouglas)
-
-    MAPREDUCE-1440. Replace the long user name in MapReduce with the local
-    name. (omalley)
-
-    MAPREDUCE-1470. Move delegation tokens from HDFS to Common so that 
-    MapReduce can use them too. (omalley)
-
-    MAPREDUCE-1425. Reduce memory usage by archive. (mahadev via szetszwo)
-
-    MAPREDUCE-1441. Trim whitespace from directory lists pulled from the
-    configuration. (Todd Lipcon via cdouglas)
-
-    MAPREDUCE-1309. Refactor Rumen trace generator to improve code structure
-    and add extensible support for log formats. (Dick King via cdouglas)
-
-    MAPREDUCE-1503. Delegation token renewing and cancelling should provide
-    meaningful exceptions when there are failures instead of returning 
-    false. (omalley)
-
-    HADOOP-6579. Upgrade commons-codec library to 1.4. (omalley)
-
-    MAPREDUCE-1423. Improve performance of CombineFileInputFormat when multiple
-    pools are configured. (Dhruba Borthakur via zshao)
-
-    MAPREDUCE-1454. Quote user supplied strings in Tracker servlets. (cdouglas)
-
-    MAPREDUCE-1408. Add customizable job submission policies to Gridmix. (Rahul
-    Singh via cdouglas)
-
-    MAPREDUCE-1527. Better warning logged when mapred.queue.names is
-    overshadowed by mapred-queues.xml. (Hong Tang via acmurthy)
-
-    MAPREDUCE-1403. Save the size and number of distributed cache artifacts in
-    the configuration. (Arun Murthy via cdouglas)
-
-    MAPREDUCE-1482. Truncate state string and diagnostic information in
-    TaskStatus. (Amar Kamat via szetszwo)
-
-    MAPREDUCE-1593.  [Rumen] Improvements to random seed generation (tamas via
-    mahadev)
-
-    MAPREDUCE-1460. Oracle support in DataDrivenDBInputFormat.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1569. Pass configuration through mocked contexts in MRUnit.
-    (Chris White via cdouglas)
-
-    MAPREDUCE-1590. Move HarFileSystem from Hadoop Common to Mapreduce tools.
-    (mahadev)
-
-    MAPREDUCE-1629. Get rid of fakeBlockLocations() on HarFileSystem, since
-    it's not used (mahadev)
-
-    MAPREDUCE-1489. DataDrivenDBInputFormat should not query the database
-    when generating only one split. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1514.  Add documentation on replication, permissions, new options,
-    limitations and internals of har.  (mahadev via szetszwo)
-
-    MAPREDUCE-1428.  Make block size and the size of archive created files
-    configurable.  (mahadev via szetszwo)
-
-    MAPREDUCE-1656. JobStory should provide queue info. (hong via mahadev)
-
-    MAPREDUCE-1466. Record number of files processed in FileInputFormat in the
-    Configuration for offline analysis. (Luke Lu and Arun Murthy via cdouglas)
-
-    MAPREDUCE-1538. TrackerDistributedCacheManager manages the
-    number of files. (Scott Chen via dhruba)
-
-    MAPREDUCE-1673. Scripts to start and stop RaidNode.
-    (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1659. RaidNode writes temp files on configured tmp directory and
-    add random numbers to their names to avoid conflicts
-    (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1221. Allow admins to control physical memory limits per-task
-    and per-node. (Scott Chen via acmurthy) 
-
-    MAPREDUCE-1065. Update mapred tutorial to use the new API. (Aaron Kimball
-    via cdouglas)
-
-    MAPREDUCE-1304. Add a task counter tracking time spent in GC. (Aaron
-    Kimball via cdouglas)
-
-    MAPREDUCE-1570. Add grouping comparators to MRUnit. (Chris White via
-    cdouglas)
-
-    MAPREDUCE-1650. Exclude Private elements from generated MapReduce
-    Javadoc. (tomwhite)
-
-    MAPREDUCE-1625. Improve grouping of packages in Javadoc. (tomwhite)
-
-    MAPREDUCE-1417. Forrest documentation should be updated to reflect 
-    the changes in MAPREDUCE-744. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1568. TrackerDistributedCacheManager should clean up cache
-    in a background thread. (Scott Chen via zshao)
-
-    MAPREDUCE-1749. Move configuration strings out of JobContext so that it
-    can be made public stable. (omalley)
-
-    MAPREDUCE-1623. Apply audience and stability notations to Hadoop
-    Map-Reduce. (tomwhite via acmurthy) 
-
-    MAPREDUCE-1751. Change MapReduce to depend on Hadoop 'common' artifacts
-    instead of 'core'. (tomwhite)
-
-    MAPREDUCE-1535.  Replace usage of FileStatus#isDir().  (Eli Collins via
-    tomwhite)
-
-    MAPREDUCE-1832. Allow file sizes less than 1MB in DFSIO benchmark. (shv)
-
-    MAPREDUCE-1404.  Move Cluster-Setup and Single-Node-Setup Docs from
-    MapReduce to Common.  (tomwhite)
-
-    MAPREDUCE-1697. Document the behavior of -file option and deprecate it
-    in favour of -files option in streaming. (Amareshwari Sriramadasu
-    via vinodkv)
-
-    MAPREDUCE-1033. Resolve location of scripts and configuration files after
-    project split. (tomwhite)
-
-    MAPREDUCE-1018. Document changes to the memory management and scheduling
-    model. (Hemanth Yamijala via vinodkv)
-
-    MAPREDUCE-1896. [Herriot] New property for multi user list. (Vinay Thota
-    via cos)
-
-    MAPREDUCE-1812. New properties for suspend and resume process. (Vinay
-    Thota via cos)
-
-  OPTIMIZATIONS
-
-    MAPREDUCE-270. Fix the tasktracker to optionally send an out-of-band
-    heartbeat on task-completion for better job-latency. (acmurthy) 
-    Configuration changes:
-      add mapreduce.tasktracker.outofband.heartbeat 
-
-    MAPREDUCE-1186. Modified code in distributed cache to set permissions
-    only on required set of localized paths.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1501. FileInputFormat supports multi-level, recursive 
-    directory listing.  (Zheng Shao via dhruba)
-
-    MAPREDUCE-1556. upgrade to Avro 1.3.0. (cutting via tomwhite)
-
-    MAPREDUCE-1613. Install/deploy source jars to Maven repo 
-    (Patrick Angeles via ddas)
-
-    MAPREDUCE-1610. Forrest documentation should be updated to reflect
-    the changes in MAPREDUCE-856. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1853. Adds caching for TaskAttemptContext in MultipleOutputs.
-    (Torsten Curdt via amareshwari)
-
-  BUG FIXES
-
-    MAPREDUCE-878. Rename fair scheduler design doc to 
-    fair-scheduler-design-doc.tex and add Apache license header (matei)
-
-    HADOOP-4687. MapReduce is split from Hadoop Core. It is a subproject under 
-    Hadoop (Owen O'Malley)
-
-    HADOOP-6096. Fix Eclipse project and classpath files following project
-    split. (tomwhite)
-
-    MAPREDUCE-419. Reconcile mapred.userlog.limit.kb defaults in configuration
-    and code. (Philip Zeyliger via cdouglas)
-
-    MAPREDUCE-2. Fixes a bug in KeyFieldBasedPartitioner in handling empty
-    keys. (Amar Kamat via sharad)
-
-    MAPREDUCE-130. Delete the jobconf copy from the log directory of the 
-    JobTracker when the job is retired. (Amar Kamat via sharad)
-
-    MAPREDUCE-657. Fix hardcoded filesystem problem in CompletedJobStatusStore.
-    (Amar Kamat via sharad)
-
-    MAPREDUCE-179. Update progress in new RecordReaders. (cdouglas)
-
-    MAPREDUCE-658. Replace NPE in distcp with a meaningful error message when
-    the source path does not exist. (Ravi Gummadi via cdouglas)
-
-    MAPREDUCE-671. Update ignore list to include untracked, generated
-    build artifacts and config files. (cdouglas)
-
-    MAPREDUCE-433. Use more reliable counters in TestReduceFetch. (cdouglas)
-
-    MAPREDUCE-124. Fix a bug in failure handling of abort task of 
-    OutputCommiter. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-694. Fix to add jsp-api jars to capacity-scheduler classpath.
-    (Giridharan Kesavan)
-    
-    MAPREDUCE-702. Fix eclipse-plugin jar target (Giridharan Kesavan) 
-
-    MAPREDUCE-522. Replace TestQueueCapacities with simpler test case to
-    test integration between capacity scheduler and MR framework.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-683. Fixes an initialization problem in the JobHistory. 
-    The initialization of JobHistoryFilesManager is now done in the 
-    JobHistory.init call. (Amar Kamat via ddas)
-
-    MAPREDUCE-708. Fixes a bug to allow updating the reason for
-    blacklisting a node on the JobTracker UI.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-709. Fixes message displayed for a blacklisted node where
-    the reason for blacklisting is due to the health check script
-    timing out. (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-676. Existing diagnostic rules fail for MAP ONLY jobs.
-    (Suhas Gogate via tomwhite)
-
-    MAPREDUCE-722. Fixes a bug with tasktracker reservations for
-    high memory jobs in capacity scheduler.
-    (Vinod Kumar Vavilapalli via yhemanth)
-
-    HADOOP-6090. Updates gridmix script to use new mapreduce api output 
-    format. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-732. Removed spurious log statements in the node
-    blacklisting logic. (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-734. Fix a ConcurrentModificationException in unreserving
-    unused reservations for a job when it completes.
-    (Arun Murthy and Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-733. Fix a RuntimeException while unreserving trackers
-    that are blacklisted for a job.
-    (Arun Murthy and Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-677. Fix timeout in TestNodeRefresh. (Amar Kamat via 
-    sharad)
-
-    MAPREDUCE-153. Fix timeout in TestJobInProgressListener. (Amar 
-    Kamat via sharad)
-
-    MAPREDUCE-742. Fix output messages and java comments in the Pi related
-    examples.  (szetszwo)
-
-    MAPREDUCE-565. Fix partitioner to work with new API. (Owen O'Malley via
-    cdouglas)
-    
-    MAPREDUCE-680. Fix so MRUnit can handle reuse of Writable objects.
-    (Aaron Kimball via johan)
-
-    MAPREDUCE-18. Puts some checks for cross checking whether a reduce
-    task gets the correct shuffle data. (Ravi Gummadi via ddas)
-
-    MAPREDUCE-771. Fix scheduling of setup and cleanup tasks to use
-    free slots instead of tasks for scheduling. (yhemanth)
-
-    MAPREDUCE-717. Fixes some corner case issues in speculative 
-    execution heuristics. (Devaraj Das)
-
-    MAPREDUCE-716. Make DBInputFormat work with Oracle. (Aaron Kimball
-    via tomwhite)
-
-    MAPREDUCE-735. Fixes a problem in the KeyFieldHelper to do with 
-    the end index for some inputs (Amar Kamat via ddas)
-
-    MAPREDUCE-682. Removes reservations on tasktrackers which are
-    blacklisted. (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-743. Fixes a problem to do with progress reporting
-    in the map phase. (Ravi Gummadi via ddas)
-
-    MAPREDUCE-765. Eliminate the deprecated warnings introduced by H-5438.
-    (He Yongqiang via szetszwo)
-
-    MAPREDUCE-383. Fix a bug in Pipes combiner due to bytes count not 
-    getting reset after the spill. (Christian Kunz via sharad)
-
-    MAPREDUCE-809. Fix job-summary logs to correctly record status of FAILED
-    and KILLED jobs.  (acmurthy)
-
-    MAPREDUCE-792. Fix unchecked warnings in DBInputFormat.  (Aaron Kimball
-    via szetszwo)
-
-    MAPREDUCE-760. Fix a timing issue in TestNodeRefresh. (Amar Kamat via 
-    sharad)
-
-    MAPREDUCE-40. Keep memory management backwards compatible for job
-    configuration parameters and limits. (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-587. Fixes a OOM issue in TestStreamingExitStatus.
-    (Amar Kamat via ddas) 
-
-    MAPREDUCE-408. Fixes an assertion problem in TestKillSubProcesses
-    (Ravi Gummadi via ddas)
-     
-    MAPREDUCE-659. Fix gridmix2 compilation. (Giridharan Kesavan)
-
-    MAPREDUCE-796. Fixes a ClassCastException in an exception log in
-    MultiThreadedMapRunner. (Amar Kamat via ddas)
-
-    MAPREDUCE-808. Fixes a serialization problem in TypedBytes.
-    (Klaas Bosteels via ddas)
-
-    MAPREDUCE-845. Fix a findbugs heap size problem in build.xml and add
-    a new property findbugs.heap.size.  (Lee Tucker via szetszwo)
-
-    MAPREDUCE-838. Fixes a problem in the way commit of task outputs
-    happens. The bug was that even if commit failed, the task would
-    be declared as successful. (Amareshwari Sriramadasu via ddas)
-
-    MAPREDUCE-813. Updates Streaming and M/R tutorial documents.
-    (Corinne Chandel via ddas)
-
-    MAPREDUCE-805. Fixes some deadlocks in the JobTracker due to the fact
-    the JobTracker lock hierarchy wasn't maintained in some JobInProgress
-    method calls. (Amar Kamat via ddas)
-    
-    MAPREDUCE-799. Fixes so all of the MRUnit self-tests run.
-    (Aaron Kimball via johan)    
-
-    MAPREDUCE-848. Fixes a problem to do with TestCapacityScheduler
-    failing (Amar Kamat via ddas)
-
-    MAPREDUCE-840. DBInputFormat leaves open transaction.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-859. Adds Avro and its dependencies required by Hadoop 
-    common. (Ravi Gummadi via sharad)
-    
-    MAPREDUCE-867. Fix ivy conf to look for avro jar from maven repo.	
-    (Giridharan Kesavan)
-
-    MAPREDUCE-877. Added avro as a dependency to contrib ivy settings.
-    (Tsz Wo (Nicholas) Sze via yhemanth)
-
-    MAPREDUCE-852. In build.xml, remove the Main-Class, which is incorrectly
-    set in tools, and rename the target "tools-jar" to "tools".  (szetszwo)
-
-    MAPREDUCE-773. Sends progress reports for compressed gzip inputs in maps.
-    Fixes a native direct buffer leak in LineRecordReader classes.
-    (Hong Tang and ddas)
-
-    MAPREDUCE-832. Reduce number of warning messages printed when
-    deprecated memory variables are used. (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-745. Fixes a testcase problem to do with generation of JobTracker
-    IDs. (Amar Kamat via ddas)
-
-    MAPREDUCE-834. Enables memory management on tasktrackers when old
-    memory management parameters are used in configuration.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-818. Fixes Counters#getGroup API. (Amareshwari Sriramadasu 
-    via sharad)
-
-    MAPREDUCE-807. Handles the AccessControlException during the deletion of
-    mapred.system.dir in the JobTracker. The JobTracker will bail out if it
-    encounters such an exception. (Amar Kamat via ddas)
-
-    MAPREDUCE-430. Fix a bug related to task getting stuck in case of 
-    OOM error. (Amar Kamat via ddas)
-
-    MAPREDUCE-871. Fix ownership of Job/Task local files to have correct 
-    group ownership according to the egid of the tasktracker.
-    (Vinod Kumar Vavilapalli via yhemanth) 
-
-    MAPREDUCE-911. Fix a bug in TestTaskFail related to speculative 
-    execution. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-687. Fix an assertion in TestMiniMRMapRedDebugScript.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-924. Fixes the TestPipes testcase to use Tool.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-903. Add Avro jar to eclipse classpath.
-    (Philip Zeyliger via tomwhite)
-
-    MAPREDUCE-943. Removes a testcase in TestNodeRefresh that doesn't make 
-    sense in the new Job recovery model. (Amar Kamat via ddas)
-
-    MAPREDUCE-764. TypedBytesInput's readRaw() does not preserve custom type
-    codes. (Klaas Bosteels via tomwhite)
-
-    HADOOP-6243. Fixes a NullPointerException in handling deprecated keys.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-968. NPE in distcp encountered when placing _logs directory on
-    S3FileSystem. (Aaron Kimball via tomwhite)
- 
-    MAPREDUCE-826. harchive doesn't use ToolRunner / harchive returns 0 even
-    if the job fails with exception (koji Noguchi via mahadev)
-
-    MAPREDUCE-839. unit test TestMiniMRChildTask fails on mac os-x (hong tang
-    via mahadev)
-
-    MAPREDUCE-112. Add counters for reduce input, output records to the new API.
-    (Jothi Padmanabhan via cdouglas)
-
-    MAPREDUCE-648. Fix two distcp bugs: (1) it should not launch a job if all
-    src paths are directories, and (2) it does not skip copying when updating
-    a single file.  (Ravi Gummadi via szetszwo)
-
-    MAPREDUCE-946. Fix a regression in LineRecordReader where the
-    maxBytesToConsume parameter is not set correctly. (cdouglas)
-
-    MAPREDUCE-977. Missing jackson jars from Eclipse template. (tomwhite)
-
-    MAPREDUCE-988. Fix a packaging issue in the contrib modules. (Hong Tang via
-    cdouglas)
-
-    MAPREDUCE-971. distcp does not always remove distcp.tmp.dir. (Aaron Kimball
-    via tomwhite)
-
-    MAPREDUCE-995. Fix a bug in JobHistory where tasks completing after the job
-    is closed cause a NPE. (Jothi Padmanabhan via cdouglas)
-
-    MAPREDUCE-953. Fix QueueManager to dump queue configuration in JSON format.
-    (V.V. Chaitanya Krishna via yhemanth)
-
-    MAPREDUCE-645. Prevent distcp from running a job when the destination is a
-    file, but the source is not. (Ravi Gummadi via cdouglas)
-
-    MAPREDUCE-1002. Flushed writer in JobQueueClient so queue information is
-    printed correctly. (V.V. Chaitanya Krishna via yhemanth)
-
-    MAPREDUCE-1003. Fix compilation problem in eclipse plugin when
-    eclipse.home is set. (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-941. Vaidya script fails on Solaris. (Chad Metcalf
-    via tomwhite)
-
-    MAPREDUCE-912. Add and standardize Apache license headers. (Chad Metcalf
-    via cdouglas)
-
-    MAPREDUCE-1022. Fix compilation of vertica testcases. (Vinod Kumar 
-    Vavilapalli via acmurthy)
-
-    MAPREDUCE-1000. Handle corrupt history files in JobHistory.initDone().
-    (Jothi Padmanabhan via sharad)
-
-    MAPREDUCE-1028. Fixed number of slots occupied by cleanup tasks to one
-    irrespective of slot size for the job.
-    (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-964. Fixed start and finish times of TaskStatus to be
-    consistent, thereby fixing inconsistencies in metering tasks.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-1076. Deprecate ClusterStatus and add javadoc in ClusterMetrics.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return
-    values of new configuration variables when deprecated variables are
-    disabled. (Sreekanth Ramakrishnan via yhemanth)
-   
-    MAPREDUCE-1030. Modified scheduling algorithm to return a map and reduce
-    task per heartbeat in the capacity scheduler.
-    (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-1071. Use DataInputStream rather than FSDataInputStream in the
-    JobHistory EventReader. (Hong Tang via cdouglas)
-
-    MAPREDUCE-986. Fix Rumen to work with truncated task lines. (Dick King via
-    cdouglas)
-
-    MAPREDUCE-1029. Fix failing TestCopyFiles by restoring the unzipping of
-    HDFS webapps from the hdfs jar. (Aaron Kimball and Jothi Padmanabhan via
-    cdouglas)
-
-    MAPREDUCE-769. Make findbugs and javac warnings to zero.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-1104. Initialize RecoveryManager in JobTracker cstr called by
-    Mumak. (Hong Tang via cdouglas)
-
-    MAPREDUCE-1061. Add unit test validating byte specifications for gridmix
-    jobs. (cdouglas)
-
-    MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
-    successful. (Dick King via cdouglas)
-
-    MAPREDUCE-1041. Make TaskInProgress::taskStatuses map package-private.
-    (Jothi Padmanabhan via cdouglas)
-
-    MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
-    (Todd Lipcon via cdouglas)
-
-    MAPREDUCE-1086. Setup Hadoop logging environment for tasks to point to
-    task related parameters. (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-1105. Remove max limit configuration in capacity scheduler in
-    favor of max capacity percentage thus allowing the limit to go over
-    queue capacity. (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-1016.  Make the job history log format JSON.  (cutting)
-
-    MAPREDUCE-1038. Weave Mumak aspects only if related files have changed.
-    (Aaron Kimball via cdouglas)
-
-    MAPREDUCE-1163. Remove unused, hard-coded paths from libhdfs. (Allen
-    Wittenauer via cdouglas)
-
-    MAPREDUCE-962. Fix a NullPointerException while killing task process 
-    trees. (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-1177. Correct setup/cleanup inversion in
-    JobTracker::getTaskReports. (Vinod Kumar Vavilapalli via cdouglas)
-
-    MAPREDUCE-1178. Fix ClassCastException in MultipleInputs by adding 
-    a DelegatingRecordReader. (Amareshwari Sriramadasu and Jay Booth 
-    via sharad)
-
-    MAPREDUCE-1068. Fix streaming job to show proper message if file is 
-    is not present. (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-1147. Add map output counters to new API. (Amar Kamat via
-    cdouglas)
-
-    MAPREDUCE-915. The debug scripts are run as the job user. (ddas)
-
-    MAPREDUCE-1007. Fix NPE in CapacityTaskScheduler.getJobs(). 
-    (V.V.Chaitanya Krishna via sharad)
-
-    MAPREDUCE-28. Refactor TestQueueManager and fix default ACLs.
-    (V.V.Chaitanya Krishna and Rahul K Singh via sharad)
-
-    MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed the
-    configured threshold. (cdouglas)
-
-    MAPREDUCE-1239. Fix contrib components build dependencies. 
-    (Giridharan Kesavan and omalley) 
-
-    MAPREDUCE-787. Fix JobSubmitter to honor user given symlink path.
-    (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-1249. Update config default value for socket read timeout to
-    match code default. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1161. Remove ineffective synchronization in NotificationTestCase.
-    (Owen O'Malley via cdouglas)
-
-    MAPREDUCE-1244. Fix eclipse-plugin's build dependencies. (gkesavan)
-
-    MAPREDUCE-1075. Fix JobTracker to not throw an NPE for a non-existent
-    queue. (V.V.Chaitanya Krishna via yhemanth)
-
-    MAPREDUCE-754. Fix NPE in expiry thread when a TT is lost. (Amar Kamat 
-    via sharad)
-
-    MAPREDUCE-1074. Document Reducer mark/reset functionality. (Jothi
-    Padmanabhan via cdouglas)
-
-    MAPREDUCE-1267. Fix typo in mapred-default.xml. (Todd Lipcon via cdouglas)
-
-    MAPREDUCE-952. Remove inadvertently reintroduced Task.Counter enum. (Jothi
-    Padmanabhan via cdouglas)
-
-    MAPREDUCE-1230. Fix handling of null records in VerticaInputFormat. (Omer
-    Trajman via cdouglas)
-
-    MAPREDUCE-1171. Allow shuffle retries and read-error reporting to be
-    configurable. (Amareshwari Sriramadasu via acmurthy)
-
-    MAPREDUCE-879. Fix broken unit test TestTaskTrackerLocalization on MacOS.
-    (Sreekanth Ramakrishnan via yhemanth)
-
-    MAPREDUCE-1124. Fix imprecise byte counts in Gridmix. (cdouglas)
-
-    MAPREDUCE-1222. Add an option to exclude numeric IP addresses in topologies
-    processed by Mumak. (Hong Tang via cdouglas)
-
-    MAPREDUCE-1284. Fix fts_open() call in task-controller that was failing
-    LinuxTaskController unit tests. (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-1143. Fix running task counters to be updated correctly
-    when speculative attempts are running for a TIP.
-    (Rahul Kumar Singh via yhemanth)
-
-    MAPREDUCE-1241. Use a default queue configuration in JobTracker when
-    mapred-queues.xml is unavailable. (Todd Lipcon via cdouglas)
-
-    MAPREDUCE-1301. Fix set up of permission checking script used in 
-    localization tests. (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1286. Remove quoting from client opts in TaskRunner. (Yuri
-    Pradkin via cdouglas)
-
-    MAPREDUCE-1059. Use distcp.bytes.per.map when adding sync markers in
-    distcp. (Aaron Kimball via cdouglas)
-
-    MAPREDUCE-1009. Update forrest documentation describing hierarchical
-    queues. (Vinod Kumar Vavilapalli via yhemanth)
-
-    MAPREDUCE-1342. Fixed deadlock in global blacklisting of tasktrackers.
-    (Amareshwari Sriramadasu via acmurthy)
-
-    MAPREDUCE-1316. Fixes a memory leak of TaskInProgress instances in
-    the jobtracker. (Amar Kamat via yhemanth)
-
-    MAPREDUCE-1359. TypedBytes TestIO doesn't mkdir its test dir first.
-    (Anatoli Fomenko via cos)
-
-    MAPREDUCE-1314. Correct errant mapreduce.x.mapreduce.x replacements from
-    bulk change. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1365. Restore accidentally renamed test in
-    TestTaskTrackerBloacklisting. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1406. Fix spelling of JobContext.MAP_COMBINE_MIN_SPILLS.
-    (cdouglas)
-
-    MAPREDUCE-1369. JUnit tests should never depend on anything in conf
-    (Anatoli Fomenko via cos)
-
-    MAPREDUCE-1412. Fix timer granularity issue causing failures in
-    TestTaskTrackerBlacklisting. (cdouglas)
-
-    MAPREDUCE-1448. Respect --config option in Mumak script. (Hong Tang via
-    cdouglas)
-
-    MAPREDUCE-1251. c++ utils doesn't compile. (Eli Collins via tomwhite)
-
-    MAPREDUCE-1522. FileInputFormat may use the default FileSystem for the
-    input path. (Tsz Wo (Nicholas), SZE via cdouglas)
-
-    MAPREDUCE-1407. Update javadoc in mapreduce.{Mapper,Reducer} to match
-    actual usage. (Benoit Sigoure via cdouglas)
-
-    MAPREDUCE-1258. Fix fair scheduler event log not logging job info.
-    (matei)
-
-    MAPREDUCE-1089. Fix NPE in fair scheduler preemption when tasks are
-    scheduled but not running. (Todd Lipcon via matei)
-
-    MAPREDUCE-1014. Fix the libraries for common and hdfs. (omalley)
-
-    MAPREDUCE-1111. JT Jetty UI not working if we run mumak.sh 
-    off packaged distribution directory. (hong tang via mahadev)
-
-    MAPREDUCE-1133. Eclipse .classpath template has outdated jar files and is
-    missing some new ones. (cos)
-
-    MAPREDUCE-1098. Fixed the distributed-cache to not do i/o while holding a
-    global lock. (Amareshwari Sriramadasu via acmurthy)
-
-    MAPREDUCE-1158. Fix JT running maps and running reduces metrics.
-    (sharad)
-
-    MAPREDUCE-1160. Reduce verbosity of log lines in some Map/Reduce classes
-    to avoid filling up jobtracker logs on a busy cluster.
-    (Ravi Gummadi and Hong Tang via yhemanth)
-
-    MAPREDUCE-1153. Fix tasktracker metrics when trackers are decommissioned.
-    (sharad)
-
-    MAPREDUCE-1128. Fix MRUnit to prohibit iterating over values twice. (Aaron
-    Kimball via cdouglas)
-
-    MAPREDUCE-665. Move libhdfs to HDFS subproject. (Eli Collins via dhruba)
-
-    MAPREDUCE-1196. Fix FileOutputCommitter to use the deprecated cleanupJob
-    api correctly. (acmurthy)
-   
-    MAPREDUCE-1244. Fix eclipse-plugin's build dependencies. (gkesavan)
-
-    MAPREDUCE-1140. Fix DistributedCache to not decrement reference counts for
-    unreferenced files in error conditions.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1245. Fix TestFairScheduler failures by instantiating lightweight
-    Jobtracker. (sharad)
-
-    MAPREDUCE-1260. Update Eclipse configuration to match changes to Ivy
-    configuration. (Edwin Chan via cos)
-
-    MAPREDUCE-1152. Distinguish between failed and killed tasks in
-    JobTrackerInstrumentation. (Sharad Agarwal via cdouglas)
-
-    MAPREDUCE-1285. In DistCp.deleteNonexisting(..), get class from the
-    parameter instead of using FileStatus.class.  (Peter Romianowski via
-    szetszwo)
-
-    MAPREDUCE-1294. Build fails to pull latest hadoop-core-* artifacts (cos)
-
-    MAPREDUCE-1213. TaskTrackers restart is faster because it deletes
-    distributed cache directory asynchronously. (Zheng Shao via dhruba)
-
-    MAPREDUCE-1265. The task attempt error log prints the name of the 
-    tasktracker machine. (Scott Chen via dhruba)
-
-    MAPREDUCE-1201. ProcfsBasedProcessTree collects CPU usage information.
-    (Scott Chen via dhruba)
-
-    MAPREDUCE-1326. fi tests don't use fi-site.xml (cos)
-
-    MAPREDUCE-1165. Replace non-portable function name with C99 equivalent.
-    (Allen Wittenauer via cdouglas)
-
-    MAPREDUCE-1331. Fixes a typo in a testcase (Devaraj Das)
-
-    MAPREDUCE-1293. AutoInputFormat doesn't work with non-default FileSystems.
-    (Andrew Hitchcock via tomwhite)
-
-    MAPREDUCE-1131. Using profilers other than hprof can cause JobClient to
-    report job failure. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1155. Streaming tests swallow exceptions.
-    (Todd Lipcon via tomwhite)
-
-    MAPREDUCE-1212. Mapreduce contrib project ivy dependencies are not included
-    in binary target. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1388. Move the HDFS RAID package from HDFS to MAPREDUCE.
-    (Eli Collins via dhruba)
-
-    MAPREDUCE-1322. Defines default value for staging directory to be user
-    based fixing a failing streaming test.
-    (Devaraj Das and Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-899. Modified LinuxTaskController to check that task-controller
-    has right permissions and ownership before performing any actions.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1443. DBInputFormat can leak connections.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1457. Fixes JobTracker to get the FileSystem object within 
-    getStagingAreaDir within a privileged block. Fixes Child.java to use the
-    appropriate UGIs while getting the TaskUmbilicalProtocol proxy and 
-    while executing the task. (Jakob Homan via ddas)
-
-    MAPREDUCE-1399. The archive command shows a null error message (nicholas
-    via mahadev)
-
-    MAPREDUCE-1305. Improve efficiency of distcp -delete. (Peter Romianowski
-    via cdouglas)
-
-    MAPREDUCE-1474. Update forrest documentation for Hadoop Archives. (Mahadev
-    Konar via cdouglas)
-
-    MAPREDUCE-1400. Use tr rather than sed to effect literal substitution in
-    the build script. (Allen Wittenauer via cdouglas)
-
-    MAPREDUCE-1358. Avoid false positives in OutputLogFilter. (Todd Lipcon via
-    cdouglas)
-
-    MAPREDUCE-1490. Fix a NullPointerException that could occur during 
-    instantiation and initialization of the DistributedRaidFileSystem. 
-    (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1476. Fix the M/R framework to not call commit for special
-    tasks like job setup/cleanup and task cleanup.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1398. Fix TaskLauncher to stop waiting for slots on a TIP that
-    is killed / failed.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1491. The parity files created by the RAID are combined
-    using Hadoop Archive Files (HAR).  (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1378. URL encode link in jobhistory.jsp to avoid errors caused by
-    unescaped characters. (E. Sammer via cdouglas)
-
-    MAPREDUCE-1519. RaidNode fails to create new parity file 
-    if an older version already exists. (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1537. Fixes a compilation problem in a testcase after commit
-    HDFS-984. (Jitendra Nath Pandey via ddas)
-
-    MAPREDUCE-1537. The patch makes the job client call the getDelegationToken
-    only when security is enabled. (Jitendra Nath Pandey via ddas)
-
-    MAPREDUCE-1510. RAID should regenerate parity files if they get deleted.
-    (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1421. Fix the LinuxTaskController tests failing on trunk after
-    the commit of MAPREDUCE-1385. (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1520. Fix TestMiniMRLocalFS failure caused by regression in
-    getting user working dir. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1512. RAID uses HarFileSystem directly instead of
-    FileSystem.get (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1435. Fix symlink handling in task work directory when
-    cleaning up, essentially to avoid following links.
-    (Ravi Gummadi via yhemanth)
-
-    MAPREDUCE-1518. RaidNode does not run the deletion check on the
-    directory that stores the parity files.  (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1573. TestStreamingAsDifferentUser fails if run as tt_user.
-    (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-927. Cleanup of task-logs should happen in TaskTracker instead
-    of the Child. (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1578. Decouple HadoopArchives vesrion from HarFileSystem version.
-    (Rodrigo Schmidt via szetszwo)
-
-    MAPREDUCE-1422. Fix cleanup of localized job directory to work if files
-    with non-deletable permissions are created within it.
-    (Amar Kamat via yhemanth)
-
-    MAPREDUCE-1306. Randomize the arrival of heartbeat responses in Mumak.
-    (Tamas Sarlos via cdouglas)
-
-    MAPREDUCE-1579. archive: check and possibly replace the space charater
-    in source paths.  (szetszwo)
-
-    MAPREDUCE-1536. DataDrivenDBInputFormat does not split date columns correctly.
-    (Aaron Kimball via enis)
-
-    MAPREDUCE-890. After HADOOP-4491, the user who started mapred system is 
-    not able to run job. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1615. Fix compilation of TestSubmitJob. (cdouglas)
-
-    MAPREDUCE-1508. Protect against NPE in TestMultipleLevelCaching. (Aaron
-    Kimball via cdouglas)
-
-    MAPREDUCE-1497. Suppress spurious findbugs warning about IndexCache
-    synchronization. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1420. Fix TestTTResourceReporting failure. (Scott Chen via
-    cdouglas)
-
-    MAPREDUCE-1480. Correctly initialize child RecordReaders in
-    CombineFileInputFormat. (Aaron Kimball via cdouglas)
-
-    MAPREDUCE-1348. Fix block forensics packaging. (Tom White via cdouglas)
-
-    MAPREDUCE-1628. HarFileSystem shows incorrect replication numbers and
-    permissions.  (szetszwo via mahadev)
-
-    MAPREDUCE-1602. Fix the error message for the case that src does not
-    exist.  (szetszwo)
-
-    MAPREDUCE-1585. Create Hadoop Archives version 2 with filenames
-    URL-encoded (rodrigo via mahadev)
-
-    MAPREDUCE-1523. Sometimes rumen trace generator fails to extract the job
-    finish time. (dick king via mahadev)
-
-    MAPREDUCE-1635. ResourceEstimator does not work after MAPREDUCE-842.
-    (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-889. binary communication formats added to Streaming by
-    HADOOP-1722 should be documented. (Klaas Bosteels via tomwhite)
-
-    MAPREDUCE-1031. ant tar target doens't seem to compile tests in contrib
-    projects. (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1692. Removed unused testcase TestStreamedMerge.
-    (Sreekanth Ramakrishnan and Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1062. Fix ReliabilityTest to work with retired jobs. (Sreekanth
-    Ramakrishnan via cdouglas)
-
-    MAPREDUCE-1409. IOExceptions thrown from FIleOutputCommitter::abortTask
-    should cause the task to fail. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev
-    targets and also fix existing warnings. (Hong Tang via yhemanth)
-
-    MAPREDUCE-1494. Ensure TestJobDirCleanup verifies the correct paths.
-    (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1622. Include missing slf4j dependencies. (cdouglas)
-
-    MAPREDUCE-1515. Accept java5.home from build.properties, not only from the
-    command line when building forrest docs. (Al Thompson via cdouglas)
-
-    MAPREDUCE-1618. Add missing javadoc to JobStatus::*JobAcls. (Amareshwari
-    Sriramadasu via cdouglas)
-
-    MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease 
-    undue load on jobtracker. (Sreekanth Ramakrishnan via sharad)
-
-    MAPREDUCE-1604. Add Forrest documentation for Job ACLs.
-    (Amareshwari Sriramadasu via yhemanth)
-
-    MAPREDUCE-1705. Archiving and Purging of HDFS parity files should 
-    handle globbed policies accurately. (Rodrigo Schmidt via dhruba)
-
-    MAPREDUCE-1612. job conf file is not accessible from job history web page.
-    (Ravi Gummadi and Sreekanth Ramakrishnan via vinodkv)
-
-    MAPREDUCE-1397. NullPointerException observed during task failures.
-    (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1728. Oracle timezone strings do not match Java.
-    (Aaron Kimball via tomwhite)
-
-    MAPREDUCE-1609. TaskTracker.localizeJob should not set permissions on 
-    job log directory recursively. (Amareshwari Sriramadasu via vinodkv)
-
-    MAPREDUCE-1657. After task logs directory is deleted, tasklog servlet 
-    displays wrong error message about job ACLs. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1727. TestJobACLs fails after HADOOP-6686. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1611. Refresh nodes and refresh queues doesnt work with service
-    authorization enabled. (Amar Kamat via vinodkv)
-
-    MAPREDUCE-1276. Correct flaws in the shuffle related to connection setup
-    and failure attribution. (Amareshwari Sriramadasu via cdouglas)
-
-    MAPREDUCE-1372. ConcurrentModificationException in JobInProgress.
-    (Dick King and Amareshwari Sriramadasu via tomwhite)
-
-    MAPREDUCE-118. Fix Job.getJobID(). (Amareshwari Sriramadasu via sharad)
-
-    MAPREDUCE-913. TaskRunner crashes with NPE resulting in held up slots,
-    UNINITIALIZED tasks and hung TaskTracker. (Amareshwari Sriramadasu and
-    Sreekanth Ramakrishnan via vinodkv)
-
-    MAPREDUCE-1725.  Fix MapReduce API incompatibilities between 0.20 and 0.21.
-    (tomwhite)
-
-    MAPREDUCE-1606. TestJobACLs may timeout as there are no slots for launching
-    JOB_CLEANUP task. (Ravi Gummadi via vinodkv)
-
-    MAPREDUCE-1765. Correct streaming documentation for StreamXmlRecordReader.
-    (Corinne Chandel via amareshwari)
-
-    MAPREDUCE-1880. Fix BigDecimal.divide(..) in the pi example.  (szetszwo)
-
-    MAPREDUCE-1885. Revert FileSystem create method that takes CreateFlags
-    (MapReduce part of HADOOP-6826). (Ravi Gummadi via tomwhite)
-
-    MAPREDUCE-1870. Harmonize MapReduce JAR library versions with Common and
-    HDFS. (tomwhite)
-
-    MAPREDUCE-1791. Remote cluster control functionality needs JavaDocs
-    improvement (Konstantin Boudnik)
-
-    MAPREDUCE-1942. 'compile-fault-inject' should never be called directly.
-    (Konstantin Boudnik)
-
-    MAPREDUCE-1876. Fixes TaskAttemptStartedEvent to correctly log event type
-    for all task types. (Amar Kamat via amareshwari)
-
-    MAPREDUCE-1926. MapReduce distribution is missing build-utils.xml.
-    (tomwhite)
-
-    MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk.
-    (Amareshwari Sriramadasu via tomwhite)
-
-    MAPREDUCE-1980. Fixes TaskAttemptUnsuccessfulCompletionEvent and
-    TaskAttemptFinishedEvent to correctly log event type for all task types.
-    (Amar Kamat via amareshwari)
-
-    MAPREDUCE-1856. Extract a subset of tests for smoke (DOA) validation (cos)
-

hadoop-mapreduce/INSTALL

@@ -1,89 +0,0 @@
-To compile  Hadoop Mapreduce next following, do the following:
-
-Step 1) Install dependencies for yarn
-
-See http://svn.apache.org/repos/asf/hadoop/common/branches/MR-279/mapreduce/yarn/README
-Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib
-
-Step 2) Checkout
-
-svn checkout http://svn.apache.org/repos/asf/hadoop/common/branches/MR-279/
-
-Step 3) Build common
-
-Go to common directory
-ant veryclean mvn-install 
-
-Step 4) Build HDFS 
-
-Go to hdfs directory
-ant veryclean mvn-install -Dresolvers=internal 
-
-Step 5) Build yarn and mapreduce
-
-Go to mapreduce directory
-export MAVEN_OPTS=-Xmx512m
-
-mvn clean install assembly:assembly
-ant veryclean jar jar-test  -Dresolvers=internal 
-
-In case you want to skip the tests run:
-
-mvn clean install assembly:assembly -DskipTests
-ant veryclean jar jar-test  -Dresolvers=internal 
-
-You will see a tarball in
-ls target/hadoop-mapreduce-1.0-SNAPSHOT-bin.tar.gz  
-
-Step 6) Untar the tarball in a clean and different directory.
-say HADOOP_YARN_INSTALL
-
-To run Hadoop Mapreduce next applications :
-
-Step 7) cd $HADOOP_YARN_INSTALL
-
-Step 8) export the following variables:
-
-HADOOP_MAPRED_HOME=
-HADOOP_COMMON_HOME=
-HADOOP_HDFS_HOME=
-YARN_HOME=directory where you untarred yarn
-HADOOP_CONF_DIR=
-YARN_CONF_DIR=$HADOOP_CONF_DIR
-
-Step 9) bin/yarn-daemon.sh start resourcemanager
-
-Step 10) bin/yarn-daemon.sh start nodemanager
-
-Step 11) bin/yarn-daemon.sh start historyserver
-
-Step 12) Create the following symlinks in hadoop-common/lib 
-
-ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar .	
-ln -s $HADOOP_YARN_INSTALL/modules/yarn-api-1.0-SNAPSHOT.jar .
-ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-common-1.0-SNAPSHOT.jar .	
-ln -s $HADOOP_YARN_INSTALL/modules/yarn-common-1.0-SNAPSHOT.jar .
-ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-core-1.0-SNAPSHOT.jar .	
-ln -s $HADOOP_YARN_INSTALL/modules/yarn-server-common-1.0-SNAPSHOT.jar .
-ln -s $HADOOP_YARN_INSTALL/modules/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar .
-ln -s $HADOOP_YARN_INSTALL/lib/protobuf-java-2.4.0a.jar .
-
-Step 13) Yarn daemons are up! But for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
-    <property>
-      <name>nodemanager.auxiluary.services</name>
-      <value>mapreduce.shuffle</value>
-    </property>
-
-    <property>
-      <name>nodemanager.aux.service.mapreduce.shuffle.class</name>
-      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
-    </property>
-
-Step 14) You are all set, an example on how to run a mapreduce job is:
-
-cd $HADOOP_MAPRED_HOME
-ant examples -Dresolvers=internal
-$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapred-examples-0.22.0-SNAPSHOT.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $HADOOP_YARN_INSTALL/hadoop-mapreduce-1.0-SNAPSHOT/modules/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar output 
-
-The output on the command line should be almost similar to what you see in the JT/TT setup (Hadoop 0.20/0.21)
-

hadoop-mapreduce/LICENSE.txt

@@ -1,244 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-
-APACHE HADOOP SUBCOMPONENTS:
-
-The Apache Hadoop project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses. 
-
-For the org.apache.hadoop.util.bloom.* classes:
-
-/**
- *
- * Copyright (c) 2005, European Commission project OneLab under contract
- * 034819 (http://www.one-lab.org)
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or 
- * without modification, are permitted provided that the following 
- * conditions are met:
- *  - Redistributions of source code must retain the above copyright 
- *    notice, this list of conditions and the following disclaimer.
- *  - Redistributions in binary form must reproduce the above copyright 
- *    notice, this list of conditions and the following disclaimer in 
- *    the documentation and/or other materials provided with the distribution.
- *  - Neither the name of the University Catholique de Louvain - UCL
- *    nor the names of its contributors may be used to endorse or 
- *    promote products derived from this software without specific prior 
- *    written permission.
- *    
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
- * POSSIBILITY OF SUCH DAMAGE.
- */

hadoop-mapreduce/NOTICE.txt

@@ -1,2 +0,0 @@
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).

hadoop-mapreduce/assembly/all.xml

@@ -1,101 +0,0 @@
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
-  <id>all</id>
-  <formats>
-    <format>tar.gz</format>
-  </formats>
-  <includeBaseDirectory>true</includeBaseDirectory>
-  <!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
-  <fileSets>
-    <fileSet>
-      <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>container-executor</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>hadoop-yarn/bin</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>bin</directory>
-      <outputDirectory>bin</outputDirectory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-    <fileSet>
-      <directory>hadoop-yarn/conf</directory>
-      <outputDirectory>conf</outputDirectory>
-      <includes>
-        <include>**/*</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <excludes>
-        <exclude>**/*.jar</exclude>
-        <exclude>**/target/**</exclude>
-        <!-- scripts to include later for setting fileMode -->
-        <exclude>**/bin/*</exclude>
-        <exclude>**/scripts/*</exclude>
-        <!-- images that we don't need (and cause problems for our tools) -->
-        <exclude>**/dt-*/images/**</exclude>
-        <!-- until the code that does this is fixed -->
-        <exclude>**/file:/**</exclude>
-        <exclude>**/SecurityAuth.audit*</exclude>
-      </excludes>
-      <includes>
-        <include>assembly/**</include>
-        <include>pom.xml</include>
-        <include>build*.xml</include>
-        <include>ivy.xml</include>
-        <include>ivy/**</include>
-        <include>INSTALL</include>
-        <include>LICENSE.txt</include>
-        <include>mr-client/**</include>
-        <include>hadoop-yarn/**</include>
-        <include>src/**</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <outputDirectory>sources</outputDirectory>
-      <includes>
-        <include>**/bin/*</include>
-        <include>**/scripts/*</include>
-      </includes>
-      <fileMode>0755</fileMode>
-    </fileSet>
-  </fileSets>
-  <moduleSets>
-    <moduleSet>
-      <excludes>
-        <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
-      </excludes>
-      <binaries>
-        <outputDirectory>modules</outputDirectory>
-        <includeDependencies>false</includeDependencies>
-        <unpack>false</unpack>
-      </binaries>
-    </moduleSet>
-  </moduleSets>
-  <dependencySets>
-    <dependencySet>
-      <useProjectArtifact>false</useProjectArtifact>
-      <outputDirectory>/lib</outputDirectory>
-      <!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
-      <excludes>
-        <exclude>org.apache.hadoop:hadoop-common</exclude>
-        <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
-      </excludes>
-    </dependencySet>
-  </dependencySets>
-</assembly>

hadoop-mapreduce/bin/mapred

@@ -1,119 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`which $0`
-bin=`dirname ${bin}`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-function print_usage(){
-  echo "Usage: mapred [--config confdir] COMMAND"
-  echo "       where COMMAND is one of:"
-  echo "  mradmin              run a Map-Reduce admin client"
-  echo "  jobtracker           run the MapReduce job Tracker node" 
-  echo "  tasktracker          run a MapReduce task Tracker node" 
-  echo "  pipes                run a Pipes job"
-  echo "  job                  manipulate MapReduce jobs"
-  echo "  queue                get information regarding JobQueues"
-  echo "  classpath            prints the class path needed for running"
-  echo "                       mapreduce subcommands"
-  echo "  groups               get the groups which users belong to"
-  echo ""
-  echo "Most commands print help when invoked w/o parameters."
-}
-
-if [ $# = 0 ]; then
-  print_usage
-  exit
-fi
-
-COMMAND=$1
-shift
-
-if [ "$COMMAND" = "mradmin" ] ; then
-  CLASS=org.apache.hadoop.mapred.tools.MRAdmin
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "jobtracker" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobTracker
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
-elif [ "$COMMAND" = "tasktracker" ] ; then
-  CLASS=org.apache.hadoop.mapred.TaskTracker
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
-elif [ "$COMMAND" = "job" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobClient
-elif [ "$COMMAND" = "queue" ] ; then
-  CLASS=org.apache.hadoop.mapred.JobQueueClient
-elif [ "$COMMAND" = "pipes" ] ; then
-  CLASS=org.apache.hadoop.mapred.pipes.Submitter
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "sampler" ] ; then
-  CLASS=org.apache.hadoop.mapred.lib.InputSampler
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "classpath" ] ; then
-  echo -n 
-elif [ "$COMMAND" = "groups" ] ; then
-  CLASS=org.apache.hadoop.mapred.tools.GetGroups
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-else
-  echo $COMMAND - invalid command
-  print_usage
-  exit
-fi
-
-# for developers, add mapred classes to CLASSPATH
-if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
-fi
-
-# for releases, add core mapred jar & webapps to CLASSPATH
-if [ -d "$HADOOP_PREFIX/share/hadoop/mapreduce/webapps" ]; then
-  CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/mapreduce
-fi
-for f in $HADOOP_MAPRED_HOME/share/hadoop-mapreduce/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
-done
-
-# add libs to CLASSPATH
-for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
-  CLASSPATH=${CLASSPATH}:$f;
-done
-
-if $cygwin; then
-  CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-
-if [ "$COMMAND" = "classpath" ] ; then
-  echo $CLASSPATH
-  exit
-fi
-
-export CLASSPATH
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"

hadoop-mapreduce/bin/mapred-config.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# included in all the mapred scripts with source command
-# should not be executed directly
-
-bin=`which "$0"`
-bin=`dirname "${bin}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
-  . "$bin/../libexec/hadoop-config.sh"
-elif [ -e "${HADOOP_COMMON_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_COMMON_HOME"/bin/hadoop-config.sh
-elif [ -e "${HADOOP_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_HOME"/bin/hadoop-config.sh
-elif [ -e "${HADOOP_MAPRED_HOME}/bin/hadoop-config.sh" ]; then
-  . "$HADOOP_MAPRED_HOME"/bin/hadoop-config.sh
-else
-  echo "Hadoop common not found."
-  exit
-fi

hadoop-mapreduce/bin/start-mapred.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop map reduce daemons.  Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-
-# start mapred daemons
-# start jobtracker first to minimize connection errors at startup
-"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker
-"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker

hadoop-mapreduce/bin/stop-mapred.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop map reduce daemons.  Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-if [ -e $bin/../libexec/mapred-config.sh ]; then
-  . $bin/../libexec/mapred-config.sh
-else
-  . "$bin/mapred-config.sh"
-fi
-
-"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker
-"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker

hadoop-mapreduce/build-utils.xml

@@ -1,33 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!--
-Contains utilities that are common for the main and contrib builds.
--->
-<project name="build-utils">
-
-  <!-- Load properties from build properties file, if available -->
-  <dirname property="build-utils.basedir" file="${ant.file.build-utils}"/>
-  <property file="${build-utils.basedir}/build.properties"/>
-
-  <target name="forrest.check" unless="forrest.home">
-    <fail message="'forrest.home' is not defined. Please pass -Dforrest.home=&lt;base of Apache Forrest installation&gt; to Ant on the command-line, or set forest.home in build properties file." />
-  </target>
-  
-</project>

hadoop-mapreduce/build.xml

@@ -1,2514 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<project name="Hadoop" default="compile" 
-   xmlns:artifact="urn:maven-artifact-ant"
-   xmlns:ivy="antlib:org.apache.ivy.ant"> 
-
-  <import file="build-utils.xml" />
-
-  <!-- Load all the default properties, and any the user wants    -->
-  <!-- to contribute (without having to type -D or edit this file -->
-  <property file="${user.home}/build.properties" />
-  <property file="${basedir}/build.properties" />
-
-  <property name="module" value="mapreduce"/> 
-  <property name="Name" value="Hadoop-Mapred"/>
-  <property name="name" value="hadoop-${module}"/>
-  <!-- Need to change aop.xml project.version prop. synchronously -->
-  <property name="_version" value="0.23.0"/>
-  <property name="version" value="${_version}-SNAPSHOT"/>
-  <property name="final.name" value="${name}-${version}"/>
-  <property name="test.final.name" value="${name}-test-${version}"/>
-  <property name="examples.final.name" value="${name}-examples-${version}"/>
-  <property name="tools.final.name" value="${name}-tools-${version}"/>
-  <property name="year" value="2009"/>
-  <property name="package.release" value="1"/>
-
-  <property name="src.dir" value="${basedir}/src"/>  	
-  <property name="mapred.src.dir" value="${src.dir}/java"/> 
-  <property name="examples.dir" value="${basedir}/src/examples"/>
-  <property name="lib.dir" value="${basedir}/lib"/>
-  <property name="conf.dir" value="${basedir}/conf"/>
-  <property name="contrib.dir" value="${basedir}/src/contrib"/>
-  <property name="docs.src" value="${basedir}/src/docs"/>
-  <property name="changes.src" value="${docs.src}/changes"/>
-  <property name="c++.src" value="${basedir}/src/c++"/>
-  <property name="c++.utils.src" value="${c++.src}/utils"/>
-  <property name="c++.pipes.src" value="${c++.src}/pipes"/>
-  <property name="c++.examples.pipes.src" value="${examples.dir}/pipes"/>
-  <property name="librecordio.src" value="${c++.src}/librecordio"/>
-  <property name="tools.src" value="${basedir}/src/tools"/>
-
-  <property name="xercescroot" value=""/> 
-  <property name="build.dir" value="${basedir}/build"/>
-  <property name="build.classes" value="${build.dir}/classes"/>
-  <property name="build.src" value="${build.dir}/src"/>
-  <property name="build.tools" value="${build.dir}/tools"/>
-  <property name="build.webapps" value="${build.dir}/webapps"/>
-  <property name="build.examples" value="${build.dir}/examples"/>
-  <property name="build.librecordio" value="${build.dir}/librecordio"/>
-
-  <!-- convert spaces to _ so that mac os doesn't break things -->
-  <exec executable="tr" inputstring="${os.name}" 
-        outputproperty="nonspace.os">
-     <arg value="[:space:]"/>
-     <arg value="_"/>
-  </exec>
-  <property name="build.platform" 
-            value="${nonspace.os}-${os.arch}-${sun.arch.data.model}"/>
-  <property name="jvm.arch" 
-            value="${sun.arch.data.model}"/>
-  <property name="build.native" value="${build.dir}/native/${build.platform}"/>
-  <property name="build.c++" value="${build.dir}/c++-build/${build.platform}"/>
-  <property name="build.c++.utils" value="${build.c++}/utils"/>
-  <property name="build.c++.pipes" value="${build.c++}/pipes"/>
-  <property name="build.c++.examples.pipes" 
-            value="${build.c++}/examples/pipes"/>
-  <property name="build.docs" value="${build.dir}/docs"/>
-  <property name="build.javadoc" value="${build.docs}/api"/>
-  <property name="build.javadoc.timestamp" value="${build.javadoc}/index.html" />
-  <property name="build.javadoc.dev" value="${build.docs}/dev-api"/>
-  <property name="build.encoding" value="ISO-8859-1"/>
-  <property name="install.c++" value="${build.dir}/c++/${build.platform}"/>
-  <property name="install.c++.examples" 
-            value="${build.dir}/c++-examples/${build.platform}"/>
-
-  <property environment="env"/>
-  <property name="test.src.dir" value="${basedir}/src/test"/>
-  <property name="test.lib.dir" value="${basedir}/src/test/lib"/>
-  <property name="test.build.dir" value="${build.dir}/test"/>
-  <property name="test.generated.dir" value="${test.build.dir}/src"/>
-  <property name="test.build.data" value="${test.build.dir}/data"/>
-  <property name="test.cache.data" value="${test.build.dir}/cache"/>
-  <property name="test.concat.data" value="${test.build.dir}/concat"/>
-  <property name="test.debug.data" value="${test.build.dir}/debug"/>
-  <property name="test.log.dir" value="${test.build.dir}/logs"/>
-  <property name="test.build.classes" value="${test.build.dir}/classes"/>
-  <property name="test.mapred.build.testjar" value="${test.build.dir}/mapred/testjar"/>
-  <property name="test.mapred.build.testshell" value="${test.build.dir}/mapred/testshell"/>
-  <property name="test.build.extraconf" value="${test.build.dir}/extraconf"/>
-  <property name="test.build.javadoc" value="${test.build.dir}/docs/api"/>
-  <property name="test.build.javadoc.dev" value="${test.build.dir}/docs/dev-api"/>
-  <property name="test.include" value="Test*"/>
-  <property name="test.classpath.id" value="test.classpath"/>
-  <property name="test.output" value="no"/>
-  <property name="test.timeout" value="900000"/>
-  <property name="test.junit.output.format" value="plain"/>
-  <property name="test.junit.fork.mode" value="perTest" />
-  <property name="test.junit.printsummary" value="yes" />
-  <property name="test.junit.haltonfailure" value="no" />
-  <property name="test.junit.maxmemory" value="512m" />
-  <property name="test.junit.jvmargs" value="-ea" />
-  <property name="test.tmp.dir" value="${env.TMP}" />
-  <property name="test.temp.dir" value="${env.TEMP}" />
-  <property name="test.conf.dir" value="${build.dir}/test/conf" />
-
-  <property name="test.tools.input.dir" value="${basedir}/src/test/tools/data" />
-
-  <property name="test.mapred.build.classes" value="${test.build.dir}/mapred/classes"/>
-  <property name="test.mapred.commit.tests.file" value="${test.src.dir}/commit-tests" />
-  <property name="test.mapred.smoke.tests.file" value="${test.src.dir}/smoke-tests" />
-  <property name="test.mapred.all.tests.file" value="${test.src.dir}/all-tests" />
-  <property name="test.exclude.file" value="${test.src.dir}/empty-file" />
-
-  <property name="librecordio.test.dir" value="${test.build.dir}/librecordio"/>
-  <property name="web.src.dir" value="${basedir}/src/web"/>
-  <property name="src.webapps" value="${basedir}/src/webapps"/>
-
-  <property name="javadoc.link.java"
-	    value="http://java.sun.com/javase/6/docs/api/"/>
-  <property name="javadoc.packages" value="org.apache.hadoop.*"/>
-  <property name="javadoc.maxmemory" value="512m" />
-
-  <property name="dist.dir" value="${build.dir}/${final.name}"/>
-
-  <property name="javac.debug" value="on"/>
-  <property name="javac.optimize" value="on"/>
-  <property name="javac.deprecation" value="off"/>
-  <property name="javac.version" value="1.6"/>
-  <property name="javac.args" value=""/>
-  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
-
-  <property name="clover.db.dir" location="${build.dir}/test/clover/db"/>
-  <property name="clover.report.dir" location="${build.dir}/test/clover/reports"/>
-
-  <property name="rat.reporting.classname" value="rat.Report"/>
-
-  <property name="jdiff.build.dir" value="${build.docs}/jdiff"/>
-  <property name="jdiff.xml.dir" value="${lib.dir}/jdiff"/>
-  <property name="jdiff.stability" value="-unstable"/>
-  <property name="jdiff.compatibility" value=""/>
-  <property name="jdiff.stable" value="0.21.0"/>
-  <property name="jdiff.stable.javadoc" 
-            value="http://hadoop.apache.org/core/docs/r${jdiff.stable}/api/"/>
-
-  <property name="scratch.dir" value="${user.home}/tmp"/>
-  <property name="svn.cmd" value="svn"/>
-  <property name="grep.cmd" value="grep"/>
-  <property name="patch.cmd" value="patch"/>
-  <property name="make.cmd" value="make"/>
-
-  <property name="findbugs.heap.size" value="512M"/>
-  <!-- task-controller properties set here -->
-  <!-- Source directory from where configure is run and files are copied
-  -->
-	
-  <property name="c++.task-controller.src" 
-    value="${basedir}/src/c++/task-controller" />
-  <!-- directory where autoconf files + temporary files and src is 
-    stored for compilation -->
-  <property name="build.c++.task-controller" 
-    value="${build.c++}/task-controller" />
-  <!-- the default install dir is build directory override it using
-   -Dtask-controller.install.dir=$HADOOP_HOME/bin -->
-  <property name="task-controller.install.dir" value="${dist.dir}/bin" />
-  <!-- end of task-controller properties -->
-	
-  <!-- IVY properteis set here -->
-  <property name="ivy.dir" location="ivy" />
-  <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
-  <property name="mvn.repo" value="http://repo2.maven.org/maven2"/>
-  <property name="ivy.jar" location="${ivy.dir}/ivy-${ivy.version}.jar"/>
-  <property name="asfrepo" value="https://repository.apache.org"/>
-  <property name="asfsnapshotrepo" value="${asfrepo}/content/repositories/snapshots"/>
-  <property name="asfstagingrepo"
-    value="${asfrepo}/service/local/staging/deploy/maven2"/>
-  <property name="ant_task.jar" location="${ivy.dir}/maven-ant-tasks-${ant-task.version}.jar"/>
-  <property name="ant_task_repo_url" value="${mvn.repo}/org/apache/maven/maven-ant-tasks/${ant-task.version}/maven-ant-tasks-${ant-task.version}.jar"/>
-  <property name="ivy_repo_url" value="${mvn.repo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>
-  <property name="ivysettings.xml" location="${ivy.dir}/ivysettings.xml" />
-  <property name="ivy.org" value="org.apache.hadoop"/>
-  <property name="build.dir" location="build" />
-  <property name="dist.dir" value="${build.dir}/${final.name}"/>
-  <property name="build.ivy.dir" location="${build.dir}/ivy" />
-  <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
-  <property name="common.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/common"/>
-  <property name="mapred.ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}/mapred"/>
-  <property name="build.ivy.report.dir" location="${build.ivy.dir}/report" />
-  <property name="build.ivy.maven.dir" location="${build.ivy.dir}/maven" />
-  <property name="build.ivy.maven.pom" location="${build.ivy.maven.dir}/hadoop-mapred-${version}.pom" />
-  <property name="build.ivy.maven.jar" location="${build.ivy.maven.dir}/hadoop-mapred-${version}.jar" />
-  <property name="hadoop-mapred.pom" location="${ivy.dir}/hadoop-mapred.xml"/>
-  <property name="hadoop-mapred-test.pom" location="${ivy.dir}/hadoop-mapred-test.xml"/>
-  <property name="hadoop-mapred-examples.pom" location="${ivy.dir}/hadoop-mapred-examples.xml"/>
-  <property name="hadoop-mapred-tools.pom" location="${ivy.dir}/hadoop-mapred-tools.xml"/>
-
-  <!--this is the naming policy for artifacts we want pulled down-->
-  <property name="ivy.artifact.retrieve.pattern" value="${ant.project.name}/[conf]/[artifact]-[revision].[ext]"/>
-
-  <!--this is how artifacts that get built are named-->
-  <property name="ivy.publish.pattern" value="hadoop-[revision]-core.[ext]"/>
-  <property name="hadoop-mapred.jar" location="${build.dir}/${final.name}.jar" />
-  <property name="hadoop-mapred-test.jar" location="${build.dir}/${test.final.name}.jar" />
-  <property name="hadoop-mapred-examples.jar" location="${build.dir}/${examples.final.name}.jar" />
-  <property name="hadoop-mapred-tools.jar" location="${build.dir}/${tools.final.name}.jar" />
-  <property name="hadoop-mapred-fi.jar" location="${build.dir}/${final.name}-fi.jar" />
-  <property name="hadoop-mapred-sources.jar" location="${build.dir}/${final.name}-sources.jar" />
-  <property name="hadoop-mapred-test-sources.jar" location="${build.dir}/${test.final.name}-sources.jar" />
-  <property name="hadoop-mapred-examples-sources.jar" location="${build.dir}/${examples.final.name}-sources.jar" />
-  <property name="hadoop-mapred-tools-sources.jar" location="${build.dir}/${tools.final.name}-sources.jar" />
-
-  <!-- jdiff.home property set -->
-  <property name="jdiff.home" value="${build.ivy.lib.dir}/${ant.project.name}/jdiff"/>
-  <property name="jdiff.jar" value="${jdiff.home}/jdiff-${jdiff.version}.jar"/>
-  <property name="xerces.jar" value="${jdiff.home}/xerces-${xerces.version}.jar"/>
-	
-  <!-- Eclipse properties -->
-  <property name="build.dir.eclipse" value="${build.dir}/eclipse"/>
-  <property name="build.dir.eclipse-main-classes" value="${build.dir.eclipse}/classes-main"/>
-  <property name="build.dir.eclipse-main-generated-classes" value="${build.dir.eclipse}/classes-main-generated"/>
-  <property name="build.dir.eclipse-test-classes" value="${build.dir.eclipse}/classes-test"/>
-  <property name="build.dir.eclipse-example-classes" value="${build.dir.eclipse}/classes-example"/>
-  <property name="build.dir.eclipse-tools-classes" value="${build.dir.eclipse}/classes-tools"/>
-  <property name="build.dir.eclipse-contrib-classes" value="${build.dir.eclipse}/classes-contrib"/>
-
-  <property name="clover.jar" location="${clover.home}/lib/clover.jar"/>
-  <available property="clover.present" file="${clover.jar}" />
-
-  <!-- check if clover reports should be generated -->
-  <condition property="clover.enabled">
-    <and>
-        <isset property="run.clover"/>
-        <isset property="clover.present"/>
-    </and>
-  </condition>
-
-  <condition property="staging">
-    <equals arg1="${repo}" arg2="staging"/>
-  </condition>
-
-  <!-- packaging properties -->
-  <property name="package.prefix" value="/usr"/>
-  <property name="package.conf.dir" value="/etc/hadoop"/>
-  <property name="package.log.dir" value="/var/log/hadoop/mapred"/>
-  <property name="package.pid.dir" value="/var/run/hadoop"/>
-  <property name="package.var.dir" value="/var/lib/hadoop"/>
-  <property name="package.share.dir" value="/share/hadoop/${module}"/>
-  <!-- Use fixed path to build rpm for avoiding rpmbuild conflict with dash path names -->
-  <property name="package.buildroot" value="/tmp/hadoop_mapred_package_build_${user.name}"/>
-  <property name="package.build.dir" value="/tmp/hadoop_mapred_package_build_${user.name}/BUILD"/>
-
-  <!-- the normal classpath -->
-  <path id="classpath">
-    <pathelement location="${build.classes}"/>
-    <pathelement location="${conf.dir}"/>
-    <path refid="ivy-mapred.classpath"/>
-  </path>
-
-  <path id="test.classpath">
-    <pathelement location="${test.build.extraconf}"/>
-    <pathelement location="${test.mapred.build.classes}" />
-    <pathelement location="${test.src.dir}"/>
-    <pathelement location="${build.dir}"/>
-    <pathelement location="${build.examples}"/>
-    <pathelement location="${build.tools}"/>
-    <pathelement path="${clover.jar}"/>
-    <pathelement location="${build.classes}"/>
-    <pathelement location="${test.conf.dir}"/>
-    <path refid="ivy-common.classpath"/>
-    <path refid="ivy-mapred.classpath"/>
-    <pathelement location="${test.mapred.build.classes}" />
-    <path refid="ivy-test.classpath"/>
-  </path>
-
-  <!-- the cluster test classpath: uses conf.dir for configuration -->
-  <path id="test.cluster.classpath">
-    <path refid="classpath"/>
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="${test.src.dir}"/>
-    <pathelement location="${build.dir}"/>
-  </path>
-
-
-  <!-- ====================================================== -->
-  <!-- Macro definitions                                      -->
-  <!-- ====================================================== -->
-  <macrodef name="macro_tar" description="Worker Macro for tar">
-    <attribute name="param.destfile"/>
-    <element name="param.listofitems"/>
-    <sequential>
-      <tar compression="gzip" longfile="gnu"
-      destfile="@{param.destfile}">
-      <param.listofitems/>
-      </tar>
-    </sequential>
-  </macrodef>
-
-  <!-- ====================================================== -->
-  <!-- Stuff needed by all targets                            -->
-  <!-- ====================================================== -->
-  <target name="init" depends="ivy-retrieve-common,ivy-retrieve-mapred">
-    <mkdir dir="${build.dir}"/>
-    <mkdir dir="${build.classes}"/>
-    <mkdir dir="${build.tools}"/>
-    <mkdir dir="${build.src}"/>
-    <mkdir dir="${build.webapps}/task/WEB-INF"/>
-    <mkdir dir="${build.webapps}/job/WEB-INF"/>
-    <mkdir dir="${build.examples}"/>
-    <mkdir dir="${build.dir}/c++"/>
- 
-    <mkdir dir="${test.build.dir}"/>
-    <mkdir dir="${test.build.classes}"/>
-    <mkdir dir="${test.build.extraconf}"/>
-    <tempfile property="touch.temp.file" destDir="${java.io.tmpdir}"/>
-    <touch millis="0" file="${touch.temp.file}">
-      <fileset dir="${conf.dir}" includes="**/*.template"/>
-      <fileset dir="${contrib.dir}" includes="**/*.template"/>
-    </touch>
-    <delete file="${touch.temp.file}"/>
-
-    <!-- copy all of the jsp and static files -->
-    <copy todir="${build.webapps}">
-      <fileset dir="${src.webapps}">
-        <exclude name="**/*.jsp" />
-        <exclude name="**/*.jspx" />
-      </fileset>
-    </copy>
-
-     <unzip src="${common.ivy.lib.dir}/hadoop-hdfs-${hadoop-hdfs.version}.jar"
-         dest="${build.dir}">
-       <patternset>
-         <include name="webapps/hdfs/**"/>
-         <include name="webapps/datanode/**"/>
-         <include name="webapps/secondary/**"/>
-       </patternset>
-     </unzip>
-
-    <copy todir="${conf.dir}" verbose="true">
-      <fileset dir="${conf.dir}" includes="**/*.template"/>
-      <mapper type="glob" from="*.template" to="*"/>
-    </copy>
-
-    <mkdir dir="${test.conf.dir}"/>
-    <copy todir="${test.conf.dir}" verbose="true">
-      <fileset dir="${conf.dir}" includes="**/*.template"/>
-      <mapper type="glob" from="*.template" to="*"/>
-    </copy>
-
-    <copy todir="${contrib.dir}" verbose="true">
-      <fileset dir="${contrib.dir}" includes="**/*.template"/>
-      <mapper type="glob" from="*.template" to="*"/>
-    </copy>
-
-  </target>
-
-  <target name="avro-generate" depends="init">
-    <mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/>
-    <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask">
-      <classpath refid="classpath" />
-    </taskdef>
-    <protocol destdir="${build.src}">
-      <fileset dir="${mapred.src.dir}">
-	<include name="**/*.avpr" />
-      </fileset>
-    </protocol>
-  </target>
-
-  <target name="compile-mapred-classes" depends="init,avro-generate">
-    <taskdef classname="org.apache.jasper.JspC" name="jsp-compile" >
-       <classpath refid="classpath"/>
-    </taskdef>
-    <jsp-compile
-     uriroot="${src.webapps}/task"
-     outputdir="${build.src}"
-     package="org.apache.hadoop.mapred"
-     webxml="${build.webapps}/task/WEB-INF/web.xml">
-    </jsp-compile>
-
-    <jsp-compile
-     uriroot="${src.webapps}/job"
-     outputdir="${build.src}"
-     package="org.apache.hadoop.mapred"
-     webxml="${build.webapps}/job/WEB-INF/web.xml">
-    </jsp-compile>
-
-    <!-- Compile Java files (excluding JSPs) checking warnings -->
-    <javac 
-     encoding="${build.encoding}" 
-     srcdir="${mapred.src.dir};${build.src}" 
-     includes="org/apache/hadoop/**/*.java"
-     destdir="${build.classes}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
-      <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath"/>
-    </javac>   
-    
-    <copy todir="${build.classes}">
-      <fileset dir="${mapred.src.dir}" includes="**/*.properties"/>
-      <fileset dir="${mapred.src.dir}" includes="**/META-INF/services/*"/>
-      <fileset dir="${mapred.src.dir}" includes="mapred-default.xml"/>
-      <fileset dir="${mapred.src.dir}" includes="mapred-queues-default.xml"/>
-    </copy>
-  </target>
-
-  <target name="compile-tools" depends="init">
-    <javac 
-     encoding="${build.encoding}" 
-     srcdir="${tools.src}"
-     includes="org/apache/hadoop/**/*.java"
-     destdir="${build.tools}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
-      <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath"/>
-    </javac>   
-  	
-    <copy todir="${build.tools}">
-      <fileset 
-        dir="${tools.src}" 
-        includes="**/*.properties"
-      />
-    </copy>
-  </target>
-
-  <target name="compile-core" depends="clover, compile-mapred-classes, compile-c++" description="Compile core only"/> 
-
-  <target name="compile-contrib" depends="compile-core,tools">
-     <subant target="compile">
-        <property name="version" value="${version}"/>
-        <property name="hadoop-common.version" value="${hadoop-common.version}"/>
-        <fileset file="${contrib.dir}/build.xml"/>
-     </subant>  	
-  </target>
-  
-  <target name="compile" depends="compile-core, compile-contrib, compile-tools" description="Compile core, contrib">
-  </target>
-
-  <target name="compile-examples" 
-          depends="compile-core,compile-tools,compile-c++-examples">
-    <javac 
-     encoding="${build.encoding}" 
-     srcdir="${examples.dir}"
-     includes="org/apache/hadoop/**/*.java"
-     destdir="${build.examples}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
-      <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath>
-        <path refid="classpath"/>
-        <pathelement location="${build.tools}"/>
-      </classpath>
-    </javac>    
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Make hadoop-mapred.jar                                          -->
-  <!-- ================================================================== -->
-  <!--                                                                    -->
-  <!-- ================================================================== -->
-  <target name="jar" depends="compile-core" description="Make hadoop-mapered.jar">
-    <jar jarfile="${hadoop-mapred.jar}"
-         basedir="${build.classes}">
-      <manifest>
-        <section name="org/apache/hadoop">
-          <attribute name="Implementation-Title" value="${ant.project.name}"/>
-          <attribute name="Implementation-Version" value="${version}"/>
-          <attribute name="Implementation-Vendor" value="Apache"/>
-        </section>
-      </manifest>
-     <!-- <fileset file="${conf.dir}/commons-logging.properties"/>
-      <fileset file="${conf.dir}/log4j.properties"/>
-      <fileset file="${conf.dir}/hadoop-metrics.properties"/> -->
-      <zipfileset dir="${build.webapps}" prefix="webapps"/>
-      <fileset file="${jar.extra.properties.list}" />
-    </jar>
-
-    <jar jarfile="${hadoop-mapred-sources.jar}">
-      <fileset dir="${mapred.src.dir}" includes="org/apache/hadoop/**/*.java" />
-      <fileset dir="${build.src}" includes="org/apache/hadoop/**/*.java" />
-    </jar>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Make the Hadoop examples jar.                                      -->
-  <!-- ================================================================== -->
-  <!--                                                                    -->
-  <!-- ================================================================== -->
-  <target name="examples" depends="jar, compile-examples" description="Make the Hadoop examples jar.">
-    <jar jarfile="${build.dir}/${examples.final.name}.jar"
-         basedir="${build.examples}">
-      <manifest>
-        <attribute name="Main-Class" 
-                   value="org/apache/hadoop/examples/ExampleDriver"/>
-      </manifest>
-    </jar>
-
-    <jar jarfile="${hadoop-mapred-examples-sources.jar}">
-      <fileset dir="${examples.dir}" includes="org/apache/hadoop/**/*.java" />
-    </jar>
-  </target>
-
-  <target name="tools" depends="jar, compile-tools" 
-          description="Make the Hadoop tools jar.">
-    <jar jarfile="${build.dir}/${tools.final.name}.jar"
-         basedir="${build.tools}">
-    </jar>
-
-    <jar jarfile="${hadoop-mapred-tools-sources.jar}">
-      <fileset dir="${tools.src}" includes="org/apache/hadoop/**/*.java" />
-    </jar>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Compile test code                                                  --> 
-  <!-- ================================================================== -->
-
-  <target name="compile-mapred-test" depends="compile-mapred-classes, compile-examples, ivy-retrieve-test">
-
-    <mkdir dir="${test.mapred.build.classes}"/>
-    <mkdir dir="${test.mapred.build.testjar}"/>
-    <mkdir dir="${test.mapred.build.testshell}"/>
-
-    <macro-compile-test
-      source.dir="${test.src.dir}/mapred;${test.src.dir}/unit"
-      dest.dir="${test.mapred.build.classes}"
-      classpath="test.classpath"/>
-
-    <javac
-      encoding="${build.encoding}"
-      srcdir="${test.src.dir}/mapred/testjar"
-      includes="*.java"
-      destdir="${test.mapred.build.testjar}"
-      debug="${javac.debug}"
-      optimize="${javac.optimize}"
-      target="${javac.version}"
-      source="${javac.version}"
-      deprecation="${javac.deprecation}">
-      <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="test.classpath"/>
-    </javac>
-
-    <delete file="${test.mapred.build.testjar}/testjob.jar"/> 
-    <jar jarfile="${test.mapred.build.testjar}/testjob.jar"
-       basedir="${test.mapred.build.testjar}">
-    </jar>
-
-    <javac 
-     encoding="${build.encoding}"
-     srcdir="${test.src.dir}/mapred/testshell"
-     includes="*.java"
-     destdir="${test.mapred.build.testshell}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
-      <compilerarg line="${javac.args} ${javac.args.warnings}"/>
-      <classpath refid="test.classpath"/>
-    </javac>
-    <delete file="${test.mapred.build.testshell}/testshell.jar"/>
-    <jar jarfile="${test.mapred.build.testshell}/testshell.jar"
-      basedir="${test.mapred.build.testshell}">
-    </jar>
-
-     <delete dir="${test.cache.data}"/>
-     <mkdir dir="${test.cache.data}"/>
-     <delete dir="${test.concat.data}"/>
-     <mkdir dir="${test.concat.data}"/>
-     <delete dir="${test.debug.data}"/>
-     <mkdir dir="${test.debug.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testscript.txt" todir="${test.debug.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.txt" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.jar" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.zip" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tar" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tgz" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/test.tar.gz" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/cli/testMRConf.xml" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/cli/data60bytes" todir="${test.cache.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/concat.bz2" todir="${test.concat.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/concat.gz" todir="${test.concat.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.bz2" todir="${test.concat.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testCompressThenConcat.txt.gz" todir="${test.concat.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.bz2" todir="${test.concat.data}"/>
-     <copy file="${test.src.dir}/mapred/org/apache/hadoop/mapred/testConcatThenCompress.txt.gz" todir="${test.concat.data}"/>
-  </target>
-
-  <macrodef name="macro-compile-test">
-    <attribute name="source.dir"/>
-    <attribute name="dest.dir"/>
-    <attribute name="classpath"/>
-    <sequential>
-      <mkdir dir="@{dest.dir}"/>
-      <javac
-        encoding="${build.encoding}"
-        srcdir="@{source.dir}"
-        includes="org/apache/hadoop/**/*.java"
-        destdir="@{dest.dir}"
-        debug="${javac.debug}"
-        optimize="${javac.optimize}"
-        target="${javac.version}"
-        source="${javac.version}"
-        deprecation="${javac.deprecation}">
-        <compilerarg line="${javac.args} ${javac.args.warnings}"/>
-        <classpath refid="@{classpath}"/>
-      </javac>
-    </sequential>
-  </macrodef>
-
-  <!-- ================================================================== -->
-  <!-- Make hadoop-mapred-test.jar                                        -->
-  <!-- ================================================================== -->
-  <!--                                                                    -->
-  <!-- ================================================================== -->
-  <target name="jar-test" depends="compile-mapred-test" description="Make hadoop-mapred-test.jar">
-    <copy todir="${test.build.classes}">
-      <fileset dir="${test.mapred.build.classes}"/>
-    </copy>
-    <subant buildpath="build.xml" target="-do-jar-test"/>
-    <jar jarfile="${hadoop-mapred-test-sources.jar}">
-      <fileset dir="${test.src.dir}/mapred" includes="org/apache/hadoop/**/*.java" />
-      <fileset dir="${test.src.dir}/unit" includes="org/apache/hadoop/**/*.java" />
-      <fileset dir="${test.src.dir}/mapred/testjar" includes="*.java" />
-      <fileset dir="${test.src.dir}/mapred/testshell" includes="*.java" />
-    </jar>
-  </target>
-
-  <target  name="-do-jar-test">
-    <jar jarfile="${build.dir}/${test.final.name}.jar"
-         basedir="${test.build.classes}">
-         <manifest>
-           <attribute name="Main-Class"
-                      value="org/apache/hadoop/test/MapredTestDriver"/>
-          <section name="org/apache/hadoop">
-            <attribute name="Implementation-Title" value="${ant.project.name}"/>
-            <attribute name="Implementation-Version" value="${version}"/>
-            <attribute name="Implementation-Vendor" value="Apache"/>
-          </section>
-         </manifest>
-    </jar>
-  </target>
-  <!-- ================================================================== -->
-  <!-- Define exclude lists for different kinds of testing                -->
-  <!-- ================================================================== -->
-  <patternset id="empty.exclude.list.id" />
-  <patternset id="commit.smoke.exclude.list.id">
-    <excludesfile name="${test.mapred.commit.tests.file}"/>
-    <excludesfile name="${test.mapred.smoke.tests.file}"/>
-  </patternset>
-
-  <macrodef name="macro-test-runner">
-    <attribute name="test.file" />
-    <attribute name="suite.type" />
-    <attribute name="classpath" />
-    <attribute name="test.dir" />
-    <attribute name="fileset.dir" />
-    <attribute name="hadoop.conf.dir.deployed" default="" />
-    <attribute name="exclude.list.id" default="empty.exclude.list.id" />
-    <sequential>
-      <delete dir="@{test.dir}/data"/>
-      <mkdir dir="@{test.dir}/data"/>
-      <delete dir="@{test.dir}/logs"/>
-      <mkdir dir="@{test.dir}/logs"/>
-      <copy file="${test.src.dir}/hadoop-policy.xml" 
-        todir="@{test.dir}/extraconf" />
-      <copy file="${test.src.dir}/fi-site.xml"
-        todir="@{test.dir}/extraconf" />
-      <junit showoutput="${test.output}"
-        printsummary="${test.junit.printsummary}"
-        haltonfailure="${test.junit.haltonfailure}"
-        fork="yes"
-        forkmode="${test.junit.fork.mode}"
-        maxmemory="${test.junit.maxmemory}"
-        dir="${basedir}" timeout="${test.timeout}"
-        errorProperty="tests.failed" failureProperty="tests.failed">
-        <jvmarg value="${test.junit.jvmargs}" />
-        <sysproperty key="java.net.preferIPv4Stack" value="true"/>
-        <sysproperty key="test.build.data" value="@{test.dir}/data"/>
-        <sysproperty key="java.security.krb5.conf"
-          value="${test.src.dir}/krb5.conf"/>
-        <sysproperty key="test.tools.input.dir" value = "${test.tools.input.dir}"/>
-        <sysproperty key="test.cache.data" value="${test.cache.data}"/>     
-        <sysproperty key="test.concat.data" value="${test.concat.data}"/>
-        <sysproperty key="test.debug.data" value="${test.debug.data}"/>
-        <sysproperty key="hadoop.log.dir" value="@{test.dir}/logs"/>
-        <sysproperty key="test.src.dir" value="@{fileset.dir}"/>
-        <sysproperty key="taskcontroller-path" value="${taskcontroller-path}"/>
-        <sysproperty key="taskcontroller-ugi" value="${taskcontroller-ugi}"/>
-        <sysproperty key="test.build.extraconf" value="@{test.dir}/extraconf" />
-        <sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
-        <sysproperty key="java.library.path"
-          value="${build.native}/lib:${lib.dir}/native/${build.platform}"/>
-        <sysproperty key="install.c++.examples" value="${install.c++.examples}"/>
-        <syspropertyset dynamic="no">
-          <propertyref name="hadoop.tmp.dir"/>
-        </syspropertyset>
-        <!-- set compile.c++ in the child jvm only if it is set -->
-        <syspropertyset dynamic="no">
-          <propertyref name="compile.c++"/>
-        </syspropertyset>
-        
-        <!-- Pass probability specifications to the spawn JVM -->
-        <syspropertyset id="FaultProbabilityProperties">
-          <propertyref regex="fi.*"/>
-        </syspropertyset>
-        <sysproperty key="test.system.hdrc.deployed.hadoopconfdir"
-                     value="@{hadoop.conf.dir.deployed}" />
-        <classpath refid="@{classpath}"/>
-        <formatter type="${test.junit.output.format}" />
-        <batchtest todir="@{test.dir}" unless="testcase">
-          <fileset dir="@{fileset.dir}/@{suite.type}"
-            excludes="**/${test.exclude}.java aop/** system/**">
-            <patternset>
-              <includesfile name="@{test.file}"/>
-              <excludesfile name="${test.exclude.file}" />
-            </patternset>
-            <patternset refid="@{exclude.list.id}"/>
-          </fileset>
-        </batchtest>
-        <batchtest todir="@{test.dir}" if="testcase">
-          <fileset dir="@{fileset.dir}/mapred" includes="**/${testcase}.java"/>
-          <fileset dir="@{fileset.dir}/unit" includes="**/${testcase}.java"/>
-          <fileset dir="@{fileset.dir}/system/test" includes="**/${testcase}.java"/>
-        </batchtest>
-        <!--batch test to test all the testcases in aop folder with fault 
-        injection-->
-        <batchtest todir="@{test.dir}" if="tests.notestcase.fi">
-          <fileset dir="@{fileset.dir}/aop"
-            includes="**/${test.include}.java"
-            excludes="**/${test.exclude}.java"
-            excludesfile="${test.exclude.file}" />
-        </batchtest>
-        <!-- batch test for testing a single test case in aop folder with
-        fault injection-->
-        <batchtest todir="@{test.dir}" if="tests.testcase.fi">
-         <fileset dir="@{fileset.dir}/aop" includes="**/${testcase}.java"/>
-        </batchtest>
-         <!--The following batch is for very special occasions only when
-         a non-FI tests are needed to be executed against FI-environment -->
-         <batchtest todir="@{test.dir}" if="tests.testcaseonly.fi">
-          <fileset dir="@{fileset.dir}/mapred" 
-            includes="**/${testcase}.java"/>
-        </batchtest>
-      </junit>
-      <antcall target="checkfailure"/>
-    </sequential>
-  </macrodef>
-
-  <target name="run-test-mapred" depends="run-commit-test, run-smoke-test,
-    run-test-mapred-excluding-commit-and-smoke, run-test-mapred-all-withtestcaseonly"
-    description="Run mapred functional and system tests">
-  </target>
-
-  <target name="run-test-mapred-all-withtestcaseonly"
-    depends="compile-mapred-test" if="testcase">
-    <macro-test-runner
-      test.file="${test.mapred.all.tests.file}"
-      suite.type="mapred"
-      classpath="${test.classpath.id}"
-      test.dir="${test.build.dir}"
-      fileset.dir="${test.src.dir}"/>
-  </target>
-
-  <target name="run-test-mapred-excluding-commit-and-smoke"
-    depends="compile-mapred-test" unless="testcase">
-    <macro-test-runner
-      test.file="${test.mapred.all.tests.file}"
-      suite.type="mapred"
-      classpath="${test.classpath.id}"
-      test.dir="${test.build.dir}"
-      fileset.dir="${test.src.dir}"
-      exclude.list.id="commit.smoke.exclude.list.id"
-    />
-   </target>
-
-   <target name="run-commit-test" depends="compile-mapred-test"
-     description="Run approximate 10-minute set of unit tests prior to commiting"
-     unless="testcase">
-     <macro-test-runner test.file="${test.mapred.commit.tests.file}" suite.type="mapred"
-      classpath="${test.classpath.id}"
-      test.dir="${test.build.dir}"
-      fileset.dir="${test.src.dir}"/>
-   </target>
- 
-   <target name="run-smoke-test" depends="compile-mapred-test"
-     description="Run approximate 30-minute set of functional tests to guarantee viability of the MapReduce"
-     unless="testcase">
-     <macro-test-runner
-       test.file="${test.mapred.smoke.tests.file}"
-       suite.type="mapred"
-       classpath="${test.classpath.id}"
-       test.dir="${test.build.dir}"
-       fileset.dir="${test.src.dir}"/>
-   </target>
-
-   <target name="run-test-unit" depends="compile-mapred-test" description="Run unit tests">
-     <macro-test-runner test.file="${test.mapred.all.tests.file}" suite.type="unit"
-      classpath="${test.classpath.id}"
-      test.dir="${test.build.dir}"
-      fileset.dir="${test.src.dir}"/>
-   </target>
-
-
-  <target name="checkfailure" if="tests.failed">
-    <touch file="${test.build.dir}/testsfailed"/>
-    <fail unless="continueOnFailure">Tests failed!</fail>
-  </target>
-
-  <target name="test-contrib" depends="compile,compile-tools,compile-mapred-test" description="Run contrib unit tests">
-    <subant target="test">
-       <property name="version" value="${version}"/>
-       <property name="clover.jar" value="${clover.jar}"/>
-       <fileset file="${contrib.dir}/build.xml"/>
-    </subant> 
-  </target>
-
-  <target name="test-core" description="Run core, hdfs and mapred unit tests">
-    <delete file="${test.build.dir}/testsfailed"/>
-    <property name="continueOnFailure" value="true"/>
-    <antcall target="run-test-mapred"/>
-    <available file="${test.build.dir}/testsfailed" property="testsfailed"/>
-    <fail if="testsfailed">Tests failed!</fail>
-  </target>
-
-  <target name="test" depends="jar-test, test-core" description="Run all unit tests">
-    <subant target="test-contrib">
-      <fileset file="${basedir}/build.xml"/>
-     </subant>
-  </target>
-
-  <!-- Run all unit tests, not just Test*, and use non-test configuration. -->
-  <target name="test-cluster" description="Run all unit tests, not just Test*, and use non-test configuration.">
-    <antcall target="test">
-      <param name="test.include" value="*"/>
-      <param name="test.classpath.id" value="test.cluster.classpath"/>
-    </antcall>
-  </target>
-
-  <target name="nightly" depends="test, tar">
-  </target>
-	
-  <!-- ================================================================== -->
-  <!-- Run optional third-party tool targets                              --> 
-  <!-- ================================================================== -->
-  <target name="checkstyle" depends="ivy-retrieve-checkstyle,check-for-checkstyle" if="checkstyle.present" description="Run optional third-party tool targets">
-       <taskdef resource="checkstyletask.properties">
-         <classpath refid="checkstyle-classpath"/>
-       </taskdef>
-  
-	<mkdir dir="${test.build.dir}"/>
-  	
-  	<checkstyle config="${test.src.dir}/checkstyle.xml"
-  		failOnViolation="false">
-      <fileset dir="${mapred.src.dir}" includes="**/*.java" excludes="**/generated/**"/>
-      <formatter type="xml" toFile="${test.build.dir}/checkstyle-errors.xml"/>
-  	</checkstyle>
-  	
-  	<xslt style="${test.src.dir}/checkstyle-noframes-sorted.xsl"
-        in="${test.build.dir}/checkstyle-errors.xml"
-        out="${test.build.dir}/checkstyle-errors.html"/>
-  </target>
-	
-  <target name="check-for-checkstyle">
-    <available property="checkstyle.present" resource="checkstyletask.properties">
-       <classpath refid="checkstyle-classpath"/>
-    </available>  	
-  </target>
-
- <target name="all-jars" depends="tools,examples,jar-test">
-   <subant target="jar">
-      <property name="version" value="${version}"/>
-      <property name="dist.dir" value="${dist.dir}"/>
-      <fileset file="${contrib.dir}/capacity-scheduler/build.xml"/>
-      <fileset file="${contrib.dir}/streaming/build.xml"/>
-      <fileset file="${contrib.dir}/gridmix/build.xml"/>
-      <fileset file="${contrib.dir}/mumak/build.xml"/>
-    </subant>
- </target>
-
- <property name="findbugs.home" value=""/>
-  <target name="findbugs" depends="check-for-findbugs, all-jars" if="findbugs.present" description="Run findbugs if present">
-    <property name="findbugs.out.dir" value="${test.build.dir}/findbugs"/>
-    <property name="findbugs.exclude.file" value="${test.src.dir}/findbugsExcludeFile.xml"/>
-    <property name="findbugs.report.htmlfile" value="${findbugs.out.dir}/hadoop-findbugs-report.html"/>
-    <property name="findbugs.report.xmlfile" value="${findbugs.out.dir}/hadoop-findbugs-report.xml"/>
-    <taskdef name="findbugs" classname="edu.umd.cs.findbugs.anttask.FindBugsTask"
-        classpath="${findbugs.home}/lib/findbugs-ant.jar" />
-
-        <mkdir dir="${findbugs.out.dir}"/>
-
-    <findbugs home="${findbugs.home}" output="xml:withMessages"
-        outputFile="${findbugs.report.xmlfile}" effort="max"
-        excludeFilter="${findbugs.exclude.file}" jvmargs="-Xmx${findbugs.heap.size}">
-      <auxClasspath>
-        <fileset dir="${lib.dir}">
-          <include name="**/*.jar"/>
-        </fileset>
-        <fileset dir="${build.ivy.lib.dir}/${ant.project.name}/common">
-          <include name="**/*.jar"/>
-        </fileset>
-      </auxClasspath>
-      <sourcePath path="${mapred.src.dir}"/>
-      <sourcePath path="${examples.dir}" />
-      <sourcePath path="${tools.src}" />
-      <sourcePath path="${basedir}/src/contrib/capacity-scheduler/src/java" />
-      <sourcePath path="${basedir}/src/contrib/streaming/src/java" />
-      <sourcePath path="${basedir}/src/contrib/gridmix/src/java" />
-      <sourcePath path="${basedir}/src/contrib/mumak/src/java" />
-      <class location="${basedir}/build/${final.name}.jar" />
-      <class location="${basedir}/build/${examples.final.name}.jar" />
-      <class location="${basedir}/build/${tools.final.name}.jar" />
-      <class location="${basedir}/build/contrib/capacity-scheduler/hadoop-${version}-capacity-scheduler.jar" />
-      <class location="${basedir}/build/contrib/streaming/hadoop-${version}-streaming.jar" />
-      <class location="${basedir}/build/contrib/gridmix/hadoop-${version}-gridmix.jar" />
-      <class location="${basedir}/build/contrib/mumak/hadoop-${version}-mumak.jar" />
-    </findbugs>
-
-        <xslt style="${findbugs.home}/src/xsl/default.xsl"
-        in="${findbugs.report.xmlfile}"
-        out="${findbugs.report.htmlfile}"/>
-  </target>
-	
-  <target name="check-for-findbugs">
-    <available property="findbugs.present"
-        file="${findbugs.home}/lib/findbugs.jar" />
-  </target>
-
-
-  <!-- ================================================================== -->
-  <!-- Documentation                                                      -->
-  <!-- ================================================================== -->
-  
-  <target name="docs" depends="forrest.check" description="Generate forrest-based documentation. To use, specify -Dforrest.home=&lt;base of Apache Forrest installation&gt; on the command line." if="forrest.home">
-  	<!-- The template file may not exist if building from a tarball -->
-  	<copy file="${conf.dir}/mapred-queues.xml.template"
-  	      tofile="${build.docs}/mapred-queues.xml" failonerror="false"/>
-    <exec dir="${docs.src}" executable="${forrest.home}/bin/forrest"
-	  failonerror="true">
-    </exec>
-    <copy todir="${build.docs}">
-      <fileset dir="${docs.src}/build/site/" />
-    </copy>
-    <copy file="${docs.src}/releasenotes.html" todir="${build.docs}"/>
-    <style basedir="${mapred.src.dir}" destdir="${build.docs}"
-           includes="mapred-default.xml" style="conf/configuration.xsl"/>
-    <antcall target="changes-to-html"/>
-    <subant target="docs">
-       <property name="build.docs" value="${build.docs}"/>
-       <fileset file="${contrib.dir}/build.xml"/>
-    </subant> 
-  </target>
-
-  <target name="javadoc-dev" depends="compile, ivy-retrieve-javadoc" description="Generate javadoc for hadoop developers">
-    <mkdir dir="${build.javadoc.dev}"/>
-    <javadoc
-      overview="${mapred.src.dir}/overview.html"
-      packagenames="org.apache.hadoop.*"
-      destdir="${build.javadoc.dev}"
-      author="true"
-      version="true"
-      use="true"
-      windowtitle="${Name} ${version} API"
-      doctitle="${Name} ${version} Developer API"
-      bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
-      maxmemory="${javadoc.maxmemory}"
-      >
-        <packageset dir="${mapred.src.dir}"/>
-    	<packageset dir="${examples.dir}"/>
-    	<packageset dir="${tools.src}"/>
-
-       <packageset dir="src/contrib/data_join/src/java"/>
-       <packageset dir="src/contrib/gridmix/src/java"/>
-       <packageset dir="src/contrib/index/src/java"/>
-       <packageset dir="src/contrib/mumak/src/java"/>
-       <packageset dir="src/contrib/streaming/src/java"/>
-       <packageset dir="src/contrib/vaidya/src/java"/>
-       <packageset dir="src/contrib/vertica/src/java"/>
-
-        <link href="${javadoc.link.java}"/>
-
-        <classpath >
-          <path refid="classpath" />
-          <fileset dir="src/contrib/">
-            <include name="*/lib/*.jar" />
-          </fileset>
-          <path refid="javadoc-classpath"/>
-          <pathelement path="${java.class.path}"/>
-          <pathelement path="${lib.dir}/hadoop-common-test-${hadoop-common.version}.jar"/>
-          <pathelement location="${build.tools}"/>
-        </classpath>
-
-       <group title="Packages" packages="org.apache.*"/>
-       <group title="Libraries" packages="org.apache.hadoop.mapred.lib*:org.apache.hadoop.mapreduce.lib*"/>
-       <group title="Tools" packages="org.apache.hadoop.tools"/>
-       <group title="Examples" packages="org.apache.hadoop.examples*"/>
-
-       <group title="contrib: DataJoin" packages="org.apache.hadoop.contrib.utils.join*"/>
-       <group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
-       <group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
-       <group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
-       <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
-       <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
-    </javadoc>
-  </target>	
-
-  <target name="javadoc-uptodate" depends="compile, ivy-retrieve-common, ivy-retrieve-javadoc">
-    <uptodate property="javadoc.is.uptodate">
-      <srcfiles dir="${src.dir}">
-        <include name="**/*.java" />
-        <include name="**/*.html" />
-      </srcfiles>
-      <mapper type="merge" to="${build.javadoc.timestamp}" />
-    </uptodate>
-  </target>
- 
-  <target name="javadoc" description="Generate javadoc" depends="javadoc-uptodate"
-       unless="javadoc.is.uptodate">
-    <mkdir dir="${build.javadoc}"/>
-    <javadoc
-      overview="${mapred.src.dir}/overview.html"
-      packagenames="org.apache.hadoop.*"
-      destdir="${build.javadoc}"
-      author="true"
-      version="true"
-      use="true"
-      windowtitle="${Name} ${version} API"
-      doctitle="${Name} ${version} API"
-      bottom="Copyright &amp;copy; ${year} The Apache Software Foundation"
-      maxmemory="${javadoc.maxmemory}"
-      >
-        <packageset dir="${mapred.src.dir}"/>
-    	<packageset dir="${examples.dir}"/>
-
-       <!-- Don't include contrib modules that use the same packages as core
-       MapReduce. This includes capacity-scheduler, dynamic-scheduler,
-       fairscheduler, mumak. See also the javadoc-dev target. -->
-       <packageset dir="src/contrib/data_join/src/java"/>
-       <packageset dir="src/contrib/gridmix/src/java"/>
-       <packageset dir="src/contrib/index/src/java"/>
-       <packageset dir="src/contrib/streaming/src/java"/>
-       <packageset dir="src/contrib/vaidya/src/java"/>
-       <packageset dir="src/contrib/vertica/src/java"/>
-       <packageset dir="${tools.src}"/>
-	
-        <link href="${javadoc.link.java}"/>
-
-        <classpath >
-          <path refid="classpath" />
-          <fileset dir="src/contrib/">
-            <include name="*/lib/*.jar" />
-          </fileset>
-          <path refid="javadoc-classpath"/>
-          <pathelement path="${java.class.path}"/>
-          <pathelement path="${lib.dir}/hadoop-common-test-${hadoop-common.version}.jar"/>
-          <pathelement location="${build.tools}"/>
-        </classpath>
-
-       <group title="Packages" packages="org.apache.*"/>
-       <group title="Libraries" packages="org.apache.hadoop.mapred.lib*:org.apache.hadoop.mapreduce.lib*"/>
-       <group title="Tools" packages="org.apache.hadoop.fs*:org.apache.hadoop.tools*:org.apache.hadoop.mapred.tool*:org.apache.hadoop.mapreduce.tool*"/>
-       <group title="Examples" packages="org.apache.hadoop.examples*"/>
-
-       <group title="contrib: DataJoin" packages="org.apache.hadoop.contrib.utils.join*"/>
-       <group title="contrib: Gridmix" packages="org.apache.hadoop.mapred.gridmix*"/>
-       <group title="contrib: Index" packages="org.apache.hadoop.contrib.index*"/>
-       <group title="contrib: Streaming" packages="org.apache.hadoop.streaming*:org.apache.hadoop.typedbytes*"/>
-       <group title="contrib: Vaidya" packages="org.apache.hadoop.vaidya*"/>
-       <group title="contrib: Vertica" packages="org.apache.hadoop.vertica*"/>
-
-       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet"
-         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar"/>
-    </javadoc>
-  </target>	
-
-  <target name="api-xml" depends="ivy-retrieve-jdiff,javadoc,write-null">
-    <javadoc maxmemory="${javadoc.maxmemory}">
-       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
-         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar:${jdiff.jar}:${xerces.jar}">
-         <param name="-apidir" value="${jdiff.xml.dir}"/>
-         <param name="-apiname" value="hadoop-mapred ${version}"/>
-         <param name="${jdiff.stability}"/>
-       </doclet>
-       <packageset dir="src/java"/>
-       <packageset dir="src/tools"/>
-       <packageset dir="${tools.src}"/>
-       <classpath >
-         <path refid="classpath" />
-         <path refid="jdiff-classpath" />
-         <pathelement path="${java.class.path}"/>
-       </classpath>
-    </javadoc>
-  </target>
-	
-  <target name="write-null">
-	<exec executable="touch">
-	   <arg value="${jdiff.home}/Null.java"/>
-        </exec>
-  </target> 
-
-  <target name="api-report" depends="ivy-retrieve-jdiff,api-xml">
-    <mkdir dir="${jdiff.build.dir}"/>
-    <javadoc destdir="${jdiff.build.dir}"
-	     sourceFiles="${jdiff.home}/Null.java"
-	     maxmemory="${javadoc.maxmemory}">
-       <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
-         path="${common.ivy.lib.dir}/hadoop-common-${hadoop-common.version}.jar:${jdiff.jar}:${xerces.jar}">
-         <param name="-oldapi" value="hadoop-mapred ${jdiff.stable}"/>
-         <param name="-newapi" value="hadoop-mapred ${version}"/>
-         <param name="-oldapidir" value="${jdiff.xml.dir}"/>
-         <param name="-newapidir" value="${jdiff.xml.dir}"/>
-         <param name="-javadocold" value="${jdiff.stable.javadoc}"/>
-         <param name="-javadocnew" value="../../api/"/>
-         <param name="-stats"/>
-         <param name="${jdiff.stability}"/>
-         <param name="${jdiff.compatibility}"/>
-       </doclet>
-       <packageset dir="src/java"/>
-       <packageset dir="src/tools"/>
-       <packageset dir="${tools.src}"/>
-       <classpath >
-         <path refid="classpath" />
-         <path refid="jdiff-classpath"/>
-         <pathelement path="${java.class.path}"/>
-       </classpath>
-    </javadoc>
-  </target>
-	
-  <target name="changes-to-html" description="Convert CHANGES.txt into an html file">
-    <mkdir dir="${build.docs}"/>
-    <exec executable="perl" input="CHANGES.txt" output="${build.docs}/changes.html" failonerror="true">
-      <arg value="${changes.src}/changes2html.pl"/>
-    </exec>
-    <copy todir="${build.docs}">
-      <fileset dir="${changes.src}" includes="*.css"/>
-    </copy>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- D I S T R I B U T I O N                                            -->
-  <!-- ================================================================== -->
-  <!--                                                                    -->
-  <!-- ================================================================== -->
-  <target name="package" depends="compile, jar, javadoc, docs, api-report, examples, tools, jar-test, package-librecordio"
-	  description="Build distribution">
-    <mkdir dir="${dist.dir}"/>
-    <mkdir dir="${dist.dir}/lib"/>
-    <mkdir dir="${dist.dir}/contrib"/>
-    <mkdir dir="${dist.dir}/bin"/>
-    <mkdir dir="${dist.dir}/docs"/>
-    <mkdir dir="${dist.dir}/docs/api"/>
-    <mkdir dir="${dist.dir}/docs/jdiff"/>
-
-    <copy todir="${dist.dir}/lib" includeEmptyDirs="false" flatten="true">
-      <fileset dir="${mapred.ivy.lib.dir}"/>
-    </copy>
-
-    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
-      <fileset dir="lib">
-        <exclude name="**/native/**"/>
-      </fileset>
-    </copy>
-
-    <subant target="compile-test">
-      <!--Pass down the version in case its needed again and the target
-      distribution directory so contribs know where to install to.-->
-      <property name="version" value="${version}"/>
-      <property name="dist.dir" value="${dist.dir}"/>
-      <fileset file="${contrib.dir}/build.xml"/>
-    </subant>  	
-    <subant target="package">
-      <!--Pass down the version in case its needed again and the target
-      distribution directory so contribs know where to install to.-->
-      <property name="version" value="${version}"/>
-      <property name="dist.dir" value="${dist.dir}"/>
-      <fileset file="${contrib.dir}/build.xml"/>
-    </subant>  	
-
-    <copy todir="${dist.dir}/webapps">
-      <fileset dir="${build.webapps}"/>
-    </copy>
-
-    <copy todir="${dist.dir}"> 
-      <fileset file="${build.dir}/${final.name}*.jar"/>
-      <fileset file="${build.dir}/${test.final.name}.jar"/>
-      <fileset file="${build.dir}/${examples.final.name}.jar"/>
-      <fileset file="${build.dir}/${tools.final.name}.jar"/> 
-    </copy>
-    
-    <copy todir="${dist.dir}/bin">
-      <fileset dir="bin"/>
-    </copy>
-
-    <copy todir="${dist.dir}/conf">
-      <fileset dir="${conf.dir}" excludes="**/*.template"/>
-    </copy>
-
-    <copy todir="${dist.dir}/docs">
-      <fileset dir="${build.docs}"/>
-    </copy>
-
-    <copy file="ivy.xml" tofile="${dist.dir}/ivy.xml"/>
-
-    <copy todir="${dist.dir}/ivy">
-      <fileset dir="ivy"/>
-    </copy>
-
-    <copy todir="${dist.dir}">
-      <fileset dir=".">
-        <include name="*.txt" />
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/src" includeEmptyDirs="true">
-      <fileset dir="src" excludes="**/*.template **/docs/build/**/*"/>
-    </copy>
-  	
-    <copy todir="${dist.dir}/c++" includeEmptyDirs="false">
-      <fileset dir="${build.dir}/c++"/>
-    </copy>
-
-    <copy todir="${dist.dir}/" file="build.xml"/>
-    <copy todir="${dist.dir}/" file="build-utils.xml"/>
-
-    <chmod perm="ugo+x" type="file" parallel="false">
-        <fileset file="${dist.dir}/src/examples/pipes/configure"/>
-        <fileset file="${dist.dir}/src/c++/utils/configure"/>
-        <fileset file="${dist.dir}/src/c++/pipes/configure"/>
-        <fileset file="${dist.dir}/src/c++/task-controller/configure"/>
-    </chmod>
-    <chmod perm="ugo+x" type="file" parallel="false">
-        <fileset dir="${dist.dir}/bin"/>
-        <fileset dir="${dist.dir}/src/contrib/">
-          <include name="*/bin/*" />
-        </fileset>
-    </chmod>
-    <chmod perm="ugo+x" type="file">
-        <fileset dir="${dist.dir}/src/c++/pipes/debug"/>
-    </chmod>
-
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Make release tarball                                               -->
-  <!-- ================================================================== -->
-  <target name="tar" depends="package" description="Make release tarball">
-    <macro_tar param.destfile="${build.dir}/${final.name}.tar.gz">
-      <param.listofitems>
-        <tarfileset dir="${build.dir}" mode="664">
-          <exclude name="${final.name}/bin/*" />
-          <exclude name="${final.name}/contrib/*/bin/*" />
-          <exclude name="${final.name}/src/examples/pipes/configure"/>
-          <exclude name="${final.name}/src/c++/utils/configure"/>
-          <exclude name="${final.name}/src/c++/pipes/configure"/>
-          <exclude name="${final.name}/src/c++/task-controller/configure"/>
-          <include name="${final.name}/**" />
-        </tarfileset>
-        <tarfileset dir="${build.dir}" mode="755">
-          <include name="${final.name}/bin/*" />
-          <include name="${final.name}/contrib/*/bin/*" />
-          <include name="${final.name}/src/examples/pipes/configure"/>
-          <include name="${final.name}/src/c++/utils/configure"/>
-          <include name="${final.name}/src/c++/pipes/configure"/>
-          <include name="${final.name}/src/c++/task-controller/configure"/>
-        </tarfileset>
-      </param.listofitems>
-    </macro_tar>
-  </target>
-
-  <target name="bin-package" depends="compile, jar, examples, tools, jar-test, package-librecordio" 
-		description="assembles artifacts for binary target">
-    <mkdir dir="${dist.dir}"/>
-    <mkdir dir="${dist.dir}/include"/>
-    <mkdir dir="${dist.dir}/lib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/contrib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/lib"/>
-    <mkdir dir="${dist.dir}/${package.share.dir}/templates"/>
-    <mkdir dir="${dist.dir}/bin"/>
-    <mkdir dir="${dist.dir}/sbin"/>
-
-    <!-- enable this if there is mapred specific dependencies
-    <copy todir="${dist.dir}/${package.share.dir}/lib" includeEmptyDirs="false" flatten="true">
-      <fileset dir="${mapred.ivy.lib.dir}"/>
-    </copy> -->
-
-    <copy todir="${dist.dir}/include" includeEmptyDirs="false">
-      <fileset dir="${build.dir}/c++/${build.platform}/include" 
-               erroronmissingdir="false">
-        <include name="**"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/lib" includeEmptyDirs="false">
-      <fileset dir="${build.dir}/c++/${build.platform}/lib"
-               erroronmissingdir="false">
-        <include name="**"/>
-      </fileset>
-    </copy>
-
-    <subant target="package">
-      <!--Pass down the version in case its needed again and the target
-      distribution directory so contribs know where to install to.-->
-      <property name="version" value="${version}"/>
-      <property name="dist.dir" value="${dist.dir}/${package.share.dir}"/>
-      <fileset file="${contrib.dir}/build.xml"/>
-    </subant>  	
-
-    <copy todir="${dist.dir}/${package.share.dir}"> 
-      <fileset file="${build.dir}/${final.name}*.jar"/>
-      <fileset file="${build.dir}/${test.final.name}.jar"/>
-      <fileset file="${build.dir}/${examples.final.name}.jar"/>
-      <fileset file="${build.dir}/${tools.final.name}.jar"/>
-    </copy>
-
-    <copy todir="${dist.dir}/bin">
-      <fileset dir="bin">
-        <include name="mapred"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/libexec">
-      <fileset dir="bin">
-        <include name="mapred-config.sh"/>
-      </fileset>
-    </copy>
-
-    <copy todir="${dist.dir}/sbin">
-      <fileset dir="bin">
-        <include name="start-*.sh"/>
-        <include name="stop-*.sh"/>
-      </fileset>
-    </copy>
-
-    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
-    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.redhat"/>
-    <copy file="${basedir}/src/packages/rpm/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.redhat"/>
-    <copy file="${basedir}/src/packages/deb/init.d/hadoop-jobtracker" tofile="${dist.dir}/sbin/hadoop-jobtracker.debian"/>
-    <copy file="${basedir}/src/packages/deb/init.d/hadoop-tasktracker" tofile="${dist.dir}/sbin/hadoop-tasktracker.debian"/>
-    
-    <copy file="${basedir}/src/packages/update-mapred-env.sh" tofile="${dist.dir}/sbin/update-mapred-env.sh"/>
-      	
-    <copy todir="${dist.dir}/etc/hadoop">
-      <fileset dir="${conf.dir}" excludes="**/*.template"/>
-    </copy>
-
-    <copy todir="${dist.dir}/${package.share.dir}/templates">
-      <fileset dir="${basedir}/src/packages/templates/conf" includes="*"/>
-    </copy>
-
-    <copy todir="${dist.dir}/${package.share.dir}/webapps">
-      <fileset dir="${build.webapps}"/>
-    </copy>
-
-    <copy todir="${dist.dir}/share/doc/hadoop/${module}">
-      <fileset dir=".">
-        <include name="*.txt" />
-      </fileset>
-    </copy>
-
-    <chmod perm="ugo+x" type="file" parallel="false">
-        <fileset dir="${dist.dir}/bin"/>
-        <fileset dir="${dist.dir}/sbin"/>
-    </chmod>  	
-  </target>
-
-  <target name="binary-system" depends="bin-package, jar-system, jar-test-system"
-     description="make system test package for deployment">
-    <!--TODO!!!!! fix this shit...-->
-    <copy todir="${system-test-build-dir}/${final.name}">
-      <fileset dir="${dist.dir}">
-      </fileset>
-    </copy>
-    <copy todir="${system-test-build-dir}/${final.name}/conf">
-      <fileset dir="${test.src.dir}/system/conf/"/>
-    </copy>
-    <copy todir="${system-test-build-dir}">
-      <fileset dir="${build.dir}">
-        <include name="${test.final.name}.jar"/>
-        <include name="${examples.final.name}.jar"/>
-      </fileset>
-    </copy>
-    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-common-${hadoop-common.version}.jar"
-      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-common-${herriot.suffix}-${hadoop-common.version}.jar"
-      overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/lib/hadoop-hdfs-${version}.jar"
-      file="${system-test-build-dir}/ivy/lib/${ant.project.name}/system/hadoop-hdfs-${herriot.suffix}-${version}.jar"
-      overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}.jar"
-      file="${system-test-build-dir}/${instrumented.final.name}.jar" overwrite="true"/>
-    <copy tofile="${system-test-build-dir}/${final.name}/${final.name}-sources.jar"
-      file="${system-test-build-dir}/${instrumented.final.name}-sources.jar" overwrite="true"/>
-    <macro_tar 
-      param.destfile="${system-test-build-dir}/${final.name}-bin.${herriot.suffix}.tar.gz">
-        <param.listofitems>
-          <tarfileset dir="${system-test-build-dir}" mode="664">
-            <exclude name="${final.name}/bin/*" />
-            <exclude name="${final.name}/src/**" />
-            <exclude name="${final.name}/docs/**" />
-            <include name="${final.name}/**" />
-          </tarfileset>
-        </param.listofitems>
-      </macro_tar>
-  </target>
-  
-  <target name="binary" depends="bin-package" description="Make tarball without source and documentation">
-    <macro_tar param.destfile="${build.dir}/${final.name}-bin.tar.gz">
-      <param.listofitems>
-        <tarfileset dir="${build.dir}" mode="664">
-          <exclude name="${final.name}/bin/*" />
-          <exclude name="${final.name}/src/**" />
-          <exclude name="${final.name}/docs/**" />
-          <include name="${final.name}/**" />
-        </tarfileset>
-        <tarfileset dir="${build.dir}" mode="755">
-          <include name="${final.name}/bin/*" />
-        </tarfileset>
-      </param.listofitems>
-    </macro_tar>
-  </target>
-
-  <target name="rpm" depends="binary" description="Make rpm package">
-    <mkdir dir="${package.buildroot}/BUILD" />
-    <mkdir dir="${package.buildroot}/RPMS" />
-    <mkdir dir="${package.buildroot}/SRPMS" />
-    <mkdir dir="${package.buildroot}/SOURCES" />
-    <mkdir dir="${package.buildroot}/SPECS" />
-    <copy todir="${package.buildroot}/SOURCES">
-      <fileset dir="${build.dir}">
-        <include name="${final.name}-bin.tar.gz" />
-      </fileset>
-    </copy>
-    <copy file="${src.dir}/packages/rpm/spec/hadoop-mapred.spec" todir="${package.buildroot}/SPECS">
-      <filterchain>
-        <replacetokens>
-          <token key="final.name" value="${final.name}" />
-          <token key="version" value="${_version}" />
-          <token key="package.release" value="${package.release}" />
-          <token key="package.build.dir" value="${package.build.dir}" />
-          <token key="package.prefix" value="${package.prefix}" />
-          <token key="package.conf.dir" value="${package.conf.dir}" />
-          <token key="package.log.dir" value="${package.log.dir}" />
-          <token key="package.pid.dir" value="${package.pid.dir}" />
-          <token key="package.var.dir" value="${package.var.dir}" />
-        </replacetokens>
-      </filterchain>
-    </copy>
-    <rpm specFile="hadoop-mapred.spec" command="-bb --target ${os.arch}" topDir="${package.buildroot}" cleanBuildDir="true" failOnError="true"/>
-    <copy todir="${build.dir}/" flatten="true">
-      <fileset dir="${package.buildroot}/RPMS">
-        <include name="**/*.rpm" />
-      </fileset>
-    </copy>
-    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
-  </target>
-
-  <target name="deb" depends="ivy-retrieve-package, binary" description="Make deb package">
-    <taskdef name="deb"
-           classname="org.vafer.jdeb.ant.DebAntTask">
-      <classpath refid="ivy-package.classpath" />
-    </taskdef>
-
-    <mkdir dir="${package.build.dir}/hadoop.control" />
-    <mkdir dir="${package.buildroot}/${package.prefix}" />
-    <copy todir="${package.buildroot}/${package.prefix}">
-      <fileset dir="${build.dir}/${final.name}">
-        <include name="**" />
-      </fileset>
-    </copy>
-    <copy todir="${package.build.dir}/hadoop.control">
-      <fileset dir="${src.dir}/packages/deb/hadoop.control">
-        <exclude name="control" />
-      </fileset>
-    </copy>
-    <copy file="${src.dir}/packages/deb/hadoop.control/control" todir="${package.build.dir}/hadoop.control">
-      <filterchain>
-        <replacetokens>
-          <token key="final.name" value="${final.name}" />
-          <token key="version" value="${_version}" />
-          <token key="package.release" value="${package.release}" />
-          <token key="package.build.dir" value="${package.build.dir}" />
-          <token key="package.prefix" value="${package.prefix}" />
-          <token key="package.conf.dir" value="${package.conf.dir}" />
-          <token key="package.log.dir" value="${package.log.dir}" />
-          <token key="package.pid.dir" value="${package.pid.dir}" />
-        </replacetokens>
-      </filterchain>
-    </copy>
-    <deb destfile="${package.buildroot}/${name}_${_version}-${package.release}_${os.arch}.deb" control="${package.build.dir}/hadoop.control">
-      <tarfileset dir="${build.dir}/${final.name}" filemode="644" prefix="${package.prefix}">
-        <exclude name="bin/*" />
-        <exclude name="${package.share.dir}/contrib/*/bin/*" />
-        <exclude name="etc" />
-        <exclude name="etc/**" />
-        <exclude name="libexec/*" />
-        <exclude name="sbin/*" />
-        <include name="**" />
-      </tarfileset>
-      <tarfileset dir="${build.dir}/${final.name}" filemode="755" prefix="${package.prefix}">
-        <include name="bin/*" />
-        <include name="sbin/*" />
-        <exclude name="sbin/*.redhat" />
-        <exclude name="sbin/*.debian" />
-        <include name="libexec/*" />
-        <include name="${package.share.dir}/contrib/*/bin/*" />
-      </tarfileset>
-      <tarfileset dir="${src.dir}/packages" filemode="755" prefix="${package.prefix}/sbin">
-        <include name="*.sh" />
-      </tarfileset>
-      <tarfileset dir="${build.dir}/${final.name}/etc/hadoop" filemode="644" prefix="${package.conf.dir}">
-        <exclude name="configuration.xsl" />
-        <exclude name="hadoop-metrics2.properties" />
-        <exclude name="core-site.xml" />
-        <exclude name="hdfs-site.xml" />
-        <exclude name="mapred-site.xml" />
-        <include name="**" />
-      </tarfileset>
-      <tarfileset dir="${basedir}/src/packages/deb/init.d" filemode="755" prefix="/etc/init.d">
-        <include name="**" />
-      </tarfileset>
-    </deb>
-    <copy todir="${build.dir}/" flatten="true">
-      <fileset dir="${package.buildroot}">
-        <include name="**/${name}*.deb" />
-      </fileset>
-    </copy>
-    <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
-  </target>
-
-  <target name="ant-task-download" description="To download mvn-ant-task">
-    <get src="${ant_task_repo_url}" dest="${ant_task.jar}" usetimestamp="true"/>
-  </target>
-
-  <target name="mvn-taskdef" depends="ant-task-download">
-     <path id="mvn-ant-task.classpath" path="${ant_task.jar}"/> 
-     <typedef resource="org/apache/maven/artifact/ant/antlib.xml" 
-         uri="urn:maven-artifact-ant"
-         classpathref="mvn-ant-task.classpath"/>
-  </target>   
-
-  <target name="clean-cache" description="Clean. Delete ivy cache">
-    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-common"/>
-    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-common-test"/>
-    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-hdfs"/>
-    <delete dir="${user.home}/.ivy2/cache/org.apache.hadoop/hadoop-hdfs-test"/>
-  </target>
-
-  <target name="mvn-install" depends="mvn-taskdef,jar,jar-test,set-version,-mvn-system-install"
-      description="To install hadoop mapreduce and test jars to local filesystem's m2 cache">
-    <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
-    <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
-    <artifact:install file="${hadoop-mapred.jar}">
-      <pom refid="hadoop.mapred"/>
-      <attach file="${hadoop-mapred-sources.jar}" classifier="sources" />
-    </artifact:install>
-    <artifact:install file="${hadoop-mapred-test.jar}">
-      <pom refid="hadoop.mapred.test"/>
-      <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources" />
-    </artifact:install>
-  </target>
-
-  <target name="mvn-deploy" depends="mvn-taskdef, jar, jar-test,
-    jar-system, jar-test-system, set-version, signanddeploy, simpledeploy"
-    description="To deploy hadoop mapredice and test jar's to apache
-    snapshot's repository"/>
-
-   <target name="signanddeploy" if="staging" depends="sign">
-     <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
-     <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
-     <artifact:pom file="${hadoop-mapred-instrumented.pom}" 
-       id="hadoop.mapred.${herriot.suffix}"/>
-     <artifact:pom file="${hadoop-mapred-instrumented-test.pom}" 
-       id="hadoop.mapred.${herriot.suffix}.test"/>
-     <artifact:install-provider artifactId="wagon-http"
-     version="${wagon-http.version}"/>
-
-     <artifact:deploy file="${hadoop-mapred.jar}">
-       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
-       <pom refid="hadoop.mapred"/>
-       <attach file="${hadoop-mapred.jar}.asc" type="jar.asc"/>
-       <attach file="${hadoop-mapred.pom}.asc" type="pom.asc"/>
-       <attach file="${hadoop-mapred-sources.jar}.asc" type="jar.asc"
-         classifier="sources" />
-       <attach file="${hadoop-mapred-sources.jar}" classifier="sources"/>
-     </artifact:deploy>
-
-     <artifact:deploy file="${hadoop-mapred-test.jar}">
-       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
-       <pom refid="hadoop.mapred.test"/>
-       <attach file="${hadoop-mapred-test.jar}.asc" type="jar.asc"/>
-       <attach file="${hadoop-mapred-test.pom}.asc" type="pom.asc"/>
-       <attach file="${hadoop-mapred-test-sources.jar}.asc" type="jar.asc"
-         classifier="sources"/>
-       <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources"/>
-     </artifact:deploy>
-
-     <artifact:deploy file="${hadoop-mapred-instrumented.jar}">
-       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
-       <pom refid="hadoop.mapred.${herriot.suffix}"/>
-       <attach file="${hadoop-mapred-instrumented.jar}.asc" type="jar.asc"/>
-       <attach file="${hadoop-mapred-instrumented.pom}.asc" type="pom.asc"/>
-       <attach file="${hadoop-mapred-instrumented-sources.jar}.asc" 
-         type="jar.asc" classifier="sources"/>
-       <attach file="${hadoop-mapred-instrumented-sources.jar}"
-         classifier="sources"/>
-     </artifact:deploy>
-
-      <artifact:deploy file="${hadoop-mapred-instrumented-test.jar}">
-       <remoteRepository id="apache.staging.https" url="${asfstagingrepo}"/>
-       <pom refid="hadoop.mapred.${herriot.suffix}.test"/>
-       <attach file="${hadoop-mapred-instrumented-test.jar}.asc" type="jar.asc"/>
-       <attach file="${hadoop-mapred-instrumented-test.pom}.asc" type="pom.asc"/>
-       <attach file="${hadoop-mapred-instrumented-test-sources.jar}.asc" 
-         type="jar.asc" classifier="sources"/>
-       <attach file="${hadoop-mapred-instrumented-test-sources.jar}"
-         classifier="sources"/>
-     </artifact:deploy>
-   </target>
-
-   <target name="sign" depends="clean-sign" if="staging">
-    <input message="password:>" addproperty="gpg.passphrase">
-     <handler classname="org.apache.tools.ant.input.SecureInputHandler" />
-    </input>
-    <macrodef name="sign-artifact" description="Signs the artifact">
-      <attribute name="input.file"/>
-      <attribute name="output.file" default="@{input.file}.asc"/>
-      <attribute name="gpg.passphrase"/>
-      <sequential>
-        <echo>Signing @{input.file} Sig File: @{output.file}</echo>
-        <exec executable="gpg" >
-          <arg value="--armor"/>
-          <arg value="--output"/>
-          <arg value="@{output.file}"/>
-          <arg value="--passphrase"/>
-          <arg value="@{gpg.passphrase}"/>
-          <arg value="--detach-sig"/>
-          <arg value="@{input.file}"/>
-        </exec>
-      </sequential>
-    </macrodef>
-    <sign-artifact input.file="${hadoop-mapred.jar}" 
-     output.file="${hadoop-mapred.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-test.jar}" 
-     output.file="${hadoop-mapred-test.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-sources.jar}" 
-     output.file="${hadoop-mapred-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-test-sources.jar}" 
-     output.file="${hadoop-mapred-test-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred.pom}" 
-     output.file="${hadoop-mapred.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-test.pom}" 
-     output.file="${hadoop-mapred-test.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented.jar}" 
-     output.file="${hadoop-mapred-instrumented.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented.pom}" 
-     output.file="${hadoop-mapred-instrumented.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented-sources.jar}" 
-     output.file="${hadoop-mapred-instrumented-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented-test.jar}" 
-     output.file="${hadoop-mapred-instrumented-test.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented-test.pom}" 
-     output.file="${hadoop-mapred-instrumented-test.pom}.asc" gpg.passphrase="${gpg.passphrase}"/>
-    <sign-artifact input.file="${hadoop-mapred-instrumented-test-sources.jar}" 
-     output.file="${hadoop-mapred-instrumented-test-sources.jar}.asc" gpg.passphrase="${gpg.passphrase}"/>
-  </target>
-
-  <target name="simpledeploy" unless="staging">
-     <artifact:pom file="${hadoop-mapred.pom}" id="hadoop.mapred"/>
-     <artifact:pom file="${hadoop-mapred-test.pom}" id="hadoop.mapred.test"/>
-     <artifact:pom file="${hadoop-mapred-instrumented.pom}" 
-       id="hadoop.mapred.${herriot.suffix}"/>
-
-     <artifact:install-provider artifactId="wagon-http" version="${wagon-http.version}"/>
-     <artifact:deploy file="${hadoop-mapred.jar}">
-         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
-         <pom refid="hadoop.mapred"/>
-         <attach file="${hadoop-mapred-sources.jar}" classifier="sources" />
-     </artifact:deploy>
-
-     <artifact:deploy file="${hadoop-mapred-test.jar}">
-         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
-         <pom refid="hadoop.mapred.test"/>
-         <attach file="${hadoop-mapred-test-sources.jar}" classifier="sources" />
-     </artifact:deploy> 
-
-     <artifact:deploy file="${hadoop-mapred-instrumented.jar}">
-         <remoteRepository id="apache.snapshots.https" url="${asfsnapshotrepo}"/>
-         <pom refid="hadoop.mapred.${herriot.suffix}"/>
-         <attach file="${hadoop-mapred-instrumented-sources.jar}" classifier="sources" />
-     </artifact:deploy>
-  </target>
-
-  <target name="set-version">
-    <delete file="${basedir}/ivy/hadoop-mapred.xml"/>
-    <delete file="${basedir}/ivy/hadoop-mapred-test.xml"/>
-    <delete file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}.xml"/>
-    <delete file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test.xml"/>
-    <copy file="${basedir}/ivy/hadoop-mapred-template.xml" tofile="${basedir}/ivy/hadoop-mapred.xml"/>
-    <copy file="${basedir}/ivy/hadoop-mapred-test-template.xml" tofile="${basedir}/ivy/hadoop-mapred-test.xml"/>
-    <copy file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-template.xml"
-      tofile="${basedir}/ivy/hadoop-mapred-${herriot.suffix}.xml"/>
-    <copy file="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test-template.xml"
-      tofile="${basedir}/ivy/hadoop-mapred-${herriot.suffix}-test.xml"/>
-    <replaceregexp byline="true">
-      <regexp pattern="@version"/>
-      <substitution expression="${version}"/>
-      <fileset dir="${basedir}/ivy">
-        <include name="hadoop-mapred.xml"/>
-        <include name="hadoop-mapred-test.xml"/>
-        <include name="hadoop-mapred-${herriot.suffix}.xml"/>
-        <include name="hadoop-mapred-${herriot.suffix}-test.xml"/>
-      </fileset>
-    </replaceregexp>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Perform audit activities for the release                           -->
-  <!-- ================================================================== -->
-  <target name="rats-taskdef" depends="ivy-retrieve-releaseaudit">
-     <typedef format="xml" resource="org/apache/rat/anttasks/antlib.xml" uri="antlib:org.apache.rat.anttasks"
-      classpathref="releaseaudit-classpath"/>
-  </target>
-
-  <!--<target name="releaseaudit" depends="package, rats-taskdef" description="Release Audit activities"> -->
-  <target name="releaseaudit" depends="package, rats-taskdef" description="Release Audit activities">
-   <rat:report xmlns:rat="antlib:org.apache.rat.anttasks">
-      <fileset dir="${dist.dir}">
-        <exclude name="CHANGES.txt"/>
-        <exclude name="**/conf/*"/>
-        <exclude name="**/docs/"/>
-        <exclude name="**/VERSION"/>
-        <exclude name="webapps/**/WEB-INF/web.xml"/>
-        <exclude name="lib/jdiff/"/>
-        <exclude name="src/test/all-tests"/>
-        <exclude name="src/test/commit-tests"/>
-        <exclude name="src/test/empty-file"/>
-        <exclude name="src/test/mapred/org/apache/hadoop/mapred/test.tgz"/>
-        <exclude name="src/test/tools/data/rumen/**/*"/>
-        <exclude name="src/test/mapred/org/apache/hadoop/mapred/*.txt"/>
-        <exclude name="src/contrib/mumak/src/test/data/*.json"/>
-        <exclude name="src/contrib/index/sample/*.txt"/>
-        <exclude name="src/test/mapred/org/apache/hadoop/cli/data60bytes"/>
-        <exclude name="src/examples/org/apache/hadoop/examples/dancing/puzzle1.dta"/>
-        <exclude name="src/contrib/eclipse-plugin/META-INF/MANIFEST.MF"/>
-        <exclude name="src/c++/librecordio/*"/>
-        <exclude name="src/c++/pipes/*"/>
-        <exclude name="src/c++/utils/*"/>
-        <exclude name="src/c++/task-controller/*"/>
-        <exclude name="src/examples/pipes/*"/>
-        <exclude name="src/c++/pipes/debug/*"/>
-      </fileset>
-    </rat:report>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Clean.  Delete the build files, and their directories              -->
-  <!-- ================================================================== -->
-  <target name="clean" depends="clean-contrib, clean-fi, clean-sign" description="Clean.  Delete the build files, and their directories">
-    <delete dir="${build.dir}"/>
-    <delete dir="${docs.src}/build"/>
-    <delete file="${hadoop-mapred.pom}"/>
-    <delete file="${hadoop-mapred-test.pom}"/>
-    <delete file="${hadoop-mapred-instrumented.pom}"/>
-    <delete file="${hadoop-mapred-instrumented-test.pom}"/>
-    <delete file="${hadoop-mapred-examples.pom}"/>
-    <delete file="${hadoop-mapred-tools.pom}"/>
-  </target>
-
-  <target name="clean-sign" description="Clean.  Delete .asc files">
-    <delete>
-      <fileset dir="." includes="**/**/*.asc"/>
-    </delete>
-  </target>
-
-  <target name="veryclean" depends="clean-cache,clean" 
-          description="veryclean.  Delete ivy and ant maven task jar">
-    <delete file="${ant_task.jar}"/>
-    <delete file="${ivy.jar}"/>
-  </target>
-
-  <!-- ================================================================== -->
-  <!-- Clean contrib target. For now, must be called explicitly           -->
-  <!-- Using subant instead of ant as a workaround for 30569              -->
-  <!-- ================================================================== -->
-  <target name="clean-contrib">
-     <subant target="clean">        
-        <fileset file="src/contrib/build.xml"/>
-     </subant>  	
-  </target>
-	
-
-<!-- ================================================================== -->
-<!-- librecordio targets.                                               -->
-<!-- ================================================================== -->		
-
-  <target name="compile-librecordio" depends="init" if="librecordio" >
-     <mkdir dir="${build.librecordio}"/>
-     <exec dir="${librecordio.src}" executable="${make.cmd}" failonerror="true">
-        <env key="XERCESCROOT" value="${xercescroot}"/>
-        <env key="LIBRECORDIO_BUILD_DIR" value="${build.librecordio}"/>
-     </exec>
-  </target>
-  	
-  <target name="test-librecordio" depends="compile-librecordio, compile-core" if="librecordio">
-    <delete dir="${librecordio.test.dir}"/>
-    <mkdir dir="${librecordio.test.dir}"/>
-    <exec dir="${librecordio.src}/test" executable="${make.cmd}" failonerror="true">
-        <env key="HADOOP_HOME" value="${basedir}"/>
-	<env key="XERCESCROOT" value="${xercescroot}"/>
-        <env key="LIBRECORDIO_BUILD_DIR" value="${build.librecordio}"/> 	
-        <env key="LIBRECORDIO_TEST_DIR" value="${librecordio.test.dir}"/>
-      		<arg value="all"/>		
-    </exec>
-  </target>
-
-  <target name="package-librecordio" depends="compile-librecordio" if="librecordio">
-    <mkdir dir="${dist.dir}/lib"/> 
-    <copy todir="${dist.dir}/lib">
-       <fileset dir="${build.librecordio}" casesensitive="yes" followsymlinks="false">
-          <exclude name="**/tests/**"/>
-          <exclude name="*.so"/> 
-          <exclude name="*.o"/>
-       </fileset>
-    </copy>
-    <chmod perm="ugo+x" type="file">
-       <fileset dir="${dist.dir}/lib"/>
-    </chmod>
-  </target>
- 
-  <target name="check-c++-configure" depends="init" if="compile.c++">
-    <condition property="need.c++.utils.configure">
-       <not> <available file="${c++.utils.src}/configure"/> </not>
-    </condition>
-    <condition property="need.c++.pipes.configure">
-       <not> <available file="${c++.pipes.src}/configure"/> </not>
-    </condition>
-    <condition property="need.c++.examples.pipes.configure">
-       <not> <available file="${c++.examples.pipes.src}/configure"/> </not>
-    </condition>
-    <condition property="need.c++.task-controller.configure">
-       <not> <available file="${c++.task-controller.src}/configure"/> </not>
-    </condition>
-  </target>
-
-  <target name="create-c++-utils-configure" depends="check-c++-configure" 
-                                            if="need.c++.utils.configure">
-    <exec executable="autoreconf" dir="${c++.utils.src}" searchpath="yes" 
-          failonerror="yes">
-       <arg value="-i"/>
-       <arg value="-f"/>
-    </exec>
-  </target>
-
-  <target name="create-c++-pipes-configure" depends="check-c++-configure" 
-                                            if="need.c++.pipes.configure">
-    <exec executable="autoreconf" dir="${c++.pipes.src}" searchpath="yes" 
-          failonerror="yes">
-       <arg value="-i"/>
-       <arg value="-f"/>
-    </exec>
-  </target>
-
-  <target name="create-c++-examples-pipes-configure" depends="check-c++-configure" 
-                                            if="need.c++.examples.pipes.configure">
-    <exec executable="autoreconf" dir="${c++.examples.pipes.src}" 
-          searchpath="yes" failonerror="yes">
-       <arg value="-i"/>
-       <arg value="-f"/>
-    </exec>
-  </target>
-
-  <target name="create-c++-task-controller-configure" depends="check-c++-configure" 
-                                            if="need.c++.task-controller.configure">
-    <exec executable="autoreconf" dir="${c++.task-controller.src}" 
-          searchpath="yes" failonerror="yes">
-       <arg value="-i"/>
-       <arg value="-f"/>
-    </exec>
-  </target>
-
-  <target name="create-c++-configure" depends="create-c++-utils-configure,
-                                               create-c++-pipes-configure,
-                                               create-c++-examples-pipes-configure,
-                                               create-c++-task-controller-configure" 
-                                      if="compile.c++">
-  </target>
-   
-  <target name="check-c++-makefiles" depends="init" if="compile.c++">
-    <condition property="need.c++.utils.makefile">
-       <not> <available file="${build.c++.utils}/Makefile"/> </not>
-    </condition>
-    <condition property="need.c++.pipes.makefile">
-       <not> <available file="${build.c++.pipes}/Makefile"/> </not>
-    </condition>
-    <condition property="need.c++.examples.pipes.makefile">
-       <not> <available file="${build.c++.examples.pipes}/Makefile"/> </not>
-    </condition>
-  </target>
-
-  <target name="create-c++-utils-makefile" depends="check-c++-makefiles" 
-                                           if="need.c++.utils.makefile">
-    <antcall target="create-c++-utils-configure"/>
-    <mkdir dir="${build.c++.utils}"/>
-    <chmod file="${c++.utils.src}/configure" perm="ugo+x"/>
-    <exec executable="${c++.utils.src}/configure" dir="${build.c++.utils}"
-          failonerror="yes">
-      <arg value="--prefix=${install.c++}"/>
-    </exec>
-  </target>
-
-  <target name="compile-c++-utils" depends="create-c++-utils-makefile"
-                                   if="compile.c++">
-    <exec executable="${make.cmd}" dir="${build.c++.utils}" searchpath="yes" 
-          failonerror="yes">
-      <arg  value="install"/>
-    </exec>
-  </target>
-
-  <target name="create-c++-pipes-makefile" depends="check-c++-makefiles" 
-                                           if="need.c++.pipes.makefile">
-    <antcall target="create-c++-pipes-configure"/>
-    <mkdir dir="${build.c++.pipes}"/>
-    <chmod file="${c++.pipes.src}/configure" perm="ugo+x"/>
-    <exec executable="${c++.pipes.src}/configure" dir="${build.c++.pipes}"
-          failonerror="yes">
-      <arg value="--prefix=${install.c++}"/>
-    </exec>
-  </target>
-
-  <target name="compile-c++-pipes" 
-          depends="create-c++-pipes-makefile,compile-c++-utils"
-          if="compile.c++">
-    <exec executable="${make.cmd}" dir="${build.c++.pipes}" searchpath="yes" 
-          failonerror="yes">
-      <arg value="install"/>
-    </exec>
-  </target>
-
-  <target name="compile-c++" 
-          depends="compile-c++-pipes"/>
-
-  <target name="create-c++-examples-pipes-makefile" 
-          depends="check-c++-makefiles" 
-          if="need.c++.examples.pipes.makefile">
-    <antcall target="create-c++-examples-pipes-configure"/>
-    <mkdir dir="${build.c++.examples.pipes}"/>
-    <chmod file="${c++.examples.pipes.src}/configure" perm="ugo+x"/>
-    <exec executable="${c++.examples.pipes.src}/configure" 
-          dir="${build.c++.examples.pipes}"
-          failonerror="yes">
-      <arg value="--prefix=${install.c++.examples}"/>
-      <arg value="--with-hadoop-utils=${install.c++}"/>
-      <arg value="--with-hadoop-pipes=${install.c++}"/>
-    </exec>
-  </target>
-
-  <target name="compile-c++-examples-pipes" 
-          depends="create-c++-examples-pipes-makefile,compile-c++-pipes"
-          if="compile.c++">
-    <exec executable="${make.cmd}" dir="${build.c++.examples.pipes}" searchpath="yes" 
-          failonerror="yes">
-      <arg  value="install"/>
-    </exec>
-  </target>
-
-  <target name="compile-c++-examples" 
-          depends="compile-c++-examples-pipes"/>
-
- <target name="clover" depends="clover.setup, clover.info" description="Instrument the Unit tests using Clover.  To use, specify -Dclover.home=&lt;base of clover installation&gt; -Drun.clover=true on the command line."/>
-
-<target name="clover.setup" if="clover.enabled">
-   <taskdef resource="cloverlib.xml" classpath="${clover.jar}"/>
-   <mkdir dir="${clover.db.dir}"/>
-   <clover-setup initString="${clover.db.dir}/hadoop_coverage.db">
-     <fileset dir="${src.dir}" includes="tools/**/* java/**/*"/>
-     <testsources dir="${test.src.dir}" />
-   </clover-setup>
-</target>
-
-<target name="clover.info" unless="clover.present">
-  <echo>
-     Clover not found. Code coverage reports disabled.
-  </echo>
-</target>
-
-<target name="clover.check">
-  <fail unless="clover.present">
-  ##################################################################
-   Clover not found.
-   Please specify -Dclover.home=&lt;base of clover installation&gt;
-   on the command line.
-  ##################################################################
-  </fail>
-</target>
-
-<target name="generate-clover-reports" depends="clover.check, clover">
-  <mkdir dir="${clover.report.dir}"/>
-  <clover-report>
-     <current outfile="${clover.report.dir}" title="${final.name}">
-     <format type="html"/>
-     </current>
-  </clover-report>
-  <clover-report>
-     <current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
-     <format type="xml"/>
-     </current>
-  </clover-report>
-</target>
-
-<target name="findbugs.check" depends="check-for-findbugs" unless="findbugs.present">
-  <fail message="'findbugs.home' is not defined. Please pass -Dfindbugs.home=&lt;base of Findbugs installation&gt; to Ant on the command-line." />
-</target>
-
-<target name="patch.check" unless="patch.file">
-  <fail message="'patch.file' is not defined. Please pass -Dpatch.file=&lt;location of patch file&gt; to Ant on the command-line." />
-</target>
-
-<target name="test-patch" depends="patch.check,findbugs.check,forrest.check">
-  <exec executable="bash" failonerror="true">
-    <arg value="${basedir}/src/test/bin/test-patch.sh"/>
-    <arg value="DEVELOPER"/>
-    <arg value="${patch.file}"/>
-    <arg value="${scratch.dir}"/>
-    <arg value="${svn.cmd}"/>
-    <arg value="${grep.cmd}"/>
-    <arg value="${patch.cmd}"/>
-    <arg value="${findbugs.home}"/>
-    <arg value="${forrest.home}"/>
-    <arg value="${basedir}"/>
-  </exec>
-</target>
-
-<target name="hudson-test-patch" depends="findbugs.check,forrest.check">
-  <exec executable="bash" failonerror="true">
-    <arg value="${basedir}/src/test/bin/test-patch.sh"/>
-    <arg value="HUDSON"/>
-    <arg value="${scratch.dir}"/>
-    <arg value="${support.dir}"/>
-    <arg value="${ps.cmd}"/>
-    <arg value="${wget.cmd}"/>
-    <arg value="${jiracli.cmd}"/>
-    <arg value="${svn.cmd}"/>
-    <arg value="${grep.cmd}"/>
-    <arg value="${patch.cmd}"/>
-    <arg value="${findbugs.home}"/>
-    <arg value="${forrest.home}"/>
-    <arg value="${eclipse.home}"/>
-    <arg value="${basedir}"/>
-    <arg value="${jira.passwd}"/>
-    <arg value="${curl.cmd}"/>
-    <arg value="${defect}"/>
-  </exec>
-</target>
-	
-  <condition property="ant-eclipse.jar.exists">
-    <available file="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar"/>
-  </condition>
-
-  <target name="ant-eclipse-download" unless="ant-eclipse.jar.exists"
-          description="Downloads the ant-eclipse binary.">
-    <get src="http://downloads.sourceforge.net/project/ant-eclipse/ant-eclipse/1.0/ant-eclipse-1.0.bin.tar.bz2"
-         dest="${build.dir}/ant-eclipse-1.0.bin.tar.bz2" usetimestamp="false" />
-
-    <untar src="${build.dir}/ant-eclipse-1.0.bin.tar.bz2"
-           dest="${build.dir}" compression="bzip2">
-      <patternset>
-        <include name="lib/ant-eclipse-1.0-jvm1.2.jar"/>
-      </patternset>
-    </untar>
-    <delete file="${build.dir}/java/ant-eclipse-1.0.bin.tar.bz2" />
-  </target>
-
-  <target name="eclipse" 
-          depends="init,ant-eclipse-download,ivy-retrieve-common,ivy-retrieve-mapred,ivy-retrieve-test"
-          description="Create eclipse project files">
-       <pathconvert property="eclipse.project">
-         <path path="${basedir}"/>
-         <regexpmapper from="^.*/([^/]+)$$" to="\1" handledirsep="yes"/>
-       </pathconvert>
-    <taskdef name="eclipse"
-             classname="prantl.ant.eclipse.EclipseTask"
-             classpath="${build.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />
-    <eclipse updatealways="true">
-      <project name="${eclipse.project}" />
-      <classpath>
-        <source path="${java.src.dir}"
-                output="${build.dir.eclipse-main-classes}" />
-        <source path="${build.src}"
-                output="${build.dir.eclipse-main-generated-classes}" />
-        <source path="${test.src.dir}/mapred"
-                output="${build.dir.eclipse-test-classes}" />
-        <source path="${test.src.dir}/aop"
-                output="${build.dir.eclipse-test-classes}" />
-        <source path="${test.src.dir}/unit"
-                output="${build.dir.eclipse-test-classes}" />
-        <source path="${examples.dir}"
-                output="${build.dir.eclipse-example-classes}" />
-        <source path="${tools.src}"
-                output="${build.dir.eclipse-tools-classes}" />
-        <source path="${contrib.dir}/block_forensics/src/java"
-                output="${build.dir.eclipse-contrib-classes}/block_forensics/main" />
-        <source path="${contrib.dir}/capacity-scheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/main" />
-        <source path="${contrib.dir}/capacity-scheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/capacity-scheduler/test" />
-        <source path="${contrib.dir}/data_join/src/java"
-                output="${build.dir.eclipse-contrib-classes}/data_join/main" />
-        <source path="${contrib.dir}/data_join/src/examples"
-                output="${build.dir.eclipse-contrib-classes}/data_join/examples" />
-        <source path="${contrib.dir}/data_join/src/test"
-                output="${build.dir.eclipse-contrib-classes}/data_join/test" />
-        <source path="${contrib.dir}/dynamic-scheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/main" />
-        <source path="${contrib.dir}/dynamic-scheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/dynamic-scheduler/test" />
-        <source path="${contrib.dir}/fairscheduler/src/java"
-                output="${build.dir.eclipse-contrib-classes}/fairscheduler/main" />
-        <source path="${contrib.dir}/fairscheduler/src/test"
-                output="${build.dir.eclipse-contrib-classes}/fairscheduler/test" />
-        <source path="${contrib.dir}/gridmix/src/java"
-                output="${build.dir.eclipse-contrib-classes}/gridmix/main" />
-        <source path="${contrib.dir}/gridmix/src/test"
-                output="${build.dir.eclipse-contrib-classes}/gridmix/test" />
-        <source path="${contrib.dir}/mumak/src/java"
-                output="${build.dir.eclipse-contrib-classes}/mumak/main" />
-        <source path="${contrib.dir}/mumak/src/test"
-                output="${build.dir.eclipse-contrib-classes}/mumak/test" />
-        <source path="${contrib.dir}/raid/src/java"
-                output="${build.dir.eclipse-contrib-classes}/raid/main" />
-        <source path="${contrib.dir}/raid/src/test"
-                output="${build.dir.eclipse-contrib-classes}/raid/test" />
-        <source path="${contrib.dir}/streaming/src/java"
-                output="${build.dir.eclipse-contrib-classes}/streaming/main" />
-        <source path="${contrib.dir}/streaming/src/test"
-                output="${build.dir.eclipse-contrib-classes}/streaming/test" />
-        <source path="${contrib.dir}/vaidya/src/java"
-                output="${build.dir.eclipse-contrib-classes}/vaidya/main" />
-        <source path="${contrib.dir}/vertica/src/java"
-                output="${build.dir.eclipse-contrib-classes}/vertica/main" />
-        <source path="${contrib.dir}/vertica/src/test"
-                output="${build.dir.eclipse-contrib-classes}/vertica/test" />
-        <output path="${build.dir.eclipse-main-classes}" />
-        <library pathref="ivy-common.classpath" exported="true" />
-        <library pathref="ivy-mapred.classpath" exported="true" />
-        <library pathref="ivy-test.classpath" exported="false" />
-        <library path="${conf.dir}" exported="false" />
-      </classpath>
-    </eclipse>
-    <copy todir="." overwrite="true">
-      <fileset dir=".eclipse.templates">
-        <exclude name="**/README.txt"/>
-      </fileset>
-      <filterset>
-        <filter token="PROJECT" value="${eclipse.project}"/>
-      </filterset>
-    </copy>
-  </target>
-
-  <target name="ivy-init-dirs">
-    <mkdir dir="${build.ivy.dir}" />
-    <mkdir dir="${build.ivy.lib.dir}" />
-    <mkdir dir="${build.ivy.report.dir}" />
-    <mkdir dir="${build.ivy.maven.dir}" />
-  </target>
-
-  <target name="ivy-probe-antlib" >
-    <condition property="ivy.found">
-      <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
-    </condition>
-  </target>
-
-  <target name="ivy-download" description="To download ivy" unless="offline">
-    <get src="${ivy_repo_url}" dest="${ivy.jar}" usetimestamp="true"/>
-  </target>
-
-  <!--
-  To avoid Ivy leaking things across big projects, always load Ivy in the same classloader.
-  Also note how we skip loading Ivy if it is already there, just to make sure all is well.
-  -->
-  <target name="ivy-init-antlib" depends="ivy-download,ivy-init-dirs,ivy-probe-antlib" unless="ivy.found">
-    <typedef uri="antlib:org.apache.ivy.ant" onerror="fail"
-      loaderRef="ivyLoader">
-      <classpath>
-        <pathelement location="${ivy.jar}"/>
-      </classpath>
-    </typedef>
-    <fail >
-      <condition >
-        <not>
-          <typefound uri="antlib:org.apache.ivy.ant" name="cleancache"/>
-        </not>
-      </condition>
-      You need Apache Ivy 2.0 or later from http://ant.apache.org/
-      It could not be loaded from ${ivy_repo_url}
-    </fail>
-  </target>
-
-
-  <property name="ivyresolvelog" value="download-only"/>
-  <property name="ivyretrievelog" value="quiet"/>
-
-  <target name="ivy-init" depends="ivy-init-antlib" >
-
-    <!--Configure Ivy by reading in the settings file
-        If anyone has already read in a settings file into this settings ID, it gets priority
-    -->
-    <ivy:configure settingsid="${ant.project.name}.ivy.settings" file="${ivysettings.xml}" override='false'/>
-  </target>
-
-  <target name="ivy-resolve" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-javadoc" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="javadoc"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-releaseaudit" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="releaseaudit"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-test" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="test"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-common" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="common"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-package" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="package"
-        log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-mapred" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="mapred"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-jdiff" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="jdiff" 
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-checkstyle" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="checkstyle"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-resolve-system" depends="ivy-init">
-    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="system"
-      log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-retrieve" depends="ivy-resolve"
-    description="Retrieve Ivy-managed artifacts">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-  </target>
-
-  <target name="ivy-retrieve-checkstyle" depends="ivy-resolve-checkstyle"
-    description="Retrieve Ivy-managed artifacts for the checkstyle configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="checkstyle-classpath" conf="checkstyle"/>
-  </target>
-
-  <target name="ivy-retrieve-jdiff" depends="ivy-resolve-jdiff"
-    description="Retrieve Ivy-managed artifacts for the javadoc configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="jdiff-classpath" conf="jdiff"/>
-  </target>
-
-  <target name="ivy-retrieve-javadoc" depends="ivy-resolve-javadoc"
-    description="Retrieve Ivy-managed artifacts for the javadoc configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="javadoc-classpath" conf="javadoc"/>
-  </target>
-
-  <target name="ivy-retrieve-test" depends="ivy-resolve-test"
-    description="Retrieve Ivy-managed artifacts for the test configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="ivy-test.classpath" conf="test"/>
-  </target>
-
-  <target name="ivy-retrieve-common" depends="ivy-resolve-common"
-    description="Retrieve Ivy-managed artifacts for the compile configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="ivy-common.classpath" conf="common"/>
-  </target>
-
-  <target name="ivy-retrieve-package" depends="ivy-resolve-package"
-    description="Retrieve Ivy-managed artifacts for the package configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-                log="${ivyretrievelog}"/>
-    <ivy:cachepath pathid="ivy-package.classpath" conf="package"/>
-  </target>
-
-  <target name="ivy-retrieve-mapred" depends="ivy-resolve-mapred"
-    description="Retrieve Ivy-managed artifacts for the mapred configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="ivy-mapred.classpath" conf="mapred"/>
-  </target>
-
-  <target name="ivy-retrieve-releaseaudit" depends="ivy-resolve-releaseaudit"
-    description="Retrieve Ivy-managed artifacts for the compile configurations">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="releaseaudit-classpath" conf="releaseaudit"/>
-  </target>
-
-  <target name="ivy-retrieve-system" depends="ivy-resolve-system"
-    description="Retrieve Ivy-managed artifacts for the system tests">
-    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
-      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
-        log="${ivyresolvelog}"/>
-    <ivy:cachepath pathid="ivy-system.classpath" conf="system"/>
-  </target>
-
-  <target name="ivy-report" depends="ivy-resolve-releaseaudit"
-    description="Generate">
-    <ivy:report todir="${build.ivy.report.dir}" settingsRef="${ant.project.name}.ivy.settings"/>
-    <echo>
-      Reports generated:${build.ivy.report.dir}
-    </echo>
-  </target>
-
-  <!-- taskcontroller targets -->
-  <target name="init-task-controller-build">
-    <antcall target="create-c++-task-controller-configure"/>
-    <mkdir dir="${build.c++.task-controller}" />
-    <copy todir="${build.c++.task-controller}">
-      <fileset dir="${c++.task-controller.src}" includes="*.c"/>
-      <fileset dir="${c++.task-controller.src}" includes="*.h"/>
-    </copy>
-    <chmod file="${c++.task-controller.src}/configure" perm="ugo+x"/> 
-    <condition property="task-controller.conf.dir.passed">
-      <not>
-        <equals arg1="${hadoop.conf.dir}" arg2="$${hadoop.conf.dir}"/>
-      </not>
-    </condition>
-  </target>
-  <target name="configure-task-controller" depends="init,
-          init-task-controller-build,
-          task-controller-configuration-with-confdir,
-          task-controller-configuration-with-no-confdir">
-  </target>
-  <target name="task-controller-configuration-with-confdir" 
-          if="task-controller.conf.dir.passed" >
-    <exec executable="${c++.task-controller.src}/configure" 
-          dir="${build.c++.task-controller}" failonerror="yes">
-      <arg value="--prefix=${task-controller.install.dir}" />
-      <arg value="--with-confdir=${hadoop.conf.dir}" />
-    </exec>
-  </target>
-  <target name="task-controller-configuration-with-no-confdir" 
-          unless="task-controller.conf.dir.passed">
-    <exec executable="${c++.task-controller.src}/configure" 
-          dir="${build.c++.task-controller}" failonerror="yes">
-      <arg value="--prefix=${task-controller.install.dir}" />
-    </exec>
-  </target>
-  <!--
-    * Create the installation directory.
-    * Do a make install.
-   -->
-  <target name="task-controller" depends="configure-task-controller">
-    <mkdir dir="${task-controller.install.dir}" />
-    <exec executable="${make.cmd}" dir="${build.c++.task-controller}" 
-        searchpath="yes" failonerror="yes">
-      <arg value="install" />
-    </exec>
-  </target>
-  <target name="test-task-controller" depends="task-controller">
-    <copy todir="${build.c++.task-controller}" verbose="true">
-      <fileset dir="${c++.task-controller.src}" includes="tests/"/>
-    </copy>
-    <exec executable="${make.cmd}" dir="${build.c++.task-controller}" 
-        searchpath="yes" failonerror="yes">
-      <arg value="clean" />
-      <arg value="test" />
-    </exec>
-    <exec executable="${build.c++.task-controller}/tests/test-task-controller"
-        dir="${build.c++.task-controller}/tests/"
-        failonerror="yes">
-    </exec>
-  </target>
-  <!-- end of task-controller targets -->
-  
-  <!-- Begining of fault-injection targets-->
-  <import file="${test.src.dir}/aop/build/aop.xml"/>
-  
-  <!-- declaring mapred.src.dir as java.src.dir for aop.xml -->
-  <property name="java.src.dir" value="${src.dir}/java"/>
-  
-  <!-- target dependency from aop.xml -->
-  <target name="-classes-compilation" 
-    depends="compile-mapred-classes, compile-mapred-test"/>
-  
-  <target name="jar-test-fault-inject" depends="jar-mapred-test-fault-inject"
-            description="Make hadoop-mapred-test-fi.jar files"/>
-  
-  <!-- target to build test-fi.jar-->
-  <target name="jar-mapred-test-fault-inject" depends="injectfaults"
-    description="Make hadoop-mapred-test-fi.jar">
-    <macro-jar-test-fault-inject target.name="jar-test" 
-      jar.final.name="test.final.name"
-      jar.final.value="${name}-test-${version}-fi"/>
-  </target>
-  
-  <!-- target to build the hadoop-fi.jar -->
-  <target name="jar-fault-inject" depends="injectfaults"
-    description="Make hadoop-fi.jar">
-    <macro-jar-fault-inject
-      target.name="jar"
-      build.dir="${build-fi.dir}"
-      jar.final.name="final.name"
-      jar.final.value="${final.name}-fi" />
-  </target>
-  
-  <!-- target to run fault injected test cases will run entire mapred test 
-       suite-->
-  <target name="run-test-mapred-fault-inject" depends="injectfaults"
-    description="Run full suite of unit tests with fault injection">
-    <macro-run-tests-fault-inject target.name="run-test-mapred"
-      testcasesonly="false"/>
-  </target>
-
-  <!-- target to run non-FI tests in a FI environment-->
-  <target name="run-fault-inject-with-testcaseonly" depends="injectfaults">
-    <fail unless="testcase">
-    Can't run this target without -Dtestcase setting!
-    </fail>
-    <macro-run-tests-fault-inject target.name="run-test-mapred"
-      testcasesonly="true"/>
-  </target>
-  <condition property="tests.notestcase">
-    <and>
-      <isfalse value="${test.fault.inject}"/>
-      <not>
-        <isset property="testcase"/>
-      </not>
-    </and>
-  </condition>
-  <condition property="tests.notestcase.fi">
-    <and>
-      <not>
-        <isset property="testcase"/>
-      </not>
-      <istrue value="${test.fault.inject}"/>
-    </and>
-  </condition>
-  <condition property="test.testcase">
-    <and>
-      <isfalse value="${test.fault.inject}"/>
-      <isset property="testcase"/>
-    </and>
-  </condition>
-  <condition property="tests.testcaseonly.fi">
-    <istrue value="${special.fi.testcasesonly}" />
-  </condition>
-  <condition property="tests.testcase.fi">
-    <and>
-      <istrue value="${test.fault.inject}" />
-      <isset property="testcase" />
-      <isfalse value="${special.fi.testcasesonly}" />
-    </and>
-  </condition>
-  <!-- End of fault injection targets-->
-  
-</project>

hadoop-mapreduce/conf/capacity-scheduler.xml.template

@@ -1,82 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- This is one of the configuration files for capacity-scheduler
-     (org.apache.hadoop.mapred.CapacityTaskScheduler), a TaskScheduler
-     for Map/Reduce system. The other configuration file is
-     conf/mapred-queues.xml which it shares with the framework for
-     configuring queues in the system. -->
-
-<!-- This file can be used to configure (1) job-initialization-poller
-     related properties and (2) the default values for various properties
-     for all the queues.-->
-
-<configuration>   
-  <!-- The default configuration settings for the capacity task scheduler --> 
-  <!-- The default values would be applied to all the queues which don't have -->    
-  <!-- the appropriate property for the particular queue configured in the -->       
-  <!-- queue-configuration file conf/mapred-queues.xml -->                           
-
-  <property>
-    <name>mapred.capacity-scheduler.default-supports-priority</name>
-    <value>false</value>
-    <description>If true, priorities of jobs will be taken into 
-      account in scheduling decisions by default in a job queue.
-    </description>
-  </property>
-  
-  <property>
-    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
-    <value>100</value>
-    <description>The percentage of the resources limited to a particular user
-      for the job queue at any given point of time by default.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user</name>
-    <value>2</value>
-    <description>The maximum number of jobs to be pre-initialized for a user
-    of the job queue.
-    </description>
-  </property>
-
-
-  <!-- Capacity scheduler Job Initialization configuration parameters -->
-  <property>
-    <name>mapred.capacity-scheduler.init-poll-interval</name>
-    <value>5000</value>
-    <description>The amount of time in miliseconds which is used to poll 
-    the job queues for jobs to initialize.
-    </description>
-  </property>
-  <property>
-    <name>mapred.capacity-scheduler.init-worker-threads</name>
-    <value>5</value>
-    <description>Number of worker threads which would be used by
-    Initialization poller to initialize jobs in a set of queue.
-    If number mentioned in property is equal to number of job queues
-    then a single thread would initialize jobs in a queue. If lesser
-    then a thread would get a set of queues assigned. If the number
-    is greater then number of threads would be equal to number of 
-    job queues.
-    </description>
-  </property>
-
-</configuration>

hadoop-mapreduce/conf/configuration.xsl

@@ -1,40 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
-<xsl:output method="html"/>
-<xsl:template match="configuration">
-<html>
-<body>
-<table border="1">
-<tr>
- <td>name</td>
- <td>value</td>
- <td>description</td>
-</tr>
-<xsl:for-each select="property">
-<tr>
-  <td><a name="{name}"><xsl:value-of select="name"/></a></td>
-  <td><xsl:value-of select="value"/></td>
-  <td><xsl:value-of select="description"/></td>
-</tr>
-</xsl:for-each>
-</table>
-</body>
-</html>
-</xsl:template>
-</xsl:stylesheet>

hadoop-mapreduce/conf/fair-scheduler.xml.template

@@ -1,12 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-  This file contains pool and user allocations for the Fair Scheduler.
-  Its format is explained in the Fair Scheduler documentation at
-  http://hadoop.apache.org/core/docs/r0.21.0/fair_scheduler.html.
-  The documentation also includes a sample config file.
--->
-
-<allocations>
-
-</allocations>

hadoop-mapreduce/conf/mapred-queues.xml.template

@@ -1,92 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<!-- This is the template for queue configuration. The format supports nesting of
-     queues within queues - a feature called hierarchical queues. All queues are
-     defined within the 'queues' tag which is the top level element for this
-     XML document. The queue acls configured here for different queues are
-     checked for authorization only if the configuration property
-     mapreduce.cluster.acls.enabled is set to true. -->
-<queues>
-
-  <!-- Configuration for a queue is specified by defining a 'queue' element. -->
-  <queue>
-
-    <!-- Name of a queue. Queue name cannot contain a ':'  -->
-    <name>default</name>
-
-    <!-- properties for a queue, typically used by schedulers,
-    can be defined here -->
-    <properties>
-    </properties>
-
-	<!-- State of the queue. If running, the queue will accept new jobs.
-         If stopped, the queue will not accept new jobs. -->
-    <state>running</state>
-
-    <!-- Specifies the ACLs to check for submitting jobs to this queue.
-         If set to '*', it allows all users to submit jobs to the queue.
-         If set to ' '(i.e. space), no user will be allowed to do this
-         operation. The default value for any queue acl is ' '.
-         For specifying a list of users and groups the format to use is
-         user1,user2 group1,group2
-
-         It is only used if authorization is enabled in Map/Reduce by setting
-         the configuration property mapreduce.cluster.acls.enabled to true.
-
-         Irrespective of this ACL configuration, the user who started the
-         cluster and cluster administrators configured via
-         mapreduce.cluster.administrators can do this operation. -->
-    <acl-submit-job> </acl-submit-job>
-
-    <!-- Specifies the ACLs to check for viewing and modifying jobs in this
-         queue. Modifications include killing jobs, tasks of jobs or changing
-         priorities.
-         If set to '*', it allows all users to view, modify jobs of the queue.
-         If set to ' '(i.e. space), no user will be allowed to do this
-         operation.
-         For specifying a list of users and groups the format to use is
-         user1,user2 group1,group2
-
-         It is only used if authorization is enabled in Map/Reduce by setting
-         the configuration property mapreduce.cluster.acls.enabled to true.
-
-         Irrespective of this ACL configuration, the user who started the
-         cluster  and cluster administrators configured via
-         mapreduce.cluster.administrators can do the above operations on all
-         the jobs in all the queues. The job owner can do all the above
-         operations on his/her job irrespective of this ACL configuration. -->
-    <acl-administer-jobs> </acl-administer-jobs>
-  </queue>
-
-  <!-- Here is a sample of a hierarchical queue configuration
-       where q2 is a child of q1. In this example, q2 is a leaf level
-       queue as it has no queues configured within it. Currently, ACLs
-       and state are only supported for the leaf level queues.
-       Note also the usage of properties for the queue q2.
-  <queue>
-    <name>q1</name>
-    <queue>
-      <name>q2</name>
-      <properties>
-        <property key="capacity" value="20"/>
-        <property key="user-limit" value="30"/>
-      </properties>
-    </queue>
-  </queue>
- -->
-</queues>

hadoop-mapreduce/conf/mapred-site.xml.template

@@ -1,8 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
-</configuration>

hadoop-mapreduce/conf/taskcontroller.cfg

@@ -1,3 +0,0 @@
-mapreduce.cluster.local.dir=#configured value of mapreduce.cluster.local.dir. It can be a list of comma separated paths.
-hadoop.log.dir=#configured value of hadoop.log.dir.
-mapreduce.tasktracker.group=#configured value of mapreduce.tasktracker.group

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml

@@ -1,103 +0,0 @@
-<?xml version="1.0"?>
-<project>
-  <parent>
-    <artifactId>hadoop-mapreduce-client</artifactId>
-    <groupId>org.apache.hadoop</groupId>
-    <version>${hadoop-mapreduce.version}</version>
-  </parent>
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>org.apache.hadoop</groupId>
-  <artifactId>hadoop-mapreduce-client-app</artifactId>
-  <name>hadoop-mapreduce-client-app</name>
-
-  <properties>
-    <install.file>${project.artifact.file}</install.file>
-    <applink.base>${project.build.directory}/${project.name}</applink.base>
-    <mr.basedir>${project.parent.parent.basedir}</mr.basedir>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-common</artifactId>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapreduce-client-common</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-common</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-nodemanager</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <!-- local name for links -->
-    <finalName>mr-app</finalName>
-    <plugins>
-      <plugin>
-        <artifactId>maven-jar-plugin</artifactId>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar</goal>
-            </goals>
-            <phase>test-compile</phase>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>build-classpath</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>build-classpath</goal>
-            </goals>
-            <configuration>
-              <outputFile>target/classes/mrapp-generated-classpath</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>create-mr-app-symlinks</id>
-            <phase>package</phase>
-            <configuration>
-              <target>
-                <symlink link="${applink.base}.jar"
-                    resource="mr-app.jar" failonerror="false"/>
-                <symlink link="${applink.base}-1.0-SNAPSHOT.jar"
-                    resource="mr-app.jar" failonerror="false"/>
-              </target>
-            </configuration>
-            <goals>
-              <goal>run</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-</project>

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/CustomOutputCommitter.java

@@ -1,67 +0,0 @@
-package org.apache.hadoop;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.OutputCommitter;
-import org.apache.hadoop.mapred.TaskAttemptContext;
-
-public class CustomOutputCommitter extends OutputCommitter {
-
-  public static final String JOB_SETUP_FILE_NAME = "_job_setup";
-  public static final String JOB_COMMIT_FILE_NAME = "_job_commit";
-  public static final String JOB_ABORT_FILE_NAME = "_job_abort";
-  public static final String TASK_SETUP_FILE_NAME = "_task_setup";
-  public static final String TASK_ABORT_FILE_NAME = "_task_abort";
-  public static final String TASK_COMMIT_FILE_NAME = "_task_commit";
-
-  @Override
-  public void setupJob(JobContext jobContext) throws IOException {
-    writeFile(jobContext.getJobConf(), JOB_SETUP_FILE_NAME);
-  }
-
-  @Override
-  public void commitJob(JobContext jobContext) throws IOException {
-    super.commitJob(jobContext);
-    writeFile(jobContext.getJobConf(), JOB_COMMIT_FILE_NAME);
-  }
-
-  @Override
-  public void abortJob(JobContext jobContext, int status) 
-  throws IOException {
-    super.abortJob(jobContext, status);
-    writeFile(jobContext.getJobConf(), JOB_ABORT_FILE_NAME);
-  }
-  
-  @Override
-  public void setupTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
-  }
-
-  @Override
-  public boolean needsTaskCommit(TaskAttemptContext taskContext)
-      throws IOException {
-    return true;
-  }
-
-  @Override
-  public void commitTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
-  }
-
-  @Override
-  public void abortTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
-  }
-
-  private void writeFile(JobConf conf , String filename) throws IOException {
-    System.out.println("writing file ----" + filename);
-    Path outputPath = FileOutputFormat.getOutputPath(conf);
-    FileSystem fs = outputPath.getFileSystem(conf);
-    fs.create(new Path(outputPath, filename)).close();
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java

@@ -1,455 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-import java.util.HashSet;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FSError;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.mapreduce.JobCounter;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-/**
- * Runs the container task locally in a thread.
- * Since all (sub)tasks share the same local directory, they must be executed
- * sequentially in order to avoid creating/deleting the same files/dirs.
- */
-public class LocalContainerLauncher extends AbstractService implements
-    ContainerLauncher {
-
-  private static final File curDir = new File(".");
-  private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
-
-  private FileContext curFC = null;
-  private final HashSet<File> localizedFiles;
-  private final AppContext context;
-  private final TaskUmbilicalProtocol umbilical;
-  private Thread eventHandlingThread;
-  private BlockingQueue<ContainerLauncherEvent> eventQueue =
-      new LinkedBlockingQueue<ContainerLauncherEvent>();
-
-  public LocalContainerLauncher(AppContext context,
-                                TaskUmbilicalProtocol umbilical) {
-    super(LocalContainerLauncher.class.getName());
-    this.context = context;
-    this.umbilical = umbilical;
-        // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us  (TODO/FIXME:  pointless to use RPC to talk to self; should create LocalTaskAttemptListener or similar:  implement umbilical protocol but skip RPC stuff)
-
-    try {
-      curFC = FileContext.getFileContext(curDir.toURI());
-    } catch (UnsupportedFileSystemException ufse) {
-      LOG.error("Local filesystem " + curDir.toURI().toString()
-                + " is unsupported?? (should never happen)");
-    }
-
-    // Save list of files/dirs that are supposed to be present so can delete
-    // any extras created by one task before starting subsequent task.  Note
-    // that there's no protection against deleted or renamed localization;
-    // users who do that get what they deserve (and will have to disable
-    // uberization in order to run correctly).
-    File[] curLocalFiles = curDir.listFiles();
-    localizedFiles = new HashSet<File>(curLocalFiles.length);
-    for (int j = 0; j < curLocalFiles.length; ++j) {
-      localizedFiles.add(curLocalFiles[j]);
-    }
-
-    // Relocalization note/future FIXME (per chrisdo, 20110315):  At moment,
-    // full localization info is in AppSubmissionContext passed from client to
-    // RM and then to NM for AM-container launch:  no difference between AM-
-    // localization and MapTask- or ReduceTask-localization, so can assume all
-    // OK.  Longer-term, will need to override uber-AM container-localization
-    // request ("needed resources") with union of regular-AM-resources + task-
-    // resources (and, if maps and reduces ever differ, then union of all three
-    // types), OR will need localizer service/API that uber-AM can request
-    // after running (e.g., "localizeForTask()" or "localizeForMapTask()").
-  }
-
-  public void start() {
-    eventHandlingThread = new Thread(new SubtaskRunner(), "uber-SubtaskRunner");
-    eventHandlingThread.start();
-    super.start();
-  }
-
-  public void stop() {
-    eventHandlingThread.interrupt();
-    super.stop();
-  }
-
-  @Override
-  public void handle(ContainerLauncherEvent event) {
-    try {
-      eventQueue.put(event);
-    } catch (InterruptedException e) {
-      throw new YarnException(e);  // FIXME? YarnException is "for runtime exceptions only"
-    }
-  }
-
-
-  /*
-   * Uber-AM lifecycle/ordering ("normal" case):
-   *
-   * - [somebody] sends TA_ASSIGNED
-   *   - handled by ContainerAssignedTransition (TaskAttemptImpl.java)
-   *     - creates "remoteTask" for us == real Task
-   *     - sends CONTAINER_REMOTE_LAUNCH
-   *     - TA: UNASSIGNED -> ASSIGNED
-   * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us)
-   *   - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask()
-   *   - sends TA_CONTAINER_LAUNCHED
-   *     [[ elsewhere...
-   *       - TA_CONTAINER_LAUNCHED handled by LaunchedContainerTransition
-   *         - registers "remoteTask" with TaskAttemptListener (== umbilical)
-   *         - NUKES "remoteTask"
-   *         - sends T_ATTEMPT_LAUNCHED (Task: SCHEDULED -> RUNNING)
-   *         - TA: ASSIGNED -> RUNNING
-   *     ]]
-   *   - runs Task (runSubMap() or runSubReduce())
-   *     - TA can safely send TA_UPDATE since in RUNNING state
-   *       [modulo possible TA-state-machine race noted below:  CHECK (TODO)]
-   */
-  private class SubtaskRunner implements Runnable {
-
-    private boolean doneWithMaps = false;
-    private int finishedSubMaps = 0;
-
-    SubtaskRunner() {
-    }
-
-    @Override
-    public void run() {
-      ContainerLauncherEvent event = null;
-
-      // _must_ either run subtasks sequentially or accept expense of new JVMs
-      // (i.e., fork()), else will get weird failures when maps try to create/
-      // write same dirname or filename:  no chdir() in Java
-      while (!Thread.currentThread().isInterrupted()) {
-        try {
-          event = eventQueue.take();
-        } catch (InterruptedException e) {  // mostly via T_KILL? JOB_KILL?
-          LOG.error("Returning, interrupted : " + e);
-          return;
-        }
-
-        LOG.info("Processing the event " + event.toString());
-
-        if (event.getType() == EventType.CONTAINER_REMOTE_LAUNCH) {
-
-          ContainerRemoteLaunchEvent launchEv =
-              (ContainerRemoteLaunchEvent)event;
-          TaskAttemptId attemptID = launchEv.getTaskAttemptID(); //FIXME:  can attemptID ever be null?  (only if retrieved over umbilical?)
-
-          Job job = context.getAllJobs().get(attemptID.getTaskId().getJobId());
-          int numMapTasks = job.getTotalMaps();
-          int numReduceTasks = job.getTotalReduces();
-
-          // YARN (tracking) Task:
-          org.apache.hadoop.mapreduce.v2.app.job.Task ytask =
-              job.getTask(attemptID.getTaskId());
-          // classic mapred Task:
-          org.apache.hadoop.mapred.Task remoteTask = launchEv.getRemoteTask();
-
-          // after "launching," send launched event to task attempt to move
-          // state from ASSIGNED to RUNNING (also nukes "remoteTask", so must
-          // do getRemoteTask() call first)
-          context.getEventHandler().handle(
-              new TaskAttemptEvent(attemptID,
-                  TaskAttemptEventType.TA_CONTAINER_LAUNCHED)); //FIXME:  race condition here?  or do we have same kind of lock on TA handler => MapTask can't send TA_UPDATE before TA_CONTAINER_LAUNCHED moves TA to RUNNING state?  (probably latter)
-
-          if (numMapTasks == 0) {
-            doneWithMaps = true;
-          }
-
-          try {
-            if (remoteTask.isMapOrReduce()) {
-              JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.getTaskId().getJobId());
-              jce.addCounterUpdate(JobCounter.TOTAL_LAUNCHED_UBERTASKS, 1);
-              if (remoteTask.isMapTask()) {
-                jce.addCounterUpdate(JobCounter.NUM_UBER_SUBMAPS, 1);
-              } else {
-                jce.addCounterUpdate(JobCounter.NUM_UBER_SUBREDUCES, 1);
-              }
-              context.getEventHandler().handle(jce);
-            }
-            runSubtask(remoteTask, ytask.getType(), attemptID, numMapTasks,
-                       (numReduceTasks > 0));
-            
-          } catch (RuntimeException re) {
-            JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.getTaskId().getJobId());
-            jce.addCounterUpdate(JobCounter.NUM_FAILED_UBERTASKS, 1);
-            context.getEventHandler().handle(jce);
-            // this is our signal that the subtask failed in some way, so
-            // simulate a failed JVM/container and send a container-completed
-            // event to task attempt (i.e., move state machine from RUNNING
-            // to FAIL_CONTAINER_CLEANUP [and ultimately to FAILED])
-            context.getEventHandler().handle(new TaskAttemptEvent(attemptID,
-                TaskAttemptEventType.TA_CONTAINER_COMPLETED));
-          } catch (IOException ioe) {
-            // if umbilical itself barfs (in error-handler of runSubMap()),
-            // we're pretty much hosed, so do what YarnChild main() does
-            // (i.e., exit clumsily--but can never happen, so no worries!)
-            LOG.fatal("oopsie...  this can never happen: "
-                + StringUtils.stringifyException(ioe));
-            System.exit(-1);
-          }
-
-        } else if (event.getType() == EventType.CONTAINER_REMOTE_CLEANUP) {
-
-          // no container to kill, so just send "cleaned" event to task attempt
-          // to move us from SUCCESS_CONTAINER_CLEANUP to SUCCEEDED state
-          // (or {FAIL|KILL}_CONTAINER_CLEANUP to {FAIL|KILL}_TASK_CLEANUP)
-          context.getEventHandler().handle(
-              new TaskAttemptEvent(event.getTaskAttemptID(),
-                  TaskAttemptEventType.TA_CONTAINER_CLEANED));
-
-        } else {
-          LOG.warn("Ignoring unexpected event " + event.toString());
-        }
-
-      }
-    }
-
-    private void runSubtask(org.apache.hadoop.mapred.Task task,
-                            final TaskType taskType,
-                            TaskAttemptId attemptID,
-                            final int numMapTasks,
-                            boolean renameOutputs)
-    throws RuntimeException, IOException {
-      org.apache.hadoop.mapred.TaskAttemptID classicAttemptID =
-          TypeConverter.fromYarn(attemptID);
-
-      try {
-        JobConf conf = new JobConf(getConfig());
-
-        // mark this as an uberized subtask so it can set task counter
-        // (longer-term/FIXME:  could redefine as job counter and send
-        // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
-        // will need new Job state-machine transition and JobImpl jobCounters
-        // map to handle)
-        conf.setBoolean("mapreduce.task.uberized", true);
-
-        // META-FIXME: do we want the extra sanity-checking (doneWithMaps,
-        // etc.), or just assume/hope the state machine(s) and uber-AM work
-        // as expected?
-        if (taskType == TaskType.MAP) {
-          if (doneWithMaps) {
-            LOG.error("CONTAINER_REMOTE_LAUNCH contains a map task ("
-                      + attemptID + "), but should be finished with maps");
-            // throw new RuntimeException()  (FIXME: what's appropriate here?)
-          }
-
-          MapTask map = (MapTask)task;
-
-          //CODE-REVIEWER QUESTION: why not task.getConf() or map.getConf() instead of conf? do we need Task's localizeConfiguration() run on this first?
-          map.run(conf, umbilical);
-
-          if (renameOutputs) {
-            renameMapOutputForReduce(conf, attemptID, map.getMapOutputFile());
-          }
-          relocalize();
-
-          if (++finishedSubMaps == numMapTasks) {
-            doneWithMaps = true;
-          }
-
-        } else /* TaskType.REDUCE */ {
-
-          if (!doneWithMaps) {
-            //check if event-queue empty?  whole idea of counting maps vs. checking event queue is a tad wacky...but could enforce ordering (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):  doesn't send reduce event until maps all done]
-            LOG.error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
-                      + attemptID + "), but not yet finished with maps");
-            // throw new RuntimeException()  (FIXME) // or push reduce event back onto end of queue? (probably former)
-          }
-
-          ReduceTask reduce = (ReduceTask)task;
-
-          // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
-          conf.set(MRConfig.MASTER_ADDRESS, "local");  // bypass shuffle
-
-          reduce.run(conf, umbilical);
-          //relocalize();  // needed only if more than one reducer supported (is MAPREDUCE-434 fixed yet?)
-        }
-
-      } catch (FSError e) {
-        LOG.fatal("FSError from child", e);
-        // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
-        umbilical.fsError(classicAttemptID, e.getMessage());
-        throw new RuntimeException();
-
-      } catch (Exception exception) {
-        LOG.warn("Exception running local (uberized) 'child' : "
-            + StringUtils.stringifyException(exception));
-        try {
-          if (task != null) {
-            // do cleanup for the task
-//          if (childUGI == null) { // no need to job into doAs block
-              task.taskCleanup(umbilical);
-//          } else {
-//            final Task taskFinal = task;
-//            childUGI.doAs(new PrivilegedExceptionAction<Object>() {
-//              @Override
-//              public Object run() throws Exception {
-//                taskFinal.taskCleanup(umbilical);
-//                return null;
-//              }
-//            });
-//          }
-          }
-        } catch (Exception e) {
-          LOG.info("Exception cleaning up: "
-              + StringUtils.stringifyException(e));
-        }
-        // Report back any failures, for diagnostic purposes
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        exception.printStackTrace(new PrintStream(baos));
-//      if (classicAttemptID != null) {
-          umbilical.reportDiagnosticInfo(classicAttemptID, baos.toString());
-//      }
-        throw new RuntimeException();
-
-      } catch (Throwable throwable) {
-        LOG.fatal("Error running local (uberized) 'child' : "
-            + StringUtils.stringifyException(throwable));
-//      if (classicAttemptID != null) {
-          Throwable tCause = throwable.getCause();
-          String cause = (tCause == null)
-              ? throwable.getMessage()
-              : StringUtils.stringifyException(tCause);
-          umbilical.fatalError(classicAttemptID, cause);
-//      }
-        throw new RuntimeException();
-
-      } finally {
-/*
-FIXME:  do we need to do any of this stuff?  (guessing not since not in own JVM)
-        RPC.stopProxy(umbilical);
-        DefaultMetricsSystem.shutdown();
-        // Shutting down log4j of the child-vm...
-        // This assumes that on return from Task.run()
-        // there is no more logging done.
-        LogManager.shutdown();
- */
-      }
-    }
-
-
-/* FIXME:  may not need renameMapOutputForReduce() anymore?  TEST!
-
-${local.dir}/usercache/$user/appcache/$appId/$contId/ == $cwd for tasks;
-contains task.sh script, which, when executed, creates symlinks and sets up env
- "$local.dir"/usercache/$user/appcache/$appId/$contId/file.out
- "$local.dir"/usercache/$user/appcache/$appId/$contId/file.out.idx (?)
- "$local.dir"/usercache/$user/appcache/$appId/output/$taskId/ is where file.out* is moved after MapTask done
-
-	OHO!  no further need for this at all?  $taskId is unique per subtask
-	now => should work fine to leave alone.  TODO:  test with teragen or
-	similar
- */
-
-    /**
-     * Within the _local_ filesystem (not HDFS), all activity takes place within
-     * a single subdir (${local.dir}/usercache/$user/appcache/$appId/$contId/),
-     * and all sub-MapTasks create the same filename ("file.out").  Rename that
-     * to something unique (e.g., "map_0.out") to avoid collisions.
-     *
-     * Longer-term, we'll modify [something] to use TaskAttemptID-based
-     * filenames instead of "file.out". (All of this is entirely internal,
-     * so there are no particular compatibility issues.)
-     */
-    private void renameMapOutputForReduce(JobConf conf, TaskAttemptId mapId,
-                                          MapOutputFile subMapOutputFile)
-    throws IOException {
-      FileSystem localFs = FileSystem.getLocal(conf);
-      // move map output to reduce input
-      Path mapOut = subMapOutputFile.getOutputFile();
-      Path reduceIn = subMapOutputFile.getInputFileForWrite(
-          TypeConverter.fromYarn(mapId).getTaskID(), localFs.getLength(mapOut));
-      if (!localFs.mkdirs(reduceIn.getParent())) {
-        throw new IOException("Mkdirs failed to create "
-            + reduceIn.getParent().toString());
-      }
-      if (!localFs.rename(mapOut, reduceIn))
-        throw new IOException("Couldn't rename " + mapOut);
-    }
-
-    /**
-     * Also within the local filesystem, we need to restore the initial state
-     * of the directory as much as possible.  Compare current contents against
-     * the saved original state and nuke everything that doesn't belong, with
-     * the exception of the renamed map outputs (see above).
-FIXME:  do we really need to worry about renamed map outputs, or already moved to output dir on commit?  if latter, fix comment
-     *
-     * Any jobs that go out of their way to rename or delete things from the
-     * local directory are considered broken and deserve what they get...
-     */
-    private void relocalize() {
-      File[] curLocalFiles = curDir.listFiles();
-      for (int j = 0; j < curLocalFiles.length; ++j) {
-        if (!localizedFiles.contains(curLocalFiles[j])) {
-          // found one that wasn't there before:  delete it
-          boolean deleted = false;
-          try {
-            if (curFC != null) {
-              // this is recursive, unlike File delete():
-              deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true);
-            }
-          } catch (IOException e) {
-            deleted = false;
-          }
-          if (!deleted) {
-            LOG.warn("Unable to delete unexpected local file/dir "
-                + curLocalFiles[j].getName() + ": insufficient permissions?");
-          }
-        }
-      }
-    }
-
-  } // end SubtaskRunner
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java

@@ -1,264 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.io.File;
-import java.net.InetSocketAddress;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Vector;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.TaskLog.LogName;
-import org.apache.hadoop.mapreduce.ID;
-import org.apache.hadoop.util.StringUtils;
-
-public class MapReduceChildJVM {
-  private static final String SYSTEM_PATH_SEPARATOR = 
-    System.getProperty("path.separator");
-
-  private static final Log LOG = LogFactory.getLog(MapReduceChildJVM.class);
-
-  private static File getTaskLogFile(String logDir, LogName filter) {
-    return new File(logDir, filter.toString());
-  }
-
-  private static String getChildEnv(JobConf jobConf, boolean isMap) {
-    if (isMap) {
-      return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV,
-          jobConf.get(JobConf.MAPRED_TASK_ENV));
-    }
-    return jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV,
-        jobConf.get(jobConf.MAPRED_TASK_ENV));
-  }
-
-  public static void setVMEnv(Map<String, String> env,
-      List<String> classPaths, String pwd, String containerLogDir,
-      String nmLdLibraryPath, Task task, CharSequence applicationTokensFile) {
-
-    JobConf conf = task.conf;
-
-    // Add classpath.
-    CharSequence cp = env.get("CLASSPATH");
-    String classpath = StringUtils.join(SYSTEM_PATH_SEPARATOR, classPaths);
-    if (null == cp) {
-      env.put("CLASSPATH", classpath);
-    } else {
-      env.put("CLASSPATH", classpath + SYSTEM_PATH_SEPARATOR + cp);
-    }
-
-    /////// Environmental variable LD_LIBRARY_PATH
-    StringBuilder ldLibraryPath = new StringBuilder();
-
-    ldLibraryPath.append(nmLdLibraryPath);
-    ldLibraryPath.append(SYSTEM_PATH_SEPARATOR);
-    ldLibraryPath.append(pwd);
-    env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
-    /////// Environmental variable LD_LIBRARY_PATH
-
-    // for the child of task jvm, set hadoop.root.logger
-    env.put("HADOOP_ROOT_LOGGER", "DEBUG,CLA"); // TODO: Debug
-
-    // TODO: The following is useful for instance in streaming tasks. Should be
-    // set in ApplicationMaster's env by the RM.
-    String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
-    if (hadoopClientOpts == null) {
-      hadoopClientOpts = "";
-    } else {
-      hadoopClientOpts = hadoopClientOpts + " ";
-    }
-    // FIXME: don't think this is also needed given we already set java
-    // properties.
-    long logSize = TaskLog.getTaskLogLength(conf);
-    Vector<String> logProps = new Vector<String>(4);
-    setupLog4jProperties(logProps, logSize, containerLogDir);
-    Iterator<String> it = logProps.iterator();
-    StringBuffer buffer = new StringBuffer();
-    while (it.hasNext()) {
-      buffer.append(" " + it.next());
-    }
-    hadoopClientOpts = hadoopClientOpts + buffer.toString();
-    
-    env.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
-
-    // add the env variables passed by the user
-    String mapredChildEnv = getChildEnv(conf, task.isMapTask());
-    if (mapredChildEnv != null && mapredChildEnv.length() > 0) {
-      String childEnvs[] = mapredChildEnv.split(",");
-      for (String cEnv : childEnvs) {
-        String[] parts = cEnv.split("="); // split on '='
-        String value = (String) env.get(parts[0]);
-        if (value != null) {
-          // replace $env with the child's env constructed by tt's
-          // example LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp
-          value = parts[1].replace("$" + parts[0], value);
-        } else {
-          // this key is not configured by the tt for the child .. get it 
-          // from the tt's env
-          // example PATH=$PATH:/tmp
-          value = System.getenv(parts[0]); // Get from NM?
-          if (value != null) {
-            // the env key is present in the tt's env
-            value = parts[1].replace("$" + parts[0], value);
-          } else {
-            // the env key is note present anywhere .. simply set it
-            // example X=$X:/tmp or X=/tmp
-            value = parts[1].replace("$" + parts[0], "");
-          }
-        }
-        env.put(parts[0], value);
-      }
-    }
-
-    //This should not be set here (If an OS check is requied. moved to ContainerLuanch)
-    // env.put("JVM_PID", "`echo $$`");
-
-    env.put(Constants.STDOUT_LOGFILE_ENV,
-        getTaskLogFile(containerLogDir, TaskLog.LogName.STDOUT).toString());
-    env.put(Constants.STDERR_LOGFILE_ENV,
-        getTaskLogFile(containerLogDir, TaskLog.LogName.STDERR).toString());
-  }
-
-  private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) {
-    if (isMapTask) {
-      return jobConf.get(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, jobConf.get(
-          JobConf.MAPRED_TASK_JAVA_OPTS,
-          JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
-    }
-    return jobConf
-        .get(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, jobConf.get(
-            JobConf.MAPRED_TASK_JAVA_OPTS,
-            JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
-  }
-
-  private static void setupLog4jProperties(Vector<String> vargs,
-      long logSize, String containerLogDir) {
-    vargs.add("-Dlog4j.configuration=container-log4j.properties");
-    vargs.add("-Dhadoop.yarn.mr.containerLogDir=" + containerLogDir);
-    vargs.add("-Dhadoop.yarn.mr.totalLogFileSize=" + logSize);
-  }
-
-  public static List<String> getVMCommand(
-      InetSocketAddress taskAttemptListenerAddr, Task task, String javaHome,
-      String workDir, String logDir, String childTmpDir, ID jvmID) {
-
-    TaskAttemptID attemptID = task.getTaskID();
-    JobConf conf = task.conf;
-
-    Vector<String> vargs = new Vector<String>(8);
-
-    vargs.add("exec");
-    vargs.add(javaHome + "/bin/java");
-
-    // Add child (task) java-vm options.
-    //
-    // The following symbols if present in mapred.{map|reduce}.child.java.opts 
-    // value are replaced:
-    // + @taskid@ is interpolated with value of TaskID.
-    // Other occurrences of @ will not be altered.
-    //
-    // Example with multiple arguments and substitutions, showing
-    // jvm GC logging, and start of a passwordless JVM JMX agent so can
-    // connect with jconsole and the likes to watch child memory, threads
-    // and get thread dumps.
-    //
-    //  <property>
-    //    <name>mapred.map.child.java.opts</name>
-    //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@taskid@.gc \
-    //           -Dcom.sun.management.jmxremote.authenticate=false \
-    //           -Dcom.sun.management.jmxremote.ssl=false \
-    //    </value>
-    //  </property>
-    //
-    //  <property>
-    //    <name>mapred.reduce.child.java.opts</name>
-    //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@taskid@.gc \
-    //           -Dcom.sun.management.jmxremote.authenticate=false \
-    //           -Dcom.sun.management.jmxremote.ssl=false \
-    //    </value>
-    //  </property>
-    //
-    String javaOpts = getChildJavaOpts(conf, task.isMapTask());
-    javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
-    String [] javaOptsSplit = javaOpts.split(" ");
-    
-    // Add java.library.path; necessary for loading native libraries.
-    //
-    // 1. We add the 'cwd' of the task to it's java.library.path to help 
-    //    users distribute native libraries via the DistributedCache.
-    // 2. The user can also specify extra paths to be added to the 
-    //    java.library.path via mapred.{map|reduce}.child.java.opts.
-    //
-    String libraryPath = workDir;
-    boolean hasUserLDPath = false;
-    for(int i=0; i<javaOptsSplit.length ;i++) { 
-      if(javaOptsSplit[i].startsWith("-Djava.library.path=")) {
-        // TODO: Does the above take care of escaped space chars
-        javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath;
-        hasUserLDPath = true;
-        break;
-      }
-    }
-    if(!hasUserLDPath) {
-      vargs.add("-Djava.library.path=" + libraryPath);
-    }
-    for (int i = 0; i < javaOptsSplit.length; i++) {
-      vargs.add(javaOptsSplit[i]);
-    }
-
-    if (childTmpDir != null) {
-      vargs.add("-Djava.io.tmpdir=" + childTmpDir);
-    }
-
-    // Setup the log4j prop
-    long logSize = TaskLog.getTaskLogLength(conf);
-    setupLog4jProperties(vargs, logSize, logDir);
-
-    if (conf.getProfileEnabled()) {
-      if (conf.getProfileTaskRange(task.isMapTask()
-                                   ).isIncluded(task.getPartition())) {
-        File prof = getTaskLogFile(logDir, TaskLog.LogName.PROFILE);
-        vargs.add(String.format(conf.getProfileParams(), prof.toString()));
-      }
-    }
-
-    // Add main class and its arguments 
-    vargs.add(YarnChild.class.getName());  // main of Child
-    // pass TaskAttemptListener's address
-    vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
-    vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
-    vargs.add(attemptID.toString());                      // pass task identifier
-
-    // Finally add the jvmID
-    vargs.add(String.valueOf(jvmID.getId()));
-    vargs.add("1>" + getTaskLogFile(logDir, TaskLog.LogName.STDERR));
-    vargs.add("2>" + getTaskLogFile(logDir, TaskLog.LogName.STDOUT));
-
-    // Final commmand
-    StringBuilder mergedCommand = new StringBuilder();
-    for (CharSequence str : vargs) {
-      mergedCommand.append(str).append(" ");
-    }
-    Vector<String> vargsFinal = new Vector<String>(1);
-    vargsFinal.add(mergedCommand.toString());
-    return vargsFinal;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapTaskAttemptImpl.java

@@ -1,65 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public class MapTaskAttemptImpl extends TaskAttemptImpl {
-
-  private final TaskSplitMetaInfo splitInfo;
-
-  public MapTaskAttemptImpl(TaskId taskId, int attempt, 
-      EventHandler eventHandler, Path jobFile, 
-      int partition, TaskSplitMetaInfo splitInfo, Configuration conf,
-      TaskAttemptListener taskAttemptListener, 
-      OutputCommitter committer, Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
-    super(taskId, attempt, eventHandler, 
-        taskAttemptListener, jobFile, partition, conf, splitInfo.getLocations(),
-        committer, jobToken, fsTokens, clock);
-    this.splitInfo = splitInfo;
-  }
-
-  @Override
-  public Task createRemoteTask() {
-    //job file name is set in TaskAttempt, setting it null here
-    MapTask mapTask =
-      new MapTask("", TypeConverter.fromYarn(getID()), partition,
-          splitInfo.getSplitIndex(), 1); // YARN doesn't have the concept of slots per task, set it as 1.
-    mapTask.setUser(conf.get(MRJobConfig.USER_NAME));
-    mapTask.setConf(conf);
-    return mapTask;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ReduceTaskAttemptImpl.java

@@ -1,64 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-
-public class ReduceTaskAttemptImpl extends TaskAttemptImpl {
-
-  private final int numMapTasks;
-
-  public ReduceTaskAttemptImpl(TaskId id, int attempt,
-      EventHandler eventHandler, Path jobFile, int partition,
-      int numMapTasks, Configuration conf,
-      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
-      Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
-    super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
-        conf, new String[] {}, committer, jobToken, fsTokens, clock);
-    this.numMapTasks = numMapTasks;
-  }
-
-  @Override
-  public Task createRemoteTask() {
-  //job file name is set in TaskAttempt, setting it null here
-    ReduceTask reduceTask =
-      new ReduceTask("", TypeConverter.fromYarn(getID()), partition,
-          numMapTasks, 1); // YARN doesn't have the concept of slots per task, set it as 1.
-  reduceTask.setUser(conf.get(MRJobConfig.USER_NAME));
-  reduceTask.setConf(conf);
-    return reduceTask;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java

@@ -1,434 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.mapred.SortedRanges.Range;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.service.CompositeService;
-
-/**
- * This class is responsible for talking to the task umblical.
- * It also converts all the old data structures
- * to yarn data structures.
- * 
- * This class HAS to be in this package to access package private 
- * methods/classes.
- */
-public class TaskAttemptListenerImpl extends CompositeService 
-    implements TaskUmbilicalProtocol, TaskAttemptListener {
-
-  private static final Log LOG = LogFactory.getLog(TaskAttemptListenerImpl.class);
-
-  private AppContext context;
-  private Server server;
-  private TaskHeartbeatHandler taskHeartbeatHandler;
-  private InetSocketAddress address;
-  private Map<WrappedJvmID, org.apache.hadoop.mapred.Task> jvmIDToAttemptMap = 
-    Collections.synchronizedMap(new HashMap<WrappedJvmID, 
-        org.apache.hadoop.mapred.Task>());
-  private JobTokenSecretManager jobTokenSecretManager = null;
-  
-  public TaskAttemptListenerImpl(AppContext context,
-      JobTokenSecretManager jobTokenSecretManager) {
-    super(TaskAttemptListenerImpl.class.getName());
-    this.context = context;
-    this.jobTokenSecretManager = jobTokenSecretManager;
-  }
-
-  @Override
-  public void init(Configuration conf) {
-   registerHeartbeatHandler();
-   super.init(conf);
-  }
-
-  @Override
-  public void start() {
-    startRpcServer();
-    super.start();
-  }
-
-  protected void registerHeartbeatHandler() {
-    taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), 
-        context.getClock());
-    addService(taskHeartbeatHandler);
-  }
-
-  protected void startRpcServer() {
-    Configuration conf = getConfig();
-    try {
-      server =
-          RPC.getServer(TaskUmbilicalProtocol.class, this, "0.0.0.0", 0, 
-              conf.getInt(AMConstants.AM_TASK_LISTENER_THREADS, 
-                  AMConstants.DEFAULT_AM_TASK_LISTENER_THREADS),
-              false, conf, jobTokenSecretManager);
-      server.start();
-      InetSocketAddress listenerAddress = server.getListenerAddress();
-      this.address =
-          NetUtils.createSocketAddr(listenerAddress.getAddress()
-              .getLocalHost().getCanonicalHostName()
-              + ":" + listenerAddress.getPort());
-    } catch (IOException e) {
-      throw new YarnException(e);
-    }
-  }
-
-  @Override
-  public void stop() {
-    stopRpcServer();
-    super.stop();
-  }
-
-  protected void stopRpcServer() {
-    server.stop();
-  }
-
-  @Override
-  public InetSocketAddress getAddress() {
-    return address;
-  }
-
-  /**
-   * Child checking whether it can commit.
-   * 
-   * <br/>
-   * Commit is a two-phased protocol. First the attempt informs the
-   * ApplicationMaster that it is
-   * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
-   * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
-   * a legacy from the centralized commit protocol handling by the JobTracker.
-   */
-  @Override
-  public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException {
-    LOG.info("Commit go/no-go request from " + taskAttemptID.toString());
-    // An attempt is asking if it can commit its output. This can be decided
-    // only by the task which is managing the multiple attempts. So redirect the
-    // request there.
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-        TypeConverter.toYarn(taskAttemptID);
-
-    taskHeartbeatHandler.receivedPing(attemptID);
-
-    Job job = context.getJob(attemptID.getTaskId().getJobId());
-    Task task = job.getTask(attemptID.getTaskId());
-    return task.canCommit(attemptID);
-  }
-
-  /**
-   * TaskAttempt is reporting that it is in commit_pending and it is waiting for
-   * the commit Response
-   * 
-   * <br/>
-   * Commit it a two-phased protocol. First the attempt informs the
-   * ApplicationMaster that it is
-   * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls
-   * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is
-   * a legacy from the centralized commit protocol handling by the JobTracker.
-   */
-  @Override
-  public void commitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu)
-          throws IOException, InterruptedException {
-    LOG.info("Commit-pending state update from " + taskAttemptID.toString());
-    // An attempt is asking if it can commit its output. This can be decided
-    // only by the task which is managing the multiple attempts. So redirect the
-    // request there.
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-        TypeConverter.toYarn(taskAttemptID);
-
-    taskHeartbeatHandler.receivedPing(attemptID);
-    //Ignorable TaskStatus? - since a task will send a LastStatusUpdate
-    context.getEventHandler().handle(
-        new TaskAttemptEvent(attemptID, 
-            TaskAttemptEventType.TA_COMMIT_PENDING));
-  }
-
-  @Override
-  public void done(TaskAttemptID taskAttemptID) throws IOException {
-    LOG.info("Done acknowledgement from " + taskAttemptID.toString());
-
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-        TypeConverter.toYarn(taskAttemptID);
-
-    taskHeartbeatHandler.receivedPing(attemptID);
-
-    context.getEventHandler().handle(
-        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_DONE));
-  }
-
-  @Override
-  public void fatalError(TaskAttemptID taskAttemptID, String msg)
-      throws IOException {
-    // This happens only in Child and in the Task.
-    LOG.fatal("Task: " + taskAttemptID + " - exited : " + msg);
-    reportDiagnosticInfo(taskAttemptID, "Error: " + msg);
-
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-        TypeConverter.toYarn(taskAttemptID);
-    context.getEventHandler().handle(
-        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
-  }
-
-  @Override
-  public void fsError(TaskAttemptID taskAttemptID, String message)
-      throws IOException {
-    // This happens only in Child.
-    LOG.fatal("Task: " + taskAttemptID + " - failed due to FSError: "
-        + message);
-    reportDiagnosticInfo(taskAttemptID, "FSError: " + message);
-
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-        TypeConverter.toYarn(taskAttemptID);
-    context.getEventHandler().handle(
-        new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
-  }
-
-  @Override
-  public void shuffleError(TaskAttemptID taskAttemptID, String message) throws IOException {
-    // TODO: This isn't really used in any MR code. Ask for removal.    
-  }
-
-  @Override
-  public MapTaskCompletionEventsUpdate getMapCompletionEvents(
-      JobID jobIdentifier, int fromEventId, int maxEvents,
-      TaskAttemptID taskAttemptID) throws IOException {
-    LOG.info("MapCompletionEvents request from " + taskAttemptID.toString()
-        + ". fromEventID " + fromEventId + " maxEvents " + maxEvents);
-
-    // TODO: shouldReset is never used. See TT. Ask for Removal.
-    boolean shouldReset = false;
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-      TypeConverter.toYarn(taskAttemptID);
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events =
-        context.getJob(attemptID.getTaskId().getJobId()).getTaskAttemptCompletionEvents(
-            fromEventId, maxEvents);
-
-    taskHeartbeatHandler.receivedPing(attemptID);
-
-    // filter the events to return only map completion events in old format
-    List<TaskCompletionEvent> mapEvents = new ArrayList<TaskCompletionEvent>();
-    for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent event : events) {
-      if (TaskType.MAP.equals(event.getAttemptId().getTaskId().getTaskType())) {
-        mapEvents.add(TypeConverter.fromYarn(event));
-      }
-    }
-    
-    return new MapTaskCompletionEventsUpdate(
-        mapEvents.toArray(new TaskCompletionEvent[0]), shouldReset);
-  }
-
-  @Override
-  public boolean ping(TaskAttemptID taskAttemptID) throws IOException {
-    LOG.info("Ping from " + taskAttemptID.toString());
-    taskHeartbeatHandler.receivedPing(TypeConverter.toYarn(taskAttemptID));
-    return true;
-  }
-
-  @Override
-  public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo)
- throws IOException {
-    LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": "
-        + diagnosticInfo);
-
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
-      TypeConverter.toYarn(taskAttemptID);
-    taskHeartbeatHandler.receivedPing(attemptID);
-
-    // This is mainly used for cases where we want to propagate exception traces
-    // of tasks that fail.
-
-    // This call exists as a hadoop mapreduce legacy wherein all changes in
-    // counters/progress/phase/output-size are reported through statusUpdate()
-    // call but not diagnosticInformation.
-    context.getEventHandler().handle(
-        new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo));
-  }
-
-  @Override
-  public boolean statusUpdate(TaskAttemptID taskAttemptID,
-      TaskStatus taskStatus) throws IOException, InterruptedException {
-    LOG.info("Status update from " + taskAttemptID.toString());
-    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID =
-        TypeConverter.toYarn(taskAttemptID);
-    taskHeartbeatHandler.receivedPing(yarnAttemptID);
-    TaskAttemptStatus taskAttemptStatus =
-        new TaskAttemptStatus();
-    taskAttemptStatus.id = yarnAttemptID;
-    // Task sends the updated progress to the TT.
-    taskAttemptStatus.progress = taskStatus.getProgress();
-    LOG.info("Progress of TaskAttempt " + taskAttemptID + " is : "
-        + taskStatus.getProgress());
-    // Task sends the diagnostic information to the TT
-    taskAttemptStatus.diagnosticInfo = taskStatus.getDiagnosticInfo();
-    // Task sends the updated state-string to the TT.
-    taskAttemptStatus.stateString = taskStatus.getStateString();
-    // Set the output-size when map-task finishes. Set by the task itself.
-    taskAttemptStatus.outputSize = taskStatus.getOutputSize();
-    // Task sends the updated phase to the TT.
-    taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
-    // Counters are updated by the task.
-    taskAttemptStatus.counters =
-        TypeConverter.toYarn(taskStatus.getCounters());
-
-    // Map Finish time set by the task (map only)
-    if (taskStatus.getIsMap() && taskStatus.getMapFinishTime() != 0) {
-      taskAttemptStatus.mapFinishTime = taskStatus.getMapFinishTime();
-    }
-
-    // Shuffle Finish time set by the task (reduce only).
-    if (!taskStatus.getIsMap() && taskStatus.getShuffleFinishTime() != 0) {
-      taskAttemptStatus.shuffleFinishTime = taskStatus.getShuffleFinishTime();
-    }
-
-    // Sort finish time set by the task (reduce only).
-    if (!taskStatus.getIsMap() && taskStatus.getSortFinishTime() != 0) {
-      taskAttemptStatus.sortFinishTime = taskStatus.getSortFinishTime();
-    }
-
-    // Not Setting the task state. Used by speculation - will be set in TaskAttemptImpl
-    //taskAttemptStatus.taskState =  TypeConverter.toYarn(taskStatus.getRunState());
-    
-    //set the fetch failures
-    if (taskStatus.getFetchFailedMaps() != null 
-        && taskStatus.getFetchFailedMaps().size() > 0) {
-      taskAttemptStatus.fetchFailedMaps = 
-        new ArrayList<org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId>();
-      for (TaskAttemptID failedMapId : taskStatus.getFetchFailedMaps()) {
-        taskAttemptStatus.fetchFailedMaps.add(
-            TypeConverter.toYarn(failedMapId));
-      }
-    }
-
- // Task sends the information about the nextRecordRange to the TT
-    
-//    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
-//    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
-//    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
-//    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
-//    // This was used by TT to do counter updates only once every minute. So this
-//    // isn't ever changed by the Task itself.
-//    taskStatus.getIncludeCounters();
-
-    context.getEventHandler().handle(
-        new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id,
-            taskAttemptStatus));
-    return true;
-  }
-
-  @Override
-  public long getProtocolVersion(String arg0, long arg1) throws IOException {
-    return TaskUmbilicalProtocol.versionID;
-  }
-
-  @Override
-  public void reportNextRecordRange(TaskAttemptID taskAttemptID, Range range)
-      throws IOException {
-    // This is used when the feature of skipping records is enabled.
-
-    // This call exists as a hadoop mapreduce legacy wherein all changes in
-    // counters/progress/phase/output-size are reported through statusUpdate()
-    // call but not the next record range information.
-    throw new IOException("Not yet implemented.");
-  }
-
-  @Override
-  public JvmTask getTask(JvmContext context) throws IOException {
-
-    // A rough imitation of code from TaskTracker.
-
-    JVMId jvmId = context.jvmId;
-    LOG.info("JVM with ID : " + jvmId + " asked for a task");
-
-    // TODO: Is it an authorised container to get a task? Otherwise return null.
-
-    // TODO: Is the request for task-launch still valid?
-
-    // TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
-    // to jobId and task-type.
-
-    WrappedJvmID wJvmID = new WrappedJvmID(jvmId.getJobId(), jvmId.isMap,
-        jvmId.getId());
-    org.apache.hadoop.mapred.Task task = jvmIDToAttemptMap.get(wJvmID);
-    if (task != null) { //there may be lag in the attempt getting added here
-      LOG.info("JVM with ID: " + jvmId + " given task: " + task.getTaskID());
-      JvmTask jvmTask = new JvmTask(task, false);
-      
-      //remove the task as it is no more needed and free up the memory
-      jvmIDToAttemptMap.remove(wJvmID);
-      
-      return jvmTask;
-    }
-    return null;
-  }
-
-  @Override
-  public void register(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
-      org.apache.hadoop.mapred.Task task, WrappedJvmID jvmID) {
-    //create the mapping so that it is easy to look up
-    //when it comes back to ask for Task.
-    jvmIDToAttemptMap.put(jvmID, task);
-    //register this attempt
-    taskHeartbeatHandler.register(attemptID);
-  }
-
-  @Override
-  public void unregister(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
-      WrappedJvmID jvmID) {
-    //remove the mapping if not already removed
-    jvmIDToAttemptMap.remove(jvmID);
-
-    //unregister this attempt
-    taskHeartbeatHandler.unregister(attemptID);
-  }
-
-  @Override
-  public ProtocolSignature getProtocolSignature(String protocol,
-      long clientVersion, int clientMethodsHash) throws IOException {
-    return ProtocolSignature.getProtocolSignature(this, 
-        protocol, clientVersion, clientMethodsHash);
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedJvmID.java

@@ -1,30 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-/**
- * A simple wrapper for increasing the visibility.
- */
-public class WrappedJvmID extends JVMId {
-
-  public WrappedJvmID(JobID jobID, boolean mapTask, int nextInt) {
-    super(jobID, mapTask, nextInt);
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedPeriodicStatsAccumulator.java

@@ -1,15 +0,0 @@
-package org.apache.hadoop.mapred;
-
-//Workaround for PeriodicStateAccumulator being package access
-public class WrappedPeriodicStatsAccumulator {
-
-  private PeriodicStatsAccumulator real;
-
-  public WrappedPeriodicStatsAccumulator(PeriodicStatsAccumulator real) {
-    this.real = real;
-  }
-  
-  public void extend(double newProgress, int newValue) {
-    real.extend(newProgress, newValue);
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedProgressSplitsBlock.java

@@ -1,52 +0,0 @@
-package org.apache.hadoop.mapred;
-
-// Workaround for ProgressSplitBlock being package access
-public class WrappedProgressSplitsBlock extends ProgressSplitsBlock {
-
-  public static final int DEFAULT_NUMBER_PROGRESS_SPLITS = 12;
-
-  private WrappedPeriodicStatsAccumulator wrappedProgressWallclockTime;
-  private WrappedPeriodicStatsAccumulator wrappedProgressCPUTime;
-  private WrappedPeriodicStatsAccumulator wrappedProgressVirtualMemoryKbytes;
-  private WrappedPeriodicStatsAccumulator wrappedProgressPhysicalMemoryKbytes;
-
-  public WrappedProgressSplitsBlock(int numberSplits) {
-    super(numberSplits);
-  }
-
-  public int[][] burst() {
-    return super.burst();
-  }
-
-  public WrappedPeriodicStatsAccumulator getProgressWallclockTime() {
-    if (wrappedProgressWallclockTime == null) {
-      wrappedProgressWallclockTime = new WrappedPeriodicStatsAccumulator(
-          progressWallclockTime);
-    }
-    return wrappedProgressWallclockTime;
-  }
-
-  public WrappedPeriodicStatsAccumulator getProgressCPUTime() {
-    if (wrappedProgressCPUTime == null) {
-      wrappedProgressCPUTime = new WrappedPeriodicStatsAccumulator(
-          progressCPUTime);
-    }
-    return wrappedProgressCPUTime;
-  }
-
-  public WrappedPeriodicStatsAccumulator getProgressVirtualMemoryKbytes() {
-    if (wrappedProgressVirtualMemoryKbytes == null) {
-      wrappedProgressVirtualMemoryKbytes = new WrappedPeriodicStatsAccumulator(
-          progressVirtualMemoryKbytes);
-    }
-    return wrappedProgressVirtualMemoryKbytes;
-  }
-
-  public WrappedPeriodicStatsAccumulator getProgressPhysicalMemoryKbytes() {
-    if (wrappedProgressPhysicalMemoryKbytes == null) {
-      wrappedProgressPhysicalMemoryKbytes = new WrappedPeriodicStatsAccumulator(
-          progressPhysicalMemoryKbytes);
-    }
-    return wrappedProgressPhysicalMemoryKbytes;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java

@@ -1,346 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSError;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.log4j.LogManager;
-
-/**
- * The main() for MapReduce task processes.
- */
-class YarnChild {
-
-  private static final Log LOG = LogFactory.getLog(YarnChild.class);
-
-  static volatile TaskAttemptID taskid = null;
-
-  public static void main(String[] args) throws Throwable {
-    LOG.debug("Child starting");
-
-    final JobConf defaultConf = new JobConf();
-    defaultConf.addResource(MRConstants.JOB_CONF_FILE);
-    UserGroupInformation.setConfiguration(defaultConf);
-
-    String host = args[0];
-    int port = Integer.parseInt(args[1]);
-    final InetSocketAddress address = new InetSocketAddress(host, port);
-    final TaskAttemptID firstTaskid = TaskAttemptID.forName(args[2]);
-    int jvmIdInt = Integer.parseInt(args[3]);
-    JVMId jvmId = new JVMId(firstTaskid.getJobID(),
-        firstTaskid.getTaskType() == TaskType.MAP, jvmIdInt);
-
-    // initialize metrics
-    DefaultMetricsSystem.initialize(
-        StringUtils.camelize(firstTaskid.getTaskType().name()) +"Task");
-
-    Token<JobTokenIdentifier> jt = loadCredentials(defaultConf, address);
-
-    // Create TaskUmbilicalProtocol as actual task owner.
-    UserGroupInformation taskOwner =
-      UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
-    taskOwner.addToken(jt);
-    final TaskUmbilicalProtocol umbilical =
-      taskOwner.doAs(new PrivilegedExceptionAction<TaskUmbilicalProtocol>() {
-      @Override
-      public TaskUmbilicalProtocol run() throws Exception {
-        return (TaskUmbilicalProtocol)RPC.getProxy(TaskUmbilicalProtocol.class,
-            TaskUmbilicalProtocol.versionID, address, defaultConf);
-      }
-    });
-
-    // report non-pid to application master
-    JvmContext context = new JvmContext(jvmId, "-1000");
-    LOG.debug("PID: " + System.getenv().get("JVM_PID"));
-    Task task = null;
-    UserGroupInformation childUGI = null;
-
-    try {
-      int idleLoopCount = 0;
-      JvmTask myTask = null;;
-      // poll for new task
-      for (int idle = 0; null == myTask; ++idle) {
-        long sleepTimeMilliSecs = Math.min(idle * 500, 1500);
-        LOG.info("Sleeping for " + sleepTimeMilliSecs
-            + "ms before retrying again. Got null now.");
-        MILLISECONDS.sleep(sleepTimeMilliSecs);
-        myTask = umbilical.getTask(context);
-      }
-      if (myTask.shouldDie()) {
-        return;
-      }
-
-      task = myTask.getTask();
-      YarnChild.taskid = task.getTaskID();
-
-      // Create the job-conf and set credentials
-      final JobConf job =
-        configureTask(task, defaultConf.getCredentials(), jt);
-
-      // Initiate Java VM metrics
-      JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
-      LOG.debug("Remote user: " + job.get("user.name"));
-      childUGI = UserGroupInformation.createRemoteUser(job.get("user.name"));
-      // Add tokens to new user so that it may execute its task correctly.
-      for(Token<?> token : UserGroupInformation.getCurrentUser().getTokens()) {
-        childUGI.addToken(token);
-      }
-
-      // Create a final reference to the task for the doAs block
-      final Task taskFinal = task;
-      childUGI.doAs(new PrivilegedExceptionAction<Object>() {
-        @Override
-        public Object run() throws Exception {
-          // use job-specified working directory
-          FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
-          taskFinal.run(job, umbilical); // run the task
-          return null;
-        }
-      });
-    } catch (FSError e) {
-      LOG.fatal("FSError from child", e);
-      umbilical.fsError(taskid, e.getMessage());
-    } catch (Exception exception) {
-      LOG.warn("Exception running child : "
-          + StringUtils.stringifyException(exception));
-      try {
-        if (task != null) {
-          // do cleanup for the task
-          if (childUGI == null) { // no need to job into doAs block
-            task.taskCleanup(umbilical);
-          } else {
-            final Task taskFinal = task;
-            childUGI.doAs(new PrivilegedExceptionAction<Object>() {
-              @Override
-              public Object run() throws Exception {
-                taskFinal.taskCleanup(umbilical);
-                return null;
-              }
-            });
-          }
-        }
-      } catch (Exception e) {
-        LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
-      }
-      // Report back any failures, for diagnostic purposes
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      exception.printStackTrace(new PrintStream(baos));
-      if (taskid != null) {
-        umbilical.reportDiagnosticInfo(taskid, baos.toString());
-      }
-    } catch (Throwable throwable) {
-      LOG.fatal("Error running child : "
-    	        + StringUtils.stringifyException(throwable));
-      if (taskid != null) {
-        Throwable tCause = throwable.getCause();
-        String cause = tCause == null
-                                 ? throwable.getMessage()
-                                 : StringUtils.stringifyException(tCause);
-        umbilical.fatalError(taskid, cause);
-      }
-    } finally {
-      RPC.stopProxy(umbilical);
-      DefaultMetricsSystem.shutdown();
-      // Shutting down log4j of the child-vm...
-      // This assumes that on return from Task.run()
-      // there is no more logging done.
-      LogManager.shutdown();
-    }
-  }
-
-  private static Token<JobTokenIdentifier> loadCredentials(JobConf conf,
-      InetSocketAddress address) throws IOException {
-    //load token cache storage
-    String tokenFileLocation =
-        System.getenv(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME);
-    String jobTokenFile =
-        new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf))
-            .toUri().getPath();
-    Credentials credentials =
-      TokenCache.loadTokens(jobTokenFile, conf);
-    LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
-        "; from file=" + jobTokenFile);
-    Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
-    jt.setService(new Text(address.getAddress().getHostAddress() + ":"
-        + address.getPort()));
-    UserGroupInformation current = UserGroupInformation.getCurrentUser();
-    current.addToken(jt);
-    for (Token<? extends TokenIdentifier> tok : credentials.getAllTokens()) {
-      current.addToken(tok);
-    }
-    // Set the credentials
-    conf.setCredentials(credentials);
-    return jt;
-  }
-
-  /**
-   * Configure mapred-local dirs. This config is used by the task for finding
-   * out an output directory.
-   */
-  private static void configureLocalDirs(Task task, JobConf job) {
-    String[] localSysDirs = StringUtils.getTrimmedStrings(
-        System.getenv(ApplicationConstants.LOCAL_DIR_ENV));
-    job.setStrings(MRConfig.LOCAL_DIR, localSysDirs);
-    LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR));
-  }
-
-  private static JobConf configureTask(Task task, Credentials credentials,
-      Token<JobTokenIdentifier> jt) throws IOException {
-    final JobConf job = new JobConf(MRConstants.JOB_CONF_FILE);
-    job.setCredentials(credentials);
-    // set tcp nodelay
-    job.setBoolean("ipc.client.tcpnodelay", true);
-    job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
-        YarnOutputFiles.class, MapOutputFile.class);
-    // set the jobTokenFile into task
-    task.setJobTokenSecret(
-        JobTokenSecretManager.createSecretKey(jt.getPassword()));
-
-    // setup the child's MRConfig.LOCAL_DIR.
-    configureLocalDirs(task, job);
-
-    // setup the child's attempt directories
-    // Do the task-type specific localization
-    task.localizeConfiguration(job);
-
-    // Set up the DistributedCache related configs
-    setupDistributedCacheConfig(job);
-
-    // Overwrite the localized task jobconf which is linked to in the current
-    // work-dir.
-    Path localTaskFile = new Path(Constants.JOBFILE);
-    writeLocalJobFile(localTaskFile, job);
-    task.setJobFile(localTaskFile.toString());
-    task.setConf(job);
-    return job;
-  }
-
-  /**
-   * Set up the DistributedCache related configs to make
-   * {@link DistributedCache#getLocalCacheFiles(Configuration)}
-   * and
-   * {@link DistributedCache#getLocalCacheArchives(Configuration)}
-   * working.
-   * @param job
-   * @throws IOException
-   */
-  private static void setupDistributedCacheConfig(final JobConf job)
-      throws IOException {
-
-    String localWorkDir = System.getenv("PWD");
-    //        ^ ^ all symlinks are created in the current work-dir
-
-    // Update the configuration object with localized archives.
-    URI[] cacheArchives = DistributedCache.getCacheArchives(job);
-    if (cacheArchives != null) {
-      List<String> localArchives = new ArrayList<String>();
-      for (int i = 0; i < cacheArchives.length; ++i) {
-        URI u = cacheArchives[i];
-        Path p = new Path(u);
-        Path name =
-            new Path((null == u.getFragment()) ? p.getName()
-                : u.getFragment());
-        String linkName = name.toUri().getPath();
-        localArchives.add(new Path(localWorkDir, linkName).toUri().getPath());
-      }
-      if (!localArchives.isEmpty()) {
-        job.set(MRJobConfig.CACHE_LOCALARCHIVES, StringUtils
-            .arrayToString(localArchives.toArray(new String[localArchives
-                .size()])));
-      }
-    }
-
-    // Update the configuration object with localized files.
-    URI[] cacheFiles = DistributedCache.getCacheFiles(job);
-    if (cacheFiles != null) {
-      List<String> localFiles = new ArrayList<String>();
-      for (int i = 0; i < cacheFiles.length; ++i) {
-        URI u = cacheFiles[i];
-        Path p = new Path(u);
-        Path name =
-            new Path((null == u.getFragment()) ? p.getName()
-                : u.getFragment());
-        String linkName = name.toUri().getPath();
-        localFiles.add(new Path(localWorkDir, linkName).toUri().getPath());
-      }
-      if (!localFiles.isEmpty()) {
-        job.set(MRJobConfig.CACHE_LOCALFILES,
-            StringUtils.arrayToString(localFiles
-                .toArray(new String[localFiles.size()])));
-      }
-    }
-  }
-
-  private static final FsPermission urw_gr =
-    FsPermission.createImmutable((short) 0640);
-
-  /**
-   * Write the task specific job-configuration file.
-   * @throws IOException
-   */
-  private static void writeLocalJobFile(Path jobFile, JobConf conf)
-      throws IOException {
-    FileSystem localFs = FileSystem.getLocal(conf);
-    localFs.delete(jobFile);
-    OutputStream out = null;
-    try {
-      out = FileSystem.create(localFs, jobFile, urw_gr);
-      conf.writeXml(out);
-    } finally {
-      IOUtils.cleanup(LOG, out);
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java

@@ -1,238 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRConfig;
-
-/**
- * Manipulate the working area for the transient store for maps and reduces.
- *
- * This class is used by map and reduce tasks to identify the directories that
- * they need to write to/read from for intermediate files. The callers of
- * these methods are from child space.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class YarnOutputFiles extends MapOutputFile {
-
-  private JobConf conf;
-
-  private static final String JOB_OUTPUT_DIR = "output";
-  private static final String SPILL_FILE_PATTERN = "%s_spill_%d.out";
-  private static final String SPILL_INDEX_FILE_PATTERN = SPILL_FILE_PATTERN
-      + ".index";
-
-  public YarnOutputFiles() {
-  }
-
-  // assume configured to $localdir/usercache/$user/appcache/$appId
-  private LocalDirAllocator lDirAlloc = 
-    new LocalDirAllocator(MRConfig.LOCAL_DIR);
-
-  private Path getAttemptOutputDir() {
-    return new Path(JOB_OUTPUT_DIR, conf.get(JobContext.TASK_ATTEMPT_ID));
-  }
-  
-  /**
-   * Return the path to local map output file created earlier
-   * 
-   * @return path
-   * @throws IOException
-   */
-  public Path getOutputFile() throws IOException {
-    Path attemptOutput =
-      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING);
-    return lDirAlloc.getLocalPathToRead(attemptOutput.toString(), conf);
-  }
-
-  /**
-   * Create a local map output file name.
-   * 
-   * @param size the size of the file
-   * @return path
-   * @throws IOException
-   */
-  public Path getOutputFileForWrite(long size) throws IOException {
-    Path attemptOutput = 
-      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING);
-    return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf);
-  }
-
-  /**
-   * Create a local map output file name on the same volume.
-   */
-  public Path getOutputFileForWriteInVolume(Path existing) {
-    Path outputDir = new Path(existing.getParent(), JOB_OUTPUT_DIR);
-    Path attemptOutputDir = new Path(outputDir,
-        conf.get(JobContext.TASK_ATTEMPT_ID));
-    return new Path(attemptOutputDir, MAP_OUTPUT_FILENAME_STRING);
-  }
-
-  /**
-   * Return the path to a local map output index file created earlier
-   * 
-   * @return path
-   * @throws IOException
-   */
-  public Path getOutputIndexFile() throws IOException {
-    Path attemptIndexOutput =
-      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING +
-                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
-    return lDirAlloc.getLocalPathToRead(attemptIndexOutput.toString(), conf);
-  }
-
-  /**
-   * Create a local map output index file name.
-   * 
-   * @param size the size of the file
-   * @return path
-   * @throws IOException
-   */
-  public Path getOutputIndexFileForWrite(long size) throws IOException {
-    Path attemptIndexOutput =
-      new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING +
-                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
-    return lDirAlloc.getLocalPathForWrite(attemptIndexOutput.toString(),
-        size, conf);
-  }
-
-  /**
-   * Create a local map output index file name on the same volume.
-   */
-  public Path getOutputIndexFileForWriteInVolume(Path existing) {
-    Path outputDir = new Path(existing.getParent(), JOB_OUTPUT_DIR);
-    Path attemptOutputDir = new Path(outputDir,
-        conf.get(JobContext.TASK_ATTEMPT_ID));
-    return new Path(attemptOutputDir, MAP_OUTPUT_FILENAME_STRING +
-                                      MAP_OUTPUT_INDEX_SUFFIX_STRING);
-  }
-
-  /**
-   * Return a local map spill file created earlier.
-   * 
-   * @param spillNumber the number
-   * @return path
-   * @throws IOException
-   */
-  public Path getSpillFile(int spillNumber) throws IOException {
-    return lDirAlloc.getLocalPathToRead(
-        String.format(SPILL_FILE_PATTERN,
-            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf);
-  }
-
-  /**
-   * Create a local map spill file name.
-   * 
-   * @param spillNumber the number
-   * @param size the size of the file
-   * @return path
-   * @throws IOException
-   */
-  public Path getSpillFileForWrite(int spillNumber, long size)
-      throws IOException {
-    return lDirAlloc.getLocalPathForWrite(
-        String.format(String.format(SPILL_FILE_PATTERN,
-            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber)), size, conf);
-  }
-
-  /**
-   * Return a local map spill index file created earlier
-   * 
-   * @param spillNumber the number
-   * @return path
-   * @throws IOException
-   */
-  public Path getSpillIndexFile(int spillNumber) throws IOException {
-    return lDirAlloc.getLocalPathToRead(
-        String.format(SPILL_INDEX_FILE_PATTERN,
-            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf);
-  }
-
-  /**
-   * Create a local map spill index file name.
-   * 
-   * @param spillNumber the number
-   * @param size the size of the file
-   * @return path
-   * @throws IOException
-   */
-  public Path getSpillIndexFileForWrite(int spillNumber, long size)
-      throws IOException {
-    return lDirAlloc.getLocalPathForWrite(
-        String.format(SPILL_INDEX_FILE_PATTERN,
-            conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), size, conf);
-  }
-
-  /**
-   * Return a local reduce input file created earlier
-   * 
-   * @param mapId a map task id
-   * @return path
-   * @throws IOException 
-   */
-  public Path getInputFile(int mapId) throws IOException {
-    throw new UnsupportedOperationException("Incompatible with LocalRunner");
-  }
-
-  /**
-   * Create a local reduce input file name.
-   * 
-   * @param mapId a map task id
-   * @param size the size of the file
-   * @return path
-   * @throws IOException
-   */
-  public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId,
-      long size) throws IOException {
-    return lDirAlloc.getLocalPathForWrite(String.format(
-        REDUCE_INPUT_FILE_FORMAT_STRING,
-        getAttemptOutputDir().toString(), mapId.getId()),
-        size, conf);
-  }
-
-  /** Removes all of the files related to a task. */
-  public void removeAll() throws IOException {
-    throw new UnsupportedOperationException("Incompatible with LocalRunner");
-  }
-
-  @Override
-  public void setConf(Configuration conf) {
-    if (conf instanceof JobConf) {
-      this.conf = (JobConf) conf;
-    } else {
-      this.conf = new JobConf(conf);
-    }
-  }
-
-  @Override
-  public Configuration getConf() {
-    return conf;
-  }
-  
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java

@@ -1,42 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.jobhistory;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class JobHistoryEvent extends AbstractEvent<EventType>{
-
-  private final JobId jobID;
-  private final HistoryEvent historyEvent;
-
-  public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent) {
-    super(historyEvent.getEventType());
-    this.jobID = jobID;
-    this.historyEvent = historyEvent;
-  }
-
-  public JobId getJobID() {
-    return jobID;
-  }
-
-  public HistoryEvent getHistoryEvent() {
-    return historyEvent;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -1,675 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.jobhistory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapreduce.JobCounter;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-/**
- * The job history events get routed to this class. This class writes the Job
- * history events to the DFS directly into a staging dir and then moved to a
- * done-dir. JobHistory implementation is in this package to access package
- * private classes.
- */
-public class JobHistoryEventHandler extends AbstractService
-    implements EventHandler<JobHistoryEvent> {
-
-  private final AppContext context;
-  private final int startCount;
-
-  //TODO Does the FS object need to be different ? 
-  private FileSystem stagingDirFS; // log Dir FileSystem
-  private FileSystem doneDirFS; // done Dir FileSystem
-
-  private Configuration conf;
-
-  private Path stagingDirPath = null;
-  private Path doneDirPrefixPath = null; // folder for completed jobs
-
-
-  private BlockingQueue<JobHistoryEvent> eventQueue =
-    new LinkedBlockingQueue<JobHistoryEvent>();
-  private Thread eventHandlingThread;
-  private volatile boolean stopped;
-  private final Object lock = new Object();
-
-  private static final Log LOG = LogFactory.getLog(
-      JobHistoryEventHandler.class);
-
-  private static final Map<JobId, MetaInfo> fileMap =
-    Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
-
-  public JobHistoryEventHandler(AppContext context, int startCount) {
-    super("JobHistoryEventHandler");
-    this.context = context;
-    this.startCount = startCount;
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.hadoop.yarn.service.AbstractService#init(org.apache.hadoop.conf.Configuration)
-   * Initializes the FileSystem and Path objects for the log and done directories.
-   * Creates these directories if they do not already exist.
-   */
-  @Override
-  public void init(Configuration conf) {
-
-    this.conf = conf;
-
-    String stagingDirStr = null;
-    String doneDirStr = null;
-    String userDoneDirStr = null;
-    try {
-      stagingDirStr = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf);
-      doneDirStr =
-          JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
-      userDoneDirStr =
-          JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
-    } catch (IOException e) {
-      LOG.error("Failed while getting the configured log directories", e);
-      throw new YarnException(e);
-    }
-
-    //Check for the existence of the history staging dir. Maybe create it. 
-    try {
-      stagingDirPath =
-          FileSystem.get(conf).makeQualified(new Path(stagingDirStr));
-      stagingDirFS = FileSystem.get(stagingDirPath.toUri(), conf);
-      mkdir(stagingDirFS, stagingDirPath, new FsPermission(
-          JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
-    } catch (IOException e) {
-      LOG.error("Failed while checking for/creating  history staging path: ["
-          + stagingDirPath + "]", e);
-      throw new YarnException(e);
-    }
-
-    //Check for the existence of intermediate done dir.
-    Path doneDirPath = null;
-    try {
-      doneDirPath = FileSystem.get(conf).makeQualified(new Path(doneDirStr));
-      doneDirFS = FileSystem.get(doneDirPath.toUri(), conf);
-      // This directory will be in a common location, or this may be a cluster
-      // meant for a single user. Creating based on the conf. Should ideally be
-      // created by the JobHistoryServer or as part of deployment.
-      if (!doneDirFS.exists(doneDirPath)) {
-      if (JobHistoryUtils.shouldCreateNonUserDirectory(conf)) {
-        LOG.info("Creating intermediate history logDir: ["
-            + doneDirPath
-            + "] + based on conf. Should ideally be created by the JobHistoryServer: "
-            + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY);
-          mkdir(
-              doneDirFS,
-              doneDirPath,
-              new FsPermission(
-            JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
-                .toShort()));
-          // TODO Temporary toShort till new FsPermission(FsPermissions)
-          // respects
-        // sticky
-      } else {
-          String message = "Not creating intermediate history logDir: ["
-                + doneDirPath
-                + "] based on conf: "
-                + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY
-                + ". Either set to true or pre-create this directory with appropriate permissions";
-        LOG.error(message);
-        throw new YarnException(message);
-      }
-      }
-    } catch (IOException e) {
-      LOG.error("Failed checking for the existance of history intermediate done directory: ["
-          + doneDirPath + "]");
-      throw new YarnException(e);
-    }
-
-    //Check/create user directory under intermediate done dir.
-    try {
-      doneDirPrefixPath =
-          FileSystem.get(conf).makeQualified(new Path(userDoneDirStr));
-      mkdir(doneDirFS, doneDirPrefixPath, new FsPermission(
-          JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS));
-    } catch (IOException e) {
-      LOG.error("Error creating user intermediate history done directory: [ "
-          + doneDirPrefixPath + "]", e);
-      throw new YarnException(e);
-    }
-
-    super.init(conf);
-  }
-
-  private void mkdir(FileSystem fs, Path path, FsPermission fsp)
-      throws IOException {
-    if (!fs.exists(path)) {
-      try {
-        fs.mkdirs(path, fsp);
-        FileStatus fsStatus = fs.getFileStatus(path);
-        LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
-            + ", Expected: " + fsp.toShort());
-        if (fsStatus.getPermission().toShort() != fsp.toShort()) {
-          LOG.info("Explicitly setting permissions to : " + fsp.toShort()
-              + ", " + fsp);
-          fs.setPermission(path, fsp);
-        }
-      } catch (FileAlreadyExistsException e) {
-        LOG.info("Directory: [" + path + "] already exists.");
-      }
-    }
-  }
-
-  @Override
-  public void start() {
-    eventHandlingThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        JobHistoryEvent event = null;
-        while (!stopped && !Thread.currentThread().isInterrupted()) {
-          try {
-            event = eventQueue.take();
-          } catch (InterruptedException e) {
-            LOG.info("EventQueue take interrupted. Returning");
-            return;
-          }
-          // If an event has been removed from the queue. Handle it.
-          // The rest of the queue is handled via stop()
-          // Clear the interrupt status if it's set before calling handleEvent
-          // and set it if it was set before calling handleEvent. 
-          // Interrupts received from other threads during handleEvent cannot be
-          // dealth with - Shell.runCommand() ignores them.
-          synchronized (lock) {
-            boolean isInterrupted = Thread.interrupted();
-            handleEvent(event);
-            if (isInterrupted) {
-              Thread.currentThread().interrupt();
-            }
-          }
-        }
-      }
-    });
-    eventHandlingThread.start();
-    super.start();
-  }
-
-  @Override
-  public void stop() {
-    LOG.info("Stopping JobHistoryEventHandler");
-    stopped = true;
-    //do not interrupt while event handling is in progress
-    synchronized(lock) {
-      eventHandlingThread.interrupt();
-    }
-
-    try {
-      eventHandlingThread.join();
-    } catch (InterruptedException ie) {
-      LOG.info("Interruped Exception while stopping", ie);
-    }
-    //write all the events remaining in queue
-    Iterator<JobHistoryEvent> it = eventQueue.iterator();
-    while(it.hasNext()) {
-      JobHistoryEvent ev = it.next();
-      LOG.info("In stop, writing event " + ev.getType());
-      handleEvent(ev);
-    }
-    
-    //close all file handles
-    for (MetaInfo mi : fileMap.values()) {
-      try {
-        mi.closeWriter();
-      } catch (IOException e) {
-        LOG.info("Exception while closing file " + e.getMessage());
-      }
-    }
-    LOG.info("Stopped JobHistoryEventHandler. super.stop()");
-    super.stop();
-  }
-
-  /**
-   * Create an event writer for the Job represented by the jobID.
-   * Writes out the job configuration to the log directory.
-   * This should be the first call to history for a job
-   * 
-   * @param jobId the jobId.
-   * @throws IOException
-   */
-  protected void setupEventWriter(JobId jobId, JobSubmittedEvent jse)
-      throws IOException {
-    if (stagingDirPath == null) {
-      LOG.error("Log Directory is null, returning");
-      throw new IOException("Missing Log Directory for History");
-    }
-
-    MetaInfo oldFi = fileMap.get(jobId);
-    Configuration conf = getConfig();
-
-    long submitTime = oldFi == null ? jse.getSubmitTime() : oldFi
-        .getJobIndexInfo().getSubmitTime();
-    
-    // TODO Ideally this should be written out to the job dir
-    // (.staging/jobid/files - RecoveryService will need to be patched)
-    Path historyFile = JobHistoryUtils.getStagingJobHistoryFile(
-        stagingDirPath, jobId, startCount);
-    String user = UserGroupInformation.getCurrentUser().getShortUserName();
-    if (user == null) {
-      throw new IOException(
-          "User is null while setting up jobhistory eventwriter");
-    }
-
-    String jobName = context.getJob(jobId).getName();
-    EventWriter writer = (oldFi == null) ? null : oldFi.writer;
- 
-    if (writer == null) {
-      try {
-        FSDataOutputStream out = stagingDirFS.create(historyFile, true);
-        writer = new EventWriter(out);
-        LOG.info("Event Writer setup for JobId: " + jobId + ", File: "
-            + historyFile);
-      } catch (IOException ioe) {
-        LOG.info("Could not create log file: [" + historyFile + "] + for job "
-            + "[" + jobName + "]");
-        throw ioe;
-      }
-    }
-    
-    Path logDirConfPath = null;
-    if (conf != null) {
-      // TODO Ideally this should be written out to the job dir
-      // (.staging/jobid/files - RecoveryService will need to be patched)
-      logDirConfPath = JobHistoryUtils.getStagingConfFile(stagingDirPath, jobId,
-          startCount);
-      FSDataOutputStream jobFileOut = null;
-      try {
-        if (logDirConfPath != null) {
-          jobFileOut = stagingDirFS.create(logDirConfPath, true);
-          conf.writeXml(jobFileOut);
-          jobFileOut.close();
-        }
-      } catch (IOException e) {
-        LOG.info("Failed to write the job configuration file", e);
-        throw e;
-      }
-    }
-    
-    MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer, submitTime,
-        user, jobName, jobId);
-    fi.getJobSummary().setJobId(jobId);
-    fi.getJobSummary().setJobSubmitTime(submitTime);
-    fileMap.put(jobId, fi);
-  }
-
-  /** Close the event writer for this id 
-   * @throws IOException */
-  public void closeWriter(JobId id) throws IOException {
-    try {
-      final MetaInfo mi = fileMap.get(id);
-      if (mi != null) {
-        mi.closeWriter();
-      }
-      
-    } catch (IOException e) {
-      LOG.error("Error closing writer for JobID: " + id);
-      throw e;
-    }
-  }
-
-  @Override
-  public void handle(JobHistoryEvent event) {
-    try {
-      eventQueue.put(event);
-    } catch (InterruptedException e) {
-      throw new YarnException(e);
-    }
-  }
-
-  protected void handleEvent(JobHistoryEvent event) {
-    synchronized (lock) {
-
-      // If this is JobSubmitted Event, setup the writer
-      if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) {
-        try {
-          JobSubmittedEvent jobSubmittedEvent =
-              (JobSubmittedEvent) event.getHistoryEvent();
-          setupEventWriter(event.getJobID(), jobSubmittedEvent);
-        } catch (IOException ioe) {
-          LOG.error("Error JobHistoryEventHandler in handleEvent: " + event,
-              ioe);
-          throw new YarnException(ioe);
-        }
-      }
-
-      // For all events
-      // (1) Write it out
-      // (2) Process it for JobSummary
-      MetaInfo mi = fileMap.get(event.getJobID());
-      try {
-        HistoryEvent historyEvent = event.getHistoryEvent();
-        mi.writeEvent(historyEvent);
-        processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID());
-        LOG.info("In HistoryEventHandler "
-            + event.getHistoryEvent().getEventType());
-      } catch (IOException e) {
-        LOG.error("Error writing History Event: " + event.getHistoryEvent(),
-            e);
-        throw new YarnException(e);
-      }
-
-      // If this is JobFinishedEvent, close the writer and setup the job-index
-      if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
-        try {
-          JobFinishedEvent jFinishedEvent =
-              (JobFinishedEvent) event.getHistoryEvent();
-          mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime());
-          mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps());
-          mi.getJobIndexInfo().setNumReduces(
-              jFinishedEvent.getFinishedReduces());
-          mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString());
-          closeEventWriter(event.getJobID());
-        } catch (IOException e) {
-          throw new YarnException(e);
-        }
-      }
-
-      if (event.getHistoryEvent().getEventType() == EventType.JOB_FAILED
-          || event.getHistoryEvent().getEventType() == EventType.JOB_KILLED) {
-        try {
-          JobUnsuccessfulCompletionEvent jucEvent = (JobUnsuccessfulCompletionEvent) event
-              .getHistoryEvent();
-          mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime());
-          mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps());
-          mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces());
-          mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus());
-          closeEventWriter(event.getJobID());
-        } catch (IOException e) {
-          throw new YarnException(e);
-        }
-      }
-    }
-  }
-
-  private void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId) {
-    // context.getJob could be used for some of this info as well.
-    switch (event.getEventType()) {
-    case JOB_SUBMITTED:
-      JobSubmittedEvent jse = (JobSubmittedEvent) event;
-      summary.setUser(jse.getUserName());
-      summary.setQueue(jse.getJobQueueName());
-      break;
-    case JOB_INITED:
-      JobInitedEvent jie = (JobInitedEvent) event;
-      summary.setJobLaunchTime(jie.getLaunchTime());
-      break;
-    case MAP_ATTEMPT_STARTED:
-      TaskAttemptStartedEvent mtase = (TaskAttemptStartedEvent) event;
-      if (summary.getFirstMapTaskLaunchTime() == 0)
-        summary.setFirstMapTaskLaunchTime(mtase.getStartTime());
-      break;
-    case REDUCE_ATTEMPT_STARTED:
-      TaskAttemptStartedEvent rtase = (TaskAttemptStartedEvent) event;
-      if (summary.getFirstReduceTaskLaunchTime() == 0)
-        summary.setFirstReduceTaskLaunchTime(rtase.getStartTime());
-      break;
-    case JOB_FINISHED:
-      JobFinishedEvent jfe = (JobFinishedEvent) event;
-      summary.setJobFinishTime(jfe.getFinishTime());
-      summary.setNumFinishedMaps(jfe.getFinishedMaps());
-      summary.setNumFailedMaps(jfe.getFailedMaps());
-      summary.setNumFinishedReduces(jfe.getFinishedReduces());
-      summary.setNumFailedReduces(jfe.getFailedReduces());
-      if (summary.getJobStatus() == null)
-        summary
-            .setJobStatus(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED
-                .toString());
-      // TODO JOB_FINISHED does not have state. Effectively job history does not
-      // have state about the finished job.
-      setSummarySlotSeconds(summary, jobId);
-      break;
-    case JOB_FAILED:
-    case JOB_KILLED:
-      JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
-      summary.setJobStatus(juce.getStatus());
-      summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
-      summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
-      summary.setJobFinishTime(juce.getFinishTime());
-      setSummarySlotSeconds(summary, jobId);
-      break;
-    }
-  }
-
-  private void setSummarySlotSeconds(JobSummary summary, JobId jobId) {
-    Counter slotMillisMapCounter =
-        context.getJob(jobId).getCounters()
-            .getCounter(JobCounter.SLOTS_MILLIS_MAPS);
-    if (slotMillisMapCounter != null) {
-      summary.setMapSlotSeconds(slotMillisMapCounter.getValue());
-    }
-    Counter slotMillisReduceCounter =
-        context.getJob(jobId).getCounters()
-            .getCounter(JobCounter.SLOTS_MILLIS_REDUCES);
-    if (slotMillisReduceCounter != null) {
-      summary.setMapSlotSeconds(slotMillisReduceCounter.getValue());
-    }
-  }
-
-  protected void closeEventWriter(JobId jobId) throws IOException {
-
-    final MetaInfo mi = fileMap.get(jobId);
-    if (mi == null) {
-      throw new IOException("No MetaInfo found for JobId: [" + jobId + "]");
-    }
-
-    if (!mi.isWriterActive()) {
-      throw new IOException(
-          "Inactive Writer: Likely received multiple JobFinished / JobUnsuccessful events for JobId: ["
-              + jobId + "]");
-    }
-
-    // Close the Writer
-    try {
-      mi.closeWriter();
-    } catch (IOException e) {
-      LOG.error("Error closing writer for JobID: " + jobId);
-      throw e;
-    }
-     
-    if (mi.getHistoryFile() == null) {
-      LOG.warn("No file for job-history with " + jobId + " found in cache!");
-    }
-    if (mi.getConfFile() == null) {
-      LOG.warn("No file for jobconf with " + jobId + " found in cache!");
-    }
-      
-    // Writing out the summary file.
-    // TODO JH enhancement - reuse this file to store additional indexing info
-    // like ACLs, etc. JHServer can use HDFS append to build an index file
-    // with more info than is available via the filename.
-    Path qualifiedSummaryDoneFile = null;
-    FSDataOutputStream summaryFileOut = null;
-    try {
-      String doneSummaryFileName = getTempFileName(JobHistoryUtils
-          .getIntermediateSummaryFileName(jobId));
-      qualifiedSummaryDoneFile = doneDirFS.makeQualified(new Path(
-          doneDirPrefixPath, doneSummaryFileName));
-      summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true);
-      summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
-      summaryFileOut.close();
-    } catch (IOException e) {
-      LOG.info("Unable to write out JobSummaryInfo to ["
-          + qualifiedSummaryDoneFile + "]", e);
-      throw e;
-    }
-
-    try {
-
-      // Move historyFile to Done Folder.
-      Path qualifiedDoneFile = null;
-      if (mi.getHistoryFile() != null) {
-        Path historyFile = mi.getHistoryFile();
-        Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
-        String doneJobHistoryFileName =
-            getTempFileName(FileNameIndexUtils.getDoneFileName(mi
-                .getJobIndexInfo()));
-        qualifiedDoneFile =
-            doneDirFS.makeQualified(new Path(doneDirPrefixPath,
-                doneJobHistoryFileName));
-        moveToDoneNow(qualifiedLogFile, qualifiedDoneFile);
-      }
-
-      // Move confFile to Done Folder
-      Path qualifiedConfDoneFile = null;
-      if (mi.getConfFile() != null) {
-        Path confFile = mi.getConfFile();
-        Path qualifiedConfFile = stagingDirFS.makeQualified(confFile);
-        String doneConfFileName =
-            getTempFileName(JobHistoryUtils
-                .getIntermediateConfFileName(jobId));
-        qualifiedConfDoneFile =
-            doneDirFS.makeQualified(new Path(doneDirPrefixPath,
-                doneConfFileName));
-        moveToDoneNow(qualifiedConfFile, qualifiedConfDoneFile);
-      }
-      
-      moveTmpToDone(qualifiedSummaryDoneFile);
-      moveTmpToDone(qualifiedConfDoneFile);
-      moveTmpToDone(qualifiedDoneFile);
-
-    } catch (IOException e) {
-      LOG.error("Error closing writer for JobID: " + jobId);
-      throw e;
-    }
-  }
-
-  private class MetaInfo {
-    private Path historyFile;
-    private Path confFile;
-    private EventWriter writer;
-    JobIndexInfo jobIndexInfo;
-    JobSummary jobSummary;
-
-    MetaInfo(Path historyFile, Path conf, EventWriter writer, long submitTime,
-             String user, String jobName, JobId jobId) {
-      this.historyFile = historyFile;
-      this.confFile = conf;
-      this.writer = writer;
-      this.jobIndexInfo = new JobIndexInfo(submitTime, -1, user, jobName, jobId, -1, -1, null);
-      this.jobSummary = new JobSummary();
-    }
-
-    Path getHistoryFile() { return historyFile; }
-
-    Path getConfFile() {return confFile; } 
-
-    JobIndexInfo getJobIndexInfo() { return jobIndexInfo; }
-
-    JobSummary getJobSummary() { return jobSummary; }
-
-    boolean isWriterActive() {return writer != null ; }
-
-    void closeWriter() throws IOException {
-      synchronized (lock) {
-      if (writer != null) {
-        writer.close();
-      }
-      writer = null;
-    }
-    }
-
-    void writeEvent(HistoryEvent event) throws IOException {
-      synchronized (lock) {
-      if (writer != null) {
-        writer.write(event);
-        writer.flush();
-      }
-    }
-  }
-  }
-
-  private void moveTmpToDone(Path tmpPath) throws IOException {
-    if (tmpPath != null) {
-      String tmpFileName = tmpPath.getName();
-      String fileName = getFileNameFromTmpFN(tmpFileName);
-      Path path = new Path(tmpPath.getParent(), fileName);
-      doneDirFS.rename(tmpPath, path);
-      LOG.info("Moved tmp to done: " + tmpPath + " to " + path);
-    }
-  }
-  
-  // TODO If the FS objects are the same, this should be a rename instead of a
-  // copy.
-  private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
-    // check if path exists, in case of retries it may not exist
-    if (stagingDirFS.exists(fromPath)) {
-      LOG.info("Moving " + fromPath.toString() + " to " + toPath.toString());
-      // TODO temporarily removing the existing dst
-      if (doneDirFS.exists(toPath)) {
-        doneDirFS.delete(toPath, true);
-      }
-      boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
-          false, conf);
-
-      if (copied)
-        LOG.info("Copied to done location: " + toPath);
-      else 
-          LOG.info("copy failed");
-      doneDirFS.setPermission(toPath, new FsPermission(
-          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
-      
-      stagingDirFS.delete(fromPath, false);
-    }
-    }
-
-  boolean pathExists(FileSystem fileSys, Path path) throws IOException {
-    return fileSys.exists(path);
-  }
-
-  private String getTempFileName(String srcFile) {
-    return srcFile + "_tmp";
-  }
-  
-  private String getFileNameFromTmpFN(String tmpFileName) {
-    //TODO. Some error checking here.
-    return tmpFileName.substring(0, tmpFileName.length()-4);
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSummary.java

@@ -1,231 +0,0 @@
-package org.apache.hadoop.mapreduce.jobhistory;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.util.StringUtils;
-
-public class JobSummary {
-  private JobId jobId;
-  private long jobSubmitTime;
-  private long jobLaunchTime;
-  private long firstMapTaskLaunchTime; // MapAttempteStarted |
-                                       // TaskAttemptStartEvent
-  private long firstReduceTaskLaunchTime; // ReduceAttemptStarted |
-                                          // TaskAttemptStartEvent
-  private long jobFinishTime;
-  private int numFinishedMaps;
-  private int numFailedMaps;
-  private int numFinishedReduces;
-  private int numFailedReduces;
-  // private int numSlotsPerMap; | Doesn't make sense with potentially different
-  // resource models
-  // private int numSlotsPerReduce; | Doesn't make sense with potentially
-  // different resource models
-  private String user;
-  private String queue;
-  private String jobStatus;
-  private long mapSlotSeconds; // TODO Not generated yet in MRV2
-  private long reduceSlotSeconds; // TODO Not generated yet MRV2
-  // private int clusterSlotCapacity;
-
-  JobSummary() {
-  }
-
-  public JobId getJobId() {
-    return jobId;
-  }
-
-  public void setJobId(JobId jobId) {
-    this.jobId = jobId;
-  }
-
-  public long getJobSubmitTime() {
-    return jobSubmitTime;
-  }
-
-  public void setJobSubmitTime(long jobSubmitTime) {
-    this.jobSubmitTime = jobSubmitTime;
-  }
-
-  public long getJobLaunchTime() {
-    return jobLaunchTime;
-  }
-
-  public void setJobLaunchTime(long jobLaunchTime) {
-    this.jobLaunchTime = jobLaunchTime;
-  }
-
-  public long getFirstMapTaskLaunchTime() {
-    return firstMapTaskLaunchTime;
-  }
-
-  public void setFirstMapTaskLaunchTime(long firstMapTaskLaunchTime) {
-    this.firstMapTaskLaunchTime = firstMapTaskLaunchTime;
-  }
-
-  public long getFirstReduceTaskLaunchTime() {
-    return firstReduceTaskLaunchTime;
-  }
-
-  public void setFirstReduceTaskLaunchTime(long firstReduceTaskLaunchTime) {
-    this.firstReduceTaskLaunchTime = firstReduceTaskLaunchTime;
-  }
-
-  public long getJobFinishTime() {
-    return jobFinishTime;
-  }
-
-  public void setJobFinishTime(long jobFinishTime) {
-    this.jobFinishTime = jobFinishTime;
-  }
-
-  public int getNumFinishedMaps() {
-    return numFinishedMaps;
-  }
-
-  public void setNumFinishedMaps(int numFinishedMaps) {
-    this.numFinishedMaps = numFinishedMaps;
-  }
-
-  public int getNumFailedMaps() {
-    return numFailedMaps;
-  }
-
-  public void setNumFailedMaps(int numFailedMaps) {
-    this.numFailedMaps = numFailedMaps;
-  }
-
-  // public int getNumSlotsPerMap() {
-  // return numSlotsPerMap;
-  // }
-  //
-  // public void setNumSlotsPerMap(int numSlotsPerMap) {
-  // this.numSlotsPerMap = numSlotsPerMap;
-  // }
-
-  public int getNumFinishedReduces() {
-    return numFinishedReduces;
-  }
-
-  public void setNumFinishedReduces(int numFinishedReduces) {
-    this.numFinishedReduces = numFinishedReduces;
-  }
-
-  public int getNumFailedReduces() {
-    return numFailedReduces;
-  }
-
-  public void setNumFailedReduces(int numFailedReduces) {
-    this.numFailedReduces = numFailedReduces;
-  }
-
-  // public int getNumSlotsPerReduce() {
-  // return numSlotsPerReduce;
-  // }
-  //
-  // public void setNumSlotsPerReduce(int numSlotsPerReduce) {
-  // this.numSlotsPerReduce = numSlotsPerReduce;
-  // }
-
-  public String getUser() {
-    return user;
-  }
-
-  public void setUser(String user) {
-    this.user = user;
-  }
-
-  public String getQueue() {
-    return queue;
-  }
-
-  public void setQueue(String queue) {
-    this.queue = queue;
-  }
-
-  public String getJobStatus() {
-    return jobStatus;
-  }
-
-  public void setJobStatus(String jobStatus) {
-    this.jobStatus = jobStatus;
-  }
-
-  public long getMapSlotSeconds() {
-    return mapSlotSeconds;
-  }
-
-  public void setMapSlotSeconds(long mapSlotSeconds) {
-    this.mapSlotSeconds = mapSlotSeconds;
-  }
-
-  public long getReduceSlotSeconds() {
-    return reduceSlotSeconds;
-  }
-
-  public void setReduceSlotSeconds(long reduceSlotSeconds) {
-    this.reduceSlotSeconds = reduceSlotSeconds;
-  }
-
-  // public int getClusterSlotCapacity() {
-  // return clusterSlotCapacity;
-  // }
-  //
-  // public void setClusterSlotCapacity(int clusterSlotCapacity) {
-  // this.clusterSlotCapacity = clusterSlotCapacity;
-  // }
-
-  public String getJobSummaryString() {
-    SummaryBuilder summary = new SummaryBuilder()
-      .add("jobId", jobId)
-      .add("submitTime", jobSubmitTime)
-      .add("launchTime", jobLaunchTime)
-      .add("firstMapTaskLaunchTime", firstMapTaskLaunchTime)
-      .add("firstReduceTaskLaunchTime", firstReduceTaskLaunchTime)
-      .add("finishTime", jobFinishTime)
-      .add("numMaps", numFinishedMaps + numFailedMaps)
-      .add("numReduces", numFinishedReduces + numFailedReduces)
-      .add("user", user)
-      .add("queue", queue)
-      .add("status", jobStatus)
-      .add("mapSlotSeconds", mapSlotSeconds)
-      .add("reduceSlotSeconds", reduceSlotSeconds);
-    return summary.toString();
-  }
-
-  static final char EQUALS = '=';
-  static final char[] charsToEscape = { StringUtils.COMMA, EQUALS,
-      StringUtils.ESCAPE_CHAR };
-  
-  static class SummaryBuilder {
-    final StringBuilder buffer = new StringBuilder();
-
-    // A little optimization for a very common case
-    SummaryBuilder add(String key, long value) {
-      return _add(key, Long.toString(value));
-    }
-
-    <T> SummaryBuilder add(String key, T value) {
-      return _add(key, StringUtils.escapeString(String.valueOf(value),
-          StringUtils.ESCAPE_CHAR, charsToEscape));
-    }
-
-    SummaryBuilder add(SummaryBuilder summary) {
-      if (buffer.length() > 0)
-        buffer.append(StringUtils.COMMA);
-      buffer.append(summary.buffer);
-      return this;
-    }
-
-    SummaryBuilder _add(String key, String value) {
-      if (buffer.length() > 0)
-        buffer.append(StringUtils.COMMA);
-      buffer.append(key).append(EQUALS).append(value);
-      return this;
-    }
-
-    @Override
-    public String toString() {
-      return buffer.toString();
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AMConstants.java

@@ -1,74 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-
-public interface AMConstants {
-
-  public static final String CONTAINERLAUNCHER_THREADPOOL_SIZE =
-      "yarn.mapreduce.containerlauncher.threadpool-size";
-
-  public static final String AM_RM_SCHEDULE_INTERVAL =
-      "yarn.appMaster.scheduler.interval";
-
-  public static final int DEFAULT_AM_RM_SCHEDULE_INTERVAL = 2000;
-
-  public static final String AM_TASK_LISTENER_THREADS =
-      MRConstants.YARN_MR_PREFIX + "task.listener.threads";
-
-  public static final int DEFAULT_AM_TASK_LISTENER_THREADS = 10;
-
-  public static final String AM_JOB_CLIENT_THREADS =
-      MRConstants.YARN_MR_PREFIX + "job.client.threads";
-
-  public static final int DEFAULT_AM_JOB_CLIENT_THREADS = 1;
-
-  public static final String SPECULATOR_CLASS =
-      MRConstants.YARN_MR_PREFIX + "speculator.class";
-
-  public static final String TASK_RUNTIME_ESTIMATOR_CLASS =
-      MRConstants.YARN_MR_PREFIX + "task.runtime.estimator.class";
-
-  public static final String TASK_ATTEMPT_PROGRESS_RUNTIME_LINEARIZER_CLASS =
-      MRConstants.YARN_MR_PREFIX + "task.runtime.linearizer.class";
-
-  public static final String EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS =
-      MRConstants.YARN_MR_PREFIX
-          + "task.runtime.estimator.exponential.smooth.lambda";
-
-  public static final String EXPONENTIAL_SMOOTHING_SMOOTH_RATE =
-      MRConstants.YARN_MR_PREFIX
-          + "task.runtime.estimator.exponential.smooth.smoothsrate";
-
-  public static final String RECOVERY_ENABLE = MRConstants.YARN_MR_PREFIX
-      + "recovery.enable";
-  
-  public static final float DEFAULT_REDUCE_RAMP_UP_LIMIT = 0.5f;
-  public static final String REDUCE_RAMPUP_UP_LIMIT = MRConstants.YARN_MR_PREFIX
-  + "reduce.rampup.limit";
-  
-  public static final float DEFAULT_REDUCE_PREEMPTION_LIMIT = 0.5f;
-  public static final String REDUCE_PREEMPTION_LIMIT = MRConstants.YARN_MR_PREFIX
-  + "reduce.preemption.limit";
-
-  public static final String NODE_BLACKLISTING_ENABLE = MRConstants.YARN_MR_PREFIX
-  + "node.blacklisting.enable";
-  
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AppContext.java

@@ -1,55 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-
-/**
- * Context interface for sharing information across components in YARN App.
- */
-@InterfaceAudience.Private
-public interface AppContext {
-
-  ApplicationId getApplicationID();
-
-  ApplicationAttemptId getApplicationAttemptId();
-
-  String getApplicationName();
-
-  long getStartTime();
-
-  CharSequence getUser();
-
-  Job getJob(JobId jobID);
-
-  Map<JobId, Job> getAllJobs();
-
-  EventHandler getEventHandler();
-
-  Clock getClock();
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java

@@ -1,576 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.LocalContainerLauncher;
-import org.apache.hadoop.mapred.TaskAttemptListenerImpl;
-import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
-import org.apache.hadoop.mapreduce.v2.app.client.MRClientService;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl;
-import org.apache.hadoop.mapreduce.v2.app.local.LocalContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
-import org.apache.hadoop.mapreduce.v2.app.recover.Recovery;
-import org.apache.hadoop.mapreduce.v2.app.recover.RecoveryService;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.speculate.DefaultSpeculator;
-import org.apache.hadoop.mapreduce.v2.app.speculate.Speculator;
-import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
-import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleaner;
-import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanerImpl;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.SystemClock;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.service.CompositeService;
-import org.apache.hadoop.yarn.service.Service;
-
-/**
- * The Map-Reduce Application Master.
- * The state machine is encapsulated in the implementation of Job interface.
- * All state changes happens via Job interface. Each event 
- * results in a Finite State Transition in Job.
- * 
- * MR AppMaster is the composition of loosely coupled services. The services 
- * interact with each other via events. The components resembles the 
- * Actors model. The component acts on received event and send out the 
- * events to other components.
- * This keeps it highly concurrent with no or minimal synchronization needs.
- * 
- * The events are dispatched by a central Dispatch mechanism. All components
- * register to the Dispatcher.
- * 
- * The information is shared across different components using AppContext.
- */
-
-public class MRAppMaster extends CompositeService {
-
-  private static final Log LOG = LogFactory.getLog(MRAppMaster.class);
-
-  private Clock clock;
-  private final long startTime = System.currentTimeMillis();
-  private String appName;
-  private final int startCount;
-  private final ApplicationId appID;
-  private final ApplicationAttemptId appAttemptID;
-  protected final MRAppMetrics metrics;
-  private Set<TaskId> completedTasksFromPreviousRun;
-  private AppContext context;
-  private Dispatcher dispatcher;
-  private ClientService clientService;
-  private ContainerAllocator containerAllocator;
-  private ContainerLauncher containerLauncher;
-  private TaskCleaner taskCleaner;
-  private Speculator speculator;
-  private TaskAttemptListener taskAttemptListener;
-  private JobTokenSecretManager jobTokenSecretManager =
-      new JobTokenSecretManager();
-
-  private Job job;
-  
-  public MRAppMaster(ApplicationId applicationId, int startCount) {
-    this(applicationId, new SystemClock(), startCount);
-  }
-
-  public MRAppMaster(ApplicationId applicationId, Clock clock, int startCount) {
-    super(MRAppMaster.class.getName());
-    this.clock = clock;
-    this.appID = applicationId;
-    this.appAttemptID = RecordFactoryProvider.getRecordFactory(null)
-        .newRecordInstance(ApplicationAttemptId.class);
-    this.appAttemptID.setApplicationId(appID);
-    this.appAttemptID.setAttemptId(startCount);
-    this.startCount = startCount;
-    this.metrics = MRAppMetrics.create();
-    LOG.info("Created MRAppMaster for application " + applicationId);
-  }
-
-  @Override
-  public void init(final Configuration conf) {
-    context = new RunningAppContext();
-
-    // Job name is the same as the app name util we support DAG of jobs
-    // for an app later
-    appName = conf.get(MRJobConfig.JOB_NAME, "<missing app name>");
-
-    if (conf.getBoolean(AMConstants.RECOVERY_ENABLE, false)
-         && startCount > 1) {
-      LOG.info("Recovery is enabled. Will try to recover from previous life.");
-      Recovery recoveryServ = new RecoveryService(appID, clock, startCount);
-      addIfService(recoveryServ);
-      dispatcher = recoveryServ.getDispatcher();
-      clock = recoveryServ.getClock();
-      completedTasksFromPreviousRun = recoveryServ.getCompletedTasks();
-    } else {
-      dispatcher = new AsyncDispatcher();
-      addIfService(dispatcher);
-    }
-
-    //service to handle requests to TaskUmbilicalProtocol
-    taskAttemptListener = createTaskAttemptListener(context);
-    addIfService(taskAttemptListener);
-
-    //service to do the task cleanup
-    taskCleaner = createTaskCleaner(context);
-    addIfService(taskCleaner);
-
-    //service to handle requests from JobClient
-    clientService = createClientService(context);
-    addIfService(clientService);
-
-    //service to log job history events
-    EventHandler<JobHistoryEvent> historyService = 
-        createJobHistoryHandler(context);
-    addIfService(historyService);
-
-    JobEventDispatcher synchronousJobEventDispatcher = new JobEventDispatcher();
-
-    //register the event dispatchers
-    dispatcher.register(JobEventType.class, synchronousJobEventDispatcher);
-    dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
-    dispatcher.register(TaskAttemptEventType.class, 
-        new TaskAttemptEventDispatcher());
-    dispatcher.register(TaskCleaner.EventType.class, taskCleaner);
-    dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class,
-        historyService);
-    
-    if (conf.getBoolean(MRJobConfig.MAP_SPECULATIVE, false)
-        || conf.getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false)) {
-      //optional service to speculate on task attempts' progress
-      speculator = createSpeculator(conf, context);
-      addIfService(speculator);
-    }
-
-    dispatcher.register(Speculator.EventType.class,
-        new SpeculatorEventDispatcher());
-
-    Credentials fsTokens = new Credentials();
-    if (UserGroupInformation.isSecurityEnabled()) {
-      // Read the file-system tokens from the localized tokens-file.
-      try {
-        Path jobSubmitDir =
-            FileContext.getLocalFSFileContext().makeQualified(
-                new Path(new File(MRConstants.JOB_SUBMIT_DIR)
-                    .getAbsolutePath()));
-        Path jobTokenFile =
-            new Path(jobSubmitDir, MRConstants.APPLICATION_TOKENS_FILE);
-        fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
-        LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
-            + jobTokenFile);
-
-        UserGroupInformation currentUser =
-            UserGroupInformation.getCurrentUser();
-        for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) {
-          LOG.info(" --- DEBUG: Token of kind " + tk.getKind()
-              + "in current ugi in the AppMaster for service "
-              + tk.getService());
-          currentUser.addToken(tk); // For use by AppMaster itself.
-        }
-      } catch (IOException e) {
-        throw new YarnException(e);
-      }
-    }
-
-    super.init(conf);
-
-    //---- start of what used to be startJobs() code:
-
-    Configuration config = getConfig();
-
-    job = createJob(config, fsTokens);
-
-    /** create a job event for job intialization */
-    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
-    /** send init to the job (this does NOT trigger job execution) */
-    synchronousJobEventDispatcher.handle(initJobEvent);
-
-    // send init to speculator. This won't yest start as dispatcher isn't
-    // started yet.
-    dispatcher.getEventHandler().handle(
-        new SpeculatorEvent(job.getID(), clock.getTime()));
-
-    // JobImpl's InitTransition is done (call above is synchronous), so the
-    // "uber-decision" (MR-1220) has been made.  Query job and switch to
-    // ubermode if appropriate (by registering different container-allocator
-    // and container-launcher services/event-handlers).
-
-    if (job.isUber()) {
-      LOG.info("MRAppMaster uberizing job " + job.getID()
-               + " in local container (\"uber-AM\").");
-    } else {
-      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
-               + "job " + job.getID() + ".");
-    }
-
-    // service to allocate containers from RM (if non-uber) or to fake it (uber)
-    containerAllocator =
-        createContainerAllocator(clientService, context, job.isUber());
-    addIfService(containerAllocator);
-    dispatcher.register(ContainerAllocator.EventType.class, containerAllocator);
-    if (containerAllocator instanceof Service) {
-      ((Service) containerAllocator).init(config);
-    }
-
-    // corresponding service to launch allocated containers via NodeManager
-    containerLauncher = createContainerLauncher(context, job.isUber());
-    addIfService(containerLauncher);
-    dispatcher.register(ContainerLauncher.EventType.class, containerLauncher);
-    if (containerLauncher instanceof Service) {
-      ((Service) containerLauncher).init(config);
-    }
-
-  } // end of init()
-
-  /** Create and initialize (but don't start) a single job. 
-   * @param fsTokens */
-  protected Job createJob(Configuration conf, Credentials fsTokens) {
-
-    // create single job
-    Job newJob = new JobImpl(appID, conf, dispatcher.getEventHandler(),
-        taskAttemptListener, jobTokenSecretManager, fsTokens, clock, startCount,
-        completedTasksFromPreviousRun, metrics);
-    ((RunningAppContext) context).jobs.put(newJob.getID(), newJob);
-
-    dispatcher.register(JobFinishEvent.Type.class,
-        new EventHandler<JobFinishEvent>() {
-          @Override
-          public void handle(JobFinishEvent event) {
-            // job has finished
-            // this is the only job, so shut down the Appmaster
-            // note in a workflow scenario, this may lead to creation of a new
-            // job (FIXME?)
-
-            // TODO:currently just wait for some time so clients can know the
-            // final states. Will be removed once RM come on.
-            try {
-              Thread.sleep(5000);
-            } catch (InterruptedException e) {
-              e.printStackTrace();
-            }
-            LOG.info("Calling stop for all the services");
-            try {
-              stop();
-            } catch (Throwable t) {
-              LOG.warn("Graceful stop failed ", t);
-            }
-            //TODO: this is required because rpc server does not shut down
-            // in spite of calling server.stop().
-            //Bring the process down by force.
-            //Not needed after HADOOP-7140
-            LOG.info("Exiting MR AppMaster..GoodBye!");
-            System.exit(0);
-          }
-        });
-
-    return newJob;
-  } // end createJob()
-
-  protected void addIfService(Object object) {
-    if (object instanceof Service) {
-      addService((Service) object);
-    }
-  }
-
-  protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
-      AppContext context) {
-    JobHistoryEventHandler eventHandler = new JobHistoryEventHandler(context, 
-        getStartCount());
-    return eventHandler;
-  }
-
-  protected Speculator createSpeculator(Configuration conf, AppContext context) {
-    Class<? extends Speculator> speculatorClass;
-
-    try {
-      speculatorClass
-          // "yarn.mapreduce.job.speculator.class"
-          = conf.getClass(AMConstants.SPECULATOR_CLASS,
-                          DefaultSpeculator.class,
-                          Speculator.class);
-      Constructor<? extends Speculator> speculatorConstructor
-          = speculatorClass.getConstructor
-               (Configuration.class, AppContext.class);
-      Speculator result = speculatorConstructor.newInstance(conf, context);
-
-      return result;
-    } catch (InstantiationException ex) {
-      LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
-      throw new YarnException(ex);
-    } catch (IllegalAccessException ex) {
-      LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
-      throw new YarnException(ex);
-    } catch (InvocationTargetException ex) {
-      LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
-      throw new YarnException(ex);
-    } catch (NoSuchMethodException ex) {
-      LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
-      throw new YarnException(ex);
-    }
-  }
-
-  protected TaskAttemptListener createTaskAttemptListener(AppContext context) {
-    TaskAttemptListener lis =
-        new TaskAttemptListenerImpl(context, jobTokenSecretManager);
-    return lis;
-  }
-
-  protected TaskCleaner createTaskCleaner(AppContext context) {
-    return new TaskCleanerImpl(context);
-  }
-
-  protected ContainerAllocator createContainerAllocator(
-      ClientService clientService, AppContext context, boolean isLocal) {
-    //return new StaticContainerAllocator(context);
-    return isLocal
-        ? new LocalContainerAllocator(clientService, context)
-        : new RMContainerAllocator(clientService, context);
-  }
-
-  protected ContainerLauncher createContainerLauncher(AppContext context,
-                                                      boolean isLocal) {
-    return isLocal
-        ? new LocalContainerLauncher(context,
-            (TaskUmbilicalProtocol) taskAttemptListener)
-        : new ContainerLauncherImpl(context);
-  }
-
-  //TODO:should have an interface for MRClientService
-  protected ClientService createClientService(AppContext context) {
-    return new MRClientService(context);
-  }
-
-  public ApplicationId getAppID() {
-    return appID;
-  }
-
-  public int getStartCount() {
-    return startCount;
-  }
-
-  public AppContext getContext() {
-    return context;
-  }
-
-  public Dispatcher getDispatcher() {
-    return dispatcher;
-  }
-
-  public Set<TaskId> getCompletedTaskFromPreviousRun() {
-    return completedTasksFromPreviousRun;
-  }
-
-  public ContainerAllocator getContainerAllocator() {
-    return containerAllocator;
-  }
-  
-  public ContainerLauncher getContainerLauncher() {
-    return containerLauncher;
-  }
-
-  public TaskAttemptListener getTaskAttemptListener() {
-    return taskAttemptListener;
-  }
-
-  class RunningAppContext implements AppContext {
-
-    private Map<JobId, Job> jobs = new ConcurrentHashMap<JobId, Job>();
-
-    @Override
-    public ApplicationAttemptId getApplicationAttemptId() {
-      return appAttemptID;
-    }
-
-    @Override
-    public ApplicationId getApplicationID() {
-      return appID;
-    }
-
-    @Override
-    public String getApplicationName() {
-      return appName;
-    }
-
-    @Override
-    public long getStartTime() {
-      return startTime;
-    }
-
-    @Override
-    public Job getJob(JobId jobID) {
-      return jobs.get(jobID);
-    }
-
-    @Override
-    public Map<JobId, Job> getAllJobs() {
-      return jobs;
-    }
-
-    @Override
-    public EventHandler getEventHandler() {
-      return dispatcher.getEventHandler();
-    }
-
-    @Override
-    public CharSequence getUser() {
-      return getConfig().get(MRJobConfig.USER_NAME);
-    }
-
-    @Override
-    public Clock getClock() {
-      return clock;
-    }
-  }
-
-  @Override
-  public void start() {
-    // metrics system init is really init & start.
-    // It's more test friendly to put it here.
-    DefaultMetricsSystem.initialize("MRAppMaster");
-
-    startJobs();
-    //start all the components
-    super.start();
-  }
-
-  /**
-   * This can be overridden to instantiate multiple jobs and create a 
-   * workflow.
-   *
-   * TODO:  Rework the design to actually support this.  Currently much of the
-   * job stuff has been moved to init() above to support uberization (MR-1220).
-   * In a typical workflow, one presumably would want to uberize only a subset
-   * of the jobs (the "small" ones), which is awkward with the current design.
-   */
-  protected void startJobs() {
-    /** create a job-start event to get this ball rolling */
-    JobEvent startJobEvent = new JobEvent(job.getID(), JobEventType.JOB_START);
-    /** send the job-start event. this triggers the job execution. */
-    dispatcher.getEventHandler().handle(startJobEvent);
-  }
-
-  private class JobEventDispatcher implements EventHandler<JobEvent> {
-    @Override
-    public void handle(JobEvent event) {
-      ((EventHandler<JobEvent>)context.getJob(event.getJobId())).handle(event);
-    }
-  }
-
-  private class TaskEventDispatcher implements EventHandler<TaskEvent> {
-    @Override
-    public void handle(TaskEvent event) {
-      Task task = context.getJob(event.getTaskID().getJobId()).getTask(
-          event.getTaskID());
-      ((EventHandler<TaskEvent>)task).handle(event);
-    }
-  }
-
-  private class TaskAttemptEventDispatcher 
-          implements EventHandler<TaskAttemptEvent> {
-    @Override
-    public void handle(TaskAttemptEvent event) {
-      Job job = context.getJob(event.getTaskAttemptID().getTaskId().getJobId());
-      Task task = job.getTask(event.getTaskAttemptID().getTaskId());
-      TaskAttempt attempt = task.getAttempt(event.getTaskAttemptID());
-      ((EventHandler<TaskAttemptEvent>) attempt).handle(event);
-    }
-  }
-
-  private class SpeculatorEventDispatcher implements
-      EventHandler<SpeculatorEvent> {
-    @Override
-    public void handle(SpeculatorEvent event) {
-      if (getConfig().getBoolean(MRJobConfig.MAP_SPECULATIVE, false)
-          || getConfig().getBoolean(MRJobConfig.REDUCE_SPECULATIVE, false)) {
-        // Speculator IS enabled, direct the event to there.
-        speculator.handle(event);
-      }
-    }
-  }
-
-  public static void main(String[] args) {
-    try {
-      //Configuration.addDefaultResource("job.xml");
-      ApplicationId applicationId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class);
-      
-      applicationId.setClusterTimestamp(Long.valueOf(args[0]));
-      applicationId.setId(Integer.valueOf(args[1]));
-      int failCount = Integer.valueOf(args[2]);
-      MRAppMaster appMaster = new MRAppMaster(applicationId, failCount);
-      YarnConfiguration conf = new YarnConfiguration(new JobConf());
-      conf.addResource(new Path(MRConstants.JOB_CONF_FILE));
-      conf.set(MRJobConfig.USER_NAME, 
-          System.getProperty("user.name")); 
-      UserGroupInformation.setConfiguration(conf);
-      appMaster.init(conf);
-      appMaster.start();
-    } catch (Throwable t) {
-      LOG.error("Caught throwable. Exiting:", t);
-      System.exit(1);
-    }
-  } 
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRClientSecurityInfo.java

@@ -1,58 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import java.lang.annotation.Annotation;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.security.SecurityInfo;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.hadoop.yarn.proto.MRClientProtocol;
-import org.apache.hadoop.yarn.security.ApplicationTokenSelector;
-
-public class MRClientSecurityInfo extends SecurityInfo {
-
-  @Override
-  public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
-    return null;
-  }
-
-  @Override
-  public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
-    if (!protocol.equals(MRClientProtocol.MRClientProtocolService.BlockingInterface.class)) {
-      return null;
-    }
-    return new TokenInfo() {
-
-      @Override
-      public Class<? extends Annotation> annotationType() {
-        return null;
-      }
-
-      @Override
-      public Class<? extends TokenSelector<? extends TokenIdentifier>>
-          value() {
-        return ApplicationTokenSelector.class;
-      }
-    };
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskAttemptListener.java

@@ -1,35 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.WrappedJvmID;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-public interface TaskAttemptListener {
-
-  InetSocketAddress getAddress();
-
-  void register(TaskAttemptId attemptID, Task task, WrappedJvmID jvmID);
-
-  void unregister(TaskAttemptId attemptID, WrappedJvmID jvmID);
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java

@@ -1,137 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-
-/**
- * This class keeps track of tasks that have already been launched. It
- * determines if a task is alive and running or marks a task as dead if it does
- * not hear from it for a long time.
- * 
- */
-public class TaskHeartbeatHandler extends AbstractService {
-
-  private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
-
-  //thread which runs periodically to see the last time since a heartbeat is
-  //received from a task.
-  private Thread lostTaskCheckerThread;
-  private volatile boolean stopped;
-  private int taskTimeOut = 5*60*1000;//5 mins
-
-  private final EventHandler eventHandler;
-  private final Clock clock;
-
-  private Map<TaskAttemptId, Long> runningAttempts 
-    = new HashMap<TaskAttemptId, Long>();
-
-  public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock) {
-    super("TaskHeartbeatHandler");
-    this.eventHandler = eventHandler;
-    this.clock = clock;
-  }
-
-  @Override
-  public void init(Configuration conf) {
-   super.init(conf);
-   taskTimeOut = conf.getInt("mapreduce.task.timeout", 5*60*1000);
-  }
-
-  @Override
-  public void start() {
-    lostTaskCheckerThread = new Thread(new PingChecker());
-    lostTaskCheckerThread.start();
-    super.start();
-  }
-
-  @Override
-  public void stop() {
-    stopped = true;
-    lostTaskCheckerThread.interrupt();
-    super.stop();
-  }
-
-  public synchronized void receivedPing(TaskAttemptId attemptID) {
-    //only put for the registered attempts
-    if (runningAttempts.containsKey(attemptID)) {
-      runningAttempts.put(attemptID, clock.getTime());
-    }
-  }
-
-  public synchronized void register(TaskAttemptId attemptID) {
-    runningAttempts.put(attemptID, clock.getTime());
-  }
-
-  public synchronized void unregister(TaskAttemptId attemptID) {
-    runningAttempts.remove(attemptID);
-  }
-
-  private class PingChecker implements Runnable {
-
-    @Override
-    public void run() {
-      while (!stopped && !Thread.currentThread().isInterrupted()) {
-        synchronized (TaskHeartbeatHandler.this) {
-          Iterator<Map.Entry<TaskAttemptId, Long>> iterator = 
-            runningAttempts.entrySet().iterator();
-
-          //avoid calculating current time everytime in loop
-          long currentTime = clock.getTime();
-
-          while (iterator.hasNext()) {
-            Map.Entry<TaskAttemptId, Long> entry = iterator.next();
-            if (currentTime > entry.getValue() + taskTimeOut) {
-              //task is lost, remove from the list and raise lost event
-              iterator.remove();
-              eventHandler.handle(
-                  new TaskAttemptDiagnosticsUpdateEvent(entry.getKey(),
-                      "AttemptID:" + entry.getKey().toString() + 
-                      " Timed out after " + taskTimeOut/1000 + " secs"));
-              eventHandler.handle(new TaskAttemptEvent(entry
-                  .getKey(), TaskAttemptEventType.TA_TIMED_OUT));
-            }
-          }
-        }
-        try {
-          Thread.sleep(taskTimeOut);
-        } catch (InterruptedException e) {
-          LOG.info("TaskHeartbeatHandler thread interrupted");
-          break;
-        }
-      }
-    }
-    
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java

@@ -1,28 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.client;
-
-import java.net.InetSocketAddress;
-
-public interface ClientService {
-
-  InetSocketAddress getBindAddress();
-
-  int getHttpPort();
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -1,392 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.client;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.security.AccessControlException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.avro.ipc.Server;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobACL;
-import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
-import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityInfo;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
-import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
-import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-
-/**
- * This module is responsible for talking to the 
- * jobclient (user facing).
- *
- */
-public class MRClientService extends AbstractService 
-    implements ClientService {
-
-  static final Log LOG = LogFactory.getLog(MRClientService.class);
-  
-  private MRClientProtocol protocolHandler;
-  private Server server;
-  private WebApp webApp;
-  private InetSocketAddress bindAddress;
-  private AppContext appContext;
-
-  public MRClientService(AppContext appContext) {
-    super("MRClientService");
-    this.appContext = appContext;
-    this.protocolHandler = new MRClientProtocolHandler();
-  }
-
-  public void start() {
-    Configuration conf = new Configuration(getConfig()); // Just for not messing up sec-info class config
-    YarnRPC rpc = YarnRPC.create(conf);
-    InetSocketAddress address = NetUtils.createSocketAddr("0.0.0.0:0");
-    InetAddress hostNameResolved = null;
-    try {
-      hostNameResolved = InetAddress.getLocalHost();
-    } catch (UnknownHostException e) {
-      throw new YarnException(e);
-    }
-
-    ClientToAMSecretManager secretManager = null;
-    if (UserGroupInformation.isSecurityEnabled()) {
-      secretManager = new ClientToAMSecretManager();
-      String secretKeyStr =
-          System
-              .getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
-      byte[] bytes = Base64.decodeBase64(secretKeyStr);
-      ApplicationTokenIdentifier identifier =
-          new ApplicationTokenIdentifier(this.appContext.getApplicationID());
-      secretManager.setMasterKey(identifier, bytes);
-      conf.setClass(
-          YarnConfiguration.YARN_SECURITY_INFO,
-          SchedulerSecurityInfo.class, SecurityInfo.class); // Same for now.
-    }
-    server =
-        rpc.getServer(MRClientProtocol.class, protocolHandler, address,
-            conf, secretManager,
-            conf.getInt(AMConstants.AM_JOB_CLIENT_THREADS, 
-                AMConstants.DEFAULT_AM_JOB_CLIENT_THREADS));
-    server.start();
-    this.bindAddress =
-        NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
-            + ":" + server.getPort());
-    LOG.info("Instantiated MRClientService at " + this.bindAddress);
-    try {
-      webApp = WebApps.$for("yarn", AppContext.class, appContext).with(conf).
-          start(new AMWebApp());
-    } catch (Exception e) {
-      LOG.error("Webapps failed to start. Ignoring for now:", e);
-    }
-    super.start();
-  }
-
-  public void stop() {
-    server.close();
-    if (webApp != null) {
-      webApp.stop();
-    }
-    super.stop();
-  }
-
-  @Override
-  public InetSocketAddress getBindAddress() {
-    return bindAddress;
-  }
-
-  @Override
-  public int getHttpPort() {
-    return webApp.port();
-  }
-
-  class MRClientProtocolHandler implements MRClientProtocol {
-
-    private RecordFactory recordFactory = 
-      RecordFactoryProvider.getRecordFactory(null);
-
-    private Job verifyAndGetJob(JobId jobID, 
-        boolean modifyAccess) throws YarnRemoteException {
-      Job job = appContext.getJob(jobID);
-      if (job == null) {
-        throw RPCUtil.getRemoteException("Unknown job " + jobID);
-      }
-      //TODO fix job acls.
-      //JobACL operation = JobACL.VIEW_JOB;
-      //if (modifyAccess) {
-      //  operation = JobACL.MODIFY_JOB;
-      //}
-      //TO disable check access ofr now.
-      //checkAccess(job, operation);
-      return job;
-    }
- 
-    private Task verifyAndGetTask(TaskId taskID, 
-        boolean modifyAccess) throws YarnRemoteException {
-      Task task = verifyAndGetJob(taskID.getJobId(), 
-          modifyAccess).getTask(taskID);
-      if (task == null) {
-        throw RPCUtil.getRemoteException("Unknown Task " + taskID);
-      }
-      return task;
-    }
-
-    private TaskAttempt verifyAndGetAttempt(TaskAttemptId attemptID, 
-        boolean modifyAccess) throws YarnRemoteException {
-      TaskAttempt attempt = verifyAndGetTask(attemptID.getTaskId(), 
-          modifyAccess).getAttempt(attemptID);
-      if (attempt == null) {
-        throw RPCUtil.getRemoteException("Unknown TaskAttempt " + attemptID);
-      }
-      return attempt;
-    }
-
-    private void checkAccess(Job job, JobACL jobOperation) 
-      throws YarnRemoteException {
-      if (!UserGroupInformation.isSecurityEnabled()) {
-        return;
-      }
-      UserGroupInformation callerUGI;
-      try {
-        callerUGI = UserGroupInformation.getCurrentUser();
-      } catch (IOException e) {
-        throw RPCUtil.getRemoteException(e);
-      }
-      if(!job.checkAccess(callerUGI, jobOperation)) {
-        throw RPCUtil.getRemoteException(new AccessControlException("User "
-            + callerUGI.getShortUserName() + " cannot perform operation "
-            + jobOperation.name() + " on " + job.getID()));
-      }
-    }
-
-    @Override
-    public GetCountersResponse getCounters(GetCountersRequest request) 
-      throws YarnRemoteException {
-      JobId jobId = request.getJobId();
-      Job job = verifyAndGetJob(jobId, false);
-      GetCountersResponse response =
-        recordFactory.newRecordInstance(GetCountersResponse.class);
-      response.setCounters(job.getCounters());
-      return response;
-    }
-    
-    @Override
-    public GetJobReportResponse getJobReport(GetJobReportRequest request) 
-      throws YarnRemoteException {
-      JobId jobId = request.getJobId();
-      Job job = verifyAndGetJob(jobId, false);
-      GetJobReportResponse response = 
-        recordFactory.newRecordInstance(GetJobReportResponse.class);
-      response.setJobReport(job.getReport());
-      return response;
-    }
-    
-    
-    @Override
-    public GetTaskAttemptReportResponse getTaskAttemptReport(
-        GetTaskAttemptReportRequest request) throws YarnRemoteException {
-      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
-      GetTaskAttemptReportResponse response =
-        recordFactory.newRecordInstance(GetTaskAttemptReportResponse.class);
-      response.setTaskAttemptReport(
-          verifyAndGetAttempt(taskAttemptId, false).getReport());
-      return response;
-    }
-
-    @Override
-    public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) 
-      throws YarnRemoteException {
-      TaskId taskId = request.getTaskId();
-      GetTaskReportResponse response = 
-        recordFactory.newRecordInstance(GetTaskReportResponse.class);
-      response.setTaskReport(verifyAndGetTask(taskId, false).getReport());
-      return response;
-    }
-
-    @Override
-    public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
-        GetTaskAttemptCompletionEventsRequest request) 
-        throws YarnRemoteException {
-      JobId jobId = request.getJobId();
-      int fromEventId = request.getFromEventId();
-      int maxEvents = request.getMaxEvents();
-      Job job = verifyAndGetJob(jobId, false);
-      
-      GetTaskAttemptCompletionEventsResponse response = 
-        recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
-      response.addAllCompletionEvents(Arrays.asList(
-          job.getTaskAttemptCompletionEvents(fromEventId, maxEvents)));
-      return response;
-    }
-    
-    @Override
-    public KillJobResponse killJob(KillJobRequest request) 
-      throws YarnRemoteException {
-      JobId jobId = request.getJobId();
-      String message = "Kill Job received from client " + jobId;
-      LOG.info(message);
-  	  verifyAndGetJob(jobId, true);
-      appContext.getEventHandler().handle(
-          new JobDiagnosticsUpdateEvent(jobId, message));
-      appContext.getEventHandler().handle(
-          new JobEvent(jobId, JobEventType.JOB_KILL));
-      KillJobResponse response = 
-        recordFactory.newRecordInstance(KillJobResponse.class);
-      return response;
-    }
-
-    @Override
-    public KillTaskResponse killTask(KillTaskRequest request) 
-      throws YarnRemoteException {
-      TaskId taskId = request.getTaskId();
-      String message = "Kill task received from client " + taskId;
-      LOG.info(message);
-      verifyAndGetTask(taskId, true);
-      appContext.getEventHandler().handle(
-          new TaskEvent(taskId, TaskEventType.T_KILL));
-      KillTaskResponse response = 
-        recordFactory.newRecordInstance(KillTaskResponse.class);
-      return response;
-    }
-    
-    @Override
-    public KillTaskAttemptResponse killTaskAttempt(
-        KillTaskAttemptRequest request) throws YarnRemoteException {
-      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
-      String message = "Kill task attempt received from client " + taskAttemptId;
-      LOG.info(message);
-      verifyAndGetAttempt(taskAttemptId, true);
-      appContext.getEventHandler().handle(
-          new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
-      appContext.getEventHandler().handle(
-          new TaskAttemptEvent(taskAttemptId, 
-              TaskAttemptEventType.TA_KILL));
-      KillTaskAttemptResponse response = 
-        recordFactory.newRecordInstance(KillTaskAttemptResponse.class);
-      return response;
-    }
-
-    @Override
-    public GetDiagnosticsResponse getDiagnostics(
-        GetDiagnosticsRequest request) throws YarnRemoteException {
-      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
-      
-      GetDiagnosticsResponse response = 
-        recordFactory.newRecordInstance(GetDiagnosticsResponse.class);
-      response.addAllDiagnostics(
-          verifyAndGetAttempt(taskAttemptId, false).getDiagnostics());
-      return response;
-    }
-
-    @Override
-    public FailTaskAttemptResponse failTaskAttempt(
-        FailTaskAttemptRequest request) throws YarnRemoteException {
-      TaskAttemptId taskAttemptId = request.getTaskAttemptId();
-      String message = "Fail task attempt received from client " + taskAttemptId;
-      LOG.info(message);
-      verifyAndGetAttempt(taskAttemptId, true);
-      appContext.getEventHandler().handle(
-          new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
-      appContext.getEventHandler().handle(
-          new TaskAttemptEvent(taskAttemptId, 
-              TaskAttemptEventType.TA_FAILMSG));
-      FailTaskAttemptResponse response = recordFactory.
-        newRecordInstance(FailTaskAttemptResponse.class);
-      return response;
-    }
-
-    @Override
-    public GetTaskReportsResponse getTaskReports(
-        GetTaskReportsRequest request) throws YarnRemoteException {
-      JobId jobId = request.getJobId();
-      TaskType taskType = request.getTaskType();
-      
-      GetTaskReportsResponse response = 
-        recordFactory.newRecordInstance(GetTaskReportsResponse.class);
-      
-      Job job = verifyAndGetJob(jobId, false);
-      LOG.info("Getting task report for " + taskType + "   " + jobId);
-      Collection<Task> tasks = job.getTasks(taskType).values();
-      LOG.info("Getting task report size " + tasks.size());
-      for (Task task : tasks) {
-        response.addTaskReport(task.getReport());
-	  }
-      return response;
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java

@@ -1,59 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.mapreduce.JobACL;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
-import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.security.UserGroupInformation;
-
-
-/**
- * Main interface to interact with the job. Provides only getters. 
- */
-public interface Job {
-
-  JobId getID();
-  String getName();
-  JobState getState();
-  JobReport getReport();
-  Counters getCounters();
-  Map<TaskId,Task> getTasks();
-  Map<TaskId,Task> getTasks(TaskType taskType);
-  Task getTask(TaskId taskID);
-  List<String> getDiagnostics();
-  int getTotalMaps();
-  int getTotalReduces();
-  int getCompletedMaps();
-  int getCompletedReduces();
-  boolean isUber();
-
-  TaskAttemptCompletionEvent[]
-      getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);
-
-  boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation);
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Task.java

@@ -1,58 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job;
-
-import java.util.Map;
-
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-
-/**
- * Read only view of Task.
- */
-public interface Task {
-  TaskId getID();
-  TaskReport getReport();
-  TaskState getState();
-  Counters getCounters();
-  float getProgress();
-  TaskType getType();
-  Map<TaskAttemptId, TaskAttempt> getAttempts();
-  TaskAttempt getAttempt(TaskAttemptId attemptID);
-
-  /** Has Task reached the final state or not.
-   */
-  boolean isFinished();
-
-  /**
-   * Can the output of the taskAttempt be committed. Note that once the task
-   * gives a go for a commit, further canCommit requests from any other attempts
-   * should return false.
-   * 
-   * @param taskAttemptID
-   * @return whether the attempt's output can be committed or not.
-   */
-  boolean canCommit(TaskAttemptId taskAttemptID);
-
-  
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java

@@ -1,66 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job;
-
-import java.util.List;
-
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-
-/**
- * Read only view of TaskAttempt.
- */
-public interface TaskAttempt {
-  TaskAttemptId getID();
-  TaskAttemptReport getReport();
-  List<String> getDiagnostics();
-  Counters getCounters();
-  float getProgress();
-  TaskAttemptState getState();
-
-  /** Has attempt reached the final state or not.
-   */
-  boolean isFinished();
-
-  /**If container Assigned then return container ID, otherwise null.
-   */
-  ContainerId getAssignedContainerID();
-
-  /**If container Assigned then return container mgr address, otherwise null.
-   */
-  String getAssignedContainerMgrAddress();
-  
-  /**If container Assigned then return the node's http address, otherwise null.
-   */
-  String getNodeHttpAddress();
-
-  /** Returns time at which container is launched. If container is not launched
-   * yet, returns 0.
-   */
-  long getLaunchTime();
-
-  /** Returns attempt's finish time. If attempt is not finished
-   *  yet, returns 0.
-   */
-  long getFinishTime();
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobCounterUpdateEvent.java

@@ -1,42 +0,0 @@
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-
-public class JobCounterUpdateEvent extends JobEvent {
-
-  List<CounterIncrementalUpdate> counterUpdates = null;
-  
-  public JobCounterUpdateEvent(JobId jobId) {
-    super(jobId, JobEventType.JOB_COUNTER_UPDATE);
-    counterUpdates = new ArrayList<JobCounterUpdateEvent.CounterIncrementalUpdate>();
-  }
-
-  public void addCounterUpdate(Enum<?> key, long incrValue) {
-    counterUpdates.add(new CounterIncrementalUpdate(key, incrValue));
-  }
-  
-  public List<CounterIncrementalUpdate> getCounterUpdates() {
-    return counterUpdates;
-  }
-  
-  public static class CounterIncrementalUpdate {
-    Enum<?> key;
-    long incrValue;
-    
-    public CounterIncrementalUpdate(Enum<?> key, long incrValue) {
-      this.key = key;
-      this.incrValue = incrValue;
-    }
-    
-    public Enum<?> getCounterKey() {
-      return key;
-    }
-
-    public long getIncrementValue() {
-      return incrValue;
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobDiagnosticsUpdateEvent.java

@@ -1,36 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-
-
-public class JobDiagnosticsUpdateEvent extends JobEvent {
-
-  private String diagnosticUpdate;
-
-  public JobDiagnosticsUpdateEvent(JobId jobID, String diagnostic) {
-    super(jobID, JobEventType.JOB_DIAGNOSTIC_UPDATE);
-    this.diagnosticUpdate = diagnostic;
-  }
-
-  public String getDiagnosticUpdate() {
-    return this.diagnosticUpdate;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEvent.java

@@ -1,41 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-
-/**
- * This class encapsulates job related events.
- *
- */
-public class JobEvent extends AbstractEvent<JobEventType> {
-
-  private JobId jobID;
-
-  public JobEvent(JobId jobID, JobEventType type) {
-    super(type);
-    this.jobID = jobID;
-  }
-
-  public JobId getJobId() {
-    return jobID;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobEventType.java

@@ -1,48 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-/**
- * Event types handled by Job.
- */
-public enum JobEventType {
-
-  //Producer:Client
-  JOB_KILL,
-
-  //Producer:MRAppMaster
-  JOB_INIT,
-  JOB_START,
-
-  //Producer:Task
-  JOB_TASK_COMPLETED,
-  JOB_MAP_TASK_RESCHEDULED,
-  JOB_TASK_ATTEMPT_COMPLETED,
-
-  //Producer:Job
-  JOB_COMPLETED,
-
-  //Producer:Any component
-  JOB_DIAGNOSTIC_UPDATE,
-  INTERNAL_ERROR,
-  JOB_COUNTER_UPDATE,
-  
-  //Producer:TaskAttemptListener
-  JOB_TASK_ATTEMPT_FETCH_FAILURE
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobFinishEvent.java

@@ -1,42 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-public class JobFinishEvent 
-          extends AbstractEvent<JobFinishEvent.Type> {
-
-  public enum Type {
-    STATE_CHANGED
-  }
-
-  private JobId jobID;
-
-  public JobFinishEvent(JobId jobID) {
-    super(Type.STATE_CHANGED);
-    this.jobID = jobID;
-  }
-
-  public JobId getJobId() {
-    return jobID;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobMapTaskRescheduledEvent.java

@@ -1,38 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-
-
-
-public class JobMapTaskRescheduledEvent extends JobEvent {
-
-  private TaskId taskID;
-
-  public JobMapTaskRescheduledEvent(TaskId taskID) {
-    super(taskID.getJobId(), JobEventType.JOB_MAP_TASK_RESCHEDULED);
-    this.taskID = taskID;
-  }
-
-  public TaskId getTaskID() {
-    return taskID;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptCompletedEvent.java

@@ -1,37 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
-
-
-public class JobTaskAttemptCompletedEvent extends JobEvent {
-
-  private TaskAttemptCompletionEvent completionEvent;
-
-  public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) {
-    super(completionEvent.getAttemptId().getTaskId().getJobId(), 
-        JobEventType.JOB_TASK_ATTEMPT_COMPLETED);
-    this.completionEvent = completionEvent;
-  }
-
-  public TaskAttemptCompletionEvent getCompletionEvent() {
-    return completionEvent;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java

@@ -1,48 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import java.util.List;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-
-
-public class JobTaskAttemptFetchFailureEvent extends JobEvent {
-
-  private final TaskAttemptId reduce;
-  private final List<TaskAttemptId> maps;
-
-  public JobTaskAttemptFetchFailureEvent(TaskAttemptId reduce, 
-      List<TaskAttemptId> maps) {
-    super(reduce.getTaskId().getJobId(), 
-        JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE);
-    this.reduce = reduce;
-    this.maps = maps;
-  }
-
-  public List<TaskAttemptId> getMaps() {
-    return maps;
-  }
-
-  public TaskAttemptId getReduce() {
-    return reduce;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskEvent.java

@@ -1,43 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-
-
-public class JobTaskEvent extends JobEvent {
-
-  private TaskId taskID;
-  private TaskState taskState;
-
-  public JobTaskEvent(TaskId taskID, TaskState taskState) {
-    super(taskID.getJobId(), JobEventType.JOB_TASK_COMPLETED);
-    this.taskID = taskID;
-    this.taskState = taskState;
-  }
-
-  public TaskId getTaskID() {
-    return taskID;
-  }
-
-  public TaskState getState() {
-    return taskState;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java

@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-
-
-
-public class TaskAttemptContainerAssignedEvent extends TaskAttemptEvent {
-
-  private final Container container;
-
-  public TaskAttemptContainerAssignedEvent(TaskAttemptId id,
-      Container container) {
-    super(id, TaskAttemptEventType.TA_ASSIGNED);
-    this.container = container;
-  }
-
-  public Container getContainer() {
-    return this.container;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptDiagnosticsUpdateEvent.java

@@ -1,37 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-
-public class TaskAttemptDiagnosticsUpdateEvent extends TaskAttemptEvent {
-
-  private String diagnosticInfo;
-
-  public TaskAttemptDiagnosticsUpdateEvent(TaskAttemptId attemptID,
-      String diagnosticInfo) {
-    super(attemptID, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE);
-    this.diagnosticInfo = diagnosticInfo;
-  }
-
-  public String getDiagnosticInfo() {
-    return diagnosticInfo;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptEvent.java

@@ -1,41 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-/**
- * This class encapsulates task attempt related events.
- *
- */
-public class TaskAttemptEvent extends AbstractEvent<TaskAttemptEventType> {
-
-  private TaskAttemptId attemptID;
-
-  public TaskAttemptEvent(TaskAttemptId id, TaskAttemptEventType type) {
-    super(type);
-    this.attemptID = id;
-  }
-
-  public TaskAttemptId getTaskAttemptID() {
-    return attemptID;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptEventType.java

@@ -1,55 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-/**
- * Event types handled by TaskAttempt.
- */
-public enum TaskAttemptEventType {
-
-  //Producer:Task
-  TA_SCHEDULE,
-  TA_RESCHEDULE,
-
-  //Producer:Client, Task
-  TA_KILL,
-
-  //Producer:ContainerAllocator
-  TA_ASSIGNED,
-  TA_CONTAINER_COMPLETED,
-
-  //Producer:ContainerLauncher
-  TA_CONTAINER_LAUNCHED,
-  TA_CONTAINER_LAUNCH_FAILED,
-  TA_CONTAINER_CLEANED,
-
-  //Producer:TaskAttemptListener
-  TA_DIAGNOSTICS_UPDATE,
-  TA_COMMIT_PENDING, 
-  TA_DONE,
-  TA_FAILMSG,
-  TA_UPDATE,
-  TA_TIMED_OUT,
-
-  //Producer:TaskCleaner
-  TA_CLEANUP_DONE,
-
-  //Producer:Job
-  TA_TOO_MANY_FETCH_FAILURE,
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java

@@ -1,61 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import java.util.List;
-
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.Phase;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-
-
-public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent {
-
-  private TaskAttemptStatus reportedTaskAttemptStatus;
-
-  public TaskAttemptStatusUpdateEvent(TaskAttemptId id,
-      TaskAttemptStatus taskAttemptStatus) {
-    super(id, TaskAttemptEventType.TA_UPDATE);
-    this.reportedTaskAttemptStatus = taskAttemptStatus;
-  }
-
-  public TaskAttemptStatus getReportedTaskAttemptStatus() {
-    return reportedTaskAttemptStatus;
-  }
-
-  /**
-   * The internal TaskAttemptStatus object corresponding to remote Task status.
-   * 
-   */
-  public static class TaskAttemptStatus {
-    public TaskAttemptId id;
-    public float progress;
-    public Counters counters;
-    public String diagnosticInfo;
-    public String stateString;
-    public Phase phase;
-    public long outputSize;
-    public List<TaskAttemptId> fetchFailedMaps;
-    public long mapFinishTime;
-    public long shuffleFinishTime;
-    public long sortFinishTime;
-    public TaskAttemptState taskState;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskEvent.java

@@ -1,40 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-
-/**
- * this class encapsulates task related events.
- *
- */
-public class TaskEvent extends AbstractEvent<TaskEventType> {
-
-  private TaskId taskID;
-
-  public TaskEvent(TaskId taskID, TaskEventType type) {
-    super(type);
-    this.taskID = taskID;
-  }
-
-  public TaskId getTaskID() {
-    return taskID;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskEventType.java

@@ -1,41 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-/**
- * Event types handled by Task.
- */
-public enum TaskEventType {
-
-  //Producer:Client, Job
-  T_KILL,
-
-  //Producer:Job
-  T_SCHEDULE,
-
-  //Producer:Speculator
-  T_ADD_SPEC_ATTEMPT,
-
-  //Producer:TaskAttempt
-  T_ATTEMPT_LAUNCHED,
-  T_ATTEMPT_COMMIT_PENDING,
-  T_ATTEMPT_FAILED,
-  T_ATTEMPT_SUCCEEDED,
-  T_ATTEMPT_KILLED
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskTAttemptEvent.java

@@ -1,37 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.event;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-
-public class TaskTAttemptEvent extends TaskEvent {
-
-  private TaskAttemptId attemptID;
-
-  public TaskTAttemptEvent(TaskAttemptId id, TaskEventType type) {
-    super(id.getTaskId(), type);
-    this.attemptID = id;
-  }
-
-  public TaskAttemptId getTaskAttemptID() {
-    return attemptID;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

@@ -1,1416 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileOutputCommitter;
-import org.apache.hadoop.mapred.JobACLsManager;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.JobACL;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.jobhistory.JobFinishedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent;
-import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
-import org.apache.hadoop.mapreduce.lib.chain.ChainReducer;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
-import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
-import org.apache.hadoop.mapreduce.task.JobContextImpl;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
-import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
-import org.apache.hadoop.yarn.state.MultipleArcTransition;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-
-/** Implementation of Job interface. Maintains the state machines of Job.
- * The read and write calls use ReadWriteLock for concurrency.
- */
-public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, 
-  EventHandler<JobEvent> {
-
-  private static final Log LOG = LogFactory.getLog(JobImpl.class);
-
-  //The maximum fraction of fetch failures allowed for a map
-  private static final double MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5;
-
-  // Maximum no. of fetch-failure notifications after which map task is failed
-  private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
-
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
-  
-  //final fields
-  private final Clock clock;
-  private final JobACLsManager aclsManager;
-  private final String username;
-  private final Map<JobACL, AccessControlList> jobACLs;
-  private final int startCount;
-  private final Set<TaskId> completedTasksFromPreviousRun;
-  private final Lock readLock;
-  private final Lock writeLock;
-  private final JobId jobId;
-  private final String jobName;
-  private final org.apache.hadoop.mapreduce.JobID oldJobId;
-  private final TaskAttemptListener taskAttemptListener;
-  private final Object tasksSyncHandle = new Object();
-  private final Set<TaskId> mapTasks = new LinkedHashSet<TaskId>();
-  private final Set<TaskId> reduceTasks = new LinkedHashSet<TaskId>();
-  private final EventHandler eventHandler;
-  private final MRAppMetrics metrics;
-
-  private boolean lazyTasksCopyNeeded = false;
-  private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
-  private Counters jobCounters = newCounters();
-    // FIXME:  
-    //
-    // Can then replace task-level uber counters (MR-2424) with job-level ones
-    // sent from LocalContainerLauncher, and eventually including a count of
-    // of uber-AM attempts (probably sent from MRAppMaster).
-  public Configuration conf;
-
-  //fields initialized in init
-  private FileSystem fs;
-  private Path remoteJobSubmitDir;
-  public Path remoteJobConfFile;
-  private JobContext jobContext;
-  private OutputCommitter committer;
-  private int allowedMapFailuresPercent = 0;
-  private int allowedReduceFailuresPercent = 0;
-  private List<TaskAttemptCompletionEvent> taskAttemptCompletionEvents;
-  private final List<String> diagnostics = new ArrayList<String>();
-  
-  //task/attempt related datastructures
-  private final Map<TaskId, Integer> successAttemptCompletionEventNoMap = 
-    new HashMap<TaskId, Integer>();
-  private final Map<TaskAttemptId, Integer> fetchFailuresMapping = 
-    new HashMap<TaskAttemptId, Integer>();
-
-  private static final DiagnosticsUpdateTransition
-      DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition();
-  private static final InternalErrorTransition
-      INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
-  private static final TaskAttemptCompletedEventTransition
-      TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION =
-          new TaskAttemptCompletedEventTransition();
-  private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION =
-      new CounterUpdateTransition();
-
-  protected static final
-    StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent> 
-       stateMachineFactory
-     = new StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent>
-              (JobState.NEW)
-
-          // Transitions from NEW state
-          .addTransition(JobState.NEW, JobState.NEW,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.NEW, JobState.NEW,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition
-              (JobState.NEW,
-              EnumSet.of(JobState.INITED, JobState.FAILED),
-              JobEventType.JOB_INIT,
-              new InitTransition())
-          .addTransition(JobState.NEW, JobState.KILLED,
-              JobEventType.JOB_KILL,
-              new KillNewJobTransition())
-          .addTransition(JobState.NEW, JobState.ERROR,
-              JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-
-          // Transitions from INITED state
-          .addTransition(JobState.INITED, JobState.INITED,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.INITED, JobState.INITED,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(JobState.INITED, JobState.RUNNING,
-              JobEventType.JOB_START,
-              new StartTransition())
-          .addTransition(JobState.INITED, JobState.KILLED,
-              JobEventType.JOB_KILL,
-              new KillInitedJobTransition())
-          .addTransition(JobState.INITED, JobState.ERROR,
-              JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-
-          // Transitions from RUNNING state
-          .addTransition(JobState.RUNNING, JobState.RUNNING,
-              JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
-              TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
-          .addTransition
-              (JobState.RUNNING,
-              EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
-              JobEventType.JOB_TASK_COMPLETED,
-              new TaskCompletedTransition())
-          .addTransition
-              (JobState.RUNNING,
-              EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
-              JobEventType.JOB_COMPLETED,
-              new JobNoTasksCompletedTransition())
-          .addTransition(JobState.RUNNING, JobState.KILL_WAIT,
-              JobEventType.JOB_KILL, new KillTasksTransition())
-          .addTransition(JobState.RUNNING, JobState.RUNNING,
-              JobEventType.JOB_MAP_TASK_RESCHEDULED,
-              new MapTaskRescheduledTransition())
-          .addTransition(JobState.RUNNING, JobState.RUNNING,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.RUNNING, JobState.RUNNING,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(JobState.RUNNING, JobState.RUNNING,
-              JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
-              new TaskAttemptFetchFailureTransition())
-          .addTransition(
-              JobState.RUNNING,
-              JobState.ERROR, JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-
-          // Transitions from KILL_WAIT state.
-          .addTransition
-              (JobState.KILL_WAIT,
-              EnumSet.of(JobState.KILL_WAIT, JobState.KILLED),
-              JobEventType.JOB_TASK_COMPLETED,
-              new KillWaitTaskCompletedTransition())
-          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
-              JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
-              TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
-          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(
-              JobState.KILL_WAIT,
-              JobState.ERROR, JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-          // Ignore-able events
-          .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
-              EnumSet.of(JobEventType.JOB_KILL,
-                         JobEventType.JOB_MAP_TASK_RESCHEDULED,
-                         JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
-
-          // Transitions from SUCCEEDED state
-          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(
-              JobState.SUCCEEDED,
-              JobState.ERROR, JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-          // Ignore-able events
-          .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
-              EnumSet.of(JobEventType.JOB_KILL,
-                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
-
-          // Transitions from FAILED state
-          .addTransition(JobState.FAILED, JobState.FAILED,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.FAILED, JobState.FAILED,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(
-              JobState.FAILED,
-              JobState.ERROR, JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-          // Ignore-able events
-          .addTransition(JobState.FAILED, JobState.FAILED,
-              EnumSet.of(JobEventType.JOB_KILL,
-                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
-
-          // Transitions from KILLED state
-          .addTransition(JobState.KILLED, JobState.KILLED,
-              JobEventType.JOB_DIAGNOSTIC_UPDATE,
-              DIAGNOSTIC_UPDATE_TRANSITION)
-          .addTransition(JobState.KILLED, JobState.KILLED,
-              JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
-          .addTransition(
-              JobState.KILLED,
-              JobState.ERROR, JobEventType.INTERNAL_ERROR,
-              INTERNAL_ERROR_TRANSITION)
-          // Ignore-able events
-          .addTransition(JobState.KILLED, JobState.KILLED,
-              EnumSet.of(JobEventType.JOB_KILL,
-                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
-
-          // No transitions from INTERNAL_ERROR state. Ignore all.
-          .addTransition(
-              JobState.ERROR,
-              JobState.ERROR,
-              EnumSet.of(JobEventType.JOB_INIT,
-                  JobEventType.JOB_KILL,
-                  JobEventType.JOB_TASK_COMPLETED,
-                  JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
-                  JobEventType.JOB_MAP_TASK_RESCHEDULED,
-                  JobEventType.JOB_DIAGNOSTIC_UPDATE,
-                  JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
-                  JobEventType.INTERNAL_ERROR))
-
-          // create the topology tables
-          .installTopology();
- 
-  private final StateMachine<JobState, JobEventType, JobEvent> stateMachine;
-
-  //changing fields while the job is running
-  private int numMapTasks;
-  private int numReduceTasks;
-  private int completedTaskCount = 0;
-  private int succeededMapTaskCount = 0;
-  private int succeededReduceTaskCount = 0;
-  private int failedMapTaskCount = 0;
-  private int failedReduceTaskCount = 0;
-  private int killedMapTaskCount = 0;
-  private int killedReduceTaskCount = 0;
-  private long submitTime;
-  private long startTime;
-  private long finishTime;
-  private float setupProgress;
-  private float cleanupProgress;
-  private boolean isUber = false;
-
-  private Credentials fsTokens;
-  private Token<JobTokenIdentifier> jobToken;
-  private JobTokenSecretManager jobTokenSecretManager;
-
-  public JobImpl(ApplicationId appID, Configuration conf,
-      EventHandler eventHandler, TaskAttemptListener taskAttemptListener,
-      JobTokenSecretManager jobTokenSecretManager,
-      Credentials fsTokenCredentials, Clock clock, int startCount, 
-      Set<TaskId> completedTasksFromPreviousRun, MRAppMetrics metrics) {
-
-    this.jobId = recordFactory.newRecordInstance(JobId.class);
-    this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
-    this.conf = conf;
-    this.metrics = metrics;
-    this.clock = clock;
-    this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
-    this.startCount = startCount;
-    jobId.setAppId(appID);
-    jobId.setId(appID.getId());
-    oldJobId = TypeConverter.fromYarn(jobId);
-    LOG.info("Job created" +
-    		" appId=" + appID + 
-    		" jobId=" + jobId + 
-    		" oldJobId=" + oldJobId);
-    
-    this.taskAttemptListener = taskAttemptListener;
-    this.eventHandler = eventHandler;
-    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
-    this.readLock = readWriteLock.readLock();
-    this.writeLock = readWriteLock.writeLock();
-
-    this.fsTokens = fsTokenCredentials;
-    this.jobTokenSecretManager = jobTokenSecretManager;
-
-    this.aclsManager = new JobACLsManager(conf);
-    this.username = System.getProperty("user.name");
-    this.jobACLs = aclsManager.constructJobACLs(conf);
-    // This "this leak" is okay because the retained pointer is in an
-    //  instance variable.
-    stateMachine = stateMachineFactory.make(this);
-  }
-
-  protected StateMachine<JobState, JobEventType, JobEvent> getStateMachine() {
-    return stateMachine;
-  }
-
-  @Override
-  public JobId getID() {
-    return jobId;
-  }
-
-  // Getter methods that make unit testing easier (package-scoped)
-  OutputCommitter getCommitter() {
-    return this.committer;
-  }
-
-  EventHandler getEventHandler() {
-    return this.eventHandler;
-  }
-
-  JobContext getJobContext() {
-    return this.jobContext;
-  }
-
-  @Override
-  public boolean checkAccess(UserGroupInformation callerUGI, 
-      JobACL jobOperation) {
-    if (!UserGroupInformation.isSecurityEnabled()) {
-      return true;
-    }
-    AccessControlList jobACL = jobACLs.get(jobOperation);
-    return aclsManager.checkAccess(callerUGI, jobOperation, username, jobACL);
-  }
-
-  @Override
-  public Task getTask(TaskId taskID) {
-    readLock.lock();
-    try {
-      return tasks.get(taskID);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public int getCompletedMaps() {
-    readLock.lock();
-    try {
-      return succeededMapTaskCount + failedMapTaskCount + killedMapTaskCount;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public int getCompletedReduces() {
-    readLock.lock();
-    try {
-      return succeededReduceTaskCount + failedReduceTaskCount 
-                  + killedReduceTaskCount;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public boolean isUber() {
-    return isUber;
-  }
-
-  @Override
-  public Counters getCounters() {
-    Counters counters = newCounters();
-    readLock.lock();
-    try {
-      incrAllCounters(counters, jobCounters);
-      return incrTaskCounters(counters, tasks.values());
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  private Counters getTypeCounters(Set<TaskId> taskIds) {
-    Counters counters = newCounters();
-    for (TaskId taskId : taskIds) {
-      Task task = tasks.get(taskId);
-      incrAllCounters(counters, task.getCounters());
-    }
-    return counters;
-  }
-
-  private Counters getMapCounters() {
-    readLock.lock();
-    try {
-      return getTypeCounters(mapTasks);
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  private Counters getReduceCounters() {
-    readLock.lock();
-    try {
-      return getTypeCounters(reduceTasks);
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  public static Counters newCounters() {
-    Counters counters = RecordFactoryProvider.getRecordFactory(null)
-        .newRecordInstance(Counters.class);
-    return counters;
-  }
-
-  public static Counters incrTaskCounters(Counters counters,
-                                          Collection<Task> tasks) {
-    for (Task task : tasks) {
-      incrAllCounters(counters, task.getCounters());
-    }
-    return counters;
-  }
-
-  public static void incrAllCounters(Counters counters, Counters other) {
-    if (other != null) {
-      for (CounterGroup otherGroup: other.getAllCounterGroups().values()) {
-        CounterGroup group = counters.getCounterGroup(otherGroup.getName());
-        if (group == null) {
-          group = RecordFactoryProvider.getRecordFactory(null)
-              .newRecordInstance(CounterGroup.class);
-          group.setName(otherGroup.getName());
-          counters.setCounterGroup(group.getName(), group);
-        }
-        group.setDisplayName(otherGroup.getDisplayName());
-        for (Counter otherCounter : otherGroup.getAllCounters().values()) {
-          Counter counter = group.getCounter(otherCounter.getName());
-          if (counter == null) {
-            counter = RecordFactoryProvider.getRecordFactory(null)
-                .newRecordInstance(Counter.class);
-            counter.setName(otherCounter.getName());
-            group.setCounter(counter.getName(), counter);
-          }
-          counter.setDisplayName(otherCounter.getDisplayName());
-          counter.setValue(counter.getValue() + otherCounter.getValue());
-        }
-      }
-    }
-  }
-
-  @Override
-  public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
-      int fromEventId, int maxEvents) {
-    TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
-    readLock.lock();
-    try {
-      if (taskAttemptCompletionEvents.size() > fromEventId) {
-        int actualMax = Math.min(maxEvents,
-            (taskAttemptCompletionEvents.size() - fromEventId));
-        events = taskAttemptCompletionEvents.subList(fromEventId,
-            actualMax + fromEventId).toArray(events);
-      }
-      return events;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public List<String> getDiagnostics() {
-    readLock.lock();
-    try {
-      return diagnostics;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public JobReport getReport() {
-    readLock.lock();
-    try {
-      JobReport report = recordFactory.newRecordInstance(JobReport.class);
-      report.setJobId(jobId);
-      report.setJobState(getState());
-      
-      // TODO - Fix to correctly setup report and to check state
-      if (report.getJobState() == JobState.NEW) {
-        return report;
-      }
-      
-      report.setStartTime(startTime);
-      report.setFinishTime(finishTime);
-      report.setSetupProgress(setupProgress);
-      report.setCleanupProgress(cleanupProgress);
-      report.setMapProgress(computeProgress(mapTasks));
-      report.setReduceProgress(computeProgress(reduceTasks));
-
-      return report;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  private float computeProgress(Set<TaskId> taskIds) {
-    readLock.lock();
-    try {
-      float progress = 0;
-      for (TaskId taskId : taskIds) {
-        Task task = tasks.get(taskId);
-        progress += task.getProgress();
-      }
-      int taskIdsSize = taskIds.size();
-      if (taskIdsSize != 0) {
-        progress = progress/taskIdsSize;
-      }
-      return progress;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public Map<TaskId, Task> getTasks() {
-    synchronized (tasksSyncHandle) {
-      lazyTasksCopyNeeded = true;
-      return Collections.unmodifiableMap(tasks);
-    }
-  }
-
-  @Override
-  public Map<TaskId,Task> getTasks(TaskType taskType) {
-    Map<TaskId, Task> localTasksCopy = tasks;
-    Map<TaskId, Task> result = new HashMap<TaskId, Task>();
-    Set<TaskId> tasksOfGivenType = null;
-    readLock.lock();
-    try {
-      if (TaskType.MAP == taskType) {
-        tasksOfGivenType = mapTasks;
-      } else {
-        tasksOfGivenType = reduceTasks;
-      }
-      for (TaskId taskID : tasksOfGivenType)
-      result.put(taskID, localTasksCopy.get(taskID));
-      return result;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public JobState getState() {
-    readLock.lock();
-    try {
-     return getStateMachine().getCurrentState();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  protected void scheduleTasks(Set<TaskId> taskIDs) {
-    for (TaskId taskID : taskIDs) {
-      eventHandler.handle(new TaskEvent(taskID, 
-          TaskEventType.T_SCHEDULE));
-    }
-  }
-
-  @Override
-  /**
-   * The only entry point to change the Job.
-   */
-  public void handle(JobEvent event) {
-    LOG.info("Processing " + event.getJobId() + " of type " + event.getType());
-    try {
-      writeLock.lock();
-      JobState oldState = getState();
-      try {
-         getStateMachine().doTransition(event.getType(), event);
-      } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
-        addDiagnostic("Invalid event " + event.getType() + 
-            " on Job " + this.jobId);
-        eventHandler.handle(new JobEvent(this.jobId,
-            JobEventType.INTERNAL_ERROR));
-      }
-      //notify the eventhandler of state change
-      if (oldState != getState()) {
-        LOG.info(jobId + "Job Transitioned from " + oldState + " to "
-                 + getState());
-      }
-    }
-    
-    finally {
-      writeLock.unlock();
-    }
-  }
-
-  //helpful in testing
-  protected void addTask(Task task) {
-    synchronized (tasksSyncHandle) {
-      if (lazyTasksCopyNeeded) {
-        Map<TaskId, Task> newTasks = new LinkedHashMap<TaskId, Task>();
-        newTasks.putAll(tasks);
-        tasks = newTasks;
-        lazyTasksCopyNeeded = false;
-      }
-    }
-    tasks.put(task.getID(), task);
-    if (task.getType() == TaskType.MAP) {
-      mapTasks.add(task.getID());
-    } else if (task.getType() == TaskType.REDUCE) {
-      reduceTasks.add(task.getID());
-    }
-    metrics.waitingTask(task);
-  }
-
-  void setFinishTime() {
-    finishTime = clock.getTime();
-  }
-
-  void logJobHistoryFinishedEvent() {
-    this.setFinishTime();
-    JobFinishedEvent jfe = createJobFinishedEvent(this);
-    LOG.info("Calling handler for JobFinishedEvent ");
-    this.getEventHandler().handle(new JobHistoryEvent(this.jobId, jfe));    
-  }
-  
-  static JobState checkJobCompleteSuccess(JobImpl job) {
-    // check for Job success
-    if (job.completedTaskCount == job.getTasks().size()) {
-      try {
-        // Commit job & do cleanup
-        job.getCommitter().commitJob(job.getJobContext());
-      } catch (IOException e) {
-        LOG.warn("Could not do commit for Job", e);
-      }
-      
-      job.logJobHistoryFinishedEvent();
-      return job.finished(JobState.SUCCEEDED);
-    }
-    return null;
-  }
-
-  JobState finished(JobState finalState) {
-    if (getState() == JobState.RUNNING) {
-      metrics.endRunningJob(this);
-    }
-    if (finishTime == 0) setFinishTime();
-    eventHandler.handle(new JobFinishEvent(jobId));
-
-    switch (finalState) {
-      case KILLED:
-        metrics.killedJob(this);
-        break;
-      case FAILED:
-        metrics.failedJob(this);
-        break;
-      case SUCCEEDED:
-        metrics.completedJob(this);
-    }
-    return finalState;
-  }
-
-  @Override
-  public String getName() {
-    return jobName;
-  }
-
-  @Override
-  public int getTotalMaps() {
-    return mapTasks.size();  //FIXME: why indirection? return numMapTasks...
-                             // unless race?  how soon can this get called?
-  }
-
-  @Override
-  public int getTotalReduces() {
-    return reduceTasks.size();  //FIXME: why indirection? return numReduceTasks
-  }
-
-  public static class InitTransition 
-      implements MultipleArcTransition<JobImpl, JobEvent, JobState> {
-
-    /**
-     * Note that this transition method is called directly (and synchronously)
-     * by MRAppMaster's init() method (i.e., no RPC, no thread-switching;
-     * just plain sequential call within AM context), so we can trigger
-     * modifications in AM state from here (at least, if AM is written that
-     * way; MR version is).
-     */
-    @Override
-    public JobState transition(JobImpl job, JobEvent event) {
-      job.submitTime = job.clock.getTime();
-      job.metrics.submittedJob(job);
-      job.metrics.preparingJob(job);
-      try {
-        setup(job);
-        job.fs = FileSystem.get(job.conf);
-
-        //log to job history
-        JobSubmittedEvent jse = new JobSubmittedEvent(job.oldJobId,
-              job.conf.get(MRJobConfig.JOB_NAME, "test"), 
-            job.conf.get(MRJobConfig.USER_NAME, "mapred"),
-            job.submitTime,
-            job.remoteJobConfFile.toString(),
-            job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "default"));
-        job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
-        //TODO JH Verify jobACLs, UserName via UGI?
-
-        TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(job, job.jobId);
-        job.numMapTasks = taskSplitMetaInfo.length;
-        job.numReduceTasks = job.conf.getInt(MRJobConfig.NUM_REDUCES, 0);
-
-        if (job.numMapTasks == 0 && job.numReduceTasks == 0) {
-          job.addDiagnostic("No of maps and reduces are 0 " + job.jobId);
-        }
-
-        checkTaskLimits();
-
-        
-        boolean newApiCommitter = false;
-        if ((job.numReduceTasks > 0 && 
-            job.conf.getBoolean("mapred.reducer.new-api", false)) ||
-              (job.numReduceTasks == 0 && 
-               job.conf.getBoolean("mapred.mapper.new-api", false)))  {
-          newApiCommitter = true;
-          LOG.info("Using mapred newApiCommitter.");
-        }
-        
-        LOG.info("OutputCommitter set in config " + job.conf.get("mapred.output.committer.class"));
-        
-        if (newApiCommitter) {
-          job.jobContext = new JobContextImpl(job.conf,
-              job.oldJobId);
-          org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = RecordFactoryProvider
-              .getRecordFactory(null)
-              .newRecordInstance(
-                  org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId.class);
-          attemptID.setTaskId(RecordFactoryProvider.getRecordFactory(null)
-              .newRecordInstance(TaskId.class));
-          attemptID.getTaskId().setJobId(job.jobId);
-          attemptID.getTaskId().setTaskType(TaskType.MAP);
-          TaskAttemptContext taskContext = new TaskAttemptContextImpl(job.conf,
-              TypeConverter.fromYarn(attemptID));
-          try {
-            OutputFormat outputFormat = ReflectionUtils.newInstance(
-                taskContext.getOutputFormatClass(), job.conf);
-            job.committer = outputFormat.getOutputCommitter(taskContext);
-          } catch(Exception e) {
-            throw new IOException("Failed to assign outputcommitter", e);
-          }
-        } else {
-          job.jobContext = new org.apache.hadoop.mapred.JobContextImpl(
-              new JobConf(job.conf), job.oldJobId);
-          job.committer = ReflectionUtils.newInstance(
-              job.conf.getClass("mapred.output.committer.class", FileOutputCommitter.class,
-              org.apache.hadoop.mapred.OutputCommitter.class), job.conf);
-        }
-        LOG.info("OutputCommitter is " + job.committer.getClass().getName());
-        
-        long inputLength = 0;
-        for (int i = 0; i < job.numMapTasks; ++i) {
-          inputLength += taskSplitMetaInfo[i].getInputDataLength();
-        }
-
-//FIXME:  need new memory criterion for uber-decision (oops, too late here; until AM-resizing supported, must depend on job client to pass fat-slot needs)
-        // these are no longer "system" settings, necessarily; user may override
-        int sysMaxMaps = job.conf.getInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 9);
-        int sysMaxReduces =
-            job.conf.getInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 1);
-        long sysMaxBytes = job.conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
-            job.conf.getLong("dfs.block.size", 64*1024*1024));  //FIXME: this is wrong; get FS from [File?]InputFormat and default block size from that
-        //long sysMemSizeForUberSlot = JobTracker.getMemSizeForReduceSlot(); // FIXME [could use default AM-container memory size...]
-
-        boolean uberEnabled =
-            job.conf.getBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
-        boolean smallNumMapTasks = (job.numMapTasks <= sysMaxMaps);
-        boolean smallNumReduceTasks = (job.numReduceTasks <= sysMaxReduces);
-        boolean smallInput = (inputLength <= sysMaxBytes);
-        boolean smallMemory = true;  //FIXME (see above)
-            // ignoring overhead due to UberTask and statics as negligible here:
-//  FIXME   && (Math.max(memoryPerMap, memoryPerReduce) <= sysMemSizeForUberSlot
-//              || sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT)
-        boolean notChainJob = !isChainJob(job.conf);
-
-        // User has overall veto power over uberization, or user can modify
-        // limits (overriding system settings and potentially shooting
-        // themselves in the head).  Note that ChainMapper/Reducer are
-        // fundamentally incompatible with MR-1220; they employ a blocking
-
-        // User has overall veto power over uberization, or user can modify
-        // limits (overriding system settings and potentially shooting
-        // themselves in the head).  Note that ChainMapper/Reducer are
-        // fundamentally incompatible with MR-1220; they employ a blocking
-        // queue between the maps/reduces and thus require parallel execution,
-        // while "uber-AM" (MR AM + LocalContainerLauncher) loops over tasks
-        // and thus requires sequential execution.
-        job.isUber = uberEnabled && smallNumMapTasks && smallNumReduceTasks
-            && smallInput && smallMemory && notChainJob;
-
-        if (job.isUber) {
-          LOG.info("Uberizing job " + job.jobId + ": " + job.numMapTasks + "m+"
-              + job.numReduceTasks + "r tasks (" + inputLength
-              + " input bytes) will run sequentially on single node.");
-              //TODO: also note which node?
-
-          // make sure reduces are scheduled only after all map are completed
-          job.conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART,
-                            1.0f);
-          // uber-subtask attempts all get launched on same node; if one fails,
-          // probably should retry elsewhere, i.e., move entire uber-AM:  ergo,
-          // limit attempts to 1 (or at most 2?  probably not...)
-          job.conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
-          job.conf.setInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 1);
-
-          // disable speculation:  makes no sense to speculate an entire job
-//        canSpeculateMaps = canSpeculateReduces = false; // [TODO: in old version, ultimately was from conf.getMapSpeculativeExecution(), conf.getReduceSpeculativeExecution()]
-        } else {
-          StringBuilder msg = new StringBuilder();
-          msg.append("Not uberizing ").append(job.jobId).append(" because:");
-          if (!uberEnabled)
-            msg.append(" not enabled;");
-          if (!smallNumMapTasks)
-            msg.append(" too many maps;");
-          if (!smallNumReduceTasks)
-            msg.append(" too many reduces;");
-          if (!smallInput)
-            msg.append(" too much input;");
-          if (!smallMemory)
-            msg.append(" too much RAM;");
-          if (!notChainJob)
-            msg.append(" chainjob");
-          LOG.info(msg.toString());
-        }
-
-        job.taskAttemptCompletionEvents =
-            new ArrayList<TaskAttemptCompletionEvent>(
-                job.numMapTasks + job.numReduceTasks + 10);
-
-        job.allowedMapFailuresPercent =
-            job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0);
-        job.allowedReduceFailuresPercent =
-            job.conf.getInt(MRJobConfig.REDUCE_FAILURES_MAXPERCENT, 0);
-
-        // do the setup
-        job.committer.setupJob(job.jobContext);
-        job.setupProgress = 1.0f;
-
-        // create the Tasks but don't start them yet
-        createMapTasks(job, inputLength, taskSplitMetaInfo);
-        createReduceTasks(job);
-
-        job.metrics.endPreparingJob(job);
-        return JobState.INITED;
-        //TODO XXX Should JobInitedEvent be generated here (instead of in StartTransition)
-
-      } catch (IOException e) {
-        LOG.warn("Job init failed", e);
-        job.addDiagnostic("Job init failed : "
-            + StringUtils.stringifyException(e));
-        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
-        job.metrics.endPreparingJob(job);
-        return job.finished(JobState.FAILED);
-      }
-    }
-
-    protected void setup(JobImpl job) throws IOException {
-
-      String oldJobIDString = job.oldJobId.toString();
-      String user = 
-        UserGroupInformation.getCurrentUser().getShortUserName();
-      Path path = MRApps.getStagingAreaDir(job.conf, user);
-      LOG.info("DEBUG --- startJobs:"
-          + " parent="
-          + path + " child="
-          + oldJobIDString);
-
-      job.remoteJobSubmitDir =
-          FileSystem.get(job.conf).makeQualified(
-              new Path(path, oldJobIDString));
-      job.remoteJobConfFile =
-          new Path(job.remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
-
-      // Prepare the TaskAttemptListener server for authentication of Containers
-      // TaskAttemptListener gets the information via jobTokenSecretManager.
-      JobTokenIdentifier identifier =
-          new JobTokenIdentifier(new Text(oldJobIDString));
-      job.jobToken =
-          new Token<JobTokenIdentifier>(identifier, job.jobTokenSecretManager);
-      job.jobToken.setService(identifier.getJobId());
-      // Add it to the jobTokenSecretManager so that TaskAttemptListener server
-      // can authenticate containers(tasks)
-      job.jobTokenSecretManager.addTokenForJob(oldJobIDString, job.jobToken);
-      LOG.info("Adding job token for " + oldJobIDString
-          + " to jobTokenSecretManager");
-
-      // Upload the jobTokens onto the remote FS so that ContainerManager can
-      // localize it to be used by the Containers(tasks)
-      Credentials tokenStorage = new Credentials();
-      TokenCache.setJobToken(job.jobToken, tokenStorage);
-
-      if (UserGroupInformation.isSecurityEnabled()) {
-        tokenStorage.addAll(job.fsTokens);
-      }
-
-      Path remoteJobTokenFile =
-          new Path(job.remoteJobSubmitDir,
-              MRConstants.APPLICATION_TOKENS_FILE);
-      tokenStorage.writeTokenStorageFile(remoteJobTokenFile, job.conf);
-      LOG.info("Writing back the job-token file on the remote file system:"
-          + remoteJobTokenFile.toString());
-    }
-
-    /**
-     * ChainMapper and ChainReducer must execute in parallel, so they're not
-     * compatible with uberization/LocalContainerLauncher (100% sequential).
-     */
-    boolean isChainJob(Configuration conf) {
-      boolean isChainJob = false;
-      try {
-        String mapClassName = conf.get(MRJobConfig.MAP_CLASS_ATTR);
-        if (mapClassName != null) {
-          Class<?> mapClass = Class.forName(mapClassName);
-          if (ChainMapper.class.isAssignableFrom(mapClass))
-            isChainJob = true;
-        }
-      } catch (ClassNotFoundException cnfe) {
-        // don't care; assume it's not derived from ChainMapper
-      }
-      try {
-        String reduceClassName = conf.get(MRJobConfig.REDUCE_CLASS_ATTR);
-        if (reduceClassName != null) {
-          Class<?> reduceClass = Class.forName(reduceClassName);
-          if (ChainReducer.class.isAssignableFrom(reduceClass))
-            isChainJob = true;
-        }
-      } catch (ClassNotFoundException cnfe) {
-        // don't care; assume it's not derived from ChainReducer
-      }
-      return isChainJob;
-    }
-
-    private void createMapTasks(JobImpl job, long inputLength,
-                                TaskSplitMetaInfo[] splits) {
-      for (int i=0; i < job.numMapTasks; ++i) {
-        TaskImpl task =
-            new MapTaskImpl(job.jobId, i,
-                job.eventHandler, 
-                job.remoteJobConfFile, 
-                job.conf, splits[i], 
-                job.taskAttemptListener, 
-                job.committer, job.jobToken, job.fsTokens.getAllTokens(), 
-                job.clock, job.completedTasksFromPreviousRun, job.startCount,
-                job.metrics);
-        job.addTask(task);
-      }
-      LOG.info("Input size for job " + job.jobId + " = " + inputLength
-          + ". Number of splits = " + splits.length);
-    }
-
-    private void createReduceTasks(JobImpl job) {
-      for (int i = 0; i < job.numReduceTasks; i++) {
-        TaskImpl task =
-            new ReduceTaskImpl(job.jobId, i,
-                job.eventHandler, 
-                job.remoteJobConfFile, 
-                job.conf, job.numMapTasks, 
-                job.taskAttemptListener, job.committer, job.jobToken,
-                job.fsTokens.getAllTokens(), job.clock, 
-                job.completedTasksFromPreviousRun, job.startCount, job.metrics);
-        job.addTask(task);
-      }
-      LOG.info("Number of reduces for job " + job.jobId + " = "
-          + job.numReduceTasks);
-    }
-
-    protected TaskSplitMetaInfo[] createSplits(JobImpl job, JobId jobId) {
-      TaskSplitMetaInfo[] allTaskSplitMetaInfo;
-      try {
-        allTaskSplitMetaInfo = SplitMetaInfoReader.readSplitMetaInfo(
-            job.oldJobId, job.fs, 
-            job.conf, 
-            job.remoteJobSubmitDir);
-      } catch (IOException e) {
-        throw new YarnException(e);
-      }
-      return allTaskSplitMetaInfo;
-    }
-
-    /**
-     * If the number of tasks are greater than the configured value
-     * throw an exception that will fail job initialization
-     */
-    private void checkTaskLimits() {
-      // no code, for now
-    }
-  } // end of InitTransition
-
-  public static class StartTransition
-  implements SingleArcTransition<JobImpl, JobEvent> {
-    /**
-     * This transition executes in the event-dispatcher thread, though it's
-     * triggered in MRAppMaster's startJobs() method.
-     */
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      job.startTime = job.clock.getTime();
-      job.scheduleTasks(job.mapTasks);  // schedule (i.e., start) the maps
-      job.scheduleTasks(job.reduceTasks);
-      JobInitedEvent jie =
-        new JobInitedEvent(job.oldJobId,
-             job.startTime,
-             job.numMapTasks, job.numReduceTasks,
-             job.getState().toString()); //Will transition to state running. Currently in INITED
-      job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
-      JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,
-          job.submitTime, job.startTime);
-      job.eventHandler.handle(new JobHistoryEvent(job.jobId, jice));
-      job.metrics.runningJob(job);
-
-			// If we have no tasks, just transition to job completed
-      if (job.numReduceTasks == 0 && job.numMapTasks == 0) {
-        job.eventHandler.handle(new JobEvent(job.jobId, JobEventType.JOB_COMPLETED));
-      }
-    }
-  }
-
-  private void abortJob(
-      org.apache.hadoop.mapreduce.JobStatus.State finalState) {
-    try {
-      committer.abortJob(jobContext, finalState);
-    } catch (IOException e) {
-      LOG.warn("Could not abortJob", e);
-    }
-    if (finishTime == 0) setFinishTime();
-    cleanupProgress = 1.0f;
-    JobUnsuccessfulCompletionEvent unsuccessfulJobEvent =
-      new JobUnsuccessfulCompletionEvent(oldJobId,
-          finishTime,
-          succeededMapTaskCount,
-          succeededReduceTaskCount,
-          finalState.toString());
-    eventHandler.handle(new JobHistoryEvent(jobId, unsuccessfulJobEvent));
-  }
-    
-  // JobFinishedEvent triggers the move of the history file out of the staging
-  // area. May need to create a new event type for this if JobFinished should 
-  // not be generated for KilledJobs, etc.
-  private static JobFinishedEvent createJobFinishedEvent(JobImpl job) {
-    JobFinishedEvent jfe = new JobFinishedEvent(
-        job.oldJobId, job.finishTime,
-        job.succeededMapTaskCount, job.succeededReduceTaskCount,
-        job.failedMapTaskCount, job.failedReduceTaskCount,
-        TypeConverter.fromYarn(job.getMapCounters()),
-        TypeConverter.fromYarn(job.getReduceCounters()),
-        TypeConverter.fromYarn(job.getCounters()));
-    return jfe;
-  }
-
-  // Task-start has been moved out of InitTransition, so this arc simply
-  // hardcodes 0 for both map and reduce finished tasks.
-  private static class KillNewJobTransition
-  implements SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      job.setFinishTime();
-      JobUnsuccessfulCompletionEvent failedEvent =
-          new JobUnsuccessfulCompletionEvent(job.oldJobId,
-              job.finishTime, 0, 0,
-              JobState.KILLED.toString());
-      job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
-      job.finished(JobState.KILLED);
-    }
-  }
-
-  private static class KillInitedJobTransition
-  implements SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
-      job.addDiagnostic("Job received Kill in INITED state.");
-      job.finished(JobState.KILLED);
-    }
-  }
-
-  private static class KillTasksTransition
-      implements SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      job.addDiagnostic("Job received Kill while in RUNNING state.");
-      for (Task task : job.tasks.values()) {
-        job.eventHandler.handle(
-            new TaskEvent(task.getID(), TaskEventType.T_KILL));
-      }
-      job.metrics.endRunningJob(job);
-    }
-  }
-
-  private static class TaskAttemptCompletedEventTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      TaskAttemptCompletionEvent tce = 
-        ((JobTaskAttemptCompletedEvent) event).getCompletionEvent();
-      // Add the TaskAttemptCompletionEvent
-      //eventId is equal to index in the arraylist
-      tce.setEventId(job.taskAttemptCompletionEvents.size());
-      job.taskAttemptCompletionEvents.add(tce);
-      
-      //make the previous completion event as obsolete if it exists
-      Object successEventNo = 
-        job.successAttemptCompletionEventNoMap.remove(tce.getAttemptId().getTaskId());
-      if (successEventNo != null) {
-        TaskAttemptCompletionEvent successEvent = 
-          job.taskAttemptCompletionEvents.get((Integer) successEventNo);
-        successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
-      }
-
-      if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) {
-        job.successAttemptCompletionEventNoMap.put(tce.getAttemptId().getTaskId(), 
-            tce.getEventId());
-      }
-    }
-  }
-
-  private static class TaskAttemptFetchFailureTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      JobTaskAttemptFetchFailureEvent fetchfailureEvent = 
-        (JobTaskAttemptFetchFailureEvent) event;
-      for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId mapId : 
-            fetchfailureEvent.getMaps()) {
-        Integer fetchFailures = job.fetchFailuresMapping.get(mapId);
-        fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1);
-        job.fetchFailuresMapping.put(mapId, fetchFailures);
-        
-        //get number of running reduces
-        int runningReduceTasks = 0;
-        for (TaskId taskId : job.reduceTasks) {
-          if (TaskState.RUNNING.equals(job.tasks.get(taskId).getState())) {
-            runningReduceTasks++;
-          }
-        }
-        
-        float failureRate = (float) fetchFailures / runningReduceTasks;
-        // declare faulty if fetch-failures >= max-allowed-failures
-        boolean isMapFaulty =
-            (failureRate >= MAX_ALLOWED_FETCH_FAILURES_FRACTION);
-        if (fetchFailures >= MAX_FETCH_FAILURES_NOTIFICATIONS && isMapFaulty) {
-          LOG.info("Too many fetch-failures for output of task attempt: " + 
-              mapId + " ... raising fetch failure to map");
-          job.eventHandler.handle(new TaskAttemptEvent(mapId, 
-              TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-          job.fetchFailuresMapping.remove(mapId);
-        }
-      }
-    }
-  }
-
-  private static class TaskCompletedTransition implements
-      MultipleArcTransition<JobImpl, JobEvent, JobState> {
-
-    @Override
-    public JobState transition(JobImpl job, JobEvent event) {
-      job.completedTaskCount++;
-      LOG.info("Num completed Tasks: " + job.completedTaskCount);
-      JobTaskEvent taskEvent = (JobTaskEvent) event;
-      Task task = job.tasks.get(taskEvent.getTaskID());
-      if (taskEvent.getState() == TaskState.SUCCEEDED) {
-        taskSucceeded(job, task);
-      } else if (taskEvent.getState() == TaskState.FAILED) {
-        taskFailed(job, task);
-      } else if (taskEvent.getState() == TaskState.KILLED) {
-        taskKilled(job, task);
-      }
-
-      return checkJobForCompletion(job);
-    }
-
-    protected JobState checkJobForCompletion(JobImpl job) {
-      //check for Job failure
-      if (job.failedMapTaskCount*100 > 
-        job.allowedMapFailuresPercent*job.numMapTasks ||
-        job.failedReduceTaskCount*100 > 
-        job.allowedReduceFailuresPercent*job.numReduceTasks) {
-        job.setFinishTime();
-
-        String diagnosticMsg = "Job failed as tasks failed. " +
-            "failedMaps:" + job.failedMapTaskCount + 
-            " failedReduces:" + job.failedReduceTaskCount;
-        LOG.info(diagnosticMsg);
-        job.addDiagnostic(diagnosticMsg);
-        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
-        return job.finished(JobState.FAILED);
-      }
-      
-      JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
-      if (jobCompleteSuccess != null) {
-        return jobCompleteSuccess;
-      }
-      
-      //return the current state, Job not finished yet
-      return job.getState();
-    }
-
-    private void taskSucceeded(JobImpl job, Task task) {
-      if (task.getType() == TaskType.MAP) {
-        job.succeededMapTaskCount++;
-      } else {
-        job.succeededReduceTaskCount++;
-      }
-      job.metrics.completedTask(task);
-    }
-  
-    private void taskFailed(JobImpl job, Task task) {
-      if (task.getType() == TaskType.MAP) {
-        job.failedMapTaskCount++;
-      } else if (task.getType() == TaskType.REDUCE) {
-        job.failedReduceTaskCount++;
-      }
-      job.addDiagnostic("Task failed " + task.getID());
-      job.metrics.failedTask(task);
-    }
-
-    private void taskKilled(JobImpl job, Task task) {
-      if (task.getType() == TaskType.MAP) {
-        job.killedMapTaskCount++;
-      } else if (task.getType() == TaskType.REDUCE) {
-        job.killedReduceTaskCount++;
-      }
-      job.metrics.killedTask(task);
-    }
-  }
-
-  // Transition class for handling jobs with no tasks
-  static class JobNoTasksCompletedTransition implements
-  MultipleArcTransition<JobImpl, JobEvent, JobState> {
-
-    @Override
-    public JobState transition(JobImpl job, JobEvent event) {
-      JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
-      if (jobCompleteSuccess != null) {
-        return jobCompleteSuccess;
-      }
-      
-      // Return the current state, Job not finished yet
-      return job.getState();
-    }
-  }
-
-  private static class MapTaskRescheduledTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      //succeeded map task is restarted back
-      job.completedTaskCount--;
-      job.succeededMapTaskCount--;
-    }
-  }
-
-  private static class KillWaitTaskCompletedTransition extends  
-      TaskCompletedTransition {
-    @Override
-    protected JobState checkJobForCompletion(JobImpl job) {
-      if (job.completedTaskCount == job.tasks.size()) {
-        job.setFinishTime();
-        job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
-        return job.finished(JobState.KILLED);
-      }
-      //return the current state, Job not finished yet
-      return job.getState();
-    }
-  }
-
-  private void addDiagnostic(String diag) {
-    diagnostics.add(diag);
-  }
-  
-  private static class DiagnosticsUpdateTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      job.addDiagnostic(((JobDiagnosticsUpdateEvent) event)
-          .getDiagnosticUpdate());
-    }
-  }
-  
-  private static class CounterUpdateTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      JobCounterUpdateEvent jce = (JobCounterUpdateEvent) event;
-      for (JobCounterUpdateEvent.CounterIncrementalUpdate ci : jce
-          .getCounterUpdates()) {
-        job.jobCounters.incrCounter(ci.getCounterKey(), ci.getIncrementValue());
-      }
-    }
-  }
-
-  private static class InternalErrorTransition implements
-      SingleArcTransition<JobImpl, JobEvent> {
-    @Override
-    public void transition(JobImpl job, JobEvent event) {
-      //TODO Is this JH event required.
-      job.setFinishTime();
-      JobUnsuccessfulCompletionEvent failedEvent =
-          new JobUnsuccessfulCompletionEvent(job.oldJobId,
-              job.finishTime, 0, 0,
-              JobState.ERROR.toString());
-      job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
-      job.finished(JobState.ERROR);
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/MapTaskImpl.java

@@ -1,97 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.impl;
-
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.MapTaskAttemptImpl;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-
-public class MapTaskImpl extends TaskImpl {
-
-  private final TaskSplitMetaInfo taskSplitMetaInfo;
-
-  public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
-      Path remoteJobConfFile, Configuration conf,
-      TaskSplitMetaInfo taskSplitMetaInfo,
-      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
-      Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock, 
-      Set<TaskId> completedTasksFromPreviousRun, int startCount,
-      MRAppMetrics metrics) {
-    super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
-        conf, taskAttemptListener, committer, jobToken, fsTokens, clock, 
-        completedTasksFromPreviousRun, startCount, metrics);
-    this.taskSplitMetaInfo = taskSplitMetaInfo;
-  }
-
-  @Override
-  protected int getMaxAttempts() {
-    return conf.getInt(MRJobConfig.MAP_MAX_ATTEMPTS, 4);
-  }
-
-  @Override
-  protected TaskAttemptImpl createAttempt() {
-    return new MapTaskAttemptImpl(getID(), nextAttemptNumber,
-        eventHandler, jobFile,
-        partition, taskSplitMetaInfo, conf, taskAttemptListener,
-        committer, jobToken, fsTokens, clock);
-  }
-
-  @Override
-  public TaskType getType() {
-    return TaskType.MAP;
-  }
-
-  protected TaskSplitMetaInfo getTaskSplitMetaInfo() {
-    return this.taskSplitMetaInfo;
-  }
-
-  /**
-   * @return a String formatted as a comma-separated list of splits.
-   */
-  @Override
-  protected String getSplitsAsString() {
-    String[] splits = getTaskSplitMetaInfo().getLocations();
-    if (splits == null || splits.length == 0)
-    return "";
-    StringBuilder sb = new StringBuilder();
-    for (int i = 0; i < splits.length; i++) {
-      if (i != 0) sb.append(",");
-      sb.append(splits[i]);
-    }
-    return sb.toString();
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ReduceTaskImpl.java

@@ -1,75 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.impl;
-
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.ReduceTaskAttemptImpl;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public class ReduceTaskImpl extends TaskImpl {
-  
-  private final int numMapTasks;
-
-  public ReduceTaskImpl(JobId jobId, int partition,
-      EventHandler eventHandler, Path jobFile, Configuration conf,
-      int numMapTasks, TaskAttemptListener taskAttemptListener,
-      OutputCommitter committer, Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock, 
-      Set<TaskId> completedTasksFromPreviousRun, int startCount,
-      MRAppMetrics metrics) {
-    super(jobId, TaskType.REDUCE, partition, eventHandler, jobFile, conf,
-        taskAttemptListener, committer, jobToken, fsTokens, clock,
-        completedTasksFromPreviousRun, startCount, metrics);
-    this.numMapTasks = numMapTasks;
-  }
-
-  @Override
-  protected int getMaxAttempts() {
-    return conf.getInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 4);
-  }
-
-  @Override
-  protected TaskAttemptImpl createAttempt() {
-    return new ReduceTaskAttemptImpl(getID(), nextAttemptNumber,
-        eventHandler, jobFile,
-        partition, numMapTasks, conf, taskAttemptListener,
-        committer, jobToken, fsTokens, clock);
-  }
-
-  @Override
-  public TaskType getType() {
-    return TaskType.REDUCE;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -1,1442 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceChildJVM;
-import org.apache.hadoop.mapred.ProgressSplitsBlock;
-import org.apache.hadoop.mapred.ShuffleHandler;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.TaskAttemptContextImpl;
-import org.apache.hadoop.mapred.WrappedJvmID;
-import org.apache.hadoop.mapred.WrappedProgressSplitsBlock;
-import org.apache.hadoop.mapreduce.JobCounter;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskCounter;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-import org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinishedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinishedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStartedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.Phase;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
-import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
-import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.RackResolver;
-
-/**
- * Implementation of TaskAttempt interface.
- */
-@SuppressWarnings("all")
-public abstract class TaskAttemptImpl implements
-    org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
-      EventHandler<TaskAttemptEvent> {
-
-  private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
-  private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
-  private static final int MAP_MEMORY_MB_DEFAULT = 1024;
-  private static final int REDUCE_MEMORY_MB_DEFAULT = 1024;
-  private final static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
-  protected final Configuration conf;
-  protected final Path jobFile;
-  protected final int partition;
-  protected final EventHandler eventHandler;
-  private final TaskAttemptId attemptId;
-  private final Clock clock;
-  private final org.apache.hadoop.mapred.JobID oldJobId;
-  private final TaskAttemptListener taskAttemptListener;
-  private final OutputCommitter committer;
-  private final Resource resourceCapability;
-  private final String[] dataLocalHosts;
-  private final List<String> diagnostics = new ArrayList<String>();
-  private final Lock readLock;
-  private final Lock writeLock;
-  private Collection<Token<? extends TokenIdentifier>> fsTokens;
-  private Token<JobTokenIdentifier> jobToken;
-  private static AtomicBoolean initialClasspathFlag = new AtomicBoolean();
-  private static String initialClasspath = null;
-  private final Object classpathLock = new Object();
-  private long launchTime;
-  private long finishTime;
-  private WrappedProgressSplitsBlock progressSplitBlock;
-
-  private static final CleanupContainerTransition CLEANUP_CONTAINER_TRANSITION =
-    new CleanupContainerTransition();
-
-  private static final DiagnosticInformationUpdater 
-    DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION 
-      = new DiagnosticInformationUpdater();
-
-  private static final StateMachineFactory
-        <TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
-        stateMachineFactory
-    = new StateMachineFactory
-             <TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
-           (TaskAttemptState.NEW)
-
-     // Transitions from the NEW state.
-     .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
-         TaskAttemptEventType.TA_SCHEDULE, new RequestContainerTransition(false))
-     .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
-         TaskAttemptEventType.TA_RESCHEDULE, new RequestContainerTransition(true))
-     .addTransition(TaskAttemptState.NEW, TaskAttemptState.KILLED,
-         TaskAttemptEventType.TA_KILL, new KilledTransition())
-     .addTransition(TaskAttemptState.NEW, TaskAttemptState.FAILED,
-         TaskAttemptEventType.TA_FAILMSG, new FailedTransition())
-
-     // Transitions from the UNASSIGNED state.
-     .addTransition(TaskAttemptState.UNASSIGNED,
-         TaskAttemptState.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED,
-         new ContainerAssignedTransition())
-     .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.KILLED,
-         TaskAttemptEventType.TA_KILL, new DeallocateContainerTransition(
-             TaskAttemptState.KILLED, true))
-     .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.FAILED,
-         TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition(
-             TaskAttemptState.FAILED, true))
-
-     // Transitions from the ASSIGNED state.
-     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.RUNNING,
-         TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
-         new LaunchedContainerTransition())
-     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.ASSIGNED,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED,
-         TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED,
-         new DeallocateContainerTransition(TaskAttemptState.FAILED, false))
-     .addTransition(TaskAttemptState.ASSIGNED, 
-         TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
-     .addTransition(TaskAttemptState.ASSIGNED, 
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
-
-     // Transitions from RUNNING state.
-     .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
-         TaskAttemptEventType.TA_UPDATE, new StatusUpdater())
-     .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // If no commit is required, task directly goes to success
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
-     // If commit is required, task goes through commit pending state.
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptEventType.TA_COMMIT_PENDING, new CommitPendingTransition())
-     // Failure handling while RUNNING
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
-      //for handling container exit without sending the done or fail msg
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-         CLEANUP_CONTAINER_TRANSITION)
-     // Timeout handling while RUNNING
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
-     // Kill handling
-     .addTransition(TaskAttemptState.RUNNING,
-         TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
-         CLEANUP_CONTAINER_TRANSITION)
-
-     // Transitions from COMMIT_PENDING state
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE,
-         new StatusUpdater())
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
-         CLEANUP_CONTAINER_TRANSITION)
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-         CLEANUP_CONTAINER_TRANSITION)
-     .addTransition(TaskAttemptState.COMMIT_PENDING,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
-
-     // Transitions from SUCCESS_CONTAINER_CLEANUP state
-     // kill and cleanup the container
-     .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-         TaskAttemptState.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED,
-         new SucceededTransition())
-     .addTransition(
-          TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-          TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-          TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-          DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-      // Ignore-able events
-     .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-         TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_FAILMSG,
-             TaskAttemptEventType.TA_TIMED_OUT,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED))
-
-     // Transitions from FAIL_CONTAINER_CLEANUP state.
-     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptState.FAIL_TASK_CLEANUP,
-         TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
-     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-      // Ignore-able events
-     .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG,
-             TaskAttemptEventType.TA_TIMED_OUT))
-
-      // Transitions from KILL_CONTAINER_CLEANUP
-     .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         TaskAttemptState.KILL_TASK_CLEANUP,
-         TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
-     .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // Ignore-able events
-     .addTransition(
-         TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         TaskAttemptState.KILL_CONTAINER_CLEANUP,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG,
-             TaskAttemptEventType.TA_TIMED_OUT))
-
-     // Transitions from FAIL_TASK_CLEANUP
-     // run the task cleanup
-     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
-         TaskAttemptState.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE,
-         new FailedTransition())
-     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
-         TaskAttemptState.FAIL_TASK_CLEANUP,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-      // Ignore-able events
-     .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
-         TaskAttemptState.FAIL_TASK_CLEANUP,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG))
-
-     // Transitions from KILL_TASK_CLEANUP
-     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
-         TaskAttemptState.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE,
-         new KilledTransition())
-     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
-         TaskAttemptState.KILL_TASK_CLEANUP,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // Ignore-able events
-     .addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
-         TaskAttemptState.KILL_TASK_CLEANUP,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG))
-
-      // Transitions from SUCCEEDED
-     .addTransition(TaskAttemptState.SUCCEEDED, //only possible for map attempts
-         TaskAttemptState.FAILED,
-         TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
-         new TooManyFetchFailureTransition())
-     .addTransition(
-         TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // Ignore-able events for SUCCEEDED state
-     .addTransition(TaskAttemptState.SUCCEEDED,
-         TaskAttemptState.SUCCEEDED,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_FAILMSG,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED))
-
-     // Transitions from FAILED state
-     .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
-       TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-       DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // Ignore-able events for FAILED state
-     .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_ASSIGNED,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG))
-
-     // Transitions from KILLED state
-     .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
-         TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
-         DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
-     // Ignore-able events for KILLED state
-     .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
-         EnumSet.of(TaskAttemptEventType.TA_KILL,
-             TaskAttemptEventType.TA_ASSIGNED,
-             TaskAttemptEventType.TA_CONTAINER_COMPLETED,
-             TaskAttemptEventType.TA_UPDATE,
-             TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
-             TaskAttemptEventType.TA_COMMIT_PENDING,
-             TaskAttemptEventType.TA_DONE,
-             TaskAttemptEventType.TA_FAILMSG))
-
-     // create the topology tables
-     .installTopology();
-
-  private final StateMachine
-         <TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
-    stateMachine;
-
-  private ContainerId containerID;
-  private String nodeHostName;
-  private String containerMgrAddress;
-  private String nodeHttpAddress;
-  private WrappedJvmID jvmID;
-  private ContainerToken containerToken;
-  private Resource assignedCapability;
-  
-  //this takes good amount of memory ~ 30KB. Instantiate it lazily
-  //and make it null once task is launched.
-  private org.apache.hadoop.mapred.Task remoteTask;
-  
-  //this is the last status reported by the REMOTE running attempt
-  private TaskAttemptStatus reportedStatus;
-
-  public TaskAttemptImpl(TaskId taskId, int i, EventHandler eventHandler,
-      TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
-      Configuration conf, String[] dataLocalHosts, OutputCommitter committer,
-      Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock) {
-    oldJobId = TypeConverter.fromYarn(taskId.getJobId());
-    this.conf = conf;
-    this.clock = clock;
-    attemptId = recordFactory.newRecordInstance(TaskAttemptId.class);
-    attemptId.setTaskId(taskId);
-    attemptId.setId(i);
-    this.taskAttemptListener = taskAttemptListener;
-
-    // Initialize reportedStatus
-    reportedStatus = new TaskAttemptStatus();
-    initTaskAttemptStatus(reportedStatus);
-
-    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
-    readLock = readWriteLock.readLock();
-    writeLock = readWriteLock.writeLock();
-
-    this.fsTokens = fsTokens;
-    this.jobToken = jobToken;
-    this.eventHandler = eventHandler;
-    this.committer = committer;
-    this.jobFile = jobFile;
-    this.partition = partition;
-
-    //TODO:create the resource reqt for this Task attempt
-    this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
-    this.resourceCapability.setMemory(getMemoryRequired(conf, taskId.getTaskType()));
-    this.dataLocalHosts = dataLocalHosts;
-    RackResolver.init(conf);
-
-    // This "this leak" is okay because the retained pointer is in an
-    //  instance variable.
-    stateMachine = stateMachineFactory.make(this);
-  }
-
-  private int getMemoryRequired(Configuration conf, TaskType taskType) {
-    int memory = 1024;
-    if (taskType == TaskType.MAP)  {
-      memory = conf.getInt(MRJobConfig.MAP_MEMORY_MB, MAP_MEMORY_MB_DEFAULT);
-    } else if (taskType == TaskType.REDUCE) {
-      memory = conf.getInt(MRJobConfig.REDUCE_MEMORY_MB, REDUCE_MEMORY_MB_DEFAULT);
-    }
-    
-    return memory;
-  }
-
-  /**
-   * Create a {@link LocalResource} record with all the given parameters.
-   * TODO: This should pave way for Builder pattern.
-   */
-  private static LocalResource createLocalResource(FileSystem fc,
-      RecordFactory recordFactory, Path file, LocalResourceType type,
-      LocalResourceVisibility visibility) throws IOException {
-    FileStatus fstat = fc.getFileStatus(file);
-    LocalResource resource =
-        recordFactory.newRecordInstance(LocalResource.class);
-    resource.setResource(ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
-        .getPath())));
-    resource.setType(type);
-    resource.setVisibility(visibility);
-    resource.setSize(fstat.getLen());
-    resource.setTimestamp(fstat.getModificationTime());
-    return resource;
-  }
-
-  /**
-   * Lock this on initialClasspath so that there is only one fork in the AM for
-   * getting the initial class-path. TODO: This should go away once we construct
-   * a parent CLC and use it for all the containers.
-   */
-  private String getInitialClasspath() throws IOException {
-    synchronized (classpathLock) {
-      if (initialClasspathFlag.get()) {
-        return initialClasspath;
-      }
-      Map<String, String> env = new HashMap<String, String>();
-      MRApps.setInitialClasspath(env);
-      initialClasspath = env.get(MRApps.CLASSPATH);
-      initialClasspathFlag.set(true);
-      return initialClasspath;
-    }
-  }
-
-
-  /**
-   * Create the {@link ContainerLaunchContext} for this attempt.
-   */
-  private ContainerLaunchContext createContainerLaunchContext() {
-
-    ContainerLaunchContext container =
-        recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
-    try {
-      FileSystem remoteFS = FileSystem.get(conf);
-
-      // //////////// Set up JobJar to be localized properly on the remote NM.
-      if (conf.get(MRJobConfig.JAR) != null) {
-        Path remoteJobJar = (new Path(remoteTask.getConf().get(
-              MRJobConfig.JAR))).makeQualified(remoteFS.getUri(), 
-                                               remoteFS.getWorkingDirectory());
-        container.setLocalResource(
-            MRConstants.JOB_JAR,
-            createLocalResource(remoteFS, recordFactory, remoteJobJar,
-                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
-        LOG.info("The job-jar file on the remote FS is "
-            + remoteJobJar.toUri().toASCIIString());
-      } else {
-        // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
-        // mapreduce jar itself which is already on the classpath.
-        LOG.info("Job jar is not present. "
-            + "Not adding any jar to the list of resources.");
-      }
-      // //////////// End of JobJar setup
-
-      // //////////// Set up JobConf to be localized properly on the remote NM.
-      Path path =
-          MRApps.getStagingAreaDir(conf, UserGroupInformation
-              .getCurrentUser().getShortUserName());
-      Path remoteJobSubmitDir =
-          new Path(path, oldJobId.toString());
-      Path remoteJobConfPath = 
-          new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
-      container.setLocalResource(
-          MRConstants.JOB_CONF_FILE,
-          createLocalResource(remoteFS, recordFactory, remoteJobConfPath,
-              LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
-      LOG.info("The job-conf file on the remote FS is "
-          + remoteJobConfPath.toUri().toASCIIString());
-      // //////////// End of JobConf setup
-
-      // Setup DistributedCache
-      setupDistributedCache(remoteFS, conf, container);
-
-      // Setup up tokens
-      Credentials taskCredentials = new Credentials();
-
-      if (UserGroupInformation.isSecurityEnabled()) {
-        // Add file-system tokens
-        for (Token<? extends TokenIdentifier> token : fsTokens) {
-          LOG.info("Putting fs-token for NM use for launching container : "
-              + token.toString());
-          taskCredentials.addToken(token.getService(), token);
-        }
-      }
-
-      // LocalStorageToken is needed irrespective of whether security is enabled
-      // or not.
-      TokenCache.setJobToken(jobToken, taskCredentials);
-
-      DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
-      LOG.info("Size of containertokens_dob is "
-          + taskCredentials.numberOfTokens());
-      taskCredentials.writeTokenStorageToStream(containerTokens_dob);
-      container.setContainerTokens(
-          ByteBuffer.wrap(containerTokens_dob.getData(), 0,
-              containerTokens_dob.getLength()));
-
-      // Add shuffle token
-      LOG.info("Putting shuffle token in serviceData");
-      DataOutputBuffer jobToken_dob = new DataOutputBuffer();
-      jobToken.write(jobToken_dob);
-      container
-          .setServiceData(
-              ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
-              ByteBuffer.wrap(jobToken_dob.getData(), 0,
-                  jobToken_dob.getLength()));
-
-      MRApps.addToClassPath(container.getAllEnv(), getInitialClasspath());
-    } catch (IOException e) {
-      throw new YarnException(e);
-    }
-    
-    container.setContainerId(containerID);
-    container.setUser(conf.get(MRJobConfig.USER_NAME)); // TODO: Fix
-
-    File workDir = new File("$PWD"); // Will be expanded by the shell.
-    String containerLogDir =
-        new File(ApplicationConstants.LOG_DIR_EXPANSION_VAR).toString();
-    String childTmpDir = new File(workDir, "tmp").toString();
-    String javaHome = "${JAVA_HOME}"; // Will be expanded by the shell.
-    String nmLdLibraryPath = "{LD_LIBRARY_PATH}"; // Expanded by the shell?
-    List<String> classPaths = new ArrayList<String>();
-
-    String localizedApplicationTokensFile =
-        new File(workDir, MRConstants.APPLICATION_TOKENS_FILE).toString();
-    classPaths.add(MRConstants.JOB_JAR);
-    classPaths.add(MRConstants.YARN_MAPREDUCE_APP_JAR_PATH);
-    classPaths.add(workDir.toString()); // TODO
-
-    // Construct the actual Container
-    container.addAllCommands(MapReduceChildJVM.getVMCommand(
-        taskAttemptListener.getAddress(), remoteTask, javaHome,
-        workDir.toString(), containerLogDir, childTmpDir, jvmID));
-
-    MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths,
-        workDir.toString(), containerLogDir, nmLdLibraryPath, remoteTask,
-        localizedApplicationTokensFile);
-
-    // Construct the actual Container
-    container.setContainerId(containerID);
-    container.setUser(conf.get(MRJobConfig.USER_NAME));
-    container.setResource(assignedCapability);
-    return container;
-  }
-
-  private static long[] parseTimeStamps(String[] strs) {
-    if (null == strs) {
-      return null;
-    }
-    long[] result = new long[strs.length];
-    for(int i=0; i < strs.length; ++i) {
-      result[i] = Long.parseLong(strs[i]);
-    }
-    return result;
-  }
-
-  private void setupDistributedCache(FileSystem remoteFS, Configuration conf, 
-      ContainerLaunchContext container) throws IOException {
-    
-    // Cache archives
-    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE, 
-        DistributedCache.getCacheArchives(conf), 
-        parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), 
-        getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
-        DistributedCache.getArchiveVisibilities(conf), 
-        DistributedCache.getArchiveClassPaths(conf));
-    
-    // Cache files
-    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE, 
-        DistributedCache.getCacheFiles(conf),
-        parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
-        getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
-        DistributedCache.getFileVisibilities(conf),
-        DistributedCache.getFileClassPaths(conf));
-  }
-
-  // TODO - Move this to MR!
-  // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], 
-  // long[], boolean[], Path[], FileType)
-  private void parseDistributedCacheArtifacts(
-      FileSystem remoteFS, ContainerLaunchContext container, LocalResourceType type,
-      URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], 
-      Path[] pathsToPutOnClasspath) throws IOException {
-
-    if (uris != null) {
-      // Sanity check
-      if ((uris.length != timestamps.length) || (uris.length != sizes.length) ||
-          (uris.length != visibilities.length)) {
-        throw new IllegalArgumentException("Invalid specification for " +
-        		"distributed-cache artifacts of type " + type + " :" +
-        		" #uris=" + uris.length +
-        		" #timestamps=" + timestamps.length +
-        		" #visibilities=" + visibilities.length
-        		);
-      }
-      
-      Map<String, Path> classPaths = new HashMap<String, Path>();
-      if (pathsToPutOnClasspath != null) {
-        for (Path p : pathsToPutOnClasspath) {
-          p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
-              remoteFS.getWorkingDirectory()));
-          classPaths.put(p.toUri().getPath().toString(), p);
-        }
-      }
-      for (int i = 0; i < uris.length; ++i) {
-        URI u = uris[i];
-        Path p = new Path(u);
-        p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
-            remoteFS.getWorkingDirectory()));
-        // Add URI fragment or just the filename
-        Path name = new Path((null == u.getFragment())
-          ? p.getName()
-          : u.getFragment());
-        if (name.isAbsolute()) {
-          throw new IllegalArgumentException("Resource name must be relative");
-        }
-        String linkName = name.toUri().getPath();
-        container.setLocalResource(
-            linkName,
-            BuilderUtils.newLocalResource(recordFactory,
-                p.toUri(), type, 
-                visibilities[i]
-                  ? LocalResourceVisibility.PUBLIC
-                  : LocalResourceVisibility.PRIVATE,
-                sizes[i], timestamps[i])
-        );
-        if (classPaths.containsKey(u.getPath())) {
-          Map<String, String> environment = container.getAllEnv();
-          MRApps.addToClassPath(environment, linkName);
-        }
-      }
-    }
-  }
-  
-  // TODO - Move this to MR!
-  private static long[] getFileSizes(Configuration conf, String key) {
-    String[] strs = conf.getStrings(key);
-    if (strs == null) {
-      return null;
-    }
-    long[] result = new long[strs.length];
-    for(int i=0; i < strs.length; ++i) {
-      result[i] = Long.parseLong(strs[i]);
-    }
-    return result;
-  }
-  
-  @Override
-  public ContainerId getAssignedContainerID() {
-    readLock.lock();
-    try {
-      return containerID;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public String getAssignedContainerMgrAddress() {
-    readLock.lock();
-    try {
-      return containerMgrAddress;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public long getLaunchTime() {
-    readLock.lock();
-    try {
-      return launchTime;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public long getFinishTime() {
-    readLock.lock();
-    try {
-      return finishTime;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  /**If container Assigned then return the node's address, otherwise null.
-   */
-  @Override
-  public String getNodeHttpAddress() {
-    readLock.lock();
-    try {
-      return nodeHttpAddress;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  protected abstract org.apache.hadoop.mapred.Task createRemoteTask();
-
-  @Override
-  public TaskAttemptId getID() {
-    return attemptId;
-  }
-
-  @Override
-  public boolean isFinished() {
-    readLock.lock();
-    try {
-      // TODO: Use stateMachine level method?
-      return (getState() == TaskAttemptState.SUCCEEDED || 
-          getState() == TaskAttemptState.FAILED ||
-          getState() == TaskAttemptState.KILLED);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public TaskAttemptReport getReport() {
-    TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class);
-    readLock.lock();
-    try {
-      result.setTaskAttemptId(attemptId);
-      //take the LOCAL state of attempt
-      //DO NOT take from reportedStatus
-      
-      result.setTaskAttemptState(getState());
-      result.setProgress(reportedStatus.progress);
-      result.setStartTime(launchTime);
-      result.setFinishTime(finishTime);
-      result.setDiagnosticInfo(reportedStatus.diagnosticInfo);
-      result.setPhase(reportedStatus.phase);
-      result.setStateString(reportedStatus.stateString);
-      result.setCounters(getCounters());
-      return result;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public List<String> getDiagnostics() {
-    List<String> result = new ArrayList<String>();
-    readLock.lock();
-    try {
-      result.addAll(diagnostics);
-      return result;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public Counters getCounters() {
-    readLock.lock();
-    try {
-      Counters counters = reportedStatus.counters;
-      if (counters == null) {
-        counters = recordFactory.newRecordInstance(Counters.class);
-//        counters.groups = new HashMap<String, CounterGroup>();
-      }
-      return counters;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public float getProgress() {
-    readLock.lock();
-    try {
-      return reportedStatus.progress;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public TaskAttemptState getState() {
-    readLock.lock();
-    try {
-      return stateMachine.getCurrentState();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public void handle(TaskAttemptEvent event) {
-    LOG.info("Processing " + event.getTaskAttemptID() +
-        " of type " + event.getType());
-    writeLock.lock();
-    try {
-      final TaskAttemptState oldState = getState();
-      try {
-        stateMachine.doTransition(event.getType(), event);
-      } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
-        eventHandler.handle(new JobDiagnosticsUpdateEvent(
-            this.attemptId.getTaskId().getJobId(), "Invalid event " + event.getType() + 
-            " on TaskAttempt " + this.attemptId));
-        eventHandler.handle(new JobEvent(this.attemptId.getTaskId().getJobId(),
-            JobEventType.INTERNAL_ERROR));
-      }
-      if (oldState != getState()) {
-          LOG.info(attemptId + " TaskAttempt Transitioned from " 
-           + oldState + " to "
-           + getState());
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  //always called in write lock
-  private void setFinishTime() {
-    //set the finish time only if launch time is set
-    if (launchTime != 0) {
-      finishTime = clock.getTime();
-    }
-  }
-  
-  private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
-    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
-    int slotMemoryReq =
-        taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
-    int simSlotsRequired =
-        slotMemoryReq
-            / (taskType == TaskType.MAP ? MAP_MEMORY_MB_DEFAULT
-                : REDUCE_MEMORY_MB_DEFAULT);
-    // Simulating MRv1 slots for counters by assuming *_MEMORY_MB_DEFAULT
-    // corresponds to a MrV1 slot.
-    // Fallow slot millis is not applicable in MRv2 - since a container is
-    // either assigned with the required memory or is not. No partial
-    // reserveations
-    long slotMillisIncrement =
-        simSlotsRequired
-            * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
-    return slotMillisIncrement;
-  }
-  
-  private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
-      TaskAttemptImpl taskAttempt) {
-    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
-    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
-    
-    long slotMillisIncrement = computeSlotMillis(taskAttempt);
-    
-    if (taskType == TaskType.MAP) {
-      jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
-      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
-    } else {
-      jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
-      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
-    }
-    return jce;
-  }
-
-  private static TaskAttemptUnsuccessfulCompletionEvent createTaskAttemptUnsuccessfulCompletionEvent(
-      TaskAttemptImpl taskAttempt, TaskAttemptState attemptState) {
-    TaskAttemptUnsuccessfulCompletionEvent tauce = new TaskAttemptUnsuccessfulCompletionEvent(
-        TypeConverter.fromYarn(taskAttempt.attemptId),
-        TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()),
-        attemptState.toString(), taskAttempt.finishTime,
-        taskAttempt.nodeHostName == null ? "UNKNOWN" : taskAttempt.nodeHostName,
-        taskAttempt.reportedStatus.diagnosticInfo.toString(),
-        taskAttempt.getProgressSplitBlock().burst());
-    return tauce;
-  }
-
-  private WrappedProgressSplitsBlock getProgressSplitBlock() {
-    readLock.lock();
-    try {
-      if (progressSplitBlock == null) {
-        progressSplitBlock = new WrappedProgressSplitsBlock(conf.getInt(
-            JHConfig.JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS_KEY,
-            WrappedProgressSplitsBlock.DEFAULT_NUMBER_PROGRESS_SPLITS));
-      }
-      return progressSplitBlock;
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  private void updateProgressSplits() {
-    double newProgress = reportedStatus.progress;
-    Counters counters = reportedStatus.counters;
-    if (counters == null)
-      return;
-
-    WrappedProgressSplitsBlock splitsBlock = getProgressSplitBlock();
-    if (splitsBlock != null) {
-      long now = clock.getTime();
-      long start = getLaunchTime(); // TODO Ensure not 0
-
-      if (start != 0 && now - start <= Integer.MAX_VALUE) {
-        splitsBlock.getProgressWallclockTime().extend(newProgress,
-            (int) (now - start));
-      }
-
-      Counter cpuCounter = counters.getCounter(
-          TaskCounter.CPU_MILLISECONDS);
-      if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
-        splitsBlock.getProgressCPUTime().extend(newProgress,
-            (int) cpuCounter.getValue());
-      }
-
-      Counter virtualBytes = counters.getCounter(
-          TaskCounter.VIRTUAL_MEMORY_BYTES);
-      if (virtualBytes != null) {
-        splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress,
-            (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
-      }
-
-      Counter physicalBytes = counters.getCounter(
-          TaskCounter.PHYSICAL_MEMORY_BYTES);
-      if (physicalBytes != null) {
-        splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress,
-            (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
-      }
-    }
-  }
-
-  private static class RequestContainerTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    private final boolean rescheduled;
-    public RequestContainerTransition(boolean rescheduled) {
-      this.rescheduled = rescheduled;
-    }
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      // Tell any speculator that we're requesting a container
-      taskAttempt.eventHandler.handle
-          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), +1));
-      //request for container
-      if (rescheduled) {
-        taskAttempt.eventHandler.handle(
-            ContainerRequestEvent.createContainerRequestEventForFailedContainer(
-                taskAttempt.attemptId, 
-                taskAttempt.resourceCapability));
-      } else {
-        int i = 0;
-        String[] racks = new String[taskAttempt.dataLocalHosts.length];
-        for (String host : taskAttempt.dataLocalHosts) {
-          racks[i++] = RackResolver.resolve(host).getNetworkLocation();
-        }
-        taskAttempt.eventHandler.handle(
-            new ContainerRequestEvent(taskAttempt.attemptId, 
-                taskAttempt.resourceCapability, 
-                taskAttempt.dataLocalHosts, racks));
-      }
-    }
-  }
-
-  private static class ContainerAssignedTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(final TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      TaskAttemptContainerAssignedEvent cEvent = 
-        (TaskAttemptContainerAssignedEvent) event;
-      taskAttempt.containerID = cEvent.getContainer().getId();
-      taskAttempt.nodeHostName = cEvent.getContainer().getNodeId().getHost();
-      taskAttempt.containerMgrAddress = cEvent.getContainer().getNodeId()
-          .toString();
-      taskAttempt.nodeHttpAddress = cEvent.getContainer().getNodeHttpAddress();
-      taskAttempt.containerToken = cEvent.getContainer().getContainerToken();
-      taskAttempt.assignedCapability = cEvent.getContainer().getResource();
-      // this is a _real_ Task (classic Hadoop mapred flavor):
-      taskAttempt.remoteTask = taskAttempt.createRemoteTask();
-      taskAttempt.jvmID = new WrappedJvmID(
-          taskAttempt.remoteTask.getTaskID().getJobID(), 
-          taskAttempt.remoteTask.isMapTask(), taskAttempt.containerID.getId());
-      
-      //launch the container
-      //create the container object to be launched for a given Task attempt
-      taskAttempt.eventHandler.handle(
-          new ContainerRemoteLaunchEvent(taskAttempt.attemptId, 
-              taskAttempt.containerID, 
-              taskAttempt.containerMgrAddress, taskAttempt.containerToken) {
-        @Override
-        public ContainerLaunchContext getContainer() {
-          return taskAttempt.createContainerLaunchContext();
-        }
-        @Override
-        public Task getRemoteTask() {  // classic mapred Task, not YARN version
-          return taskAttempt.remoteTask;
-        }
-      });
-
-      // send event to speculator that our container needs are satisfied
-      taskAttempt.eventHandler.handle
-          (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
-    }
-  }
-
-  private static class DeallocateContainerTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    private final TaskAttemptState finalState;
-    private final boolean withdrawsContainerRequest;
-    DeallocateContainerTransition
-        (TaskAttemptState finalState, boolean withdrawsContainerRequest) {
-      this.finalState = finalState;
-      this.withdrawsContainerRequest = withdrawsContainerRequest;
-    }
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      //set the finish time
-      taskAttempt.setFinishTime();
-      //send the deallocate event to ContainerAllocator
-      taskAttempt.eventHandler.handle(
-          new ContainerAllocatorEvent(taskAttempt.attemptId,
-          ContainerAllocator.EventType.CONTAINER_DEALLOCATE));
-
-      // send event to speculator that we withdraw our container needs, if
-      //  we're transitioning out of UNASSIGNED
-      if (withdrawsContainerRequest) {
-        taskAttempt.eventHandler.handle
-            (new SpeculatorEvent(taskAttempt.getID().getTaskId(), -1));
-      }
-
-      switch(finalState) {
-        case FAILED:
-          taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-              taskAttempt.attemptId,
-              TaskEventType.T_ATTEMPT_FAILED));
-          break;
-        case KILLED:
-          taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-              taskAttempt.attemptId,
-              TaskEventType.T_ATTEMPT_KILLED));
-          break;
-      }
-      if (taskAttempt.getLaunchTime() != 0) {
-        TaskAttemptUnsuccessfulCompletionEvent tauce =
-            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
-                finalState);
-        taskAttempt.eventHandler
-            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
-        taskAttempt.eventHandler.handle(new JobHistoryEvent(
-            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
-      } else {
-        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
-            + taskAttempt.getID());
-      }
-    }
-  }
-
-  private static class LaunchedContainerTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      //set the launch time
-      taskAttempt.launchTime = taskAttempt.clock.getTime();
-      // register it to TaskAttemptListener so that it start listening
-      // for it
-      taskAttempt.taskAttemptListener.register(
-          taskAttempt.attemptId, taskAttempt.remoteTask, taskAttempt.jvmID);
-      //TODO Resolve to host / IP in case of a local address.
-      InetSocketAddress nodeHttpInetAddr =
-          NetUtils.createSocketAddr(taskAttempt.nodeHttpAddress); // TODO:
-                                                                  // Costly?
-      JobCounterUpdateEvent jce =
-          new JobCounterUpdateEvent(taskAttempt.attemptId.getTaskId()
-              .getJobId());
-      jce.addCounterUpdate(
-          taskAttempt.attemptId.getTaskId().getTaskType() == TaskType.MAP ? 
-              JobCounter.TOTAL_LAUNCHED_MAPS: JobCounter.TOTAL_LAUNCHED_REDUCES
-              , 1);
-      taskAttempt.eventHandler.handle(jce);
-      
-      TaskAttemptStartedEvent tase =
-        new TaskAttemptStartedEvent(TypeConverter.fromYarn(taskAttempt.attemptId),
-            TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()),
-            taskAttempt.launchTime,
-            nodeHttpInetAddr.getHostName(), nodeHttpInetAddr.getPort());
-      taskAttempt.eventHandler.handle
-          (new JobHistoryEvent(taskAttempt.attemptId.getTaskId().getJobId(), tase));
-      taskAttempt.eventHandler.handle
-          (new SpeculatorEvent
-              (taskAttempt.attemptId, true, taskAttempt.clock.getTime()));
-      //make remoteTask reference as null as it is no more needed
-      //and free up the memory
-      taskAttempt.remoteTask = null;
-      
-      //tell the Task that attempt has started
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId, 
-         TaskEventType.T_ATTEMPT_LAUNCHED));
-    }
-  }
-   
-  private static class CommitPendingTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId, 
-         TaskEventType.T_ATTEMPT_COMMIT_PENDING));
-    }
-  }
-
-  private static class TaskCleanupTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      TaskAttemptContext taskContext =
-        new TaskAttemptContextImpl(new JobConf(taskAttempt.conf),
-            TypeConverter.fromYarn(taskAttempt.attemptId));
-      taskAttempt.eventHandler.handle(new TaskCleanupEvent(
-          taskAttempt.attemptId,
-          taskAttempt.committer,
-          taskContext));
-    }
-  }
-
-  private static class SucceededTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      //set the finish time
-      taskAttempt.setFinishTime();
-      String taskType = 
-          TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()).toString();
-      LOG.info("In TaskAttemptImpl taskType: " + taskType);
-      long slotMillis = computeSlotMillis(taskAttempt);
-      JobCounterUpdateEvent jce =
-          new JobCounterUpdateEvent(taskAttempt.attemptId.getTaskId()
-              .getJobId());
-      jce.addCounterUpdate(
-        taskAttempt.attemptId.getTaskId().getTaskType() == TaskType.MAP ? 
-          JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
-          slotMillis);
-      taskAttempt.eventHandler.handle(jce);
-      taskAttempt.logAttemptFinishedEvent(TaskAttemptState.SUCCEEDED);
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId,
-          TaskEventType.T_ATTEMPT_SUCCEEDED));
-      taskAttempt.eventHandler.handle
-      (new SpeculatorEvent
-          (taskAttempt.reportedStatus, taskAttempt.clock.getTime()));
-   }
-  }
-
-  private static class FailedTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
-      // set the finish time
-      taskAttempt.setFinishTime();
-      
-      if (taskAttempt.getLaunchTime() != 0) {
-        taskAttempt.eventHandler
-            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
-        TaskAttemptUnsuccessfulCompletionEvent tauce =
-            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
-                TaskAttemptState.FAILED);
-        taskAttempt.eventHandler.handle(new JobHistoryEvent(
-            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
-        // taskAttempt.logAttemptFinishedEvent(TaskAttemptState.FAILED); Not
-        // handling failed map/reduce events.
-      }else {
-        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
-            + taskAttempt.getID());
-      }
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
-    }
-  }
-
-  private void logAttemptFinishedEvent(TaskAttemptState state) {
-    //Log finished events only if an attempt started.
-    if (getLaunchTime() == 0) return; 
-    if (attemptId.getTaskId().getTaskType() == TaskType.MAP) {
-      MapAttemptFinishedEvent mfe =
-         new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
-         TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
-         state.toString(),
-         this.reportedStatus.mapFinishTime,
-         finishTime, this.nodeHostName == null ? "UNKNOWN" : this.nodeHostName,
-         this.reportedStatus.stateString,
-         TypeConverter.fromYarn(getCounters()),
-         getProgressSplitBlock().burst());
-         eventHandler.handle(
-           new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe));
-    } else {
-       ReduceAttemptFinishedEvent rfe =
-         new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
-         TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
-         state.toString(),
-         this.reportedStatus.shuffleFinishTime,
-         this.reportedStatus.sortFinishTime,
-         finishTime, this.containerMgrAddress == null ? "UNKNOWN" : this.containerMgrAddress,
-         this.reportedStatus.stateString,
-         TypeConverter.fromYarn(getCounters()),
-         getProgressSplitBlock().burst());
-         eventHandler.handle(
-           new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe));
-    }
-  }
-
-  private static class TooManyFetchFailureTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
-      //add to diagnostic
-      taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
-      //set the finish time
-      taskAttempt.setFinishTime();
-      
-      if (taskAttempt.getLaunchTime() != 0) {
-        taskAttempt.eventHandler
-            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
-        TaskAttemptUnsuccessfulCompletionEvent tauce =
-            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
-                TaskAttemptState.FAILED);
-        taskAttempt.eventHandler.handle(new JobHistoryEvent(
-            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
-      }else {
-        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
-            + taskAttempt.getID());
-      }
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
-    }
-  }
-
-  private static class KilledTransition implements
-      SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt,
-        TaskAttemptEvent event) {
-      //set the finish time
-      taskAttempt.setFinishTime();
-      if (taskAttempt.getLaunchTime() != 0) {
-        taskAttempt.eventHandler
-            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
-        TaskAttemptUnsuccessfulCompletionEvent tauce =
-            createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
-                TaskAttemptState.KILLED);
-        taskAttempt.eventHandler.handle(new JobHistoryEvent(
-            taskAttempt.attemptId.getTaskId().getJobId(), tauce));
-      }else {
-        LOG.debug("Not generating HistoryFinish event since start event not generated for taskAttempt: "
-            + taskAttempt.getID());
-      }
-//      taskAttempt.logAttemptFinishedEvent(TaskAttemptState.KILLED); Not logging Map/Reduce attempts in case of failure.
-      taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
-          taskAttempt.attemptId,
-          TaskEventType.T_ATTEMPT_KILLED));
-    }
-  }
-
-  private static class CleanupContainerTransition implements
-       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      // unregister it to TaskAttemptListener so that it stops listening
-      // for it
-      taskAttempt.taskAttemptListener.unregister(
-          taskAttempt.attemptId, taskAttempt.jvmID);
-      //send the cleanup event to containerLauncher
-      taskAttempt.eventHandler.handle(new ContainerLauncherEvent(
-          taskAttempt.attemptId, 
-          taskAttempt.containerID, taskAttempt.containerMgrAddress,
-          taskAttempt.containerToken,
-          ContainerLauncher.EventType.CONTAINER_REMOTE_CLEANUP));
-    }
-  }
-
-  private void addDiagnosticInfo(String diag) {
-    if (diag != null && !diag.equals("")) {
-      diagnostics.add(diag);
-    }
-  }
-
-  private static class StatusUpdater 
-       implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      // Status update calls don't really change the state of the attempt.
-      TaskAttemptStatus newReportedStatus =
-          ((TaskAttemptStatusUpdateEvent) event)
-              .getReportedTaskAttemptStatus();
-      // Now switch the information in the reportedStatus
-      taskAttempt.reportedStatus = newReportedStatus;
-      taskAttempt.reportedStatus.taskState = taskAttempt.getState();
-
-      // send event to speculator about the reported status
-      taskAttempt.eventHandler.handle
-          (new SpeculatorEvent
-              (taskAttempt.reportedStatus, taskAttempt.clock.getTime()));
-      
-      //add to diagnostic
-      taskAttempt.addDiagnosticInfo(newReportedStatus.diagnosticInfo);
-      taskAttempt.updateProgressSplits();
-      
-      //if fetch failures are present, send the fetch failure event to job
-      //this only will happen in reduce attempt type
-      if (taskAttempt.reportedStatus.fetchFailedMaps != null && 
-          taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) {
-        taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent(
-            taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps));
-      }
-    }
-  }
-
-  private static class DiagnosticInformationUpdater 
-        implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-    @Override
-    public void transition(TaskAttemptImpl taskAttempt, 
-        TaskAttemptEvent event) {
-      TaskAttemptDiagnosticsUpdateEvent diagEvent =
-          (TaskAttemptDiagnosticsUpdateEvent) event;
-      LOG.info("Diagnostics report from " + taskAttempt.attemptId + ": "
-          + diagEvent.getDiagnosticInfo());
-      taskAttempt.addDiagnosticInfo(diagEvent.getDiagnosticInfo());
-    }
-  }
-
-  private void initTaskAttemptStatus(TaskAttemptStatus result) {
-    result.progress = 0.0f;
-    result.diagnosticInfo = "";
-    result.phase = Phase.STARTING;
-    result.stateString = "NEW";
-    result.taskState = TaskAttemptState.NEW;
-    Counters counters = recordFactory.newRecordInstance(Counters.class);
-//    counters.groups = new HashMap<String, CounterGroup>();
-    result.counters = counters;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java

@@ -1,887 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.job.impl;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
-import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
-import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobMapTaskRescheduledEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
-import org.apache.hadoop.yarn.state.MultipleArcTransition;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-
-/**
- * Implementation of Task interface.
- */
-public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
-
-  private static final Log LOG = LogFactory.getLog(TaskImpl.class);
-
-  protected final Configuration conf;
-  protected final Path jobFile;
-  protected final OutputCommitter committer;
-  protected final int partition;
-  protected final TaskAttemptListener taskAttemptListener;
-  protected final EventHandler eventHandler;
-  private final TaskId taskId;
-  private Map<TaskAttemptId, TaskAttempt> attempts;
-  private final int maxAttempts;
-  protected final Clock clock;
-  private final Lock readLock;
-  private final Lock writeLock;
-  private final MRAppMetrics metrics;
-  private long scheduledTime;
-  
-  private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-  
-  protected Collection<Token<? extends TokenIdentifier>> fsTokens;
-  protected Token<JobTokenIdentifier> jobToken;
-  
-  // counts the number of attempts that are either running or in a state where
-  //  they will come to be running when they get a Container
-  private int numberUncompletedAttempts = 0;
-
-  private boolean historyTaskStartGenerated = false;
-  
-  private static final SingleArcTransition<TaskImpl, TaskEvent> 
-     ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
-  private static final SingleArcTransition<TaskImpl, TaskEvent> 
-     KILL_TRANSITION = new KillTransition();
-
-  private static final StateMachineFactory
-               <TaskImpl, TaskState, TaskEventType, TaskEvent> 
-            stateMachineFactory 
-           = new StateMachineFactory<TaskImpl, TaskState, TaskEventType, TaskEvent>
-               (TaskState.NEW)
-
-    // define the state machine of Task
-
-    // Transitions from NEW state
-    .addTransition(TaskState.NEW, TaskState.SCHEDULED, 
-        TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
-    .addTransition(TaskState.NEW, TaskState.KILLED, 
-        TaskEventType.T_KILL, new KillNewTransition())
-
-    // Transitions from SCHEDULED state
-      //when the first attempt is launched, the task state is set to RUNNING
-     .addTransition(TaskState.SCHEDULED, TaskState.RUNNING, 
-         TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition())
-     .addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT, 
-         TaskEventType.T_KILL, KILL_TRANSITION)
-     .addTransition(TaskState.SCHEDULED, TaskState.SCHEDULED, 
-         TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION)
-     .addTransition(TaskState.SCHEDULED, 
-        EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED), 
-        TaskEventType.T_ATTEMPT_FAILED, 
-        new AttemptFailedTransition())
- 
-    // Transitions from RUNNING state
-    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
-        TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
-    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
-        TaskEventType.T_ATTEMPT_COMMIT_PENDING,
-        new AttemptCommitPendingTransition())
-    .addTransition(TaskState.RUNNING, TaskState.RUNNING,
-        TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
-    .addTransition(TaskState.RUNNING, TaskState.SUCCEEDED, 
-        TaskEventType.T_ATTEMPT_SUCCEEDED,
-        new AttemptSucceededTransition())
-    .addTransition(TaskState.RUNNING, TaskState.RUNNING, 
-        TaskEventType.T_ATTEMPT_KILLED,
-        ATTEMPT_KILLED_TRANSITION)
-    .addTransition(TaskState.RUNNING, 
-        EnumSet.of(TaskState.RUNNING, TaskState.FAILED), 
-        TaskEventType.T_ATTEMPT_FAILED,
-        new AttemptFailedTransition())
-    .addTransition(TaskState.RUNNING, TaskState.KILL_WAIT, 
-        TaskEventType.T_KILL, KILL_TRANSITION)
-
-    // Transitions from KILL_WAIT state
-    .addTransition(TaskState.KILL_WAIT,
-        EnumSet.of(TaskState.KILL_WAIT, TaskState.KILLED),
-        TaskEventType.T_ATTEMPT_KILLED,
-        new KillWaitAttemptKilledTransition())
-    // Ignore-able transitions.
-    .addTransition(
-        TaskState.KILL_WAIT,
-        TaskState.KILL_WAIT,
-        EnumSet.of(TaskEventType.T_KILL,
-            TaskEventType.T_ATTEMPT_LAUNCHED,
-            TaskEventType.T_ATTEMPT_COMMIT_PENDING,
-            TaskEventType.T_ATTEMPT_FAILED,
-            TaskEventType.T_ATTEMPT_SUCCEEDED,
-            TaskEventType.T_ADD_SPEC_ATTEMPT))
-
-    // Transitions from SUCCEEDED state
-    .addTransition(TaskState.SUCCEEDED, //only possible for map tasks
-        EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED),
-        TaskEventType.T_ATTEMPT_FAILED, new MapRetroactiveFailureTransition())
-    // Ignore-able transitions.
-    .addTransition(
-        TaskState.SUCCEEDED, TaskState.SUCCEEDED,
-        EnumSet.of(TaskEventType.T_KILL,
-            TaskEventType.T_ADD_SPEC_ATTEMPT,
-            TaskEventType.T_ATTEMPT_LAUNCHED,
-            TaskEventType.T_ATTEMPT_KILLED))
-
-    // Transitions from FAILED state        
-    .addTransition(TaskState.FAILED, TaskState.FAILED,
-        EnumSet.of(TaskEventType.T_KILL,
-                   TaskEventType.T_ADD_SPEC_ATTEMPT))
-
-    // Transitions from KILLED state
-    .addTransition(TaskState.KILLED, TaskState.KILLED,
-        EnumSet.of(TaskEventType.T_KILL,
-                   TaskEventType.T_ADD_SPEC_ATTEMPT))
-
-    // create the topology tables
-    .installTopology();
-
-  private final StateMachine<TaskState, TaskEventType, TaskEvent>
-    stateMachine;
-  
-  protected int nextAttemptNumber;
-
-  //should be set to one which comes first
-  //saying COMMIT_PENDING
-  private TaskAttemptId commitAttempt;
-
-  private TaskAttemptId successfulAttempt;
-
-  private int failedAttempts;
-  private int finishedAttempts;//finish are total of success, failed and killed
-
-  @Override
-  public TaskState getState() {
-    return stateMachine.getCurrentState();
-  }
-
-  public TaskImpl(JobId jobId, TaskType taskType, int partition,
-      EventHandler eventHandler, Path remoteJobConfFile, Configuration conf,
-      TaskAttemptListener taskAttemptListener, OutputCommitter committer,
-      Token<JobTokenIdentifier> jobToken,
-      Collection<Token<? extends TokenIdentifier>> fsTokens, Clock clock,
-      Set<TaskId> completedTasksFromPreviousRun, int startCount,
-      MRAppMetrics metrics) {
-    this.conf = conf;
-    this.clock = clock;
-    this.jobFile = remoteJobConfFile;
-    ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
-    readLock = readWriteLock.readLock();
-    writeLock = readWriteLock.writeLock();
-    this.attempts = Collections.emptyMap();
-    // This overridable method call is okay in a constructor because we
-    //  have a convention that none of the overrides depends on any
-    //  fields that need initialization.
-    maxAttempts = getMaxAttempts();
-    taskId = recordFactory.newRecordInstance(TaskId.class);
-    taskId.setJobId(jobId);
-    taskId.setId(partition);
-    taskId.setTaskType(taskType);
-    this.partition = partition;
-    this.taskAttemptListener = taskAttemptListener;
-    this.eventHandler = eventHandler;
-    this.committer = committer;
-    this.fsTokens = fsTokens;
-    this.jobToken = jobToken;
-    this.metrics = metrics;
-
-    if (completedTasksFromPreviousRun != null
-        && completedTasksFromPreviousRun.contains(taskId)) {
-      LOG.info("Task is from previous run " + taskId);
-      startCount = startCount - 1;
-    }
-
-    //attempt ids are generated based on MR app startCount so that attempts
-    //from previous lives don't overstep the current one.
-    //this assumes that a task won't have more than 1000 attempts in its single 
-    //life
-    nextAttemptNumber = (startCount - 1) * 1000;
-
-    // This "this leak" is okay because the retained pointer is in an
-    //  instance variable.
-    stateMachine = stateMachineFactory.make(this);
-  }
-
-  @Override
-  public Map<TaskAttemptId, TaskAttempt> getAttempts() {
-    readLock.lock();
-
-    try {
-      if (attempts.size() <= 1) {
-        return attempts;
-      }
-      
-      Map<TaskAttemptId, TaskAttempt> result
-          = new LinkedHashMap<TaskAttemptId, TaskAttempt>();
-      result.putAll(attempts);
-
-      return result;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public TaskAttempt getAttempt(TaskAttemptId attemptID) {
-    readLock.lock();
-    try {
-      return attempts.get(attemptID);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public TaskId getID() {
-    return taskId;
-  }
-
-  @Override
-  public boolean isFinished() {
-    readLock.lock();
-    try {
-     // TODO: Use stateMachine level method?
-      return (getState() == TaskState.SUCCEEDED ||
-          getState() == TaskState.FAILED ||
-          getState() == TaskState.KILLED);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public TaskReport getReport() {
-    TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
-    readLock.lock();
-    try {
-      report.setTaskId(taskId);
-      report.setStartTime(getLaunchTime());
-      report.setFinishTime(getFinishTime());
-      report.setTaskState(getState());
-      report.setProgress(getProgress());
-      report.setCounters(getCounters());
-
-      for (TaskAttempt attempt : attempts.values()) {
-        if (TaskAttemptState.RUNNING.equals(attempt.getState())) {
-          report.addRunningAttempt(attempt.getID());
-        }
-      }
-
-      report.setSuccessfulAttempt(successfulAttempt);
-      
-      for (TaskAttempt att : attempts.values()) {
-        String prefix = "AttemptID:" + att.getID() + " Info:";
-        for (CharSequence cs : att.getDiagnostics()) {
-          report.addDiagnostics(prefix + cs);
-          
-        }
-      }
-      return report;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public Counters getCounters() {
-    Counters counters = null;
-    readLock.lock();
-    try {
-      TaskAttempt bestAttempt = selectBestAttempt();
-      if (bestAttempt != null) {
-        counters = bestAttempt.getCounters();
-      } else {
-        counters = recordFactory.newRecordInstance(Counters.class);
-//        counters.groups = new HashMap<CharSequence, CounterGroup>();
-      }
-      return counters;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public float getProgress() {
-    readLock.lock();
-    try {
-      TaskAttempt bestAttempt = selectBestAttempt();
-      if (bestAttempt == null) {
-        return 0;
-      }
-      return bestAttempt.getProgress();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  //this is always called in read/write lock
-  private long getLaunchTime() {
-    long launchTime = 0;
-    for (TaskAttempt at : attempts.values()) {
-      //select the least launch time of all attempts
-      if (launchTime == 0  || launchTime > at.getLaunchTime()) {
-        launchTime = at.getLaunchTime();
-      }
-    }
-    if (launchTime == 0) {
-      return this.scheduledTime;
-    }
-    return launchTime;
-  }
-
-  //this is always called in read/write lock
-  //TODO Verify behaviour is Task is killed (no finished attempt)
-  private long getFinishTime() {
-    if (!isFinished()) {
-      return 0;
-    }
-    long finishTime = 0;
-    for (TaskAttempt at : attempts.values()) {
-      //select the max finish time of all attempts
-      if (finishTime < at.getFinishTime()) {
-        finishTime = at.getFinishTime();
-      }
-    }
-    return finishTime;
-  }
-
-  private long getFinishTime(TaskAttemptId taId) {
-    if (taId == null) {
-      return clock.getTime();
-    }
-    long finishTime = 0;
-    for (TaskAttempt at : attempts.values()) {
-      //select the max finish time of all attempts
-      if (at.getID().equals(taId)) {
-        return at.getFinishTime();
-      }
-    }
-    return finishTime;
-  }
-  
-  private TaskState finished(TaskState finalState) {
-    if (getState() == TaskState.RUNNING) {
-      metrics.endRunningTask(this);
-    }
-    return finalState;
-  }
-
-  //select the nextAttemptNumber with best progress
-  // always called inside the Read Lock
-  private TaskAttempt selectBestAttempt() {
-    float progress = 0f;
-    TaskAttempt result = null;
-    for (TaskAttempt at : attempts.values()) {
-      if (result == null) {
-        result = at; //The first time around
-      }
-      //TODO: consider the nextAttemptNumber only if it is not failed/killed ?
-      // calculate the best progress
-      if (at.getProgress() > progress) {
-        result = at;
-        progress = at.getProgress();
-      }
-    }
-    return result;
-  }
-
-  @Override
-  public boolean canCommit(TaskAttemptId taskAttemptID) {
-    readLock.lock();
-    boolean canCommit = false;
-    try {
-      if (commitAttempt != null) {
-        canCommit = taskAttemptID.equals(commitAttempt);
-        LOG.info("Result of canCommit for " + taskAttemptID + ":" + canCommit);
-      }
-    } finally {
-      readLock.unlock();
-    }
-    return canCommit;
-  }
-
-  protected abstract TaskAttemptImpl createAttempt();
-
-  // No override of this method may require that the subclass be initialized.
-  protected abstract int getMaxAttempts();
-
-  protected TaskAttempt getSuccessfulAttempt() {
-    readLock.lock();
-    try {
-      if (null == successfulAttempt) {
-        return null;
-      }
-      return attempts.get(successfulAttempt);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  // This is always called in the Write Lock
-  private void addAndScheduleAttempt() {
-    TaskAttempt attempt = createAttempt();
-    LOG.info("Created attempt " + attempt.getID());
-    switch (attempts.size()) {
-      case 0:
-        attempts = Collections.singletonMap(attempt.getID(), attempt);
-        break;
-        
-      case 1:
-        Map newAttempts
-            = new LinkedHashMap<TaskAttemptId, TaskAttempt>(maxAttempts);
-        newAttempts.putAll(attempts);
-        attempts = newAttempts;
-        attempts.put(attempt.getID(), attempt);
-        break;
-
-      default:
-        attempts.put(attempt.getID(), attempt);
-        break;
-    }
-    ++nextAttemptNumber;
-    ++numberUncompletedAttempts;
-    //schedule the nextAttemptNumber
-    if (failedAttempts > 0) {
-      eventHandler.handle(new TaskAttemptEvent(attempt.getID(),
-        TaskAttemptEventType.TA_RESCHEDULE));
-    } else {
-      eventHandler.handle(new TaskAttemptEvent(attempt.getID(),
-          TaskAttemptEventType.TA_SCHEDULE));
-    }
-  }
-
-  @Override
-  public void handle(TaskEvent event) {
-    LOG.info("Processing " + event.getTaskID() + " of type " + event.getType());
-    try {
-      writeLock.lock();
-      TaskState oldState = getState();
-      try {
-        stateMachine.doTransition(event.getType(), event);
-      } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
-        internalError(event.getType());
-      }
-      if (oldState != getState()) {
-        LOG.info(taskId + " Task Transitioned from " + oldState + " to "
-            + getState());
-      }
-
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void internalError(TaskEventType type) {
-    eventHandler.handle(new JobDiagnosticsUpdateEvent(
-        this.taskId.getJobId(), "Invalid event " + type + 
-        " on Task " + this.taskId));
-    eventHandler.handle(new JobEvent(this.taskId.getJobId(),
-        JobEventType.INTERNAL_ERROR));
-  }
-
-  // always called inside a transition, in turn inside the Write Lock
-  private void handleTaskAttemptCompletion(TaskAttemptId attemptId,
-      TaskAttemptCompletionEventStatus status) {
-    finishedAttempts++;
-    TaskAttempt attempt = attempts.get(attemptId);
-    //raise the completion event only if the container is assigned
-    // to nextAttemptNumber
-    if (attempt.getNodeHttpAddress() != null) {
-      TaskAttemptCompletionEvent tce = recordFactory.newRecordInstance(TaskAttemptCompletionEvent.class);
-      tce.setEventId(-1);
-      //TODO: XXXXXX  hardcoded port
-      tce.setMapOutputServerAddress("http://" + attempt.getNodeHttpAddress().split(":")[0] + ":8080");
-      tce.setStatus(status);
-      tce.setAttemptId(attempt.getID());
-      int runTime = 0;
-      if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
-        runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
-      tce.setAttemptRunTime(runTime);
-      
-      //raise the event to job so that it adds the completion event to its
-      //data structures
-      eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
-    }
-  }
-
-  private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskState taskState) {
-    TaskFinishedEvent tfe =
-      new TaskFinishedEvent(TypeConverter.fromYarn(task.taskId),
-        task.getFinishTime(task.successfulAttempt),
-        TypeConverter.fromYarn(task.taskId.getTaskType()),
-        taskState.toString(),
-        TypeConverter.fromYarn(task.getCounters()));
-    return tfe;
-  }
-  
-  private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskState taskState, TaskAttemptId taId) {
-    StringBuilder errorSb = new StringBuilder();
-    if (diag != null) {
-      for (String d : diag) {
-        errorSb.append(", ").append(d);
-      }
-    }
-    TaskFailedEvent taskFailedEvent = new TaskFailedEvent(
-        TypeConverter.fromYarn(task.taskId),
-     // Hack since getFinishTime needs isFinished to be true and that doesn't happen till after the transition.
-        task.getFinishTime(taId),
-        TypeConverter.fromYarn(task.getType()),
-        errorSb.toString(),
-        taskState.toString(),
-        taId == null ? null : TypeConverter.fromYarn(taId));
-    return taskFailedEvent;
-  }
-
-  /**
-  * @return a String representation of the splits.
-  *
-  * Subclasses can override this method to provide their own representations
-  * of splits (if any).
-  *
-  */
-  protected String getSplitsAsString(){
-	  return "";
-  }
-
-  private static class InitialScheduleTransition
-    implements SingleArcTransition<TaskImpl, TaskEvent> {
-
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      task.addAndScheduleAttempt();
-      task.scheduledTime = task.clock.getTime();
-      TaskStartedEvent tse = new TaskStartedEvent(
-          TypeConverter.fromYarn(task.taskId), task.getLaunchTime(),
-          TypeConverter.fromYarn(task.taskId.getTaskType()),
-          task.getSplitsAsString());
-      task.eventHandler
-          .handle(new JobHistoryEvent(task.taskId.getJobId(), tse));
-      task.historyTaskStartGenerated = true;
-    }
-  }
-
-  // Used when creating a new attempt while one is already running.
-  //  Currently we do this for speculation.  In the future we may do this
-  //  for tasks that failed in a way that might indicate application code
-  //  problems, so we can take later failures in parallel and flush the
-  //  job quickly when this happens.
-  private static class RedundantScheduleTransition
-    implements SingleArcTransition<TaskImpl, TaskEvent> {
-
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      LOG.info("Scheduling a redundant attempt for task " + task.taskId);
-      task.addAndScheduleAttempt();
-    }
-  }
-
-  private static class AttemptCommitPendingTransition 
-          implements SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      TaskTAttemptEvent ev = (TaskTAttemptEvent) event;
-      // The nextAttemptNumber is commit pending, decide on set the commitAttempt
-      TaskAttemptId attemptID = ev.getTaskAttemptID();
-      if (task.commitAttempt == null) {
-        // TODO: validate attemptID
-        task.commitAttempt = attemptID;
-        LOG.info(attemptID + " given a go for committing the task output.");
-      } else {
-        // Don't think this can be a pluggable decision, so simply raise an
-        // event for the TaskAttempt to delete its output.
-        LOG.info(task.commitAttempt
-            + " already given a go for committing the task output, so killing "
-            + attemptID);
-        task.eventHandler.handle(new TaskAttemptEvent(
-            attemptID, TaskAttemptEventType.TA_KILL));
-      }
-    }
-  }
-
-  private static class AttemptSucceededTransition 
-      implements SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      task.handleTaskAttemptCompletion(
-          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
-          TaskAttemptCompletionEventStatus.SUCCEEDED);
-      --task.numberUncompletedAttempts;
-      task.successfulAttempt = ((TaskTAttemptEvent) event).getTaskAttemptID();
-      task.eventHandler.handle(new JobTaskEvent(
-          task.taskId, TaskState.SUCCEEDED));
-      LOG.info("Task succeeded with attempt " + task.successfulAttempt);
-      // issue kill to all other attempts
-      if (task.historyTaskStartGenerated) {
-        TaskFinishedEvent tfe = createTaskFinishedEvent(task,
-            TaskState.SUCCEEDED);
-        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
-            tfe));
-      }
-      for (TaskAttempt attempt : task.attempts.values()) {
-        if (attempt.getID() != task.successfulAttempt &&
-            // This is okay because it can only talk us out of sending a
-            //  TA_KILL message to an attempt that doesn't need one for
-            //  other reasons.
-            !attempt.isFinished()) {
-          LOG.info("Issuing kill to other attempt " + attempt.getID());
-          task.eventHandler.handle(
-              new TaskAttemptEvent(attempt.getID(), 
-                  TaskAttemptEventType.TA_KILL));
-        }
-      }
-      task.finished(TaskState.SUCCEEDED);
-    }
-  }
-
-  private static class AttemptKilledTransition implements
-      SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      task.handleTaskAttemptCompletion(
-          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
-          TaskAttemptCompletionEventStatus.KILLED);
-      --task.numberUncompletedAttempts;
-      if (task.successfulAttempt == null) {
-        task.addAndScheduleAttempt();
-      }
-    }
-  }
-
-
-  private static class KillWaitAttemptKilledTransition implements
-      MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
-
-    protected TaskState finalState = TaskState.KILLED;
-
-    @Override
-    public TaskState transition(TaskImpl task, TaskEvent event) {
-      task.handleTaskAttemptCompletion(
-          ((TaskTAttemptEvent) event).getTaskAttemptID(), 
-          TaskAttemptCompletionEventStatus.KILLED);
-      // check whether all attempts are finished
-      if (task.finishedAttempts == task.attempts.size()) {
-        if (task.historyTaskStartGenerated) {
-        TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
-              finalState, null); // TODO JH verify failedAttempt null
-        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
-            taskFailedEvent)); 
-        } else {
-          LOG.debug("Not generating HistoryFinish event since start event not" +
-          		" generated for task: " + task.getID());
-        }
-
-        task.eventHandler.handle(
-            new JobTaskEvent(task.taskId, finalState));
-        return finalState;
-      }
-      return task.getState();
-    }
-  }
-
-  private static class AttemptFailedTransition implements
-    MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
-
-    @Override
-    public TaskState transition(TaskImpl task, TaskEvent event) {
-      task.failedAttempts++;
-      TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
-      TaskAttempt attempt = task.attempts.get(castEvent.getTaskAttemptID());
-      if (attempt.getAssignedContainerMgrAddress() != null) {
-        //container was assigned
-        task.eventHandler.handle(new ContainerFailedEvent(attempt.getID(), 
-            attempt.getAssignedContainerMgrAddress()));
-      }
-      
-      if (task.failedAttempts < task.maxAttempts) {
-        task.handleTaskAttemptCompletion(
-            ((TaskTAttemptEvent) event).getTaskAttemptID(), 
-            TaskAttemptCompletionEventStatus.FAILED);
-        // we don't need a new event if we already have a spare
-        if (--task.numberUncompletedAttempts == 0
-            && task.successfulAttempt == null) {
-          task.addAndScheduleAttempt();
-        }
-      } else {
-        task.handleTaskAttemptCompletion(
-            ((TaskTAttemptEvent) event).getTaskAttemptID(), 
-            TaskAttemptCompletionEventStatus.TIPFAILED);
-        TaskTAttemptEvent ev = (TaskTAttemptEvent) event;
-        TaskAttemptId taId = ev.getTaskAttemptID();
-        
-        if (task.historyTaskStartGenerated) {
-        TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, attempt.getDiagnostics(),
-            TaskState.FAILED, taId);
-        task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
-            taskFailedEvent));
-        } else {
-          LOG.debug("Not generating HistoryFinish event since start event not" +
-          		" generated for task: " + task.getID());
-        }
-        task.eventHandler.handle(
-            new JobTaskEvent(task.taskId, TaskState.FAILED));
-        return task.finished(TaskState.FAILED);
-      }
-      return getDefaultState(task);
-    }
-
-    protected TaskState getDefaultState(Task task) {
-      return task.getState();
-    }
-
-    protected void unSucceed(TaskImpl task) {
-      ++task.numberUncompletedAttempts;
-      task.successfulAttempt = null;
-    }
-  }
-
-  private static class MapRetroactiveFailureTransition
-      extends AttemptFailedTransition {
-
-    @Override
-    public TaskState transition(TaskImpl task, TaskEvent event) {
-      //verify that this occurs only for map task
-      //TODO: consider moving it to MapTaskImpl
-      if (!TaskType.MAP.equals(task.getType())) {
-        LOG.error("Unexpected event for REDUCE task " + event.getType());
-        task.internalError(event.getType());
-      }
-      
-      // tell the job about the rescheduling
-      task.eventHandler.handle(
-          new JobMapTaskRescheduledEvent(task.taskId));
-      // super.transition is mostly coded for the case where an
-      //  UNcompleted task failed.  When a COMPLETED task retroactively
-      //  fails, we have to let AttemptFailedTransition.transition
-      //  believe that there's no redundancy.
-      unSucceed(task);
-      return super.transition(task, event);
-    }
-
-    @Override
-    protected TaskState getDefaultState(Task task) {
-      return TaskState.SCHEDULED;
-    }
-  }
-
-  private static class KillNewTransition 
-    implements SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      
-      if (task.historyTaskStartGenerated) {
-      TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
-            TaskState.KILLED, null); // TODO Verify failedAttemptId is null
-      task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
-          taskFailedEvent));
-      }else {
-        LOG.debug("Not generating HistoryFinish event since start event not" +
-        		" generated for task: " + task.getID());
-      }
-
-      task.eventHandler.handle(
-          new JobTaskEvent(task.taskId, TaskState.KILLED));
-      task.metrics.endWaitingTask(task);
-    }
-  }
-
-  private void killUnfinishedAttempt(TaskAttempt attempt, String logMsg) {
-    if (attempt != null && !attempt.isFinished()) {
-      eventHandler.handle(
-          new TaskAttemptEvent(attempt.getID(),
-              TaskAttemptEventType.TA_KILL));
-    }
-  }
-
-  private static class KillTransition 
-    implements SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      // issue kill to all non finished attempts
-      for (TaskAttempt attempt : task.attempts.values()) {
-        task.killUnfinishedAttempt
-            (attempt, "Task KILL is received. Killing attempt!");
-      }
-
-      task.numberUncompletedAttempts = 0;
-    }
-  }
-
-  static class LaunchTransition
-      implements SingleArcTransition<TaskImpl, TaskEvent> {
-    @Override
-    public void transition(TaskImpl task, TaskEvent event) {
-      task.metrics.launchedTask(task);
-      task.metrics.runningTask(task);
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java

@@ -1,31 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.launcher;
-
-
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public interface ContainerLauncher 
-    extends EventHandler<ContainerLauncherEvent> {
-
-  enum EventType {
-    CONTAINER_REMOTE_LAUNCH,
-    CONTAINER_REMOTE_CLEANUP
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherEvent.java

@@ -1,114 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.launcher;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-public class ContainerLauncherEvent 
-    extends AbstractEvent<ContainerLauncher.EventType> {
-
-  private TaskAttemptId taskAttemptID;
-  private ContainerId containerID;
-  private String containerMgrAddress;
-  private ContainerToken containerToken;
-
-  public ContainerLauncherEvent(TaskAttemptId taskAttemptID, 
-      ContainerId containerID,
-      String containerMgrAddress,
-      ContainerToken containerToken,
-      ContainerLauncher.EventType type) {
-    super(type);
-    this.taskAttemptID = taskAttemptID;
-    this.containerID = containerID;
-    this.containerMgrAddress = containerMgrAddress;
-    this.containerToken = containerToken;
-  }
-
-  public TaskAttemptId getTaskAttemptID() {
-    return this.taskAttemptID;
-  }
-
-  public ContainerId getContainerID() {
-    return containerID;
-  }
-
-  public String getContainerMgrAddress() {
-    return containerMgrAddress;
-  }
-
-  public ContainerToken getContainerToken() {
-    return containerToken;
-  }
-
-  @Override
-  public String toString() {
-    return super.toString() + " for taskAttempt " + taskAttemptID;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result
-        + ((containerID == null) ? 0 : containerID.hashCode());
-    result = prime * result
-        + ((containerMgrAddress == null) ? 0 : containerMgrAddress.hashCode());
-    result = prime * result
-        + ((containerToken == null) ? 0 : containerToken.hashCode());
-    result = prime * result
-        + ((taskAttemptID == null) ? 0 : taskAttemptID.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    ContainerLauncherEvent other = (ContainerLauncherEvent) obj;
-    if (containerID == null) {
-      if (other.containerID != null)
-        return false;
-    } else if (!containerID.equals(other.containerID))
-      return false;
-    if (containerMgrAddress == null) {
-      if (other.containerMgrAddress != null)
-        return false;
-    } else if (!containerMgrAddress.equals(other.containerMgrAddress))
-      return false;
-    if (containerToken == null) {
-      if (other.containerToken != null)
-        return false;
-    } else if (!containerToken.equals(other.containerToken))
-      return false;
-    if (taskAttemptID == null) {
-      if (other.taskAttemptID != null)
-        return false;
-    } else if (!taskAttemptID.equals(other.taskAttemptID))
-      return false;
-    return true;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java

@@ -1,279 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.launcher;
-
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityInfo;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.ContainerManager;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-/**
- * This class is responsible for launching of containers.
- */
-public class ContainerLauncherImpl extends AbstractService implements
-    ContainerLauncher {
-
-  private static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
-
-  private AppContext context;
-  private ThreadPoolExecutor launcherPool;
-  private Thread eventHandlingThread;
-  private BlockingQueue<ContainerLauncherEvent> eventQueue =
-      new LinkedBlockingQueue<ContainerLauncherEvent>();
-  private RecordFactory recordFactory;
-  //have a cache/map of UGIs so as to avoid creating too many RPC
-  //client connection objects to the same NodeManager
-  private Map<String, UserGroupInformation> ugiMap = 
-    new HashMap<String, UserGroupInformation>();
-
-  public ContainerLauncherImpl(AppContext context) {
-    super(ContainerLauncherImpl.class.getName());
-    this.context = context;
-  }
-
-  @Override
-  public synchronized void init(Configuration conf) {
-    // Clone configuration for this component so that the SecurityInfo setting
-    // doesn't affect the original configuration
-    Configuration myLocalConfig = new Configuration(conf);
-    myLocalConfig.setClass(
-        YarnConfiguration.YARN_SECURITY_INFO,
-        ContainerManagerSecurityInfo.class, SecurityInfo.class);
-    this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
-    super.init(myLocalConfig);
-  }
-
-  public void start() {
-    launcherPool =
-        new ThreadPoolExecutor(getConfig().getInt(
-            AMConstants.CONTAINERLAUNCHER_THREADPOOL_SIZE, 10),
-            Integer.MAX_VALUE, 1, TimeUnit.HOURS,
-            new LinkedBlockingQueue<Runnable>());
-    launcherPool.prestartAllCoreThreads(); // Wait for work.
-    eventHandlingThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        ContainerLauncherEvent event = null;
-        while (!Thread.currentThread().isInterrupted()) {
-          try {
-            event = eventQueue.take();
-          } catch (InterruptedException e) {
-            LOG.error("Returning, interrupted : " + e);
-            return;
-          }
-          // the events from the queue are handled in parallel
-          // using a thread pool
-          launcherPool.execute(new EventProcessor(event));
-
-          // TODO: Group launching of multiple containers to a single
-          // NodeManager into a single connection
-        }
-      }
-    });
-    eventHandlingThread.start();
-    super.start();
-  }
-
-  public void stop() {
-    eventHandlingThread.interrupt();
-    launcherPool.shutdown();
-    super.stop();
-  }
-
-  protected ContainerManager getCMProxy(ContainerId containerID,
-      final String containerManagerBindAddr, ContainerToken containerToken)
-      throws IOException {
-
-    UserGroupInformation user = UserGroupInformation.getCurrentUser();
-
-    // TODO: Synchronization problems!!
-    if (UserGroupInformation.isSecurityEnabled()) {
-      if(!ugiMap.containsKey(containerManagerBindAddr)) {
-        Token<ContainerTokenIdentifier> token =
-          new Token<ContainerTokenIdentifier>(
-              containerToken.getIdentifier().array(),
-              containerToken.getPassword().array(), new Text(
-                  containerToken.getKind()), new Text(
-                      containerToken.getService()));
-        //the user in createRemoteUser in this context is not important
-        user = UserGroupInformation.createRemoteUser(containerManagerBindAddr);
-        user.addToken(token);
-        ugiMap.put(containerManagerBindAddr, user);
-      } else {
-        user = ugiMap.get(containerManagerBindAddr);    
-      }
-    }
-    ContainerManager proxy =
-        user.doAs(new PrivilegedAction<ContainerManager>() {
-          @Override
-          public ContainerManager run() {
-            YarnRPC rpc = YarnRPC.create(getConfig());
-            return (ContainerManager) rpc.getProxy(ContainerManager.class,
-                NetUtils.createSocketAddr(containerManagerBindAddr),
-                getConfig());
-          }
-        });
-    return proxy;
-  }
-
-  /**
-   * Setup and start the container on remote nodemanager.
-   */
-  private class EventProcessor implements Runnable {
-    private ContainerLauncherEvent event;
-
-    EventProcessor(ContainerLauncherEvent event) {
-      this.event = event;
-    }
-
-    @Override
-    public void run() {
-      LOG.info("Processing the event " + event.toString());
-
-      // Load ContainerManager tokens before creating a connection.
-      // TODO: Do it only once per NodeManager.
-      final String containerManagerBindAddr = event.getContainerMgrAddress();
-      ContainerId containerID = event.getContainerID();
-      ContainerToken containerToken = event.getContainerToken();
-
-      switch(event.getType()) {
-
-      case CONTAINER_REMOTE_LAUNCH:
-        ContainerRemoteLaunchEvent launchEv = (ContainerRemoteLaunchEvent) event;
-
-        TaskAttemptId taskAttemptID = launchEv.getTaskAttemptID();
-        try {
-          
-          ContainerManager proxy = 
-            getCMProxy(containerID, containerManagerBindAddr, containerToken);
-          
-          // Construct the actual Container
-          ContainerLaunchContext containerLaunchContext =
-              launchEv.getContainer();
-
-          // Now launch the actual container
-          StartContainerRequest startRequest = recordFactory
-              .newRecordInstance(StartContainerRequest.class);
-          startRequest.setContainerLaunchContext(containerLaunchContext);
-          proxy.startContainer(startRequest);
-
-          LOG.info("Returning from container-launch for " + taskAttemptID);
-
-          // after launching, send launched event to task attempt to move
-          // it from ASSIGNED to RUNNING state
-          context.getEventHandler().handle(
-              new TaskAttemptEvent(taskAttemptID,
-                  TaskAttemptEventType.TA_CONTAINER_LAUNCHED));
-        } catch (Throwable t) {
-          String message = "Container launch failed for " + containerID
-              + " : " + StringUtils.stringifyException(t);
-          LOG.error(message);
-          context.getEventHandler().handle(
-              new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID, message));
-          context.getEventHandler().handle(
-              new TaskAttemptEvent(taskAttemptID,
-                  TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
-        }
-
-        break;
-
-      case CONTAINER_REMOTE_CLEANUP:
-        // We will have to remove the launch (meant "cleanup"? FIXME) event if it is still in eventQueue
-        // and not yet processed
-        if (eventQueue.contains(event)) {
-          eventQueue.remove(event); // TODO: Any synchro needed?
-          //deallocate the container
-          context.getEventHandler().handle(
-              new ContainerAllocatorEvent(event.getTaskAttemptID(),
-              ContainerAllocator.EventType.CONTAINER_DEALLOCATE));
-        } else {
-          try {
-            ContainerManager proxy = 
-              getCMProxy(containerID, containerManagerBindAddr, containerToken);
-            // TODO:check whether container is launched
-
-            // kill the remote container if already launched
-            StopContainerRequest stopRequest = recordFactory
-                .newRecordInstance(StopContainerRequest.class);
-            stopRequest.setContainerId(event.getContainerID());
-            proxy.stopContainer(stopRequest);
-
-          } catch (Throwable t) {
-            //ignore the cleanup failure
-            LOG.warn("cleanup failed for container " + event.getContainerID() ,
-                t);
-          }
-
-          // after killing, send killed event to taskattempt
-          context.getEventHandler().handle(
-              new TaskAttemptEvent(event.getTaskAttemptID(),
-                  TaskAttemptEventType.TA_CONTAINER_CLEANED));
-        }
-        break;
-      }
-    }
-    
-  }
-
-  @Override
-  public void handle(ContainerLauncherEvent event) {
-    try {
-      eventQueue.put(event);
-    } catch (InterruptedException e) {
-      throw new YarnException(e);
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java

@@ -1,40 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.launcher;
-
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
-
-public abstract class ContainerRemoteLaunchEvent extends ContainerLauncherEvent {
-
-  public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID,
-      ContainerId containerID, String containerMgrAddress,
-      ContainerToken containerToken) {
-    super(taskAttemptID, containerID, containerMgrAddress,
-        containerToken,
-        ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH);
-  }
-  public abstract ContainerLaunchContext getContainer();
-
-  public abstract Task getRemoteTask();
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java

@@ -1,100 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.local;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.JobCounter;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * Allocates containers locally. Doesn't allocate a real container;
- * instead sends an allocated event for all requests.
- */
-public class LocalContainerAllocator extends RMCommunicator
-    implements ContainerAllocator {
-
-  private static final Log LOG =
-      LogFactory.getLog(LocalContainerAllocator.class);
-
-  private final EventHandler eventHandler;
-  private final ApplicationId appID;
-  private AtomicInteger containerCount = new AtomicInteger();
-
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
-
-  public LocalContainerAllocator(ClientService clientService,
-                                 AppContext context) {
-    super(clientService, context);
-    this.eventHandler = context.getEventHandler();
-    this.appID = context.getApplicationID();
-  }
-
-  @Override
-  public void handle(ContainerAllocatorEvent event) {
-    if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
-      LOG.info("Processing the event " + event.toString());
-      ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
-      cID.setAppId(appID);
-      // use negative ids to denote that these are local. Need a better way ??
-      cID.setId((-1) * containerCount.getAndIncrement());
-      
-      Container container = recordFactory.newRecordInstance(Container.class);
-      container.setId(cID);
-      NodeId nodeId = Records.newRecord(NodeId.class);
-      nodeId.setHost("localhost");
-      nodeId.setPort(1234);
-      container.setNodeId(nodeId);
-      container.setContainerToken(null);
-      container.setNodeHttpAddress("localhost:9999");
-      // send the container-assigned event to task attempt
-
-      if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
-        JobCounterUpdateEvent jce =
-            new JobCounterUpdateEvent(event.getAttemptID().getTaskId()
-                .getJobId());
-        // TODO Setting OTHER_LOCAL_MAP for now.
-        jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
-        eventHandler.handle(jce);
-      }
-      eventHandler.handle(new TaskAttemptContainerAssignedEvent(
-          event.getAttemptID(), container));
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/metrics/MRAppMetrics.java

@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapreduce.v2.app.metrics;
-
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterInt;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-
-@Metrics(about="MR App Metrics", context="mapred")
-public class MRAppMetrics {
-  @Metric MutableCounterInt jobsSubmitted;
-  @Metric MutableCounterInt jobsCompleted;
-  @Metric MutableCounterInt jobsFailed;
-  @Metric MutableCounterInt jobsKilled;
-  @Metric MutableGaugeInt jobsPreparing;
-  @Metric MutableGaugeInt jobsRunning;
-
-  @Metric MutableCounterInt mapsLaunched;
-  @Metric MutableCounterInt mapsCompleted;
-  @Metric MutableCounterInt mapsFailed;
-  @Metric MutableCounterInt mapsKilled;
-  @Metric MutableGaugeInt mapsRunning;
-  @Metric MutableGaugeInt mapsWaiting;
-
-  @Metric MutableCounterInt reducesLaunched;
-  @Metric MutableCounterInt reducesCompleted;
-  @Metric MutableCounterInt reducesFailed;
-  @Metric MutableCounterInt reducesKilled;
-  @Metric MutableGaugeInt reducesRunning;
-  @Metric MutableGaugeInt reducesWaiting;
-  
-  public static MRAppMetrics create() {
-    return create(DefaultMetricsSystem.instance());
-  }
-
-  public static MRAppMetrics create(MetricsSystem ms) {
-    JvmMetrics.create("MRAppMaster", null, ms);
-    return ms.register(new MRAppMetrics());
-  }
-
-  // potential instrumentation interface methods
-
-  public void submittedJob(Job job) {
-    jobsSubmitted.incr();
-  }
-
-  public void completedJob(Job job) {
-    jobsCompleted.incr();
-  }
-
-  public void failedJob(Job job) {
-    jobsFailed.incr();
-  }
-
-  public void killedJob(Job job) {
-    jobsKilled.incr();
-  }
-
-  public void preparingJob(Job job) {
-    jobsPreparing.incr();
-  }
-
-  public void endPreparingJob(Job job) {
-    jobsPreparing.decr();
-  }
-
-  public void runningJob(Job job) {
-    jobsRunning.incr();
-  }
-
-  public void endRunningJob(Job job) {
-    jobsRunning.decr();
-  }
-
-  public void launchedTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsLaunched.incr();
-        break;
-      case REDUCE:
-        reducesLaunched.incr();
-        break;
-    }
-    endWaitingTask(task);
-  }
-
-  public void completedTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsCompleted.incr();
-        break;
-      case REDUCE:
-        reducesCompleted.incr();
-        break;
-    }
-  }
-
-  public void failedTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsFailed.incr();
-        break;
-      case REDUCE:
-        reducesFailed.incr();
-        break;
-    }
-  }
-
-  public void killedTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsKilled.incr();
-        break;
-      case REDUCE:
-        reducesKilled.incr();
-        break;
-    }
-  }
-
-  public void runningTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsRunning.incr();
-        break;
-      case REDUCE:
-        reducesRunning.incr();
-        break;
-    }
-  }
-
-  public void endRunningTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsRunning.decr();
-        break;
-      case REDUCE:
-        reducesRunning.decr();
-        break;
-    }
-  }
-
-  public void waitingTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsWaiting.incr();
-        break;
-      case REDUCE:
-        reducesWaiting.incr();
-    }
-  }
-
-  public void endWaitingTask(Task task) {
-    switch (task.getType()) {
-      case MAP:
-        mapsWaiting.decr();
-        break;
-      case REDUCE:
-        reducesWaiting.decr();
-        break;
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/ControlledClock.java

@@ -1,43 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.mapreduce.v2.app.recover;
-
-import org.apache.hadoop.yarn.Clock;
-
-class ControlledClock implements Clock {
-  private long time = -1;
-  private final Clock actualClock;
-  ControlledClock(Clock actualClock) {
-    this.actualClock = actualClock;
-  }
-  synchronized void setTime(long time) {
-    this.time = time;
-  }
-  synchronized void reset() {
-    time = -1;
-  }
-
-  @Override
-  public synchronized long getTime() {
-    if (time != -1) {
-      return time;
-    }
-    return actualClock.getTime();
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/Recovery.java

@@ -1,34 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.recover;
-
-import java.util.Set;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.event.Dispatcher;
-
-public interface Recovery {
-
-  Dispatcher getDispatcher();
-
-  Clock getClock();
-  
-  Set<TaskId> getCompletedTasks();
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java

@@ -1,368 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.recover;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.Phase;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
-import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
-import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleaner;
-import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.service.CompositeService;
-import org.apache.hadoop.yarn.service.Service;
-
-/*
- * Recovers the completed tasks from the previous life of Application Master.
- * The completed tasks are deciphered from the history file of the previous life.
- * Recovery service intercepts and replay the events for completed tasks.
- * While recovery is in progress, the scheduling of new tasks are delayed by 
- * buffering the task schedule events.
- * The recovery service controls the clock while recovery is in progress.
- */
-
-//TODO:
-//task cleanup for all non completed tasks
-//change job output committer to have 
-//    - atomic job output promotion
-//    - recover output of completed tasks
-
-public class RecoveryService extends CompositeService implements Recovery {
-
-  private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
-  private static final Log LOG = LogFactory.getLog(RecoveryService.class);
-
-  private final ApplicationId appID;
-  private final Dispatcher dispatcher;
-  private final ControlledClock clock;
-  private final int startCount;
-
-  private JobInfo jobInfo = null;
-  private final Map<TaskId, TaskInfo> completedTasks =
-    new HashMap<TaskId, TaskInfo>();
-
-  private final List<TaskEvent> pendingTaskScheduleEvents =
-    new ArrayList<TaskEvent>();
-
-  private volatile boolean recoveryMode = false;
-
-  public RecoveryService(ApplicationId appID, Clock clock, int startCount) {
-    super("RecoveringDispatcher");
-    this.appID = appID;
-    this.startCount = startCount;
-    this.dispatcher = new RecoveryDispatcher();
-    this.clock = new ControlledClock(clock);
-      addService((Service) dispatcher);
-  }
-
-  @Override
-  public void init(Configuration conf) {
-    super.init(conf);
-    // parse the history file
-    try {
-      parse();
-      if (completedTasks.size() > 0) {
-        recoveryMode = true;
-        LOG.info("SETTING THE RECOVERY MODE TO TRUE. NO OF COMPLETED TASKS " + 
-            "TO RECOVER " + completedTasks.size());
-        LOG.info("Job launch time " + jobInfo.getLaunchTime());
-        clock.setTime(jobInfo.getLaunchTime());
-      }
-    } catch (IOException e) {
-      LOG.warn(e);
-      LOG.warn("Could not parse the old history file. Aborting recovery. "
-          + "Starting afresh.");
-    }
-  }
-
-  @Override
-  public Dispatcher getDispatcher() {
-    return dispatcher;
-  }
-
-  @Override
-  public Clock getClock() {
-    return clock;
-  }
-
-  @Override
-  public Set<TaskId> getCompletedTasks() {
-    return completedTasks.keySet();
-  }
-
-  private void parse() throws IOException {
-    // TODO: parse history file based on startCount
-    String jobName = TypeConverter.fromYarn(appID).toString();
-    String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(getConfig());
-    FSDataInputStream in = null;
-    Path historyFile = null;
-    Path histDirPath = FileContext.getFileContext(getConfig()).makeQualified(
-        new Path(jobhistoryDir));
-    FileContext fc = FileContext.getFileContext(histDirPath.toUri(),
-        getConfig());
-    historyFile = fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(
-        histDirPath, jobName, startCount - 1));          //read the previous history file
-    in = fc.open(historyFile);
-    JobHistoryParser parser = new JobHistoryParser(in);
-    jobInfo = parser.parse();
-    Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
-        .getAllTasks();
-    for (TaskInfo taskInfo : taskInfos.values()) {
-      if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
-        completedTasks
-            .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo);
-        LOG.info("Read from history task "
-            + TypeConverter.toYarn(taskInfo.getTaskId()));
-      }
-    }
-    LOG.info("Read completed tasks from history "
-        + completedTasks.size());
-  }
-
-  class RecoveryDispatcher extends AsyncDispatcher {
-    private final EventHandler actualHandler;
-    private final EventHandler handler;
-
-    RecoveryDispatcher() {
-      actualHandler = super.getEventHandler();
-      handler = new InterceptingEventHandler(actualHandler);
-    }
-
-    @Override
-    public void dispatch(Event event) {
-      if (recoveryMode) {
-        if (event.getType() == TaskAttemptEventType.TA_CONTAINER_LAUNCHED) {
-          TaskAttemptInfo attInfo = getTaskAttemptInfo(((TaskAttemptEvent) event)
-              .getTaskAttemptID());
-          LOG.info("Attempt start time " + attInfo.getStartTime());
-          clock.setTime(attInfo.getStartTime());
-
-        } else if (event.getType() == TaskAttemptEventType.TA_DONE
-            || event.getType() == TaskAttemptEventType.TA_FAILMSG
-            || event.getType() == TaskAttemptEventType.TA_KILL) {
-          TaskAttemptInfo attInfo = getTaskAttemptInfo(((TaskAttemptEvent) event)
-              .getTaskAttemptID());
-          LOG.info("Attempt finish time " + attInfo.getFinishTime());
-          clock.setTime(attInfo.getFinishTime());
-        }
-
-        else if (event.getType() == TaskEventType.T_ATTEMPT_FAILED
-            || event.getType() == TaskEventType.T_ATTEMPT_KILLED
-            || event.getType() == TaskEventType.T_ATTEMPT_SUCCEEDED) {
-          TaskTAttemptEvent tEvent = (TaskTAttemptEvent) event;
-          LOG.info("Recovered Task attempt " + tEvent.getTaskAttemptID());
-          TaskInfo taskInfo = completedTasks.get(tEvent.getTaskAttemptID()
-              .getTaskId());
-          taskInfo.getAllTaskAttempts().remove(
-              TypeConverter.fromYarn(tEvent.getTaskAttemptID()));
-          // remove the task info from completed tasks if all attempts are
-          // recovered
-          if (taskInfo.getAllTaskAttempts().size() == 0) {
-            completedTasks.remove(tEvent.getTaskAttemptID().getTaskId());
-            // checkForRecoveryComplete
-            LOG.info("CompletedTasks() " + completedTasks.size());
-            if (completedTasks.size() == 0) {
-              recoveryMode = false;
-              clock.reset();
-              LOG.info("Setting the recovery mode to false. " +
-                 "Recovery is complete!");
-
-              // send all pending tasks schedule events
-              for (TaskEvent tEv : pendingTaskScheduleEvents) {
-                actualHandler.handle(tEv);
-              }
-
-            }
-          }
-        }
-      }
-      super.dispatch(event);
-    }
-
-    @Override
-    public EventHandler getEventHandler() {
-      return handler;
-    }
-  }
-
-  private TaskAttemptInfo getTaskAttemptInfo(TaskAttemptId id) {
-    TaskInfo taskInfo = completedTasks.get(id.getTaskId());
-    return taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn(id));
-  }
-
-  private class InterceptingEventHandler implements EventHandler {
-    EventHandler actualHandler;
-
-    InterceptingEventHandler(EventHandler actualHandler) {
-      this.actualHandler = actualHandler;
-    }
-
-    @Override
-    public void handle(Event event) {
-      if (!recoveryMode) {
-        // delegate to the dispatcher one
-        actualHandler.handle(event);
-        return;
-      }
-
-      else if (event.getType() == TaskEventType.T_SCHEDULE) {
-        TaskEvent taskEvent = (TaskEvent) event;
-        // delay the scheduling of new tasks till previous ones are recovered
-        if (completedTasks.get(taskEvent.getTaskID()) == null) {
-          LOG.debug("Adding to pending task events "
-              + taskEvent.getTaskID());
-          pendingTaskScheduleEvents.add(taskEvent);
-          return;
-        }
-      }
-
-      else if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
-        TaskAttemptId aId = ((ContainerAllocatorEvent) event).getAttemptID();
-        TaskAttemptInfo attInfo = getTaskAttemptInfo(aId);
-        LOG.debug("CONTAINER_REQ " + aId);
-        sendAssignedEvent(aId, attInfo);
-        return;
-      }
-
-      else if (event.getType() == TaskCleaner.EventType.TASK_CLEAN) {
-        TaskAttemptId aId = ((TaskCleanupEvent) event).getAttemptID();
-        LOG.debug("TASK_CLEAN");
-        actualHandler.handle(new TaskAttemptEvent(aId,
-            TaskAttemptEventType.TA_CLEANUP_DONE));
-        return;
-      }
-
-      else if (event.getType() == ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH) {
-        TaskAttemptId aId = ((ContainerRemoteLaunchEvent) event)
-            .getTaskAttemptID();
-        TaskAttemptInfo attInfo = getTaskAttemptInfo(aId);
-        actualHandler.handle(new TaskAttemptEvent(aId,
-            TaskAttemptEventType.TA_CONTAINER_LAUNCHED));
-        // send the status update event
-        sendStatusUpdateEvent(aId, attInfo);
-
-        TaskAttemptState state = TaskAttemptState.valueOf(attInfo.getTaskStatus());
-        switch (state) {
-        case SUCCEEDED:
-          // send the done event
-          LOG.info("Sending done event to " + aId);
-          actualHandler.handle(new TaskAttemptEvent(aId,
-              TaskAttemptEventType.TA_DONE));
-          break;
-        case KILLED:
-          LOG.info("Sending kill event to " + aId);
-          actualHandler.handle(new TaskAttemptEvent(aId,
-              TaskAttemptEventType.TA_KILL));
-          break;
-        default:
-          LOG.info("Sending fail event to " + aId);
-          actualHandler.handle(new TaskAttemptEvent(aId,
-              TaskAttemptEventType.TA_FAILMSG));
-          break;
-        }
-        return;
-      }
-
-      // delegate to the actual handler
-      actualHandler.handle(event);
-    }
-
-    private void sendStatusUpdateEvent(TaskAttemptId yarnAttemptID,
-        TaskAttemptInfo attemptInfo) {
-      LOG.info("Sending status update event to " + yarnAttemptID);
-      TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
-      taskAttemptStatus.id = yarnAttemptID;
-      taskAttemptStatus.progress = 1.0f;
-      taskAttemptStatus.diagnosticInfo = "";
-      taskAttemptStatus.stateString = attemptInfo.getTaskStatus(); 
-      // taskAttemptStatus.outputSize = attemptInfo.getOutputSize();
-      taskAttemptStatus.phase = Phase.CLEANUP;
-      org.apache.hadoop.mapreduce.Counters cntrs = attemptInfo.getCounters();
-      if (cntrs == null) {
-        taskAttemptStatus.counters = null;
-      } else {
-        taskAttemptStatus.counters = TypeConverter.toYarn(attemptInfo
-            .getCounters());
-      }
-      actualHandler.handle(new TaskAttemptStatusUpdateEvent(
-          taskAttemptStatus.id, taskAttemptStatus));
-    }
-
-    private void sendAssignedEvent(TaskAttemptId yarnAttemptID,
-        TaskAttemptInfo attemptInfo) {
-      LOG.info("Sending assigned event to " + yarnAttemptID);
-      ContainerId cId = recordFactory
-          .newRecordInstance(ContainerId.class);
-      Container container = recordFactory
-          .newRecordInstance(Container.class);
-      container.setId(cId);
-      container.setNodeId(recordFactory
-          .newRecordInstance(NodeId.class));
-      container.setContainerToken(null);
-      container.setNodeHttpAddress(attemptInfo.getHostname() + ":" + 
-          attemptInfo.getHttpPort());
-      actualHandler.handle(new TaskAttemptContainerAssignedEvent(yarnAttemptID,
-          container));
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerAllocator.java

@@ -1,32 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public interface ContainerAllocator extends EventHandler<ContainerAllocatorEvent>{
-
-  enum EventType {
-
-    CONTAINER_REQ,
-    CONTAINER_DEALLOCATE,
-    CONTAINER_FAILED
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerAllocatorEvent.java

@@ -1,38 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-public class ContainerAllocatorEvent extends 
-    AbstractEvent<ContainerAllocator.EventType> {
-  
-  private TaskAttemptId attemptID;
-
-  public ContainerAllocatorEvent(TaskAttemptId attemptID,
-      ContainerAllocator.EventType type) {
-    super(type);
-    this.attemptID = attemptID;
-  }
-
-  public TaskAttemptId getAttemptID() {
-    return attemptID;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerFailedEvent.java

@@ -1,18 +0,0 @@
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-
-public class ContainerFailedEvent extends ContainerAllocatorEvent {
-
-  private final String contMgrAddress;
-  
-  public ContainerFailedEvent(TaskAttemptId attemptID, String contMgrAddr) {
-    super(attemptID, ContainerAllocator.EventType.CONTAINER_FAILED);
-    this.contMgrAddress = contMgrAddr;
-  }
-
-  public String getContMgrAddress() {
-    return contMgrAddress;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ContainerRequestEvent.java

@@ -1,68 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-
-public class ContainerRequestEvent extends ContainerAllocatorEvent {
-  
-  private final Resource capability;
-  private final String[] hosts;
-  private final String[] racks;
-  private boolean earlierAttemptFailed = false;
-
-  public ContainerRequestEvent(TaskAttemptId attemptID, 
-      Resource capability,
-      String[] hosts, String[] racks) {
-    super(attemptID, ContainerAllocator.EventType.CONTAINER_REQ);
-    this.capability = capability;
-    this.hosts = hosts;
-    this.racks = racks;
-  }
-  
-  ContainerRequestEvent(TaskAttemptId attemptID, Resource capability) {
-    this(attemptID, capability, new String[0], new String[0]);
-    this.earlierAttemptFailed = true;
-  }
-  
-  public static ContainerRequestEvent createContainerRequestEventForFailedContainer(
-      TaskAttemptId attemptID, 
-      Resource capability) {
-    //ContainerRequest for failed events does not consider rack / node locality?
-    return new ContainerRequestEvent(attemptID, capability);
-  }
-
-  public Resource getCapability() {
-    return capability;
-  }
-
-  public String[] getHosts() {
-    return hosts;
-  }
-  
-  public String[] getRacks() {
-    return racks;
-  }
-  
-  public boolean getEarlierAttemptFailed() {
-    return earlierAttemptFailed;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java

@@ -1,280 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.JobState;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityInfo;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.AMRMProtocol;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.SchedulerSecurityInfo;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-/**
- * Registers/unregisters to RM and sends heartbeats to RM.
- */
-public class RMCommunicator extends AbstractService  {
-  private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
-  private int rmPollInterval;//millis
-  protected ApplicationId applicationId;
-  protected ApplicationAttemptId applicationAttemptId;
-  private volatile boolean stopped;
-  protected Thread allocatorThread;
-  protected EventHandler eventHandler;
-  protected AMRMProtocol scheduler;
-  private final ClientService clientService;
-  private int lastResponseID;
-  private Resource minContainerCapability;
-  private Resource maxContainerCapability;
-
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
-  
-  private final AppContext context;
-  private Job job;
-
-  public RMCommunicator(ClientService clientService, AppContext context) {
-    super("RMCommunicator");
-    this.clientService = clientService;
-    this.context = context;
-    this.eventHandler = context.getEventHandler();
-    this.applicationId = context.getApplicationID();
-    this.applicationAttemptId = context.getApplicationAttemptId();
-  }
-
-  @Override
-  public void init(Configuration conf) {
-    super.init(conf);
-    rmPollInterval =
-        conf.getInt(AMConstants.AM_RM_SCHEDULE_INTERVAL,
-            AMConstants.DEFAULT_AM_RM_SCHEDULE_INTERVAL);
-  }
-
-  @Override
-  public void start() {
-    scheduler= createSchedulerProxy();
-    //LOG.info("Scheduler is " + scheduler);
-    register();
-    startAllocatorThread();
-    JobID id = TypeConverter.fromYarn(context.getApplicationID());
-    JobId jobId = TypeConverter.toYarn(id);
-    job = context.getJob(jobId);
-    super.start();
-  }
-
-  protected AppContext getContext() {
-    return context;
-  }
-
-  protected Job getJob() {
-    return job;
-  }
-
-  protected void register() {
-    //Register
-    String host = 
-      clientService.getBindAddress().getAddress().getHostAddress();
-    try {
-      RegisterApplicationMasterRequest request =
-        recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
-      request.setApplicationAttemptId(applicationAttemptId);
-      request.setHost(host);
-      request.setRpcPort(clientService.getBindAddress().getPort());
-      request.setTrackingUrl(host + ":" + clientService.getHttpPort());
-      RegisterApplicationMasterResponse response = 
-        scheduler.registerApplicationMaster(request);
-      minContainerCapability = response.getMinimumResourceCapability();
-      maxContainerCapability = response.getMaximumResourceCapability();
-      LOG.info("minContainerCapability: " + minContainerCapability.getMemory());
-      LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
-    } catch (Exception are) {
-      LOG.info("Exception while registering", are);
-      throw new YarnException(are);
-    }
-  }
-
-  protected void unregister() {
-    try {
-      String finalState = "RUNNING";
-      if (job.getState() == JobState.SUCCEEDED) {
-        finalState = "SUCCEEDED";
-      } else if (job.getState() == JobState.KILLED) {
-        finalState = "KILLED";
-      } else if (job.getState() == JobState.FAILED
-          || job.getState() == JobState.ERROR) {
-        finalState = "FAILED";
-      }
-      StringBuffer sb = new StringBuffer();
-      for (String s : job.getDiagnostics()) {
-        sb.append(s).append("\n");
-      }
-      LOG.info("Setting job diagnostics to " + sb.toString());
-      
-      String historyUrl = JobHistoryUtils.getHistoryUrl(getConfig(), 
-          context.getApplicationID());
-      LOG.info("History url is " + historyUrl);
-
-      FinishApplicationMasterRequest request =
-          recordFactory.newRecordInstance(FinishApplicationMasterRequest.class);
-      request.setAppAttemptId(this.applicationAttemptId);
-      request.setFinalState(finalState.toString());
-      request.setDiagnostics(sb.toString());
-      request.setTrackingUrl(historyUrl);
-      scheduler.finishApplicationMaster(request);
-    } catch(Exception are) {
-      LOG.info("Exception while unregistering ", are);
-    }
-  }
-
-  protected Resource getMinContainerCapability() {
-    return minContainerCapability;
-  }
-  
-  protected Resource getMaxContainerCapability() {
-    return maxContainerCapability;
-  }
-
-  @Override
-  public void stop() {
-    stopped = true;
-    allocatorThread.interrupt();
-    try {
-      allocatorThread.join();
-    } catch (InterruptedException ie) {
-      LOG.info("InterruptedException while stopping", ie);
-    }
-    unregister();
-    super.stop();
-  }
-
-  protected void startAllocatorThread() {
-    allocatorThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        while (!stopped && !Thread.currentThread().isInterrupted()) {
-          try {
-            Thread.sleep(rmPollInterval);
-            try {
-              heartbeat();
-            } catch (Exception e) {
-              LOG.error("ERROR IN CONTACTING RM. ", e);
-              // TODO: for other exceptions
-            }
-          } catch (InterruptedException e) {
-            LOG.info("Allocated thread interrupted. Returning.");
-            return;
-          }
-        }
-      }
-    });
-    allocatorThread.start();
-  }
-
-  protected AMRMProtocol createSchedulerProxy() {
-    final YarnRPC rpc = YarnRPC.create(getConfig());
-    final Configuration conf = new Configuration(getConfig());
-    final String serviceAddr = conf.get(
-        YarnConfiguration.SCHEDULER_ADDRESS,
-        YarnConfiguration.DEFAULT_SCHEDULER_BIND_ADDRESS);
-
-    UserGroupInformation currentUser;
-    try {
-      currentUser = UserGroupInformation.getCurrentUser();
-    } catch (IOException e) {
-      throw new YarnException(e);
-    }
-
-    if (UserGroupInformation.isSecurityEnabled()) {
-      conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
-          SchedulerSecurityInfo.class, SecurityInfo.class);
-
-      String tokenURLEncodedStr = System.getenv().get(
-          ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
-      LOG.debug("AppMasterToken is " + tokenURLEncodedStr);
-      Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
-
-      try {
-        token.decodeFromUrlString(tokenURLEncodedStr);
-      } catch (IOException e) {
-        throw new YarnException(e);
-      }
-
-      currentUser.addToken(token);
-    }
-
-    return currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
-      @Override
-      public AMRMProtocol run() {
-        return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
-            NetUtils.createSocketAddr(serviceAddr), conf);
-      }
-    });
-  }
-
-  protected synchronized void heartbeat() throws Exception {
-    AllocateRequest allocateRequest =
-        recordFactory.newRecordInstance(AllocateRequest.class);
-    allocateRequest.setApplicationAttemptId(applicationAttemptId);
-    allocateRequest.setResponseId(lastResponseID);
-    allocateRequest.addAllAsks(new ArrayList<ResourceRequest>());
-    allocateRequest.addAllReleases(new ArrayList<ContainerId>());
-    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
-    AMResponse response = allocateResponse.getAMResponse();
-    if (response.getReboot()) {
-      LOG.info("Event from RM: shutting down Application Master");
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

@@ -1,784 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobCounter;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.util.RackResolver;
-
-/**
- * Allocates the container from the ResourceManager scheduler.
- */
-public class RMContainerAllocator extends RMContainerRequestor
-    implements ContainerAllocator {
-
-  private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
-  
-  public static final 
-  float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f;
-  
-  private static final Priority PRIORITY_FAST_FAIL_MAP;
-  private static final Priority PRIORITY_REDUCE;
-  private static final Priority PRIORITY_MAP;
-  
-  static {
-    PRIORITY_FAST_FAIL_MAP = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
-    PRIORITY_FAST_FAIL_MAP.setPriority(5);
-    PRIORITY_REDUCE = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
-    PRIORITY_REDUCE.setPriority(10);
-    PRIORITY_MAP = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Priority.class);
-    PRIORITY_MAP.setPriority(20);
-  }
-  
-  /*
-  Vocabulory Used: 
-  pending -> requests which are NOT yet sent to RM
-  scheduled -> requests which are sent to RM but not yet assigned
-  assigned -> requests which are assigned to a container
-  completed -> request corresponding to which container has completed
-  
-  Lifecycle of map
-  scheduled->assigned->completed
-  
-  Lifecycle of reduce
-  pending->scheduled->assigned->completed
-  
-  Maps are scheduled as soon as their requests are received. Reduces are 
-  added to the pending and are ramped up (added to scheduled) based 
-  on completed maps and current availability in the cluster.
-  */
-  
-  //reduces which are not yet scheduled
-  private final LinkedList<ContainerRequest> pendingReduces = 
-    new LinkedList<ContainerRequest>();
-
-  //holds information about the assigned containers to task attempts
-  private final AssignedRequests assignedRequests = new AssignedRequests();
-  
-  //holds scheduled requests to be fulfilled by RM
-  private final ScheduledRequests scheduledRequests = new ScheduledRequests();
-  
-  private int containersAllocated = 0;
-  private int containersReleased = 0;
-  private int hostLocalAssigned = 0;
-  private int rackLocalAssigned = 0;
-  
-  private boolean recalculateReduceSchedule = false;
-  private int mapResourceReqt;//memory
-  private int reduceResourceReqt;//memory
-  private int completedMaps = 0;
-  private int completedReduces = 0;
-  
-  private boolean reduceStarted = false;
-  private float maxReduceRampupLimit = 0;
-  private float maxReducePreemptionLimit = 0;
-  private float reduceSlowStart = 0;
-
-  public RMContainerAllocator(ClientService clientService, AppContext context) {
-    super(clientService, context);
-  }
-
-  @Override
-  public void init(Configuration conf) {
-    super.init(conf);
-    reduceSlowStart = conf.getFloat(
-        MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 
-        DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART);
-    maxReduceRampupLimit = conf.getFloat(
-        AMConstants.REDUCE_RAMPUP_UP_LIMIT, 
-        AMConstants.DEFAULT_REDUCE_RAMP_UP_LIMIT);
-    maxReducePreemptionLimit = conf.getFloat(
-        AMConstants.REDUCE_PREEMPTION_LIMIT,
-        AMConstants.DEFAULT_REDUCE_PREEMPTION_LIMIT);
-    RackResolver.init(conf);
-  }
-
-  @Override
-  protected synchronized void heartbeat() throws Exception {
-    LOG.info("Before Scheduling: " + getStat());
-    List<Container> allocatedContainers = getResources();
-    LOG.info("After Scheduling: " + getStat());
-    if (allocatedContainers.size() > 0) {
-      LOG.info("Before Assign: " + getStat());
-      scheduledRequests.assign(allocatedContainers);
-      LOG.info("After Assign: " + getStat());
-    }
-    
-    if (recalculateReduceSchedule) {
-      preemptReducesIfNeeded();
-      scheduleReduces();
-      recalculateReduceSchedule = false;
-    }
-  }
-
-  @Override
-  public void stop() {
-    super.stop();
-    LOG.info("Final Stats: " + getStat());
-  }
-
-  @Override
-  public synchronized void handle(ContainerAllocatorEvent event) {
-    LOG.info("Processing the event " + event.toString());
-    recalculateReduceSchedule = true;
-    if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
-      ContainerRequestEvent reqEvent = (ContainerRequestEvent) event;
-      if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
-        if (mapResourceReqt == 0) {
-          mapResourceReqt = reqEvent.getCapability().getMemory();
-          int minSlotMemSize = getMinContainerCapability().getMemory();
-          mapResourceReqt = (int) Math.ceil((float) mapResourceReqt/minSlotMemSize) * minSlotMemSize;
-          LOG.info("mapResourceReqt:"+mapResourceReqt);
-          if (mapResourceReqt > getMaxContainerCapability().getMemory()) {
-            String diagMsg = "MAP capability required is more than the supported " +
-            "max container capability in the cluster. Killing the Job. mapResourceReqt: " + 
-            mapResourceReqt + " maxContainerCapability:" + getMaxContainerCapability().getMemory();
-            LOG.info(diagMsg);
-            eventHandler.handle(new JobDiagnosticsUpdateEvent(
-                getJob().getID(), diagMsg));
-            eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.JOB_KILL));
-          }
-        }
-        //set the rounded off memory
-        reqEvent.getCapability().setMemory(mapResourceReqt);
-        scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
-      } else {
-        if (reduceResourceReqt == 0) {
-          reduceResourceReqt = reqEvent.getCapability().getMemory();
-          int minSlotMemSize = getMinContainerCapability().getMemory();
-          //round off on slotsize
-          reduceResourceReqt = (int) Math.ceil((float) reduceResourceReqt/minSlotMemSize) * minSlotMemSize;
-          LOG.info("reduceResourceReqt:"+reduceResourceReqt);
-          if (reduceResourceReqt > getMaxContainerCapability().getMemory()) {
-            String diagMsg = "REDUCE capability required is more than the supported " +
-            "max container capability in the cluster. Killing the Job. reduceResourceReqt: " + 
-            reduceResourceReqt + " maxContainerCapability:" + getMaxContainerCapability().getMemory();
-            LOG.info(diagMsg);
-            eventHandler.handle(new JobDiagnosticsUpdateEvent(
-                getJob().getID(), diagMsg));
-            eventHandler.handle(new JobEvent(getJob().getID(), JobEventType.JOB_KILL));
-          }
-        }
-        //set the rounded off memory
-        reqEvent.getCapability().setMemory(reduceResourceReqt);
-        if (reqEvent.getEarlierAttemptFailed()) {
-          //add to the front of queue for fail fast
-          pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE));
-        } else {
-          pendingReduces.add(new ContainerRequest(reqEvent, PRIORITY_REDUCE));//reduces are added to pending and are slowly ramped up
-        }
-      }
-      
-    } else if (
-        event.getType() == ContainerAllocator.EventType.CONTAINER_DEALLOCATE) {
-      TaskAttemptId aId = event.getAttemptID();
-      
-      boolean removed = scheduledRequests.remove(aId);
-      if (!removed) {
-        ContainerId containerId = assignedRequests.get(aId);
-        if (containerId != null) {
-          removed = true;
-          assignedRequests.remove(aId);
-          containersReleased++;
-          release(containerId);
-        }
-      }
-      if (!removed) {
-        LOG.error("Could not deallocate container for task attemptId " + 
-            aId);
-      }
-    } else if (
-        event.getType() == ContainerAllocator.EventType.CONTAINER_FAILED) {
-      ContainerFailedEvent fEv = (ContainerFailedEvent) event;
-      String host = getHost(fEv.getContMgrAddress());
-      containerFailedOnHost(host);
-    }
-  }
-
-  private static String getHost(String contMgrAddress) {
-    String host = contMgrAddress;
-    String[] hostport = host.split(":");
-    if (hostport.length == 2) {
-      host = hostport[0];
-    }
-    return host;
-  }
-
-  private void preemptReducesIfNeeded() {
-    if (reduceResourceReqt == 0) {
-      return; //no reduces
-    }
-    //check if reduces have taken over the whole cluster and there are 
-    //unassigned maps
-    if (scheduledRequests.maps.size() > 0) {
-      int memLimit = getMemLimit();
-      int availableMemForMap = memLimit - ((assignedRequests.reduces.size() -
-          assignedRequests.preemptionWaitingReduces.size()) * reduceResourceReqt);
-      //availableMemForMap must be sufficient to run atleast 1 map
-      if (availableMemForMap < mapResourceReqt) {
-        //to make sure new containers are given to maps and not reduces
-        //ramp down all scheduled reduces if any
-        //(since reduces are scheduled at higher priority than maps)
-        LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size());
-        for (ContainerRequest req : scheduledRequests.reduces.values()) {
-          pendingReduces.add(req);
-        }
-        scheduledRequests.reduces.clear();
-        
-        //preempt for making space for atleast one map
-        int premeptionLimit = Math.max(mapResourceReqt, 
-            (int) (maxReducePreemptionLimit * memLimit));
-        
-        int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt, 
-            premeptionLimit);
-        
-        int toPreempt = (int) Math.ceil((float) preemptMem/reduceResourceReqt);
-        toPreempt = Math.min(toPreempt, assignedRequests.reduces.size());
-        
-        LOG.info("Going to preempt " + toPreempt);
-        assignedRequests.preemptReduce(toPreempt);
-      }
-    }
-  }
-
-  private void scheduleReduces() {
-    
-    if (pendingReduces.size() == 0) {
-      return;
-    }
-    
-    LOG.info("Recalculating schedule...");
-    
-    //if all maps are assigned, then ramp up all reduces irrespective of the 
-    //headroom
-    if (scheduledRequests.maps.size() == 0 && pendingReduces.size() > 0) {
-      LOG.info("All maps assigned. Ramping up all remaining reduces:" + pendingReduces.size());
-      for (ContainerRequest req : pendingReduces) {
-        scheduledRequests.addReduce(req);
-      }
-      pendingReduces.clear();
-      return;
-    }
-    
-    
-    int totalMaps = assignedRequests.maps.size() + completedMaps + scheduledRequests.maps.size();
-    
-    //check for slow start
-    if (!reduceStarted) {//not set yet
-      int completedMapsForReduceSlowstart = (int)Math.ceil(reduceSlowStart * 
-                      totalMaps);
-      if(completedMaps < completedMapsForReduceSlowstart) {
-        LOG.info("Reduce slow start threshold not met. " +
-              "completedMapsForReduceSlowstart " + completedMapsForReduceSlowstart);
-        return;
-      } else {
-        LOG.info("Reduce slow start threshold reached. Scheduling reduces.");
-        reduceStarted = true;
-      }
-    }
-    
-    float completedMapPercent = 0f;
-    if (totalMaps != 0) {//support for 0 maps
-      completedMapPercent = (float)completedMaps/totalMaps;
-    } else {
-      completedMapPercent = 1;
-    }
-    
-    int netScheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt
-        + assignedRequests.maps.size() * mapResourceReqt;
-
-    int netScheduledReduceMem = scheduledRequests.reduces.size()
-        * reduceResourceReqt + assignedRequests.reduces.size()
-        * reduceResourceReqt;
-
-    int finalMapMemLimit = 0;
-    int finalReduceMemLimit = 0;
-    
-    // ramp up the reduces based on completed map percentage
-    int totalMemLimit = getMemLimit();
-    int idealReduceMemLimit = Math.min((int)(completedMapPercent * totalMemLimit),
-        (int) (maxReduceRampupLimit * totalMemLimit));
-    int idealMapMemLimit = totalMemLimit - idealReduceMemLimit;
-
-    // check if there aren't enough maps scheduled, give the free map capacity
-    // to reduce
-    if (idealMapMemLimit > netScheduledMapMem) {
-      int unusedMapMemLimit = idealMapMemLimit - netScheduledMapMem;
-      finalReduceMemLimit = idealReduceMemLimit + unusedMapMemLimit;
-      finalMapMemLimit = totalMemLimit - finalReduceMemLimit;
-    } else {
-      finalMapMemLimit = idealMapMemLimit;
-      finalReduceMemLimit = idealReduceMemLimit;
-    }
-    
-    LOG.info("completedMapPercent " + completedMapPercent +
-        " totalMemLimit:" + totalMemLimit +
-        " finalMapMemLimit:" + finalMapMemLimit +
-        " finalReduceMemLimit:" + finalReduceMemLimit + 
-        " netScheduledMapMem:" + netScheduledMapMem +
-        " netScheduledReduceMem:" + netScheduledReduceMem);
-    
-    int rampUp = (finalReduceMemLimit - netScheduledReduceMem)
-        / reduceResourceReqt;
-    
-    if (rampUp > 0) {
-      rampUp = Math.min(rampUp, pendingReduces.size());
-      LOG.info("Ramping up " + rampUp);
-      //more reduce to be scheduled
-      for (int i = 0; i < rampUp; i++) {
-        ContainerRequest request = pendingReduces.removeFirst();
-        scheduledRequests.addReduce(request);
-      }
-    } else if (rampUp < 0){
-      int rampDown = -1 * rampUp;
-      rampDown = Math.min(rampDown, scheduledRequests.reduces.size());
-      LOG.info("Ramping down " + rampDown);
-      //remove from the scheduled and move back to pending
-      for (int i = 0; i < rampDown; i++) {
-        ContainerRequest request = scheduledRequests.removeReduce();
-        pendingReduces.add(request);
-      }
-    }
-  }
-
-  /**
-   * Synchronized to avoid findbugs warnings
-   */
-  private synchronized String getStat() {
-    return "PendingReduces:" + pendingReduces.size() +
-        " ScheduledMaps:" + scheduledRequests.maps.size() +
-        " ScheduledReduces:" + scheduledRequests.reduces.size() +
-        " AssignedMaps:" + assignedRequests.maps.size() + 
-        " AssignedReduces:" + assignedRequests.reduces.size() +
-        " completedMaps:" + completedMaps +
-        " completedReduces:" + completedReduces +
-        " containersAllocated:" + containersAllocated +
-        " containersReleased:" + containersReleased +
-        " hostLocalAssigned:" + hostLocalAssigned + 
-        " rackLocalAssigned:" + rackLocalAssigned +
-        " availableResources(headroom):" + getAvailableResources();
-  }
-  
-  private List<Container> getResources() throws Exception {
-    int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
-    AMResponse response = makeRemoteRequest();
-    int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
-    List<Container> newContainers = response.getNewContainerList();
-    List<Container> finishedContainers = response.getFinishedContainerList();
-    if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
-      //something changed
-      recalculateReduceSchedule = true;
-    }
-    
-    List<Container> allocatedContainers = new ArrayList<Container>();
-    for (Container cont : newContainers) {
-        allocatedContainers.add(cont);
-        LOG.debug("Received new Container :" + cont);
-    }
-    for (Container cont : finishedContainers) {
-      LOG.info("Received completed container " + cont);
-      TaskAttemptId attemptID = assignedRequests.get(cont.getId());
-      if (attemptID == null) {
-        LOG.error("Container complete event for unknown container id "
-            + cont.getId());
-      } else {
-        assignedRequests.remove(attemptID);
-        if (attemptID.getTaskId().getTaskType().equals(TaskType.MAP)) {
-          completedMaps++;
-        } else {
-          completedReduces++;
-        }
-        // send the container completed event to Task attempt
-        eventHandler.handle(new TaskAttemptEvent(attemptID,
-            TaskAttemptEventType.TA_CONTAINER_COMPLETED));
-        // Send the diagnostics
-        String diagnostics = cont.getContainerStatus().getDiagnostics();
-        eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
-            diagnostics));
-      }
-    }
-    return newContainers;
-  }
-
-  private int getMemLimit() {
-    int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
-    return headRoom + assignedRequests.maps.size() * mapResourceReqt + 
-       assignedRequests.reduces.size() * reduceResourceReqt;
-  }
-  
-  private class ScheduledRequests {
-    
-    private final LinkedList<TaskAttemptId> earlierFailedMaps = 
-      new LinkedList<TaskAttemptId>();
-    
-    /** Maps from a host to a list of Map tasks with data on the host */
-    private final Map<String, LinkedList<TaskAttemptId>> mapsHostMapping = 
-      new HashMap<String, LinkedList<TaskAttemptId>>();
-    private final Map<String, LinkedList<TaskAttemptId>> mapsRackMapping = 
-      new HashMap<String, LinkedList<TaskAttemptId>>();
-    private final Map<TaskAttemptId, ContainerRequest> maps = 
-      new LinkedHashMap<TaskAttemptId, ContainerRequest>();
-    
-    private final LinkedHashMap<TaskAttemptId, ContainerRequest> reduces = 
-      new LinkedHashMap<TaskAttemptId, ContainerRequest>();
-    
-    boolean remove(TaskAttemptId tId) {
-      ContainerRequest req = null;
-      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
-        req = maps.remove(tId);
-      } else {
-        req = reduces.remove(tId);
-      }
-      
-      if (req == null) {
-        return false;
-      } else {
-        decContainerReq(req);
-        return true;
-      }
-    }
-    
-    ContainerRequest removeReduce() {
-      Iterator<Entry<TaskAttemptId, ContainerRequest>> it = reduces.entrySet().iterator();
-      if (it.hasNext()) {
-        Entry<TaskAttemptId, ContainerRequest> entry = it.next();
-        it.remove();
-        decContainerReq(entry.getValue());
-        return entry.getValue();
-      }
-      return null;
-    }
-    
-    void addMap(ContainerRequestEvent event) {
-      ContainerRequest request = null;
-      
-      if (event.getEarlierAttemptFailed()) {
-        earlierFailedMaps.add(event.getAttemptID());
-        request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP);
-      } else {
-        for (String host : event.getHosts()) {
-          //host comes from data splitLocations which are hostnames. Containers
-          // use IP addresses.
-          //TODO Temporary fix for locality. Use resolvers from h-common. 
-          // Cache to make this more efficient ?
-          InetAddress addr = null;
-          try {
-            addr = InetAddress.getByName(host);
-          } catch (UnknownHostException e) {
-            LOG.warn("Unable to resolve host to IP for host [: " + host + "]");
-          }
-          if (addr != null) //Fallback to host if resolve fails.
-            host = addr.getHostAddress();
-          LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
-          if (list == null) {
-            list = new LinkedList<TaskAttemptId>();
-            mapsHostMapping.put(host, list);
-          }
-          list.add(event.getAttemptID());
-          LOG.info("Added attempt req to host " + host);
-       }
-       for (String rack: event.getRacks()) {
-         LinkedList<TaskAttemptId> list = mapsRackMapping.get(rack);
-         if (list == null) {
-           list = new LinkedList<TaskAttemptId>();
-           mapsRackMapping.put(rack, list);
-         }
-         list.add(event.getAttemptID());
-         LOG.info("Added attempt req to rack " + rack);
-       }
-       request = new ContainerRequest(event, PRIORITY_MAP);
-      }
-      maps.put(event.getAttemptID(), request);
-      addContainerReq(request);
-    }
-    
-    
-    void addReduce(ContainerRequest req) {
-      reduces.put(req.attemptID, req);
-      addContainerReq(req);
-    }
-    
-    private void assign(List<Container> allocatedContainers) {
-      Iterator<Container> it = allocatedContainers.iterator();
-      LOG.info("Got allocated containers " + allocatedContainers.size());
-      containersAllocated += allocatedContainers.size();
-      while (it.hasNext()) {
-        Container allocated = it.next();
-        LOG.info("Assigning container " + allocated);
-        ContainerRequest assigned = assign(allocated);
-          
-        if (assigned != null) {
-          // Update resource requests
-          decContainerReq(assigned);
-
-          // send the container-assigned event to task attempt
-          eventHandler.handle(new TaskAttemptContainerAssignedEvent(
-              assigned.attemptID, allocated));
-
-          assignedRequests.add(allocated.getId(), assigned.attemptID);
-          
-          LOG.info("Assigned container (" + allocated + ") " +
-              " to task " + assigned.attemptID +
-              " on node " + allocated.getNodeId().toString());
-        } else {
-          //not assigned to any request, release the container
-          LOG.info("Releasing unassigned and invalid container " + allocated
-              + ". RM has gone crazy, someone go look!"
-              + " Hey RM, if you are so rich, go donate to non-profits!");
-          containersReleased++;
-          release(allocated.getId());
-        }
-      }
-    }
-    
-    private ContainerRequest assign(Container allocated) {
-      ContainerRequest assigned = null;
-      
-      if (mapResourceReqt != reduceResourceReqt) {
-        //assign based on size
-        LOG.info("Assigning based on container size");
-        if (allocated.getResource().getMemory() == mapResourceReqt) {
-          assigned = assignToFailedMap(allocated);
-          if (assigned == null) {
-            assigned = assignToMap(allocated);
-          }
-        } else if (allocated.getResource().getMemory() == reduceResourceReqt) {
-          assigned = assignToReduce(allocated);
-        }
-        
-        return assigned;
-      }
-      
-      //container can be given to either map or reduce
-      //assign based on priority
-      
-      //try to assign to earlierFailedMaps if present
-      assigned = assignToFailedMap(allocated);
-      
-      //Assign to reduces before assigning to maps ?
-      if (assigned == null) {
-        assigned = assignToReduce(allocated);
-      }
-      
-      //try to assign to maps if present
-      if (assigned == null) {
-        assigned = assignToMap(allocated);
-      }
-      
-      return assigned;
-    }
-    
-    
-    private ContainerRequest assignToFailedMap(Container allocated) {
-      //try to assign to earlierFailedMaps if present
-      ContainerRequest assigned = null;
-      while (assigned == null && earlierFailedMaps.size() > 0 && 
-          allocated.getResource().getMemory() >= mapResourceReqt) {
-        TaskAttemptId tId = earlierFailedMaps.removeFirst();
-        if (maps.containsKey(tId)) {
-          assigned = maps.remove(tId);
-          JobCounterUpdateEvent jce =
-            new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
-          jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
-          eventHandler.handle(jce);
-          LOG.info("Assigned from earlierFailedMaps");
-          break;
-        }
-      }
-      return assigned;
-    }
-    
-    private ContainerRequest assignToReduce(Container allocated) {
-      ContainerRequest assigned = null;
-      //try to assign to reduces if present
-      if (assigned == null && reduces.size() > 0
-          && allocated.getResource().getMemory() >= reduceResourceReqt) {
-        TaskAttemptId tId = reduces.keySet().iterator().next();
-        assigned = reduces.remove(tId);
-        LOG.info("Assigned to reduce");
-      }
-      return assigned;
-    }
-    
-    private ContainerRequest assignToMap(Container allocated) {
-    //try to assign to maps if present 
-      //first by host, then by rack, followed by *
-      ContainerRequest assigned = null;
-      while (assigned == null && maps.size() > 0
-          && allocated.getResource().getMemory() >= mapResourceReqt) {
-        String host = getHost(allocated.getNodeId().toString());
-        LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
-        while (list != null && list.size() > 0) {
-          LOG.info("Host matched to the request list " + host);
-          TaskAttemptId tId = list.removeFirst();
-          if (maps.containsKey(tId)) {
-            assigned = maps.remove(tId);
-            JobCounterUpdateEvent jce =
-              new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
-            jce.addCounterUpdate(JobCounter.DATA_LOCAL_MAPS, 1);
-            eventHandler.handle(jce);
-            hostLocalAssigned++;
-            LOG.info("Assigned based on host match " + host);
-            break;
-          }
-        }
-        if (assigned == null) {
-          String rack = RackResolver.resolve(host).getNetworkLocation();
-          list = mapsRackMapping.get(rack);
-          while (list != null && list.size() > 0) {
-            TaskAttemptId tId = list.removeFirst();
-            if (maps.containsKey(tId)) {
-              assigned = maps.remove(tId);
-              JobCounterUpdateEvent jce =
-                new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
-              jce.addCounterUpdate(JobCounter.RACK_LOCAL_MAPS, 1);
-              eventHandler.handle(jce);
-              rackLocalAssigned++;
-              LOG.info("Assigned based on rack match " + rack);
-              break;
-            }
-          }
-          if (assigned == null && maps.size() > 0) {
-            TaskAttemptId tId = maps.keySet().iterator().next();
-            assigned = maps.remove(tId);
-            JobCounterUpdateEvent jce =
-              new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
-            jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
-            eventHandler.handle(jce);
-            LOG.info("Assigned based on * match");
-            break;
-          }
-        }
-      }
-      return assigned;
-    }
-  }
-
-  private class AssignedRequests {
-    private final Map<ContainerId, TaskAttemptId> containerToAttemptMap =
-      new HashMap<ContainerId, TaskAttemptId>();
-    private final LinkedHashMap<TaskAttemptId, ContainerId> maps = 
-      new LinkedHashMap<TaskAttemptId, ContainerId>();
-    private final LinkedHashMap<TaskAttemptId, ContainerId> reduces = 
-      new LinkedHashMap<TaskAttemptId, ContainerId>();
-    private final Set<TaskAttemptId> preemptionWaitingReduces = 
-      new HashSet<TaskAttemptId>();
-    
-    void add(ContainerId containerId, TaskAttemptId tId) {
-      LOG.info("Assigned container " + containerId.toString()
-          + " to " + tId);
-      containerToAttemptMap.put(containerId, tId);
-      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
-        maps.put(tId, containerId);
-      } else {
-        reduces.put(tId, containerId);
-      }
-    }
-
-    void preemptReduce(int toPreempt) {
-      List<TaskAttemptId> reduceList = new ArrayList(reduces.keySet());
-      //sort reduces on progress
-      Collections.sort(reduceList,
-          new Comparator<TaskAttemptId>() {
-        @Override
-        public int compare(TaskAttemptId o1, TaskAttemptId o2) {
-          float p = getJob().getTask(o1.getTaskId()).getAttempt(o1).getProgress() -
-              getJob().getTask(o2.getTaskId()).getAttempt(o2).getProgress();
-          return p >= 0 ? 1 : -1;
-        }
-      });
-      
-      for (int i = 0; i < toPreempt && reduceList.size() > 0; i++) {
-        TaskAttemptId id = reduceList.remove(0);//remove the one on top
-        LOG.info("Preempting " + id);
-        preemptionWaitingReduces.add(id);
-        eventHandler.handle(new TaskAttemptEvent(id, TaskAttemptEventType.TA_KILL));
-      }
-    }
-    
-    boolean remove(TaskAttemptId tId) {
-      ContainerId containerId = null;
-      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
-        containerId = maps.remove(tId);
-      } else {
-        containerId = reduces.remove(tId);
-        if (containerId != null) {
-          boolean preempted = preemptionWaitingReduces.remove(tId);
-          if (preempted) {
-            LOG.info("Reduce preemption successful " + tId);
-          }
-        }
-      }
-      
-      if (containerId != null) {
-        containerToAttemptMap.remove(containerId);
-        return true;
-      }
-      return false;
-    }
-    
-    TaskAttemptId get(ContainerId cId) {
-      return containerToAttemptMap.get(cId);
-    }
-
-    ContainerId get(TaskAttemptId tId) {
-      if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
-        return maps.get(tId);
-      } else {
-        return reduces.get(tId);
-      }
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java

@@ -1,274 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.rm;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-
-/**
- * Keeps the data structures to send container requests to RM.
- */
-public abstract class RMContainerRequestor extends RMCommunicator {
-  
-  private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class);
-  static final String ANY = "*";
-
-  private int lastResponseID;
-  private Resource availableResources;
-
-  private final RecordFactory recordFactory =
-      RecordFactoryProvider.getRecordFactory(null);
-  //Key -> Priority
-  //Value -> Map
-  //Key->ResourceName (e.g., hostname, rackname, *)
-  //Value->Map
-  //Key->Resource Capability
-  //Value->ResourceReqeust
-  private final Map<Priority, Map<String, Map<Resource, ResourceRequest>>>
-  remoteRequestsTable =
-      new TreeMap<Priority, Map<String, Map<Resource, ResourceRequest>>>();
-
-  private final Set<ResourceRequest> ask = new TreeSet<ResourceRequest>();
-  private final Set<ContainerId> release = new TreeSet<ContainerId>(); 
-
-  private boolean nodeBlacklistingEnabled;
-  private int maxTaskFailuresPerNode;
-  private final Map<String, Integer> nodeFailures = new HashMap<String, Integer>();
-  private final Set<String> blacklistedNodes = new HashSet<String>();
-
-  public RMContainerRequestor(ClientService clientService, AppContext context) {
-    super(clientService, context);
-  }
-
-  static class ContainerRequest {
-    final TaskAttemptId attemptID;
-    final Resource capability;
-    final String[] hosts;
-    final String[] racks;
-    //final boolean earlierAttemptFailed;
-    final Priority priority;
-    public ContainerRequest(ContainerRequestEvent event, Priority priority) {
-      this.attemptID = event.getAttemptID();
-      this.capability = event.getCapability();
-      this.hosts = event.getHosts();
-      this.racks = event.getRacks();
-      //this.earlierAttemptFailed = event.getEarlierAttemptFailed();
-      this.priority = priority;
-    }
-  }
-
-  @Override
-  public void init(Configuration conf) {
-    super.init(conf);
-    nodeBlacklistingEnabled = 
-      conf.getBoolean(AMConstants.NODE_BLACKLISTING_ENABLE, true);
-    LOG.info("nodeBlacklistingEnabled:" + nodeBlacklistingEnabled);
-    maxTaskFailuresPerNode = 
-      conf.getInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 3);
-    LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
-  }
-
-  protected abstract void heartbeat() throws Exception;
-
-  protected AMResponse makeRemoteRequest() throws YarnRemoteException {
-    AllocateRequest allocateRequest = recordFactory
-        .newRecordInstance(AllocateRequest.class);
-    allocateRequest.setApplicationAttemptId(applicationAttemptId);
-    allocateRequest.setResponseId(lastResponseID);
-    allocateRequest.addAllAsks(new ArrayList<ResourceRequest>(ask));
-    allocateRequest.addAllReleases(new ArrayList<ContainerId>(release));
-    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
-    AMResponse response = allocateResponse.getAMResponse();
-    lastResponseID = response.getResponseId();
-    availableResources = response.getAvailableResources();
-
-    LOG.info("getResources() for " + applicationId + ":" + " ask="
-        + ask.size() + " release= " + release.size() + " newContainers="
-        + response.getNewContainerCount() + " finishedContainers="
-        + response.getFinishedContainerCount()
-        + " resourcelimit=" + availableResources);
-
-    ask.clear();
-    release.clear();
-    return response;
-  }
-
-  protected void containerFailedOnHost(String hostName) {
-    if (!nodeBlacklistingEnabled) {
-      return;
-    }
-    if (blacklistedNodes.contains(hostName)) {
-      LOG.info("Host " + hostName + " is already blacklisted.");
-      return; //already blacklisted
-    }
-    Integer failures = nodeFailures.remove(hostName);
-    failures = failures == null ? 0 : failures;
-    failures++;
-    LOG.info(failures + " failures on node " + hostName);
-    if (failures >= maxTaskFailuresPerNode) {
-      blacklistedNodes.add(hostName);
-      LOG.info("Blacklisted host " + hostName);
-      
-      //remove all the requests corresponding to this hostname
-      for (Map<String, Map<Resource, ResourceRequest>> remoteRequests 
-          : remoteRequestsTable.values()){
-        //remove from host
-        Map<Resource, ResourceRequest> reqMap = remoteRequests.remove(hostName);
-        if (reqMap != null) {
-          for (ResourceRequest req : reqMap.values()) {
-            ask.remove(req);
-          }
-        }
-        //TODO: remove from rack
-      }
-    } else {
-      nodeFailures.put(hostName, failures);
-    }
-  }
-
-  protected Resource getAvailableResources() {
-    return availableResources;
-  }
-  
-  protected void addContainerReq(ContainerRequest req) {
-    // Create resource requests
-    for (String host : req.hosts) {
-      // Data-local
-      addResourceRequest(req.priority, host, req.capability);
-    }
-
-    // Nothing Rack-local for now
-    for (String rack : req.racks) {
-      addResourceRequest(req.priority, rack, req.capability);
-    }
-
-    // Off-switch
-    addResourceRequest(req.priority, ANY, req.capability); 
-  }
-
-  protected void decContainerReq(ContainerRequest req) {
-    // Update resource requests
-    for (String hostName : req.hosts) {
-      decResourceRequest(req.priority, hostName, req.capability);
-    }
-    
-    for (String rack : req.racks) {
-      decResourceRequest(req.priority, rack, req.capability);
-    }
-   
-    decResourceRequest(req.priority, ANY, req.capability);
-  }
-
-  private void addResourceRequest(Priority priority, String resourceName,
-      Resource capability) {
-    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
-      this.remoteRequestsTable.get(priority);
-    if (remoteRequests == null) {
-      remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
-      this.remoteRequestsTable.put(priority, remoteRequests);
-      LOG.info("Added priority=" + priority);
-    }
-    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
-    if (reqMap == null) {
-      reqMap = new HashMap<Resource, ResourceRequest>();
-      remoteRequests.put(resourceName, reqMap);
-    }
-    ResourceRequest remoteRequest = reqMap.get(capability);
-    if (remoteRequest == null) {
-      remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
-      remoteRequest.setPriority(priority);
-      remoteRequest.setHostName(resourceName);
-      remoteRequest.setCapability(capability);
-      remoteRequest.setNumContainers(0);
-      reqMap.put(capability, remoteRequest);
-    }
-    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
-
-    // Note this down for next interaction with ResourceManager
-    ask.add(remoteRequest);
-    LOG.info("addResourceRequest:" + " applicationId=" + applicationId.getId()
-        + " priority=" + priority.getPriority() + " resourceName=" + resourceName
-        + " numContainers=" + remoteRequest.getNumContainers() + " #asks="
-        + ask.size());
-  }
-
-  private void decResourceRequest(Priority priority, String resourceName,
-      Resource capability) {
-    Map<String, Map<Resource, ResourceRequest>> remoteRequests =
-      this.remoteRequestsTable.get(priority);
-    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
-    ResourceRequest remoteRequest = reqMap.get(capability);
-
-    LOG.info("BEFORE decResourceRequest:" + " applicationId=" + applicationId.getId()
-        + " priority=" + priority.getPriority() + " resourceName=" + resourceName
-        + " numContainers=" + remoteRequest.getNumContainers() + " #asks="
-        + ask.size());
-
-    remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
-    if (remoteRequest.getNumContainers() == 0) {
-      reqMap.remove(capability);
-      if (reqMap.size() == 0) {
-        remoteRequests.remove(resourceName);
-      }
-      if (remoteRequests.size() == 0) {
-        remoteRequestsTable.remove(priority);
-      }
-      //remove from ask if it may have
-      ask.remove(remoteRequest);
-    } else {
-      ask.add(remoteRequest);//this will override the request if ask doesn't
-      //already have it.
-    }
-
-    LOG.info("AFTER decResourceRequest:" + " applicationId="
-             + applicationId.getId() + " priority=" + priority.getPriority()
-             + " resourceName=" + resourceName + " numContainers="
-             + remoteRequest.getNumContainers() + " #asks=" + ask.size());
-  }
-
-  protected void release(ContainerId containerId) {
-    release.add(containerId);
-  }
-  
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DataStatistics.java

@@ -1,78 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-public class DataStatistics {
-  private int count = 0;
-  private double sum = 0;
-  private double sumSquares = 0;
-
-  public DataStatistics() {
-  }
-
-  public DataStatistics(double initNum) {
-    this.count = 1;
-    this.sum = initNum;
-    this.sumSquares = initNum * initNum;
-  }
-
-  public synchronized void add(double newNum) {
-    this.count++;
-    this.sum += newNum;
-    this.sumSquares += newNum * newNum;
-  }
-
-  public synchronized void updateStatistics(double old, double update) {
-	this.sum += update - old;
-	this.sumSquares += (update * update) - (old * old);
-  }
-
-  public synchronized double mean() {
-    return count == 0 ? 0.0 : sum/count;
-  }
-
-  public synchronized double var() {
-    // E(X^2) - E(X)^2
-    if (count <= 1) {
-      return 0.0;
-    }
-    double mean = mean();
-    return Math.max((sumSquares/count) - mean * mean, 0.0d);
-  }
-
-  public synchronized double std() {
-    return Math.sqrt(this.var());
-  }
-
-  public synchronized double outlier(float sigma) {
-    if (count != 0.0) {
-      return mean() + std() * sigma;
-    }
-
-    return 0.0;
-  }
-
-  public synchronized double count() {
-    return count;
-  }
-
-  public String toString() {
-    return "DataStatistics: count is " + count + ", sum is " + sum +
-    ", sumSquares is " + sumSquares + " mean is " + mean() + " std() is " + std();
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java

@@ -1,512 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.yarn.Clock;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-
-public class DefaultSpeculator extends AbstractService implements
-    Speculator {
-
-  private static final long ON_SCHEDULE = Long.MIN_VALUE;
-  private static final long ALREADY_SPECULATING = Long.MIN_VALUE + 1;
-  private static final long TOO_NEW = Long.MIN_VALUE + 2;
-  private static final long PROGRESS_IS_GOOD = Long.MIN_VALUE + 3;
-  private static final long NOT_RUNNING = Long.MIN_VALUE + 4;
-  private static final long TOO_LATE_TO_SPECULATE = Long.MIN_VALUE + 5;
-
-  private static final long SOONEST_RETRY_AFTER_NO_SPECULATE = 1000L * 1L;
-  private static final long SOONEST_RETRY_AFTER_SPECULATE = 1000L * 15L;
-
-  private static final double PROPORTION_RUNNING_TASKS_SPECULATABLE = 0.1;
-  private static final double PROPORTION_TOTAL_TASKS_SPECULATABLE = 0.01;
-  private static final int  MINIMUM_ALLOWED_SPECULATIVE_TASKS = 10;
-
-  private static final Log LOG = LogFactory.getLog(DefaultSpeculator.class);
-
-  private final ConcurrentMap<TaskId, Boolean> runningTasks
-      = new ConcurrentHashMap<TaskId, Boolean>();
-
-  private final Map<Task, AtomicBoolean> pendingSpeculations
-      = new ConcurrentHashMap<Task, AtomicBoolean>();
-
-  // These are the current needs, not the initial needs.  For each job, these
-  //  record the number of attempts that exist and that are actively
-  //  waiting for a container [as opposed to running or finished]
-  private final ConcurrentMap<JobId, AtomicInteger> mapContainerNeeds
-      = new ConcurrentHashMap<JobId, AtomicInteger>();
-  private final ConcurrentMap<JobId, AtomicInteger> reduceContainerNeeds
-      = new ConcurrentHashMap<JobId, AtomicInteger>();
-
-  private final Set<TaskId> mayHaveSpeculated = new HashSet();
-
-  private final Configuration conf;
-  private AppContext context;
-  private Thread speculationBackgroundThread = null;
-  private BlockingQueue<SpeculatorEvent> eventQueue
-      = new LinkedBlockingQueue<SpeculatorEvent>();
-  private TaskRuntimeEstimator estimator;
-
-  private BlockingQueue<Object> scanControl = new LinkedBlockingQueue<Object>();
-
-  private final Clock clock;
-
-  private final EventHandler<TaskEvent> eventHandler;
-
-  public DefaultSpeculator(Configuration conf, AppContext context) {
-    this(conf, context, context.getClock());
-  }
-
-  public DefaultSpeculator(Configuration conf, AppContext context, Clock clock) {
-    this(conf, context, getEstimator(conf, context), clock);
-  }
-  
-  static private TaskRuntimeEstimator getEstimator
-      (Configuration conf, AppContext context) {
-    TaskRuntimeEstimator estimator;
-    
-    try {
-      // "yarn.mapreduce.job.task.runtime.estimator.class"
-      Class<? extends TaskRuntimeEstimator> estimatorClass
-          = conf.getClass(AMConstants.TASK_RUNTIME_ESTIMATOR_CLASS,
-                          LegacyTaskRuntimeEstimator.class,
-                          TaskRuntimeEstimator.class);
-
-      Constructor<? extends TaskRuntimeEstimator> estimatorConstructor
-          = estimatorClass.getConstructor();
-
-      estimator = estimatorConstructor.newInstance();
-
-      estimator.contextualize(conf, context);
-    } catch (InstantiationException ex) {
-      LOG.error("Can't make a speculation runtime extimator" + ex);
-      throw new YarnException(ex);
-    } catch (IllegalAccessException ex) {
-      LOG.error("Can't make a speculation runtime extimator" + ex);
-      throw new YarnException(ex);
-    } catch (InvocationTargetException ex) {
-      LOG.error("Can't make a speculation runtime extimator" + ex);
-      throw new YarnException(ex);
-    } catch (NoSuchMethodException ex) {
-      LOG.error("Can't make a speculation runtime extimator" + ex);
-      throw new YarnException(ex);
-    }
-    
-  return estimator;
-  }
-
-  // This constructor is designed to be called by other constructors.
-  //  However, it's public because we do use it in the test cases.
-  // Normally we figure out our own estimator.
-  public DefaultSpeculator
-      (Configuration conf, AppContext context,
-       TaskRuntimeEstimator estimator, Clock clock) {
-    super(DefaultSpeculator.class.getName());
-
-    this.conf = conf;
-    this.context = context;
-    this.estimator = estimator;
-    this.clock = clock;
-    this.eventHandler = context.getEventHandler();
-  }
-
-/*   *************************************************************    */
-
-  // This is the task-mongering that creates the two new threads -- one for
-  //  processing events from the event queue and one for periodically
-  //  looking for speculation opportunities
-
-  @Override
-  public void start() {
-    Runnable speculationBackgroundCore
-        = new Runnable() {
-            @Override
-            public void run() {
-              while (!Thread.currentThread().isInterrupted()) {
-                long backgroundRunStartTime = clock.getTime();
-                try {
-                  int speculations = computeSpeculations();
-                  long mininumRecomp
-                      = speculations > 0 ? SOONEST_RETRY_AFTER_SPECULATE
-                                         : SOONEST_RETRY_AFTER_NO_SPECULATE;
-
-                  long wait = Math.max(mininumRecomp,
-                        clock.getTime() - backgroundRunStartTime);
-
-                  if (speculations > 0) {
-                    LOG.info("We launched " + speculations
-                        + " speculations.  Sleeping " + wait + " milliseconds.");
-                  }
-
-                  Object pollResult
-                      = scanControl.poll(wait, TimeUnit.MILLISECONDS);
-                } catch (InterruptedException e) {
-                  LOG.error("Background thread returning, interrupted : " + e);
-                  e.printStackTrace(System.out);
-                  return;
-                }
-              }
-            }
-          };
-    speculationBackgroundThread = new Thread
-        (speculationBackgroundCore, "DefaultSpeculator background processing");
-    speculationBackgroundThread.start();
-
-    super.start();
-  }
-
-  @Override
-  public void stop() {
-    // this could be called before background thread is established
-    if (speculationBackgroundThread != null) {
-      speculationBackgroundThread.interrupt();
-    }
-    super.stop();
-  }
-
-  @Override
-  public void handleAttempt(TaskAttemptStatus status) {
-    long timestamp = clock.getTime();
-    statusUpdate(status, timestamp);
-  }
-
-  // This section is not part of the Speculator interface; it's used only for
-  //  testing
-  public boolean eventQueueEmpty() {
-    return eventQueue.isEmpty();
-  }
-
-  // This interface is intended to be used only for test cases.
-  public void scanForSpeculations() {
-    LOG.info("We got asked to run a debug speculation scan.");
-    // debug
-    System.out.println("We got asked to run a debug speculation scan.");
-    System.out.println("There are " + scanControl.size()
-        + " events stacked already.");
-    scanControl.add(new Object());
-    Thread.yield();
-  }
-
-
-/*   *************************************************************    */
-
-  // This section contains the code that gets run for a SpeculatorEvent
-
-  private AtomicInteger containerNeed(TaskId taskID) {
-    JobId jobID = taskID.getJobId();
-    TaskType taskType = taskID.getTaskType();
-
-    ConcurrentMap<JobId, AtomicInteger> relevantMap
-        = taskType == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;
-
-    AtomicInteger result = relevantMap.get(jobID);
-
-    if (result == null) {
-      relevantMap.putIfAbsent(jobID, new AtomicInteger(0));
-      result = relevantMap.get(jobID);
-    }
-
-    return result;
-  }
-
-  private synchronized void processSpeculatorEvent(SpeculatorEvent event) {
-    switch (event.getType()) {
-      case ATTEMPT_STATUS_UPDATE:
-        statusUpdate(event.getReportedStatus(), event.getTimestamp());
-        break;
-
-      case TASK_CONTAINER_NEED_UPDATE:
-      {
-        AtomicInteger need = containerNeed(event.getTaskID());
-        need.addAndGet(event.containersNeededChange());
-        break;
-      }
-
-      case ATTEMPT_START:
-      {
-        LOG.info("ATTEMPT_START " + event.getTaskID());
-        estimator.enrollAttempt
-            (event.getReportedStatus(), event.getTimestamp());
-        break;
-      }
-      
-      case JOB_CREATE:
-      {
-        LOG.info("JOB_CREATE " + event.getJobID());
-        estimator.contextualize(getConfig(), context);
-        break;
-      }
-    }
-  }
-
-  /**
-   * Absorbs one TaskAttemptStatus
-   *
-   * @param reportedStatus the status report that we got from a task attempt
-   *        that we want to fold into the speculation data for this job
-   * @param timestamp the time this status corresponds to.  This matters
-   *        because statuses contain progress.
-   */
-  protected void statusUpdate(TaskAttemptStatus reportedStatus, long timestamp) {
-
-    String stateString = reportedStatus.taskState.toString();
-
-    TaskAttemptId attemptID = reportedStatus.id;
-    TaskId taskID = attemptID.getTaskId();
-    Job job = context.getJob(taskID.getJobId());
-
-    if (job == null) {
-      return;
-    }
-
-    Task task = job.getTask(taskID);
-
-    if (task == null) {
-      return;
-    }
-
-    estimator.updateAttempt(reportedStatus, timestamp);
-
-    // If the task is already known to be speculation-bait, don't do anything
-    if (pendingSpeculations.get(task) != null) {
-      if (pendingSpeculations.get(task).get()) {
-        return;
-      }
-    }
-
-    if (stateString.equals(TaskAttemptState.RUNNING.name())) {
-      runningTasks.putIfAbsent(taskID, Boolean.TRUE);
-    } else {
-      runningTasks.remove(taskID, Boolean.TRUE);
-    }
-  }
-
-/*   *************************************************************    */
-
-// This is the code section that runs periodically and adds speculations for
-//  those jobs that need them.
-
-
-  // This can return a few magic values for tasks that shouldn't speculate:
-  //  returns ON_SCHEDULE if thresholdRuntime(taskID) says that we should not
-  //     considering speculating this task
-  //  returns ALREADY_SPECULATING if that is true.  This has priority.
-  //  returns TOO_NEW if our companion task hasn't gotten any information
-  //  returns PROGRESS_IS_GOOD if the task is sailing through
-  //  returns NOT_RUNNING if the task is not running
-  //
-  // All of these values are negative.  Any value that should be allowed to
-  //  speculate is 0 or positive.
-  private long speculationValue(TaskId taskID, long now) {
-    Job job = context.getJob(taskID.getJobId());
-    Task task = job.getTask(taskID);
-    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
-    long acceptableRuntime = Long.MIN_VALUE;
-    long result = Long.MIN_VALUE;
-
-    if (!mayHaveSpeculated.contains(taskID)) {
-      acceptableRuntime = estimator.thresholdRuntime(taskID);
-      if (acceptableRuntime == Long.MAX_VALUE) {
-        return ON_SCHEDULE;
-      }
-    }
-
-    TaskAttemptId runningTaskAttemptID = null;
-
-    int numberRunningAttempts = 0;
-
-    for (TaskAttempt taskAttempt : attempts.values()) {
-      if (taskAttempt.getState() == TaskAttemptState.RUNNING
-          || taskAttempt.getState() == TaskAttemptState.ASSIGNED) {
-        if (++numberRunningAttempts > 1) {
-          return ALREADY_SPECULATING;
-        }
-        runningTaskAttemptID = taskAttempt.getID();
-
-        long estimatedRunTime = estimator.estimatedRuntime(runningTaskAttemptID);
-
-        long taskAttemptStartTime
-            = estimator.attemptEnrolledTime(runningTaskAttemptID);
-        if (taskAttemptStartTime > now) {
-          // This background process ran before we could process the task
-          //  attempt status change that chronicles the attempt start
-          return TOO_NEW;
-        }
-
-        long estimatedEndTime = estimatedRunTime + taskAttemptStartTime;
-
-        long estimatedReplacementEndTime
-            = now + estimator.estimatedNewAttemptRuntime(taskID);
-
-        if (estimatedEndTime < now) {
-          return PROGRESS_IS_GOOD;
-        }
-
-        if (estimatedReplacementEndTime >= estimatedEndTime) {
-          return TOO_LATE_TO_SPECULATE;
-        }
-
-        result = estimatedEndTime - estimatedReplacementEndTime;
-      }
-    }
-
-    // If we are here, there's at most one task attempt.
-    if (numberRunningAttempts == 0) {
-      return NOT_RUNNING;
-    }
-
-
-
-    if (acceptableRuntime == Long.MIN_VALUE) {
-      acceptableRuntime = estimator.thresholdRuntime(taskID);
-      if (acceptableRuntime == Long.MAX_VALUE) {
-        return ON_SCHEDULE;
-      }
-    }
-
-    return result;
-  }
-
-  //Add attempt to a given Task.
-  protected void addSpeculativeAttempt(TaskId taskID) {
-    LOG.info
-        ("DefaultSpeculator.addSpeculativeAttempt -- we are speculating " + taskID);
-    eventHandler.handle(new TaskEvent(taskID, TaskEventType.T_ADD_SPEC_ATTEMPT));
-    mayHaveSpeculated.add(taskID);
-  }
-
-  @Override
-  public void handle(SpeculatorEvent event) {
-    processSpeculatorEvent(event);
-  }
-
-
-  private int maybeScheduleAMapSpeculation() {
-    return maybeScheduleASpeculation(TaskType.MAP);
-  }
-
-  private int maybeScheduleAReduceSpeculation() {
-    return maybeScheduleASpeculation(TaskType.REDUCE);
-  }
-
-  private int maybeScheduleASpeculation(TaskType type) {
-    int successes = 0;
-
-    long now = clock.getTime();
-
-    ConcurrentMap<JobId, AtomicInteger> containerNeeds
-        = type == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;
-
-    for (ConcurrentMap.Entry<JobId, AtomicInteger> jobEntry : containerNeeds.entrySet()) {
-      // This race conditon is okay.  If we skip a speculation attempt we
-      //  should have tried because the event that lowers the number of
-      //  containers needed to zero hasn't come through, it will next time.
-      // Also, if we miss the fact that the number of containers needed was
-      //  zero but increased due to a failure it's not too bad to launch one
-      //  container prematurely.
-      if (jobEntry.getValue().get() > 0) {
-        continue;
-      }
-
-      int numberSpeculationsAlready = 0;
-      int numberRunningTasks = 0;
-
-      // loop through the tasks of the kind
-      Job job = context.getJob(jobEntry.getKey());
-
-      Map<TaskId, Task> tasks = job.getTasks(type);
-
-      int numberAllowedSpeculativeTasks
-          = (int) Math.max(MINIMUM_ALLOWED_SPECULATIVE_TASKS,
-                           PROPORTION_TOTAL_TASKS_SPECULATABLE * tasks.size());
-
-      TaskId bestTaskID = null;
-      long bestSpeculationValue = -1L;
-
-      // this loop is potentially pricey.
-      // TODO track the tasks that are potentially worth looking at
-      for (Map.Entry<TaskId, Task> taskEntry : tasks.entrySet()) {
-        long mySpeculationValue = speculationValue(taskEntry.getKey(), now);
-
-        if (mySpeculationValue == ALREADY_SPECULATING) {
-          ++numberSpeculationsAlready;
-        }
-
-        if (mySpeculationValue != NOT_RUNNING) {
-          ++numberRunningTasks;
-        }
-
-        if (mySpeculationValue > bestSpeculationValue) {
-          bestTaskID = taskEntry.getKey();
-          bestSpeculationValue = mySpeculationValue;
-        }
-      }
-      numberAllowedSpeculativeTasks
-          = (int) Math.max(numberAllowedSpeculativeTasks,
-                           PROPORTION_RUNNING_TASKS_SPECULATABLE * numberRunningTasks);
-
-      // If we found a speculation target, fire it off
-      if (bestTaskID != null
-          && numberAllowedSpeculativeTasks > numberSpeculationsAlready) {
-        addSpeculativeAttempt(bestTaskID);
-        ++successes;
-      }
-    }
-
-    return successes;
-  }
-
-  private int computeSpeculations() {
-    // We'll try to issue one map and one reduce speculation per job per run
-    return maybeScheduleAMapSpeculation() + maybeScheduleAReduceSpeculation();
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/ExponentiallySmoothedTaskRuntimeEstimator.java

@@ -1,195 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-
-/**
- * This estimator exponentially smooths the rate of progress versus wallclock
- * time.  Conceivably we could write an estimator that smooths time per
- * unit progress, and get different results.
- */
-public class ExponentiallySmoothedTaskRuntimeEstimator extends StartEndTimesBase {
-
-  private final ConcurrentMap<TaskAttemptId, AtomicReference<EstimateVector>> estimates
-      = new ConcurrentHashMap<TaskAttemptId, AtomicReference<EstimateVector>>();
-
-  private SmoothedValue smoothedValue;
-
-  private long lambda;
-
-  public enum SmoothedValue {
-    RATE, TIME_PER_UNIT_PROGRESS
-  }
-
-  ExponentiallySmoothedTaskRuntimeEstimator
-      (long lambda, SmoothedValue smoothedValue) {
-    super();
-    this.smoothedValue = smoothedValue;
-    this.lambda = lambda;
-  }
-
-  public ExponentiallySmoothedTaskRuntimeEstimator() {
-    super();
-  }
-
-  // immutable
-  private class EstimateVector {
-    final double value;
-    final float basedOnProgress;
-    final long atTime;
-
-    EstimateVector(double value, float basedOnProgress, long atTime) {
-      this.value = value;
-      this.basedOnProgress = basedOnProgress;
-      this.atTime = atTime;
-    }
-
-    EstimateVector incorporate(float newProgress, long newAtTime) {
-      if (newAtTime <= atTime || newProgress < basedOnProgress) {
-        return this;
-      }
-
-      double oldWeighting
-          = value < 0.0
-              ? 0.0 : Math.exp(((double) (newAtTime - atTime)) / lambda);
-
-      double newRead = (newProgress - basedOnProgress) / (newAtTime - atTime);
-
-      if (smoothedValue == SmoothedValue.TIME_PER_UNIT_PROGRESS) {
-        newRead = 1.0 / newRead;
-      }
-
-      return new EstimateVector
-          (value * oldWeighting + newRead * (1.0 - oldWeighting),
-           newProgress, newAtTime);
-    }
-  }
-
-  private void incorporateReading
-      (TaskAttemptId attemptID, float newProgress, long newTime) {
-    //TODO: Refactor this method, it seems more complicated than necessary.
-    AtomicReference<EstimateVector> vectorRef = estimates.get(attemptID);
-
-    if (vectorRef == null) {
-      estimates.putIfAbsent(attemptID, new AtomicReference<EstimateVector>(null));
-      incorporateReading(attemptID, newProgress, newTime);
-      return;
-    }
-
-    EstimateVector oldVector = vectorRef.get();
-
-    if (oldVector == null) {
-      if (vectorRef.compareAndSet(null,
-             new EstimateVector(-1.0, 0.0F, Long.MIN_VALUE))) {
-        return;
-      }
-
-      incorporateReading(attemptID, newProgress, newTime);
-      return;
-    }
-
-    while (!vectorRef.compareAndSet
-            (oldVector, oldVector.incorporate(newProgress, newTime))) {
-      oldVector = vectorRef.get();
-    }
-  }
-
-  private EstimateVector getEstimateVector(TaskAttemptId attemptID) {
-    AtomicReference<EstimateVector> vectorRef = estimates.get(attemptID);
-
-    if (vectorRef == null) {
-      return null;
-    }
-
-    return vectorRef.get();
-  }
-
-  private static final long DEFAULT_EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS
-      = 1000L * 60;
-
-  @Override
-  public void contextualize(Configuration conf, AppContext context) {
-    super.contextualize(conf, context);
-
-    lambda
-        = conf.getLong(AMConstants.EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS,
-            DEFAULT_EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS);
-    smoothedValue
-        = conf.getBoolean(AMConstants.EXPONENTIAL_SMOOTHING_SMOOTH_RATE, true)
-            ? SmoothedValue.RATE : SmoothedValue.TIME_PER_UNIT_PROGRESS;
-  }
-
-  @Override
-  public long estimatedRuntime(TaskAttemptId id) {
-    Long startTime = startTimes.get(id);
-
-    if (startTime == null) {
-      return -1L;
-    }
-
-    EstimateVector vector = getEstimateVector(id);
-
-    if (vector == null) {
-      return -1L;
-    }
-
-    long sunkTime = vector.atTime - startTime;
-
-    double value = vector.value;
-    float progress = vector.basedOnProgress;
-
-    if (value == 0) {
-      return -1L;
-    }
-
-    double rate = smoothedValue == SmoothedValue.RATE ? value : 1.0 / value;
-
-    if (rate == 0.0) {
-      return -1L;
-    }
-
-    double remainingTime = (1.0 - progress) / rate;
-
-    return sunkTime + (long)remainingTime;
-  }
-
-  @Override
-  public long runtimeEstimateVariance(TaskAttemptId id) {
-    return -1L;
-  }
-
-  @Override
-  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
-    super.updateAttempt(status, timestamp);
-    TaskAttemptId attemptID = status.id;
-
-    float progress = status.progress;
-
-    incorporateReading(attemptID, progress, timestamp);
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/LegacyTaskRuntimeEstimator.java

@@ -1,150 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-
-
-
-
-public class LegacyTaskRuntimeEstimator extends StartEndTimesBase {
-
-  private final Map<TaskAttempt, AtomicLong> attemptRuntimeEstimates
-      = new ConcurrentHashMap<TaskAttempt, AtomicLong>();
-  private final ConcurrentHashMap<TaskAttempt, AtomicLong> attemptRuntimeEstimateVariances
-      = new ConcurrentHashMap<TaskAttempt, AtomicLong>();
-
-  @Override
-  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
-    super.updateAttempt(status, timestamp);
-    
-
-    TaskAttemptId attemptID = status.id;
-    TaskId taskID = attemptID.getTaskId();
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-
-    if (job == null) {
-      return;
-    }
-
-    Task task = job.getTask(taskID);
-
-    if (task == null) {
-      return;
-    }
-
-    TaskAttempt taskAttempt = task.getAttempt(attemptID);
-
-    if (taskAttempt == null) {
-      return;
-    }
-
-    Long boxedStart = startTimes.get(attemptID);
-    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
-
-    // We need to do two things.
-    //  1: If this is a completion, we accumulate statistics in the superclass
-    //  2: If this is not a completion, we learn more about it.
-
-    // This is not a completion, but we're cooking.
-    //
-    if (taskAttempt.getState() == TaskAttemptState.RUNNING) {
-      // See if this task is already in the registry
-      AtomicLong estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
-      AtomicLong estimateVarianceContainer
-          = attemptRuntimeEstimateVariances.get(taskAttempt);
-
-      if (estimateContainer == null) {
-        if (attemptRuntimeEstimates.get(taskAttempt) == null) {
-          attemptRuntimeEstimates.put(taskAttempt, new AtomicLong());
-
-          estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
-        }
-      }
-
-      if (estimateVarianceContainer == null) {
-        attemptRuntimeEstimateVariances.putIfAbsent(taskAttempt, new AtomicLong());
-        estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
-      }
-
-
-      long estimate = -1;
-      long varianceEstimate = -1;
-
-      // This code assumes that we'll never consider starting a third
-      //  speculative task attempt if two are already running for this task
-      if (start > 0 && timestamp > start) {
-        estimate = (long) ((timestamp - start) / Math.max(0.0001, status.progress));
-        varianceEstimate = (long) (estimate * status.progress / 10);
-      }
-      if (estimateContainer != null) {
-        estimateContainer.set(estimate);
-      }
-      if (estimateVarianceContainer != null) {
-        estimateVarianceContainer.set(varianceEstimate);
-      }
-    }
-  }
-
-  private long storedPerAttemptValue
-       (Map<TaskAttempt, AtomicLong> data, TaskAttemptId attemptID) {
-    TaskId taskID = attemptID.getTaskId();
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-
-    Task task = job.getTask(taskID);
-
-    if (task == null) {
-      return -1L;
-    }
-
-    TaskAttempt taskAttempt = task.getAttempt(attemptID);
-
-    if (taskAttempt == null) {
-      return -1L;
-    }
-
-    AtomicLong estimate = data.get(taskAttempt);
-
-    return estimate == null ? -1L : estimate.get();
-
-  }
-
-  @Override
-  public long estimatedRuntime(TaskAttemptId attemptID) {
-    return storedPerAttemptValue(attemptRuntimeEstimates, attemptID);
-  }
-
-  @Override
-  public long runtimeEstimateVariance(TaskAttemptId attemptID) {
-    return storedPerAttemptValue(attemptRuntimeEstimateVariances, attemptID);
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/NullTaskRuntimesEngine.java

@@ -1,72 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-
-
-/*
- * This class is provided solely as an exemplae of the values that mean
- *  that nothing needs to be computed.  It's not currently used.
- */
-public class NullTaskRuntimesEngine implements TaskRuntimeEstimator {
-  @Override
-  public void enrollAttempt(TaskAttemptStatus status, long timestamp) {
-    // no code
-  }
-
-  @Override
-  public long attemptEnrolledTime(TaskAttemptId attemptID) {
-    return Long.MAX_VALUE;
-  }
-
-  @Override
-  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
-    // no code
-  }
-
-  @Override
-  public void contextualize(Configuration conf, AppContext context) {
-    // no code
-  }
-
-  @Override
-  public long thresholdRuntime(TaskId id) {
-    return Long.MAX_VALUE;
-  }
-
-  @Override
-  public long estimatedRuntime(TaskAttemptId id) {
-    return -1L;
-  }
-  @Override
-  public long estimatedNewAttemptRuntime(TaskId id) {
-    return -1L;
-  }
-
-  @Override
-  public long runtimeEstimateVariance(TaskAttemptId id) {
-    return -1L;
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/Speculator.java

@@ -1,45 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.yarn.event.EventHandler;
-
-/**
- * Speculator component. Task Attempts' status updates are sent to this
- * component. Concrete implementation runs the speculative algorithm and
- * sends the TaskEventType.T_ADD_ATTEMPT.
- *
- * An implementation also has to arrange for the jobs to be scanned from
- * time to time, to launch the speculations.
- */
-public interface Speculator
-              extends EventHandler<SpeculatorEvent> {
-
-  enum EventType {
-    ATTEMPT_STATUS_UPDATE,
-    ATTEMPT_START,
-    TASK_CONTAINER_NEED_UPDATE,
-    JOB_CREATE
-  }
-
-  // This will be implemented if we go to a model where the events are
-  //  processed within the TaskAttempts' state transitions' code.
-  public void handleAttempt(TaskAttemptStatus status);
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/SpeculatorEvent.java

@@ -1,86 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-
-public class SpeculatorEvent extends AbstractEvent<Speculator.EventType> {
-
-  // valid for ATTEMPT_STATUS_UPDATE
-  private TaskAttemptStatus reportedStatus;
-
-  // valid for TASK_CONTAINER_NEED_UPDATE
-  private TaskId taskID;
-  private int containersNeededChange;
-  
-  // valid for CREATE_JOB
-  private JobId jobID;
-
-  public SpeculatorEvent(JobId jobID, long timestamp) {
-    super(Speculator.EventType.JOB_CREATE, timestamp);
-    this.jobID = jobID;
-  }
-
-  public SpeculatorEvent(TaskAttemptStatus reportedStatus, long timestamp) {
-    super(Speculator.EventType.ATTEMPT_STATUS_UPDATE, timestamp);
-    this.reportedStatus = reportedStatus;
-  }
-
-  public SpeculatorEvent(TaskAttemptId attemptID, boolean flag, long timestamp) {
-    super(Speculator.EventType.ATTEMPT_START, timestamp);
-    this.reportedStatus = new TaskAttemptStatus();
-    this.reportedStatus.id = attemptID;
-    this.taskID = attemptID.getTaskId();
-  }
-
-  /*
-   * This c'tor creates a TASK_CONTAINER_NEED_UPDATE event .
-   * We send a +1 event when a task enters a state where it wants a container,
-   *  and a -1 event when it either gets one or withdraws the request.
-   * The per job sum of all these events is the number of containers requested
-   *  but not granted.  The intent is that we only do speculations when the
-   *  speculation wouldn't compete for containers with tasks which need
-   *  to be run.
-   */
-  public SpeculatorEvent(TaskId taskID, int containersNeededChange) {
-    super(Speculator.EventType.TASK_CONTAINER_NEED_UPDATE);
-    this.taskID = taskID;
-    this.containersNeededChange = containersNeededChange;
-  }
-
-  public TaskAttemptStatus getReportedStatus() {
-    return reportedStatus;
-  }
-
-  public int containersNeededChange() {
-    return containersNeededChange;
-  }
-
-  public TaskId getTaskID() {
-    return taskID;
-  }
-  
-  public JobId getJobID() {
-    return jobID;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java

@@ -1,213 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-
-abstract class StartEndTimesBase implements TaskRuntimeEstimator {
-  static final float MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE
-      = 0.05F;
-  static final int MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
-      = 1;
-
-  protected Configuration conf = null;
-  protected AppContext context = null;
-
-  protected final Map<TaskAttemptId, Long> startTimes
-      = new ConcurrentHashMap<TaskAttemptId, Long>();
-
-  // XXXX This class design assumes that the contents of AppContext.getAllJobs
-  //   never changes.  Is that right?
-  //
-  // This assumption comes in in several places, mostly in data structure that
-  //   can grow without limit if a AppContext gets new Job's when the old ones
-  //   run out.  Also, these mapper statistics blocks won't cover the Job's
-  //   we don't know about.
-  protected final Map<Job, DataStatistics> mapperStatistics
-      = new HashMap<Job, DataStatistics>();
-  protected final Map<Job, DataStatistics> reducerStatistics
-      = new HashMap<Job, DataStatistics>();
-
-
-  private final Map<Job, Float> slowTaskRelativeTresholds
-      = new HashMap<Job, Float>();
-
-  protected final Set<Task> doneTasks = new HashSet<Task>();
-
-  @Override
-  public void enrollAttempt(TaskAttemptStatus status, long timestamp) {
-    startTimes.put(status.id,timestamp);
-  }
-
-  @Override
-  public long attemptEnrolledTime(TaskAttemptId attemptID) {
-    Long result = startTimes.get(attemptID);
-
-    return result == null ? Long.MAX_VALUE : result;
-  }
-
-
-  @Override
-  public void contextualize(Configuration conf, AppContext context) {
-    this.conf = conf;
-    this.context = context;
-
-    Map<JobId, Job> allJobs = context.getAllJobs();
-
-    for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) {
-      final Job job = entry.getValue();
-      mapperStatistics.put(job, new DataStatistics());
-      reducerStatistics.put(job, new DataStatistics());
-      slowTaskRelativeTresholds.put
-          (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f));
-    }
-  }
-
-  protected DataStatistics dataStatisticsForTask(TaskId taskID) {
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-
-    if (job == null) {
-      return null;
-    }
-
-    Task task = job.getTask(taskID);
-
-    if (task == null) {
-      return null;
-    }
-
-    return task.getType() == TaskType.MAP
-            ? mapperStatistics.get(job)
-            : task.getType() == TaskType.REDUCE
-                ? reducerStatistics.get(job)
-                : null;
-  }
-
-  @Override
-  public long thresholdRuntime(TaskId taskID) {
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-
-    TaskType type = taskID.getTaskType();
-
-    DataStatistics statistics
-        = dataStatisticsForTask(taskID);
-
-    int completedTasksOfType
-        = type == TaskType.MAP
-            ? job.getCompletedMaps() : job.getCompletedReduces();
-
-    int totalTasksOfType
-        = type == TaskType.MAP
-            ? job.getTotalMaps() : job.getTotalReduces();
-
-    if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
-        || (((float)completedTasksOfType) / totalTasksOfType)
-              < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) {
-      return Long.MAX_VALUE;
-    }
-
-    long result =  statistics == null
-        ? Long.MAX_VALUE
-        : (long)statistics.outlier(slowTaskRelativeTresholds.get(job));
-    return result;
-  }
-
-  @Override
-  public long estimatedNewAttemptRuntime(TaskId id) {
-    DataStatistics statistics = dataStatisticsForTask(id);
-
-    if (statistics == null) {
-      return -1L;
-    }
-
-    return (long)statistics.mean();
-  }
-
-  @Override
-  public void updateAttempt(TaskAttemptStatus status, long timestamp) {
-
-    TaskAttemptId attemptID = status.id;
-    TaskId taskID = attemptID.getTaskId();
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-
-    if (job == null) {
-      return;
-    }
-
-    Task task = job.getTask(taskID);
-
-    if (task == null) {
-      return;
-    }
-
-    Long boxedStart = startTimes.get(attemptID);
-    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
-    
-    TaskAttempt taskAttempt = task.getAttempt(attemptID);
-
-    if (taskAttempt.getState() == TaskAttemptState.SUCCEEDED) {
-      boolean isNew = false;
-      // is this  a new success?
-      synchronized (doneTasks) {
-        if (!doneTasks.contains(task)) {
-          doneTasks.add(task);
-          isNew = true;
-        }
-      }
-
-      // It's a new completion
-      // Note that if a task completes twice [because of a previous speculation
-      //  and a race, or a success followed by loss of the machine with the
-      //  local data] we only count the first one.
-      if (isNew) {
-        long finish = timestamp;
-        if (start > 1L && finish > 1L && start <= finish) {
-          long duration = finish - start;
-
-          DataStatistics statistics
-          = dataStatisticsForTask(taskID);
-
-          if (statistics != null) {
-            statistics.add(duration);
-          }
-        }
-      }
-    }
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/TaskRuntimeEstimator.java

@@ -1,90 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
-
-
-
-public interface TaskRuntimeEstimator {
-  public void enrollAttempt(TaskAttemptStatus reportedStatus, long timestamp);
-
-  public long attemptEnrolledTime(TaskAttemptId attemptID);
-
-  public void updateAttempt(TaskAttemptStatus reportedStatus, long timestamp);
-
-  public void contextualize(Configuration conf, AppContext context);
-
-  /**
-   *
-   * Find a maximum reasonable execution wallclock time.  Includes the time
-   * already elapsed.
-   *
-   * Find a maximum reasonable execution time.  Includes the time
-   * already elapsed.  If the projected total execution time for this task
-   * ever exceeds its reasonable execution time, we may speculate it.
-   *
-   * @param id the {@link TaskID} of the task we are asking about
-   * @return the task's maximum reasonable runtime, or MAX_VALUE if
-   *         we don't have enough information to rule out any runtime,
-   *         however long.
-   *
-   */
-  public long thresholdRuntime(TaskId id);
-
-  /**
-   *
-   * Estimate a task attempt's total runtime.  Includes the time already
-   * elapsed.
-   *
-   * @param id the {@link TaskAttemptID} of the attempt we are asking about
-   * @return our best estimate of the attempt's runtime, or {@code -1} if
-   *         we don't have enough information yet to produce an estimate.
-   *
-   */
-  public long estimatedRuntime(TaskAttemptId id);
-
-  /**
-   *
-   * Estimates how long a new attempt on this task will take if we start
-   *  one now
-   *
-   * @param id the {@link TaskID} of the task we are asking about
-   * @return our best estimate of a new attempt's runtime, or {@code -1} if
-   *         we don't have enough information yet to produce an estimate.
-   *
-   */
-  public long estimatedNewAttemptRuntime(TaskId id);
-
-  /**
-   *
-   * Computes the width of the error band of our estimate of the task
-   *  runtime as returned by {@link estimatedRuntime}
-   *
-   * @param id the {@link TaskAttemptID} of the attempt we are asking about
-   * @return our best estimate of the attempt's runtime, or {@code -1} if
-   *         we don't have enough information yet to produce an estimate.
-   *
-   */
-  public long runtimeEstimateVariance(TaskAttemptId id);
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/TaskSpeculationPredicate.java

@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.speculate;
-
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.Job;
-import org.apache.hadoop.mapreduce.v2.app.job.Task;
-
-
-public class TaskSpeculationPredicate {
-  boolean canSpeculate(AppContext context, TaskId taskID) {
-    // This class rejects speculating any task that already has speculations,
-    //  or isn't running.
-    //  Subclasses should call TaskSpeculationPredicate.canSpeculate(...) , but
-    //  can be even more restrictive.
-    JobId jobID = taskID.getJobId();
-    Job job = context.getJob(jobID);
-    Task task = job.getTask(taskID);
-    return task.getAttempts().size() == 1;
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleaner.java

@@ -1,28 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.taskclean;
-
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public interface TaskCleaner extends EventHandler<TaskCleanupEvent> {
-
-  enum EventType {
-    TASK_CLEAN
-  }
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanerImpl.java

@@ -1,108 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.taskclean;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.service.AbstractService;
-
-public class TaskCleanerImpl extends AbstractService implements TaskCleaner {
-
-  private static final Log LOG = LogFactory.getLog(TaskCleanerImpl.class);
-
-  private final AppContext context;
-  private ThreadPoolExecutor launcherPool;
-  private Thread eventHandlingThread;
-  private BlockingQueue<TaskCleanupEvent> eventQueue =
-      new LinkedBlockingQueue<TaskCleanupEvent>();
-
-  public TaskCleanerImpl(AppContext context) {
-    super("TaskCleaner");
-    this.context = context;
-  }
-
-  public void start() {
-    launcherPool = new ThreadPoolExecutor(1, 5, 1, 
-        TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
-    eventHandlingThread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        TaskCleanupEvent event = null;
-        while (!Thread.currentThread().isInterrupted()) {
-          try {
-            event = eventQueue.take();
-          } catch (InterruptedException e) {
-            LOG.error("Returning, interrupted : " + e);
-            return;
-          }
-          // the events from the queue are handled in parallel
-          // using a thread pool
-          launcherPool.execute(new EventProcessor(event));        }
-      }
-    });
-    eventHandlingThread.start();
-    super.start();
-  }
-
-  public void stop() {
-    eventHandlingThread.interrupt();
-    launcherPool.shutdown();
-    super.stop();
-  }
-
-  private class EventProcessor implements Runnable {
-    private TaskCleanupEvent event;
-
-    EventProcessor(TaskCleanupEvent event) {
-      this.event = event;
-    }
-
-    @Override
-    public void run() {
-      LOG.info("Processing the event " + event.toString());
-      try {
-        event.getCommitter().abortTask(event.getAttemptContext());
-      } catch (Exception e) {
-        LOG.warn("Task cleanup failed for attempt " + event.getAttemptID(), e);
-      }
-      context.getEventHandler().handle(
-          new TaskAttemptEvent(event.getAttemptID(), 
-              TaskAttemptEventType.TA_CLEANUP_DONE));
-    }
-  }
-
-  @Override
-  public void handle(TaskCleanupEvent event) {
-    try {
-      eventQueue.put(event);
-    } catch (InterruptedException e) {
-      throw new YarnException(e);
-    }
-  }
-
-}

hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/taskclean/TaskCleanupEvent.java

@@ -1,56 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app.taskclean;
-
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-/**
- * This class encapsulates task cleanup event.
- *
- */
-public class TaskCleanupEvent extends AbstractEvent<TaskCleaner.EventType> {
-
-  private final TaskAttemptId attemptID;
-  private final OutputCommitter committer;
-  private final TaskAttemptContext attemptContext;
-
-  public TaskCleanupEvent(TaskAttemptId attemptID, OutputCommitter committer, 
-      TaskAttemptContext attemptContext) {
-    super(TaskCleaner.EventType.TASK_CLEAN);
-    this.attemptID = attemptID;
-    this.committer = committer;
-    this.attemptContext = attemptContext;
-  }
-
-  public TaskAttemptId getAttemptID() {
-    return attemptID;
-  }
-
-  public OutputCommitter getCommitter() {
-    return committer;
-  }
-
-  public TaskAttemptContext getAttemptContext() {
-    return attemptContext;
-  }
-
-}