Merge trunk into HA branch

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1170378 13f79535-47bb-0310-9956-ffa450edef68

dev-support/smart-apply-patch.sh

@@ -39,40 +39,68 @@ fi
 # Come up with a list of changed files into $TMP
 TMP=/tmp/tmp.paths.$$
 TOCLEAN="$TOCLEAN $TMP"
-grep '^+++\|^---' $PATCH_FILE | cut -c '5-' | grep -v /dev/null | sort | uniq > $TMP
-
-# Assume p0 to start
-PLEVEL=0
-
-# if all of the lines start with a/ or b/, then this is a git patch that
-# was generated without --no-prefix
-if ! grep -qv '^a/\|^b/' $TMP ; then
-  echo Looks like this is a git patch. Stripping a/ and b/ prefixes
-  echo and incrementing PLEVEL
-  PLEVEL=$[$PLEVEL + 1]
-  sed -i -e 's,^[ab]/,,' $TMP
-fi
 
-PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
+if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
+  PLEVEL=0
+  #if the patch applied at P0 there is the possability that all we are doing
+  # is adding new files and they would apply anywhere. So try to guess the
+  # correct place to put those files.
 
-# if we are at the project root then nothing more to do
-if [[ -d hadoop-common-project ]]; then
-  echo Looks like this is being run at project root
+  TMP2=/tmp/tmp.paths.2.$$
+  TOCLEAN="$TOCLEAN $TMP2"
 
-# if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this is
-# relative to the hadoop root instead of the subproject root, so we need
-# to chop off another layer
-elif [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
+  grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
 
-  echo Looks like this is relative to project root. Increasing PLEVEL
-  PLEVEL=$[$PLEVEL + 1]
+  #first off check that all of the files do not exist
+  FOUND_ANY=0
+  for CHECK_FILE in $(cat $TMP2)
+  do
+    if [[ -f $CHECK_FILE ]]; then
+      FOUND_ANY=1
+    fi
+  done
 
-elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
-  echo Looks like this is a cross-subproject patch. Try applying from the project root
-  exit 1
+  if [[ "$FOUND_ANY" = "0" ]]; then
+    #all of the files are new files so we have to guess where the correct place to put it is.
+
+    # if all of the lines start with a/ or b/, then this is a git patch that
+    # was generated without --no-prefix
+    if ! grep -qv '^a/\|^b/' $TMP2 ; then
+      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+      echo and incrementing PLEVEL
+      PLEVEL=$[$PLEVEL + 1]
+      sed -i -e 's,^[ab]/,,' $TMP2
+    fi
+
+    PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 | sort | uniq)
+
+    # if we are at the project root then nothing more to do
+    if [[ -d hadoop-common-project ]]; then
+      echo Looks like this is being run at project root
+
+    # if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this is
+    # relative to the hadoop root instead of the subproject root, so we need
+    # to chop off another layer
+    elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
+
+      echo Looks like this is relative to project root. Increasing PLEVEL
+      PLEVEL=$[$PLEVEL + 1]
+
+    elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
+      echo Looks like this is a cross-subproject patch. Try applying from the project root
+      cleanup 1
+    fi
+  fi
+elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=1
+elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=2
+else
+  echo "The patch does not appear to apply with p0 to p2";
+  cleanup 1;
 fi
 
 echo Going to apply patch with: $PATCH -p$PLEVEL
 $PATCH -p$PLEVEL -E < $PATCH_FILE
 
-cleanup 0
+cleanup $?

dev-support/test-patch.sh

@@ -64,6 +64,7 @@ printUsage() {
   echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
   echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
+  echo "--run-tests            Run all tests below the base directory"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -130,6 +131,9 @@ parseArgs() {
     --dirty-workspace)
       DIRTY_WORKSPACE=true
       ;;
+    --run-tests)
+      RUN_TESTS=true
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
@@ -249,6 +253,18 @@ setup () {
   echo "======================================================================"
   echo ""
   echo ""
+  if [[ ! -d hadoop-common-project ]]; then
+    cd $bindir/..
+    echo "Compiling $(pwd)"
+    echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1"
+    $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1
+    if [[ $? != 0 ]] ; then
+      echo "Top-level trunk compilation is broken?"
+      cleanupAndExit 1
+    fi
+    cd -
+  fi
+  echo "Compiling $(pwd)"
   echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
   $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
@@ -550,8 +566,10 @@ checkFindbugsWarnings () {
     $FINDBUGS_HOME/bin/convertXmlToText -html \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
+    if [[ $newFindbugsWarnings > 0 ]] ; then
+      JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/$(basename $BASEDIR)/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
 $JIRA_COMMENT_FOOTER"
+    fi
   done
 
   ### if current warnings greater than OK_FINDBUGS_WARNINGS
@@ -580,26 +598,12 @@ runTests () {
   echo ""
   echo ""
   
-  failed_tests=""
-  modules=$(findModules)
-  for module in $modules;
-  do
-    pushd $module
-      echo "    Running tests in $module"
-      ### Kill any rogue build processes from the last attempt
-      $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-      echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
-      $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
-      if [[ $? != 0 ]] ; then
-        ### Find and format names of failed tests
-        module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
-        failed_tests="${failed_tests}
-${module_failed_tests}"
-      fi
-    popd
-  done
-  echo $failed_tests
+  echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+  $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
+  if [[ $? != 0 ]] ; then
+    ### Find and format names of failed tests
+    failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
+  fi
   
   if [[ -n "$failed_tests" ]] ; then
   
@@ -615,36 +619,6 @@ $failed_tests"
   return 0
 }
 
-###############################################################################
-### Find the modules changed by the patch
-
-findModules () {
-  # Come up with a list of changed files into $TMP
-  TMP=/tmp/tmp.paths.$$
-  $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP
-
-  # if all of the lines start with a/ or b/, then this is a git patch that
-  # was generated without --no-prefix
-  if ! $GREP -qv '^a/\|^b/' $TMP ; then
-    sed -i -e 's,^[ab]/,,' $TMP
-  fi
-
-  PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
-
-  # if all of the lines start with hadoop-common-project/, hadoop-hdfs-project/, or hadoop-mapreduce-project/, this is
-  # relative to the hadoop root instead of the subproject root
-  if [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$ ]]; then
-    echo $PREFIX_DIRS
-    return 0
-  elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project' ; then
-    echo $PREFIX_DIRS
-    return 0
-  fi
-  
-  # No modules found. Running from current directory.
-  echo .
-}
-
 ###############################################################################
 ### Run the test-contrib target
 runContribTests () {
@@ -820,8 +794,8 @@ checkFindbugsWarnings
 (( RESULT = RESULT + $? ))
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
-### Do not call these when run by a developer 
-if [[ $JENKINS == "true" ]] ; then
+### Run tests for Jenkins or if explictly asked for by a developer
+if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
   runContribTests

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -78,6 +78,13 @@
         <include>*-site.xml</include>
       </includes>
     </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/packages/templates/conf</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/templates/conf</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+    </fileSet>
     <fileSet>
       <directory>${project.build.directory}</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}</outputDirectory>

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -5,13 +5,19 @@ Trunk (unreleased changes)
   IMPROVEMENTS
 
     HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
-  HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
+
+    HADOOP-7524. Change RPC to allow multiple protocols including multuple
+                 versions of the same protocol (sanjay Radia)
+
+    HADOOP-7607. Simplify the RPC proxy cleanup process. (atm)
 
   BUGS
 
     HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required
                  by Jersey (Alejandro Abdelnur via atm)
 
+    HADOOP-7610. Fix for hadoop debian package (Eric Yang via gkesavan)
+
 Release 0.23.0 - Unreleased
 
   INCOMPATIBLE CHANGES
@@ -371,6 +377,12 @@ Release 0.23.0 - Unreleased
     HADOOP-7507. Allow ganglia metrics to include the metrics system tags
                  in the gmetric names. (Alejandro Abdelnur via todd)
 
+    HADOOP-7612. Change test-patch to run tests for all nested modules.
+    (tomwhite)
+
+    HADOOP-7599. Script improvements to setup a secure Hadoop cluster
+    (Eric Yang via ddas)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
@@ -563,6 +575,14 @@ Release 0.23.0 - Unreleased
     HADOOP-7593. Fix AssertionError in TestHttpServer.testMaxThreads().
     (Uma Maheswara Rao G via szetszwo)
 
+    HADOOP-7598. Fix smart-apply-patch.sh to handle patching from a sub
+    directory correctly. (Robert Evans via acmurthy) 
+
+    HADOOP-7328. When a serializer class is missing, return null, not throw
+    an NPE. (Harsh J Chouraria via todd)
+
+    HADOOP-7626. Bugfix for a config generator (Eric Yang via ddas)
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES
@@ -1075,6 +1095,9 @@ Release 0.22.0 - Unreleased
     HADOOP-7390. VersionInfo not generated properly in git after unsplit. (todd
     via atm)
 
+    HADOOP-7568. SequenceFile should not print into stdout.
+    (Plamen Jeliazkov via shv)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java

@@ -1668,7 +1668,6 @@ public class SequenceFile {
       try {
         seek(start);
         this.end = this.in.getPos() + length;
-        System.out.println("Setting end to " + end);
         // if it wrapped around, use the max
         if (end < length) {
           end = Long.MAX_VALUE;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java

@@ -27,10 +27,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization;
 import org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
 
 /**
  * <p>
@@ -50,14 +50,15 @@ public class SerializationFactory extends Configured {
    * <p>
    * Serializations are found by reading the <code>io.serializations</code>
    * property from <code>conf</code>, which is a comma-delimited list of
-   * classnames. 
+   * classnames.
    * </p>
    */
   public SerializationFactory(Configuration conf) {
     super(conf);
-    for (String serializerName : conf.getStrings("io.serializations", 
-      new String[]{WritableSerialization.class.getName(), 
-        AvroSpecificSerialization.class.getName(), 
+    for (String serializerName : conf.getStrings(
+      CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
+      new String[]{WritableSerialization.class.getName(),
+        AvroSpecificSerialization.class.getName(),
         AvroReflectSerialization.class.getName()})) {
       add(conf, serializerName);
     }
@@ -67,27 +68,35 @@ public class SerializationFactory extends Configured {
   private void add(Configuration conf, String serializationName) {
     try {
       Class<? extends Serialization> serializionClass =
-	(Class<? extends Serialization>) conf.getClassByName(serializationName);
+        (Class<? extends Serialization>) conf.getClassByName(serializationName);
       serializations.add((Serialization)
-	  ReflectionUtils.newInstance(serializionClass, getConf()));
+      ReflectionUtils.newInstance(serializionClass, getConf()));
     } catch (ClassNotFoundException e) {
       LOG.warn("Serialization class not found: ", e);
     }
   }
 
   public <T> Serializer<T> getSerializer(Class<T> c) {
-    return getSerialization(c).getSerializer(c);
+    Serialization<T> serializer = getSerialization(c);
+    if (serializer != null) {
+      return serializer.getSerializer(c);
+    }
+    return null;
   }
 
   public <T> Deserializer<T> getDeserializer(Class<T> c) {
-    return getSerialization(c).getDeserializer(c);
+    Serialization<T> serializer = getSerialization(c);
+    if (serializer != null) {
+      return serializer.getDeserializer(c);
+    }
+    return null;
   }
 
   @SuppressWarnings("unchecked")
   public <T> Serialization<T> getSerialization(Class<T> c) {
     for (Serialization serialization : serializations) {
       if (serialization.accept(c)) {
-	return (Serialization<T>) serialization;
+        return (Serialization<T>) serialization;
       }
     }
     return null;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AvroRpcEngine.java

@@ -131,7 +131,7 @@ public class AvroRpcEngine implements RpcEngine {
     }
 
     public void close() throws IOException {
-      ENGINE.stopProxy(tunnel);
+      RPC.stopProxy(tunnel);
     }
   }
 
@@ -152,15 +152,6 @@ public class AvroRpcEngine implements RpcEngine {
        false);
   }
 
-  /** Stop this proxy. */
-  public void stopProxy(Object proxy) {
-    try {
-      ((Invoker)Proxy.getInvocationHandler(proxy)).close();
-    } catch (IOException e) {
-      LOG.warn("Error while stopping "+proxy, e);
-    }
-  }
-
   private class Invoker implements InvocationHandler, Closeable {
     private final ClientTransceiver tx;
     private final SpecificRequestor requestor;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ipc;
 
+import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Proxy;
 import java.lang.reflect.Method;
 
@@ -26,6 +27,7 @@ import java.net.InetSocketAddress;
 import java.net.NoRouteToHostException;
 import java.net.SocketTimeoutException;
 import java.io.*;
+import java.io.Closeable;
 import java.util.Map;
 import java.util.HashMap;
 
@@ -80,12 +82,8 @@ public class RPC {
   private RPC() {}                                  // no public ctor
 
   // cache of RpcEngines by protocol
-  private static final Map<Class,RpcEngine> PROTOCOL_ENGINES
-    = new HashMap<Class,RpcEngine>();
-
-  // track what RpcEngine is used by a proxy class, for stopProxy()
-  private static final Map<Class,RpcEngine> PROXY_ENGINES
-    = new HashMap<Class,RpcEngine>();
+  private static final Map<Class<?>,RpcEngine> PROTOCOL_ENGINES
+    = new HashMap<Class<?>,RpcEngine>();
 
   private static final String ENGINE_PROP = "rpc.engine";
 
@@ -96,32 +94,23 @@ public class RPC {
    * @param engine the RpcEngine impl
    */
   public static void setProtocolEngine(Configuration conf,
-                                Class protocol, Class engine) {
+                                Class<?> protocol, Class<?> engine) {
     conf.setClass(ENGINE_PROP+"."+protocol.getName(), engine, RpcEngine.class);
   }
 
   // return the RpcEngine configured to handle a protocol
-  private static synchronized RpcEngine getProtocolEngine(Class protocol,
+  private static synchronized RpcEngine getProtocolEngine(Class<?> protocol,
                                                           Configuration conf) {
     RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
     if (engine == null) {
       Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
                                     WritableRpcEngine.class);
       engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
-      if (protocol.isInterface())
-        PROXY_ENGINES.put(Proxy.getProxyClass(protocol.getClassLoader(),
-                                              protocol),
-                          engine);
       PROTOCOL_ENGINES.put(protocol, engine);
     }
     return engine;
   }
 
-  // return the RpcEngine that handles a proxy object
-  private static synchronized RpcEngine getProxyEngine(Object proxy) {
-    return PROXY_ENGINES.get(proxy.getClass());
-  }
-
   /**
    * A version mismatch for the RPC protocol.
    */
@@ -477,13 +466,30 @@ public class RPC {
   }
 
   /**
-   * Stop this proxy and release its invoker's resource
-   * @param proxy the proxy to be stopped
+   * Stop this proxy and release its invoker's resource by getting the
+   * invocation handler for the given proxy object and calling
+   * {@link Closeable#close} if that invocation handler implements
+   * {@link Closeable}.
+   * 
+   * @param proxy the RPC proxy object to be stopped
    */
   public static void stopProxy(Object proxy) {
-    RpcEngine rpcEngine;
-    if (proxy!=null && (rpcEngine = getProxyEngine(proxy)) != null) {
-      rpcEngine.stopProxy(proxy);
+    InvocationHandler invocationHandler = null;
+    try {
+      invocationHandler = Proxy.getInvocationHandler(proxy);
+    } catch (IllegalArgumentException e) {
+      LOG.error("Tried to call RPC.stopProxy on an object that is not a proxy.", e);
+    }
+    if (proxy != null && invocationHandler != null &&
+        invocationHandler instanceof Closeable) {
+      try {
+        ((Closeable)invocationHandler).close();
+      } catch (IOException e) {
+        LOG.error("Stopping RPC invocation handler caused exception", e);
+      }
+    } else {
+      LOG.error("Could not get invocation handler " + invocationHandler +
+          " for proxy " + proxy + ", or invocation handler is not closeable.");
     }
   }
 
@@ -532,7 +538,7 @@ public class RPC {
   }
 
   /** Construct a server for a protocol implementation instance. */
-  public static Server getServer(Class protocol,
+  public static Server getServer(Class<?> protocol,
                                  Object instance, String bindAddress,
                                  int port, Configuration conf) 
     throws IOException {
@@ -543,7 +549,7 @@ public class RPC {
    * @deprecated secretManager should be passed.
    */
   @Deprecated
-  public static Server getServer(Class protocol,
+  public static Server getServer(Class<?> protocol,
                                  Object instance, String bindAddress, int port,
                                  int numHandlers,
                                  boolean verbose, Configuration conf) 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java

@@ -41,9 +41,6 @@ public interface RpcEngine {
                   UserGroupInformation ticket, Configuration conf,
                   SocketFactory factory, int rpcTimeout) throws IOException;
 
-  /** Stop this proxy. */
-  void stopProxy(Object proxy);
-
   /** Expert: Make multiple, parallel calls to a set of servers. */
   Object[] call(Method method, Object[][] params, InetSocketAddress[] addrs,
                 UserGroupInformation ticket, Configuration conf)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java

@@ -30,6 +30,7 @@ import java.io.*;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.io.Closeable;
 import java.util.Map;
 import java.util.HashMap;
 
@@ -219,7 +220,7 @@ public class WritableRpcEngine implements RpcEngine {
 
   private static ClientCache CLIENTS=new ClientCache();
   
-  private static class Invoker implements InvocationHandler {
+  private static class Invoker implements InvocationHandler, Closeable {
     private Client.ConnectionId remoteId;
     private Client client;
     private boolean isClosed = false;
@@ -250,7 +251,7 @@ public class WritableRpcEngine implements RpcEngine {
     }
     
     /* close the IPC client that's responsible for this invoker's RPCs */ 
-    synchronized private void close() {
+    synchronized public void close() {
       if (!isClosed) {
         isClosed = true;
         CLIENTS.stopClient(client);
@@ -281,15 +282,6 @@ public class WritableRpcEngine implements RpcEngine {
             factory, rpcTimeout));
     return new ProtocolProxy<T>(protocol, proxy, true);
   }
-
-  /**
-   * Stop this proxy and release its invoker's resource
-   * @param proxy the proxy to be stopped
-   */
-  public void stopProxy(Object proxy) {
-    ((Invoker)Proxy.getInvocationHandler(proxy)).close();
-  }
-
   
   /** Expert: Make multiple, parallel calls to a set of servers. */
   public Object[] call(Method method, Object[][] params,

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-datanode

@@ -39,6 +39,14 @@ fi
 
 . /lib/lsb/init-functions
 
+if [ -n "$HADOOP_SECURE_DN_USER" ]; then
+  DN_USER="root"
+  IDENT_USER=${HADOOP_SECURE_DN_USER}
+else
+  DN_USER="hdfs"
+  IDENT_USER=${DN_USER}
+fi
+
 # Are we running from init?
 run_by_init() {
     ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
@@ -67,13 +75,14 @@ check_privsep_dir() {
 }
 
 export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+export HADOOP_PREFIX="/usr"
 
 case "$1" in
   start)
 	check_privsep_dir
 	check_for_no_start
 	log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
 	    log_end_msg 0
 	else
 	    log_end_msg 1
@@ -81,7 +90,7 @@ case "$1" in
 	;;
   stop)
 	log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode"
-	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid; then
+	if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid; then
 	    log_end_msg 0
 	else
 	    log_end_msg 1
@@ -91,9 +100,9 @@ case "$1" in
   restart)
 	check_privsep_dir
 	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
+	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
 	check_for_no_start log_end_msg
-	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+	if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
 	    log_end_msg 0
 	else
 	    log_end_msg 1
@@ -104,14 +113,14 @@ case "$1" in
 	check_privsep_dir
 	log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode"
 	set +e
-	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid
+	start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid
 	RET="$?"
 	set -e
 	case $RET in
 	    0)
 		# old daemon stopped
 		check_for_no_start log_end_msg
-		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
+		if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid -c ${DN_USER} -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then
 		    log_end_msg 0
 		else
 		    log_end_msg 1
@@ -131,7 +140,7 @@ case "$1" in
 	;;
 
   status)
-	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
+	status_of_proc -p ${HADOOP_PID_DIR}/hadoop-${IDENT_USER}-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $?
 	;;
 
   *)

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-jobtracker

@@ -67,6 +67,7 @@ check_privsep_dir() {
 }
 
 export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+export HADOOP_PREFIX="/usr"
 
 case "$1" in
   start)

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-namenode

@@ -67,10 +67,11 @@ check_privsep_dir() {
 }
 
 format() {
-    su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format' hdfs
+    sudo -u hdfs ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format
 }
 
 export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+export HADOOP_PREFIX="/usr"
 
 case "$1" in
   start)

hadoop-common-project/hadoop-common/src/main/packages/deb/init.d/hadoop-tasktracker

@@ -67,6 +67,7 @@ check_privsep_dir() {
 }
 
 export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+export HADOOP_PREFIX="/usr"
 
 case "$1" in
   start)

hadoop-common-project/hadoop-common/src/main/packages/hadoop-create-user.sh

@@ -14,9 +14,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
+this="${BASH_SOURCE-$0}"
+bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+this="$bin/$script"
 
 if [ "$HADOOP_HOME" != "" ]; then
   echo "Warning: \$HADOOP_HOME is deprecated."
@@ -29,30 +30,86 @@ usage() {
   echo "
 usage: $0 <parameters>
   Require parameter:
-     -u <username>                                 Create user on HDFS
+     --config /etc/hadoop                                  Location of Hadoop configuration file
+     -u <username>                                         Create user on HDFS
   Optional parameters:
-     -h                                            Display this message
+     -h                                                    Display this message
+     --kerberos-realm=KERBEROS.EXAMPLE.COM                 Set Kerberos realm
+     --super-user=hdfs                                     Set super user id
+     --super-user-keytab=/etc/security/keytabs/hdfs.keytab Set super user keytab location
   "
   exit 1
 }
 
-# Parse script parameters
-if [ $# != 2 ] ; then
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'kerberos-realm:' \
+  -l 'super-user:' \
+  -l 'super-user-keytab:' \
+  -o 'h' \
+  -o 'u' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
     usage
     exit 1
 fi
 
-while getopts "hu:" OPTION
-do
-  case $OPTION in
-    u)
-      SETUP_USER=$2; shift 2
-      ;; 
-    h)
+create_user() {
+  if [ "${SETUP_USER}" = "" ]; then
+    break
+  fi
+  HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
+  export HADOOP_PREFIX
+  export HADOOP_CONF_DIR
+  export JAVA_HOME
+  export SETUP_USER=${SETUP_USER}
+  export SETUP_PATH=/user/${SETUP_USER}
+
+  if [ ! "${KERBEROS_REALM}" = "" ]; then
+    # locate kinit cmd
+    if [ -e /etc/lsb-release ]; then
+      KINIT_CMD="/usr/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
+    else
+      KINIT_CMD="/usr/kerberos/bin/kinit -kt ${HDFS_USER_KEYTAB} ${HADOOP_HDFS_USER}"
+    fi
+    su -c "${KINIT_CMD}" ${HADOOP_HDFS_USER} 
+  fi
+
+  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}" ${HADOOP_HDFS_USER}
+  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}" ${HADOOP_HDFS_USER}
+  su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 711 ${SETUP_PATH}" ${HADOOP_HDFS_USER}
+
+  if [ "$?" == "0" ]; then
+    echo "User directory has been setup: ${SETUP_PATH}"
+  fi
+}
+
+eval set -- "${OPTS}"
+while true; do
+  case "$1" in
+    -u)
+      shift
+      ;;
+    --kerberos-realm)
+      KERBEROS_REALM=$2; shift 2
+      ;;
+    --super-user)
+      HADOOP_HDFS_USER=$2; shift 2
+      ;;
+    --super-user-keytab)
+      HDFS_USER_KEYTAB=$2; shift 2
+      ;;
+    -h)
       usage
       ;; 
     --)
-      shift ; break
+      while shift; do
+        SETUP_USER=$1
+        create_user
+      done
+      break
       ;;
     *)
       echo "Unknown option: $1"
@@ -62,15 +119,3 @@ do
   esac
 done 
 
-# Create user directory on HDFS
-export SETUP_USER
-export SETUP_PATH=/user/${SETUP_USER}
-export HADOOP_PREFIX
-export HADOOP_CONF_DIR
-
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}' hdfs
-
-if [ "$?" == "0" ]; then
-  echo "User directory has been setup: ${SETUP_PATH}"
-fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh

@@ -18,35 +18,60 @@
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
-. "$bin"/../libexec/hadoop-config.sh
+this="${BASH_SOURCE-$0}"
+export HADOOP_PREFIX=`dirname "$this"`/..
 
 usage() {
   echo "
 usage: $0 <parameters>
 
   Optional parameters:
-     --auto                                                          Setup automatically
-     --default                                                       Generate default config
-     --conf-dir=/etc/hadoop                                          Set config directory
+     --auto                                                          Setup path and configuration automatically
+     --default                                                       Setup configuration as default
+     --conf-dir=/etc/hadoop                                          Set configuration directory
      --datanode-dir=/var/lib/hadoop/hdfs/datanode                    Set datanode directory
+     --group=hadoop                                                  Set Hadoop group name
      -h                                                              Display this message
-     --jobtracker-url=hostname:9001                                  Set jobtracker url
+     --hdfs-user=hdfs                                                Set HDFS user
+     --jobtracker-host=hostname                                      Set jobtracker host
+     --namenode-host=hostname                                        Set namenode host
+     --secondarynamenode-host=hostname                               Set secondary namenode host
+     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
+     --kinit-location=/usr/kerberos/bin/kinit                        Set kinit location
+     --keytab-dir=/etc/security/keytabs                              Set keytab directory
      --log-dir=/var/log/hadoop                                       Set log directory
-     --hdfs-dir=/var/lib/hadoop/hdfs                                 Set hdfs directory
+     --pid-dir=/var/run/hadoop                                       Set pid directory
+     --hdfs-dir=/var/lib/hadoop/hdfs                                 Set HDFS directory
+     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
      --mapred-dir=/var/lib/hadoop/mapred                             Set mapreduce directory
+     --mapreduce-user=mr                                             Set mapreduce user
+     --mapreduce-user-keytab=/home/mr/hdfs.keytab                    Set mapreduce user key tab
      --namenode-dir=/var/lib/hadoop/hdfs/namenode                    Set namenode directory
-     --namenode-url=hdfs://hostname:9000/                            Set namenode url
      --replication=3                                                 Set replication factor
      --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler  Set task scheduler
+     --datanodes=hostname1,hostname2,...                             SET the datanodes
+     --tasktrackers=hostname1,hostname2,...                          SET the tasktrackers
   "
   exit 1
 }
 
+check_permission() {
+  TARGET=$1
+  OWNER="0"
+  RESULT=0
+  while [ "$TARGET" != "/" ]; do
+    PARENT=`dirname $TARGET`
+    NAME=`basename $TARGET`
+    OWNER=`ls -ln $PARENT | grep $NAME| awk '{print $3}'`
+    if [ "$OWNER" != "0" ]; then
+      RESULT=1
+      break
+    fi
+    TARGET=`dirname $TARGET`
+  done
+  return $RESULT
+}
+
 template_generator() {
   REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
   cat $1 |
@@ -64,17 +89,30 @@ OPTS=$(getopt \
   -n $0 \
   -o '' \
   -l 'auto' \
+  -l 'java-home:' \
   -l 'conf-dir:' \
   -l 'default' \
+  -l 'group:' \
   -l 'hdfs-dir:' \
   -l 'namenode-dir:' \
   -l 'datanode-dir:' \
   -l 'mapred-dir:' \
-  -l 'namenode-url:' \
-  -l 'jobtracker-url:' \
+  -l 'namenode-host:' \
+  -l 'secondarynamenode-host:' \
+  -l 'jobtracker-host:' \
   -l 'log-dir:' \
+  -l 'pid-dir:' \
   -l 'replication:' \
   -l 'taskscheduler:' \
+  -l 'hdfs-user:' \
+  -l 'hdfs-user-keytab:' \
+  -l 'mapreduce-user:' \
+  -l 'mapreduce-user-keytab:' \
+  -l 'keytab-dir:' \
+  -l 'kerberos-realm:' \
+  -l 'kinit-location:' \
+  -l 'datanodes:' \
+  -l 'tasktrackers:' \
   -o 'h' \
   -- "$@") 
   
@@ -84,8 +122,7 @@ fi
 
 # Make sure the HADOOP_LOG_DIR is not picked up from user environment.
 unset HADOOP_LOG_DIR
-
-# Parse script parameters  
+  
 eval set -- "${OPTS}"
 while true ; do
   case "$1" in
@@ -94,6 +131,10 @@ while true ; do
       AUTOMATED=1
       shift
       ;; 
+    --java-home)
+      JAVA_HOME=$2; shift 2
+      AUTOMATED=1
+      ;; 
     --conf-dir)
       HADOOP_CONF_DIR=$2; shift 2
       AUTOMATED=1
@@ -101,6 +142,10 @@ while true ; do
     --default)
       AUTOMATED=1; shift
       ;;
+    --group)
+      HADOOP_GROUP=$2; shift 2
+      AUTOMATED=1
+      ;;
     -h)
       usage
       ;; 
@@ -120,11 +165,15 @@ while true ; do
       HADOOP_MAPRED_DIR=$2; shift 2
       AUTOMATED=1
       ;; 
-    --namenode-url)
+    --namenode-host)
       HADOOP_NN_HOST=$2; shift 2
       AUTOMATED=1
       ;; 
-    --jobtracker-url)
+    --secondarynamenode-host)
+      HADOOP_SNN_HOST=$2; shift 2
+      AUTOMATED=1
+      ;; 
+    --jobtracker-host)
       HADOOP_JT_HOST=$2; shift 2
       AUTOMATED=1
       ;; 
@@ -132,6 +181,10 @@ while true ; do
       HADOOP_LOG_DIR=$2; shift 2
       AUTOMATED=1
       ;; 
+    --pid-dir)
+      HADOOP_PID_DIR=$2; shift 2
+      AUTOMATED=1
+      ;; 
     --replication)
       HADOOP_REPLICATION=$2; shift 2
       AUTOMATED=1
@@ -139,7 +192,46 @@ while true ; do
     --taskscheduler)
       HADOOP_TASK_SCHEDULER=$2; shift 2
       AUTOMATED=1
-      ;; 
+      ;;
+    --hdfs-user)
+      HADOOP_HDFS_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-user)
+      HADOOP_MR_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --keytab-dir)
+      KEYTAB_DIR=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --hdfs-user-keytab)
+      HDFS_KEYTAB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-user-keytab)
+      MR_KEYTAB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --kerberos-realm)
+      KERBEROS_REALM=$2; shift 2
+      SECURITY_TYPE="kerberos"
+      AUTOMATED=1
+      ;;
+    --kinit-location)
+      KINIT=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --datanodes)
+      DATANODES=$2; shift 2
+      AUTOMATED=1
+      DATANODES=$(echo $DATANODES | tr ',' ' ')
+      ;;
+    --tasktrackers)
+      TASKTRACKERS=$2; shift 2
+      AUTOMATED=1
+      TASKTRACKERS=$(echo $TASKTRACKERS | tr ',' ' ')
+      ;;
     --)
       shift ; break
       ;;
@@ -151,21 +243,40 @@ while true ; do
   esac
 done 
 
-# Fill in default values, if parameters have not been defined.
 AUTOSETUP=${AUTOSETUP:-1}
 JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://`hostname`:9000/}
+HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
+HADOOP_NN_HOST=${HADOOP_NN_HOST:-`hostname`}
 HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
 HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`:9001}
+HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`}
 HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs}
 HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
 HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop}
+HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/log/hadoop}
 HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
 HADOOP_REPLICATION=${HADOOP_RELICATION:-3}
 HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
+HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
+HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
+KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
+HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
+MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
+KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
+SECURITY_TYPE=${SECURITY_TYPE:-simple}
+KINIT=${KINIT:-/usr/kerberos/bin/kinit}
+if [ "${SECURITY_TYPE}" = "kerberos" ]; then
+  TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
+  HADOOP_DN_ADDR="0.0.0.0:1019"
+  HADOOP_DN_HTTP_ADDR="0.0.0.0:1022"
+  SECURITY="true"
+else
+  TASK_CONTROLLER="org.apache.hadoop.mapred.DefaultTaskController"
+  HADDOP_DN_ADDR="0.0.0.0:50010"
+  HADOOP_DN_HTTP_ADDR="0.0.0.0:50075"
+  SECURITY="false"
+fi
 
-# Interactive setup wizard
 if [ "${AUTOMATED}" != "1" ]; then
   echo "Setup Hadoop Configuration"
   echo
@@ -173,18 +284,18 @@ if [ "${AUTOMATED}" != "1" ]; then
   read USER_HADOOP_CONF_DIR
   echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) "
   read USER_HADOOP_LOG_DIR
-  echo -n "What is the url of the namenode? (${HADOOP_NN_HOST}) "
+  echo -n "Where would you like to put pid directory? (${HADOOP_PID_DIR}) "
+  read USER_HADOOP_PID_DIR
+  echo -n "What is the host of the namenode? (${HADOOP_NN_HOST}) "
   read USER_HADOOP_NN_HOST
   echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) "
   read USER_HADOOP_NN_DIR
   echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) "
   read USER_HADOOP_DN_DIR
-  echo -n "What is the url of the jobtracker? (${HADOOP_JT_HOST}) "
+  echo -n "What is the host of the jobtracker? (${HADOOP_JT_HOST}) "
   read USER_HADOOP_JT_HOST
   echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) "
   read USER_HADOOP_MAPRED_DIR
-  echo -n "Which taskscheduler would you like? (${HADOOP_TASK_SCHEDULER}) "
-  read USER_HADOOP_TASK_SCHEDULER
   echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) "
   read USER_JAVA_HOME
   echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) "
@@ -199,16 +310,18 @@ if [ "${AUTOMATED}" != "1" ]; then
   HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR}
   HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
   HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR}
+  HADOOP_PID_DIR=${USER_HADOOP_PID_DIR:-$HADOOP_PID_DIR}
   HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR}
   AUTOSETUP=${USER_AUTOSETUP:-y}
   echo "Review your choices:"
   echo
   echo "Config directory            : ${HADOOP_CONF_DIR}"
   echo "Log directory               : ${HADOOP_LOG_DIR}"
-  echo "Namenode url                : ${HADOOP_NN_HOST}"
+  echo "PID directory               : ${HADOOP_PID_DIR}"
+  echo "Namenode host               : ${HADOOP_NN_HOST}"
   echo "Namenode directory          : ${HADOOP_NN_DIR}"
   echo "Datanode directory          : ${HADOOP_DN_DIR}"
-  echo "Jobtracker url              : ${HADOOP_JT_HOST}"
+  echo "Jobtracker host             : ${HADOOP_JT_HOST}"
   echo "Mapreduce directory         : ${HADOOP_MAPRED_DIR}"
   echo "Task scheduler              : ${HADOOP_TASK_SCHEDULER}"
   echo "JAVA_HOME directory         : ${JAVA_HOME}"
@@ -222,61 +335,180 @@ if [ "${AUTOMATED}" != "1" ]; then
   fi
 fi
 
-if [ "${AUTOSETUP}" == "1" ]; then
-  # If user wants to setup local system automatically,
-  # set config file generation location to HADOOP_CONF_DIR.
-  DEST=${HADOOP_CONF_DIR}
-else
-  # If user is only interested to generate config file locally,
-  # place config files in the current working directory.
-  DEST=`pwd`
-fi
-
-# remove existing config file, they are existed in current directory.
-rm -f ${DEST}/core-site.xml >/dev/null
-rm -f ${DEST}/hdfs-site.xml >/dev/null
-rm -f ${DEST}/mapred-site.xml >/dev/null
-rm -f ${DEST}/hadoop-env.sh >/dev/null
-
-# Generate config file with specified parameters.
-template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/core-site.xml ${DEST}/core-site.xml
-template_generator ${HADOOP_PREFIX}/share/hadoop/hdfs/templates/hdfs-site.xml ${DEST}/hdfs-site.xml
-template_generator ${HADOOP_PREFIX}/share/hadoop/mapreduce/templates/mapred-site.xml ${DEST}/mapred-site.xml
-template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template ${DEST}/hadoop-env.sh
-
-chown root:hadoop ${DEST}/hadoop-env.sh
-chmod 755 ${DEST}/hadoop-env.sh
-
-# Setup directory path and copy config files, if AUTOSETUP is chosen.
 if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then
-  mkdir -p ${HADOOP_HDFS_DIR}
-  mkdir -p ${HADOOP_NN_DIR}
-  mkdir -p ${HADOOP_DN_DIR}
-  mkdir -p ${HADOOP_MAPRED_DIR}
+  if [ -d ${KEYTAB_DIR} ]; then
+    chmod 700 ${KEYTAB_DIR}/*
+    chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[jt]t.service.keytab
+    chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${KEYTAB_DIR}/[dns]n.service.keytab
+  fi
+  chmod 755 -R ${HADOOP_PREFIX}/sbin/*hadoop*
+  chmod 755 -R ${HADOOP_PREFIX}/bin/hadoop
+  chmod 755 -R ${HADOOP_PREFIX}/libexec/hadoop-config.sh
+  mkdir -p /home/${HADOOP_MR_USER}
+  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} /home/${HADOOP_MR_USER}
+  HDFS_DIR=`echo ${HADOOP_HDFS_DIR} | sed -e 's/,/ /g'`
+  mkdir -p ${HDFS_DIR}
+  if [ -e ${HADOOP_NN_DIR} ]; then
+    rm -rf ${HADOOP_NN_DIR}
+  fi
+  DATANODE_DIR=`echo ${HADOOP_DN_DIR} | sed -e 's/,/ /g'`
+  mkdir -p ${DATANODE_DIR}
+  MAPRED_DIR=`echo ${HADOOP_MAPRED_DIR} | sed -e 's/,/ /g'`
+  mkdir -p ${MAPRED_DIR}
   mkdir -p ${HADOOP_CONF_DIR}
+  check_permission ${HADOOP_CONF_DIR}
+  if [ $? == 1 ]; then
+    echo "Full path to ${HADOOP_CONF_DIR} should be owned by root."
+    exit 1
+  fi
+
   mkdir -p ${HADOOP_LOG_DIR}
-  mkdir -p ${HADOOP_LOG_DIR}/hdfs
-  mkdir -p ${HADOOP_LOG_DIR}/mapred
-  chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  chown hdfs:hadoop ${HADOOP_NN_DIR}
-  chown hdfs:hadoop ${HADOOP_DN_DIR}
-  chown mapred:hadoop ${HADOOP_MAPRED_DIR}
-  chown root:hadoop ${HADOOP_LOG_DIR}
+  #create the log sub dir for diff users
+  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+  mkdir -p ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
+
+  mkdir -p ${HADOOP_PID_DIR}
+  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HDFS_DIR}
+  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${DATANODE_DIR}
+  chmod 700 -R ${DATANODE_DIR}
+  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${MAPRED_DIR}
+  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}
   chmod 775 ${HADOOP_LOG_DIR}
-  chown hdfs:hadoop ${HADOOP_LOG_DIR}/hdfs
-  chown mapred:hadoop ${HADOOP_LOG_DIR}/mapred
+  chmod 775 ${HADOOP_PID_DIR}
+  chown root:${HADOOP_GROUP} ${HADOOP_PID_DIR}
+
+  #change the permission and the owner
+  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+  chown ${HADOOP_HDFS_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+  chmod 755 ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
+  chown ${HADOOP_MR_USER}:${HADOOP_GROUP} ${HADOOP_LOG_DIR}/${HADOOP_MR_USER}
+
+  if [ -e ${HADOOP_CONF_DIR}/core-site.xml ]; then
+    mv -f ${HADOOP_CONF_DIR}/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/hdfs-site.xml ]; then
+    mv -f ${HADOOP_CONF_DIR}/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/mapred-site.xml ]; then
+    mv -f ${HADOOP_CONF_DIR}/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/hadoop-env.sh ]; then
+    mv -f ${HADOOP_CONF_DIR}/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/hadoop-policy.xml ]; then
+    mv -f ${HADOOP_CONF_DIR}/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/mapred-queue-acls.xml ]; then
+    mv -f ${HADOOP_CONF_DIR}/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/commons-logging.properties ]; then
+    mv -f ${HADOOP_CONF_DIR}/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/taskcontroller.cfg  ]; then
+    mv -f ${HADOOP_CONF_DIR}/taskcontroller.cfg  ${HADOOP_CONF_DIR}/taskcontroller.cfg.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/slaves  ]; then
+    mv -f ${HADOOP_CONF_DIR}/slaves  ${HADOOP_CONF_DIR}/slaves.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/dfs.include  ]; then
+    mv -f ${HADOOP_CONF_DIR}/dfs.include  ${HADOOP_CONF_DIR}/dfs.include.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/dfs.exclude  ]; then
+    mv -f ${HADOOP_CONF_DIR}/dfs.exclude  ${HADOOP_CONF_DIR}/dfs.exclude.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/mapred.include  ]; then
+    mv -f ${HADOOP_CONF_DIR}/mapred.include  ${HADOOP_CONF_DIR}/mapred.include.bak
+  fi
+  if [ -e ${HADOOP_CONF_DIR}/mapred.exclude  ]; then
+    mv -f ${HADOOP_CONF_DIR}/mapred.exclude  ${HADOOP_CONF_DIR}/mapred.exclude.bak
+  fi
+
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
+
+  #set the owner of the hadoop dir to root
+  chown root ${HADOOP_PREFIX}
+  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
+  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
+  
+  #set taskcontroller
+  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
+  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
+  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
+  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
+
+
+  #generate the slaves file and include and exclude files for hdfs and mapred
+  echo '' > ${HADOOP_CONF_DIR}/slaves
+  echo '' > ${HADOOP_CONF_DIR}/dfs.include
+  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
+  echo '' > ${HADOOP_CONF_DIR}/mapred.include
+  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
+  for dn in $DATANODES
+  do
+    echo $dn >> ${HADOOP_CONF_DIR}/slaves
+    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
+  done
+  for tt in $TASKTRACKERS
+  do
+    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
+  done
+
   echo "Configuration setup is completed."
   if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then
     echo "Proceed to run hadoop-setup-hdfs.sh on namenode."
   fi
 else
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/core-site.xml ${HADOOP_CONF_DIR}/core-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hdfs-site.xml ${HADOOP_CONF_DIR}/hdfs-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-site.xml ${HADOOP_CONF_DIR}/mapred-site.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-env.sh ${HADOOP_CONF_DIR}/hadoop-env.sh
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-policy.xml ${HADOOP_CONF_DIR}/hadoop-policy.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/commons-logging.properties ${HADOOP_CONF_DIR}/commons-logging.properties
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/mapred-queue-acls.xml ${HADOOP_CONF_DIR}/mapred-queue-acls.xml
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/taskcontroller.cfg ${HADOOP_CONF_DIR}/taskcontroller.cfg
+  template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties
+  if [ ! -e ${HADOOP_CONF_DIR}/capacity-scheduler.xml ]; then
+    template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/capacity-scheduler.xml ${HADOOP_CONF_DIR}/capacity-scheduler.xml
+  fi
+  
+  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
+  chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
+  #set taskcontroller
+  chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/taskcontroller.cfg
+  chmod 400 ${HADOOP_CONF_DIR}/taskcontroller.cfg
+  chown root:${HADOOP_GROUP} ${HADOOP_PREFIX}/bin/task-controller
+  chmod 6050 ${HADOOP_PREFIX}/bin/task-controller
+  
+  #generate the slaves file and include and exclude files for hdfs and mapred
+  echo '' > ${HADOOP_CONF_DIR}/slaves
+  echo '' > ${HADOOP_CONF_DIR}/dfs.include
+  echo '' > ${HADOOP_CONF_DIR}/dfs.exclude
+  echo '' > ${HADOOP_CONF_DIR}/mapred.include
+  echo '' > ${HADOOP_CONF_DIR}/mapred.exclude
+  for dn in $DATANODES
+  do
+    echo $dn >> ${HADOOP_CONF_DIR}/slaves
+    echo $dn >> ${HADOOP_CONF_DIR}/dfs.include
+  done
+  for tt in $TASKTRACKERS
+  do
+    echo $tt >> ${HADOOP_CONF_DIR}/mapred.include
+  done
+  
   echo
-  echo "Configuration file has been generated, please copy:"
+  echo "Configuration file has been generated in:"
   echo
-  echo "core-site.xml"
-  echo "hdfs-site.xml"
-  echo "mapred-site.xml"
-  echo "hadoop-env.sh"
+  echo "${HADOOP_CONF_DIR}/core-site.xml"
+  echo "${HADOOP_CONF_DIR}/hdfs-site.xml"
+  echo "${HADOOP_CONF_DIR}/mapred-site.xml"
+  echo "${HADOOP_CONF_DIR}/hadoop-env.sh"
   echo
   echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode."
 fi

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-hdfs.sh

@@ -18,37 +18,65 @@
 bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 
-if [ "$HADOOP_HOME" != "" ]; then
-  echo "Warning: \$HADOOP_HOME is deprecated."
-  echo
-fi
-
 . "$bin"/../libexec/hadoop-config.sh
 
 usage() {
   echo "
 usage: $0 <parameters>
-  Require parameter:
-     -c <clusterid>                                Set cluster identifier for HDFS
+
   Optional parameters:
-     -h                                            Display this message
+     --format                                                        Force namenode format
+     --group=hadoop                                                  Set Hadoop group
+     -h                                                              Display this message
+     --hdfs-user=hdfs                                                Set HDFS user
+     --kerberos-realm=KERBEROS.EXAMPLE.COM                           Set Kerberos realm
+     --hdfs-user-keytab=/home/hdfs/hdfs.keytab                       Set HDFS user key tab
+     --mapreduce-user=mr                                             Set mapreduce user
   "
   exit 1
 }
 
-if [ $# != 2 ] ; then
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'format' \
+  -l 'hdfs-user:' \
+  -l 'hdfs-user-keytab:' \
+  -l 'mapreduce-user:' \
+  -l 'kerberos-realm:' \
+  -o 'h' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
     usage
-    exit 1
 fi
 
-while getopts "hc:" OPTION
-do
-  case $OPTION in
-    c)
-      SETUP_CLUSTER=$2; shift 2
+eval set -- "${OPTS}"
+while true ; do
+  case "$1" in
+    --format)
+      FORMAT_NAMENODE=1; shift
+      AUTOMATED=1
       ;;
-    h)
-      usage
+    --group)
+      HADOOP_GROUP=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --hdfs-user)
+      HADOOP_HDFS_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-user)
+      HADOOP_MR_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --hdfs-user-keytab)
+      HDFS_KEYTAB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --kerberos-realm)
+      KERBEROS_REALM=$2; shift 2
+      AUTOMATED=1
       ;;
     --)
       shift ; break
@@ -61,30 +89,56 @@ do
   esac
 done
 
-export HADOOP_PREFIX
-export HADOOP_CONF_DIR
-export SETUP_CLUSTER
+HADOOP_GROUP=${HADOOP_GROUP:-hadoop}
+HADOOP_HDFS_USER=${HADOOP_HDFS_USER:-hdfs}
+HADOOP_MAPREDUCE_USER=${HADOOP_MR_USER:-mapred}
+
+if [ "${KERBEROS_REALM}" != "" ]; then
+  # Determine kerberos location base on Linux distro.
+  if [ -e /etc/lsb-release ]; then
+    KERBEROS_BIN=/usr/bin
+  else
+    KERBEROS_BIN=/usr/kerberos/bin
+  fi
+  kinit_cmd="${KERBEROS_BIN}/kinit -k -t ${HDFS_KEYTAB} ${HADOOP_HDFS_USER}"
+  su -c "${kinit_cmd}" ${HADOOP_HDFS_USER}
+fi
 
-# Start namenode and initialize file system structure
 echo "Setup Hadoop Distributed File System"
 echo
-echo "Formatting namenode"
-echo
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format -clusterid ${SETUP_CLUSTER}' hdfs
-echo
+
+# Format namenode
+if [ "${FORMAT_NAMENODE}" == "1" ]; then
+  echo "Formatting namenode"
+  echo
+  su -c "echo Y | ${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format" ${HADOOP_HDFS_USER}
+  echo
+fi
+
+# Start namenode process
 echo "Starting namenode process"
 echo
-/etc/init.d/hadoop-namenode start
+if [ -e ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh ]; then
+  DAEMON_PATH=${HADOOP_PREFIX}/sbin
+else
+  DAEMON_PATH=${HADOOP_PREFIX}/bin
+fi
+su -c "${DAEMON_PATH}/hadoop-daemon.sh --config ${HADOOP_CONF_DIR} start namenode" ${HADOOP_HDFS_USER}
 echo
 echo "Initialize HDFS file system: "
 echo
 
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs
+#create the /user dir 
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user" ${HADOOP_HDFS_USER}
+
+#create /tmp and give it 777
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp" ${HADOOP_HDFS_USER}
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp" ${HADOOP_HDFS_USER}
+
+#create /mapred
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /mapred" ${HADOOP_HDFS_USER}
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 700 /mapred" ${HADOOP_HDFS_USER}
+su -c "${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown ${HADOOP_MAPREDUCE_USER}:system /mapred" ${HADOOP_HDFS_USER}
 
 if [ $? -eq 0 ]; then
   echo "Completed."

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-single-node.sh

@@ -17,16 +17,16 @@
 
 # Script for setup HDFS file system for single node deployment
 
-bin=`which $0`
-bin=`dirname ${bin}`
+bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 
-export HADOOP_PREFIX=${bin}/..
-
-if [ -e /etc/hadoop/hadoop-env.sh ]; then
-  . /etc/hadoop/hadoop-env.sh
+if [ "$HADOOP_HOME" != "" ]; then
+  echo "Warning: \$HADOOP_HOME is deprecated."
+  echo
 fi
 
+. "$bin"/../libexec/hadoop-config.sh
+
 usage() {
   echo "
 usage: $0 <parameters>
@@ -38,7 +38,19 @@ usage: $0 <parameters>
   exit 1
 }
 
-# Parse script parameters
+template_generator() {
+  REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})'
+  cat $1 |
+  while read line ; do
+    while [[ "$line" =~ $REGEX ]] ; do
+      LHS=${BASH_REMATCH[1]}
+      RHS="$(eval echo "\"$LHS\"")"
+      line=${line//$LHS/$RHS}
+    done
+    echo $line >> $2
+  done
+}
+
 OPTS=$(getopt \
   -n $0 \
   -o '' \
@@ -49,6 +61,10 @@ if [ $? != 0 ] ; then
     usage
 fi
 
+if [ -e /etc/hadoop/hadoop-env.sh ]; then
+  . /etc/hadoop/hadoop-env.sh
+fi
+
 eval set -- "${OPTS}"
 while true ; do
   case "$1" in
@@ -69,7 +85,6 @@ while true ; do
   esac
 done
 
-# Interactive setup wizard
 if [ "${AUTOMATED}" != "1" ]; then
   echo "Welcome to Hadoop single node setup wizard"
   echo
@@ -119,68 +134,59 @@ SET_REBOOT=${SET_REBOOT:-y}
 /etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null
 /etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null
 
-# Default settings
-JAVA_HOME=${JAVA_HOME:-/usr/java/default}
-HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://localhost:9000/}
-HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
-HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
-HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost:9001}
-HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs}
-HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
-HADOOP_LOG_DIR="/var/log/hadoop"
-HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
-HADOOP_REPLICATION=${HADOOP_RELICATION:-1}
-HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler}
-
-# Setup config files
 if [ "${SET_CONFIG}" == "y" ]; then
+  JAVA_HOME=${JAVA_HOME:-/usr/java/default}
+  HADOOP_NN_HOST=${HADOOP_NN_HOST:-localhost}
+  HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode}
+  HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode}
+  HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost}
+  HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs}
+  HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred}
+  HADOOP_PID_DIR=${HADOOP_PID_DIR:-/var/run/hadoop}
+  HADOOP_LOG_DIR="/var/log/hadoop"
+  HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop}
+  HADOOP_REPLICATION=${HADOOP_RELICATION:-1}
   ${HADOOP_PREFIX}/sbin/hadoop-setup-conf.sh --auto \
+    --hdfs-user=hdfs \
+    --mapreduce-user=mapred \
     --conf-dir=${HADOOP_CONF_DIR} \
     --datanode-dir=${HADOOP_DN_DIR} \
     --hdfs-dir=${HADOOP_HDFS_DIR} \
-    --jobtracker-url=${HADOOP_JT_HOST} \
+    --jobtracker-host=${HADOOP_JT_HOST} \
     --log-dir=${HADOOP_LOG_DIR} \
+    --pid-dir=${HADOOP_PID_DIR} \
     --mapred-dir=${HADOOP_MAPRED_DIR} \
     --namenode-dir=${HADOOP_NN_DIR} \
-    --namenode-url=${HADOOP_NN_HOST} \
+    --namenode-host=${HADOOP_NN_HOST} \
     --replication=${HADOOP_REPLICATION}
 fi
 
-export HADOOP_CONF_DIR
-
-# Format namenode
 if [ ! -e ${HADOOP_NN_DIR} ]; then
   rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
   mkdir -p ${HADOOP_HDFS_DIR}
   chmod 755 ${HADOOP_HDFS_DIR}
   chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs
+  /etc/init.d/hadoop-namenode format
 elif [ "${SET_FORMAT}" == "y" ]; then
   rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null
   mkdir -p ${HADOOP_HDFS_DIR}
   chmod 755 ${HADOOP_HDFS_DIR}
   chown hdfs:hadoop ${HADOOP_HDFS_DIR}
-  rm -rf /var/lib/hadoop/hdfs/namenode
-  su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs
+  rm -rf ${HADOOP_NN_DIR}
+  /etc/init.d/hadoop-namenode format
 fi
 
-# Start hdfs service
 /etc/init.d/hadoop-namenode start
 /etc/init.d/hadoop-datanode start
 
-# Initialize file system structure
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs
-su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /user/mapred' hdfs
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chown mapred:mapred /user/mapred' hdfs
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -mkdir /tmp' hdfs
+su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} dfs -chmod 777 /tmp' hdfs
 
-# Start mapreduce service
 /etc/init.d/hadoop-jobtracker start
 /etc/init.d/hadoop-tasktracker start
 
-# Toggle service startup on reboot
 if [ "${SET_REBOOT}" == "y" ]; then
   if [ -e /etc/debian_version ]; then
     ln -sf ../init.d/hadoop-namenode /etc/rc2.d/S90hadoop-namenode
@@ -203,7 +209,6 @@ if [ "${SET_REBOOT}" == "y" ]; then
   fi
 fi
 
-# Shutdown service, if user choose to stop services after setup
 if [ "${STARTUP}" != "y" ]; then
   /etc/init.d/hadoop-namenode stop
   /etc/init.d/hadoop-datanode stop

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-datanode

@@ -27,10 +27,15 @@ source /etc/default/hadoop-env.sh
 RETVAL=0
 PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid"
 desc="Hadoop datanode daemon"
+HADOOP_PREFIX="/usr"
 
 start() {
   echo -n $"Starting $desc (hadoop-datanode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
+  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
+    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
+  else
+    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode
+  fi
   RETVAL=$?
   echo
   [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-datanode
@@ -39,7 +44,11 @@ start() {
 
 stop() {
   echo -n $"Stopping $desc (hadoop-datanode): "
-  daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
+  if [ -n "$HADOOP_SECURE_DN_USER" ]; then
+    daemon ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
+  else
+    daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode
+  fi
   RETVAL=$?
   sleep 5
   echo

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-jobtracker

@@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
 RETVAL=0
 PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid"
 desc="Hadoop jobtracker daemon"
+export HADOOP_PREFIX="/usr"
 
 start() {
   echo -n $"Starting $desc (hadoop-jobtracker): "

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-namenode

@@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
 RETVAL=0
 PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid"
 desc="Hadoop namenode daemon"
+export HADOOP_PREFIX="/usr"
 
 start() {
   echo -n $"Starting $desc (hadoop-namenode): "

hadoop-common-project/hadoop-common/src/main/packages/rpm/init.d/hadoop-tasktracker

@@ -27,6 +27,7 @@ source /etc/default/hadoop-env.sh
 RETVAL=0
 PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid"
 desc="Hadoop tasktracker daemon"
+export HADOOP_PREFIX="/usr"
 
 start() {
   echo -n $"Starting $desc (hadoop-tasktracker): "

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/capacity-scheduler.xml

@@ -0,0 +1,178 @@
+<?xml version="1.0"?>
+
+<!-- This is the configuration file for the resource manager in Hadoop. -->
+<!-- You can configure various scheduling parameters related to queues. -->
+<!-- The properties for a queue follow a naming convention,such as, -->
+<!-- mapred.capacity-scheduler.queue.<queue-name>.property-name. -->
+
+<configuration>
+
+  <property>
+    <name>mapred.capacity-scheduler.maximum-system-jobs</name>
+    <value>3000</value>
+    <description>Maximum number of jobs in the system which can be initialized,
+     concurrently, by the CapacityScheduler.
+    </description>    
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.capacity</name>
+    <value>100</value>
+    <description>Percentage of the number of slots in the cluster that are
+      to be available for jobs in this queue.
+    </description>    
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.maximum-capacity</name>
+    <value>-1</value>
+    <description>
+	maximum-capacity defines a limit beyond which a queue cannot use the capacity of the cluster.
+	This provides a means to limit how much excess capacity a queue can use. By default, there is no limit.
+	The maximum-capacity of a queue can only be greater than or equal to its minimum capacity.
+        Default value of -1 implies a queue can use complete capacity of the cluster.
+
+        This property could be to curtail certain jobs which are long running in nature from occupying more than a 
+        certain percentage of the cluster, which in the absence of pre-emption, could lead to capacity guarantees of 
+        other queues being affected.
+        
+        One important thing to note is that maximum-capacity is a percentage , so based on the cluster's capacity
+        the max capacity would change. So if large no of nodes or racks get added to the cluster , max Capacity in 
+        absolute terms would increase accordingly.
+    </description>    
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.supports-priority</name>
+    <value>false</value>
+    <description>If true, priorities of jobs will be taken into 
+      account in scheduling decisions.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.minimum-user-limit-percent</name>
+    <value>100</value>
+    <description> Each queue enforces a limit on the percentage of resources 
+    allocated to a user at any given time, if there is competition for them. 
+    This user limit can vary between a minimum and maximum value. The former
+    depends on the number of users who have submitted jobs, and the latter is
+    set to this property value. For example, suppose the value of this 
+    property is 25. If two users have submitted jobs to a queue, no single 
+    user can use more than 50% of the queue resources. If a third user submits
+    a job, no single user can use more than 33% of the queue resources. With 4 
+    or more users, no user can use more than 25% of the queue's resources. A 
+    value of 100 implies no user limits are imposed. 
+    </description>
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.user-limit-factor</name>
+    <value>1</value>
+    <description>The multiple of the queue capacity which can be configured to 
+    allow a single user to acquire more slots. 
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks</name>
+    <value>200000</value>
+    <description>The maximum number of tasks, across all jobs in the queue, 
+    which can be initialized concurrently. Once the queue's jobs exceed this 
+    limit they will be queued on disk.  
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user</name>
+    <value>100000</value>
+    <description>The maximum number of tasks per-user, across all the of the 
+    user's jobs in the queue, which can be initialized concurrently. Once the 
+    user's jobs exceed this limit they will be queued on disk.  
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.queue.default.init-accept-jobs-factor</name>
+    <value>10</value>
+    <description>The multipe of (maximum-system-jobs * queue-capacity) used to 
+    determine the number of jobs which are accepted by the scheduler.  
+    </description>
+  </property>
+
+  <!-- The default configuration settings for the capacity task scheduler -->
+  <!-- The default values would be applied to all the queues which don't have -->
+  <!-- the appropriate property for the particular queue -->
+  <property>
+    <name>mapred.capacity-scheduler.default-supports-priority</name>
+    <value>false</value>
+    <description>If true, priorities of jobs will be taken into 
+      account in scheduling decisions by default in a job queue.
+    </description>
+  </property>
+  
+  <property>
+    <name>mapred.capacity-scheduler.default-minimum-user-limit-percent</name>
+    <value>100</value>
+    <description>The percentage of the resources limited to a particular user
+      for the job queue at any given point of time by default.
+    </description>
+  </property>
+
+
+  <property>
+    <name>mapred.capacity-scheduler.default-user-limit-factor</name>
+    <value>1</value>
+    <description>The default multiple of queue-capacity which is used to 
+    determine the amount of slots a single user can consume concurrently.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-queue</name>
+    <value>200000</value>
+    <description>The default maximum number of tasks, across all jobs in the 
+    queue, which can be initialized concurrently. Once the queue's jobs exceed 
+    this limit they will be queued on disk.  
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-user</name>
+    <value>100000</value>
+    <description>The default maximum number of tasks per-user, across all the of 
+    the user's jobs in the queue, which can be initialized concurrently. Once 
+    the user's jobs exceed this limit they will be queued on disk.  
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.capacity-scheduler.default-init-accept-jobs-factor</name>
+    <value>10</value>
+    <description>The default multipe of (maximum-system-jobs * queue-capacity) 
+    used to determine the number of jobs which are accepted by the scheduler.  
+    </description>
+  </property>
+
+  <!-- Capacity scheduler Job Initialization configuration parameters -->
+  <property>
+    <name>mapred.capacity-scheduler.init-poll-interval</name>
+    <value>5000</value>
+    <description>The amount of time in miliseconds which is used to poll 
+    the job queues for jobs to initialize.
+    </description>
+  </property>
+  <property>
+    <name>mapred.capacity-scheduler.init-worker-threads</name>
+    <value>5</value>
+    <description>Number of worker threads which would be used by
+    Initialization poller to initialize jobs in a set of queue.
+    If number mentioned in property is equal to number of job queues
+    then a single thread would initialize jobs in a queue. If lesser
+    then a thread would get a set of queues assigned. If the number
+    is greater then number of threads would be equal to number of 
+    job queues.
+    </description>
+  </property>
+
+</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/commons-logging.properties

@@ -0,0 +1,7 @@
+#Logging Implementation
+
+#Log4J
+org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
+
+#JDK Logger
+#org.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/core-site.xml

@@ -1,27 +1,78 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
 
 <!-- Put site-specific property overrides in this file. -->
 
 <configuration>
+
+  <property>
+    <name>local.realm</name>
+    <value>${KERBEROS_REALM}</value>
+  </property>
+
+  <!-- file system properties -->
+
   <property>
     <name>fs.default.name</name>
-    <value>${HADOOP_NN_HOST}</value>
+    <value>hdfs://${HADOOP_NN_HOST}:8020</value>
+    <description>The name of the default file system.  Either the
+      literal string "local" or a host:port for NDFS.
+    </description>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>fs.trash.interval</name>
+    <value>360</value>
+    <description>Number of minutes between trash checkpoints.
+      If zero, the trash feature is disabled.
+    </description>
+  </property>
+  
+  <property>
+    <name>hadoop.security.auth_to_local</name>
+    <value>
+	RULE:[2:$1@$0]([jt]t@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
+        RULE:[2:$1@$0]([nd]n@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
+        RULE:[2:$1@$0](mapred@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
+        RULE:[2:$1@$0](hdfs@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
+        RULE:[2:$1@$0](mapredqa@.*${KERBEROS_REALM})s/.*/${HADOOP_MR_USER}/
+        RULE:[2:$1@$0](hdfsqa@.*${KERBEROS_REALM})s/.*/${HADOOP_HDFS_USER}/
+        DEFAULT
+    </value>
+    <description></description>
+  </property>
+
+  <property>
+    <name>hadoop.security.authentication</name>
+    <value>${SECURITY_TYPE}</value>
+    <description>
+      Set the authentication for the cluster. Valid values are: simple or
+      kerberos.
+    </description>
   </property>
+
+  <property>
+    <name>hadoop.security.authorization</name>
+    <value>${SECURITY}</value>
+    <description>
+      Enable authorization for different protocols.
+    </description>
+  </property>
+
+  <property>
+    <name>hadoop.security.groups.cache.secs</name>
+    <value>14400</value>
+  </property>
+
+  <property>
+    <name>hadoop.kerberos.kinit.command</name>
+    <value>${KINIT}</value>
+  </property>
+
+  <property>
+    <name>hadoop.http.filter.initializers</name>
+    <value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
+  </property>
+
 </configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-env.sh

@@ -0,0 +1,54 @@
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME.  All others are
+# optional.  When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.
+export JAVA_HOME=${JAVA_HOME}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
+
+# Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
+for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
+  if [ "$HADOOP_CLASSPATH" ]; then
+    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
+  else
+    export HADOOP_CLASSPATH=$f
+  fi
+done
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+#export HADOOP_HEAPSIZE=
+#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+# Extra Java runtime options.  Empty by default.
+export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true $HADOOP_OPTS"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
+HADOOP_JOBTRACKER_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dmapred.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
+HADOOP_TASKTRACKER_OPTS="-Dsecurity.audit.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
+HADOOP_DATANODE_OPTS="-Dsecurity.audit.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dsecurity.audit.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx128m ${HADOOP_CLIENT_OPTS}"
+#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData ${HADOOP_JAVA_PLATFORM_OPTS}"
+
+# On secure datanodes, user to run the datanode as after dropping privileges
+export HADOOP_SECURE_DN_USER=${HADOOP_HDFS_USER}
+
+# Where log files are stored.  $HADOOP_HOME/logs by default.
+export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+
+# The directory where pid files are stored. /tmp by default.
+export HADOOP_PID_DIR=${HADOOP_PID_DIR}
+export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml

@@ -0,0 +1,118 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>security.client.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for ClientProtocol, which is used by user code
+    via the DistributedFileSystem.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.client.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
+    for block recovery.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for DatanodeProtocol, which is used by datanodes to
+    communicate with the namenode.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.inter.datanode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
+    for updating generation timestamp.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.namenode.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for NamenodeProtocol, the protocol used by the secondary
+    namenode to communicate with the namenode.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.inter.tracker.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for InterTrackerProtocol, used by the tasktrackers to
+    communicate with the jobtracker.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.job.submission.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for JobSubmissionProtocol, used by job clients to
+    communciate with the jobtracker for job submission, querying job status etc.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.task.umbilical.protocol.acl</name>
+    <value>*</value>
+    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
+    tasks to communicate with the parent tasktracker.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+ <property>
+    <name>security.admin.operations.protocol.acl</name>
+    <value>${HADOOP_HDFS_USER}</value>
+    <description>ACL for AdminOperationsProtocol. Used for admin commands.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+  <property>
+    <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+    <value>${HADOOP_HDFS_USER}</value>
+    <description>ACL for RefreshUserMappingsProtocol. Used to refresh
+    users mappings. The ACL is a comma-separated list of user and
+    group names. The user and group list is separated by a blank. For
+    e.g. "alice,bob users,wheel".  A special value of "*" means all
+    users are allowed.</description>
+  </property>
+
+<property>
+    <name>security.refresh.policy.protocol.acl</name>
+    <value>${HADOOP_HDFS_USER}</value>
+    <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
+    dfsadmin and mradmin commands to refresh the security policy in-effect.
+    The ACL is a comma-separated list of user and group names. The user and
+    group list is separated by a blank. For e.g. "alice,bob users,wheel".
+    A special value of "*" means all users are allowed.</description>
+  </property>
+
+
+
+</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml

@@ -0,0 +1,225 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+<!-- file system properties -->
+
+  <property>
+    <name>dfs.name.dir</name>
+    <value>${HADOOP_NN_DIR}</value>
+    <description>Determines where on the local filesystem the DFS name node
+      should store the name table.  If this is a comma-delimited list
+      of directories then the name table is replicated in all of the
+      directories, for redundancy. </description>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>dfs.data.dir</name>
+    <value>${HADOOP_DN_DIR}</value>
+    <description>Determines where on the local filesystem an DFS data node
+       should store its blocks.  If this is a comma-delimited
+       list of directories, then data will be stored in all named
+       directories, typically on different devices.
+       Directories that do not exist are ignored.
+    </description>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>dfs.safemode.threshold.pct</name>
+    <value>1.0f</value>
+    <description>
+        Specifies the percentage of blocks that should satisfy 
+        the minimal replication requirement defined by dfs.replication.min.
+        Values less than or equal to 0 mean not to start in safe mode.
+        Values greater than 1 will make safe mode permanent.
+        </description>
+  </property>
+
+  <property>
+    <name>dfs.datanode.address</name>
+    <value>${HADOOP_DN_ADDR}</value>
+  </property>
+
+  <property>
+    <name>dfs.datanode.http.address</name>
+    <value>${HADOOP_DN_HTTP_ADDR}</value>
+  </property>
+
+  <property>
+    <name>dfs.http.address</name>
+    <value>${HADOOP_NN_HOST}:50070</value>
+    <description>The name of the default file system.  Either the
+       literal string "local" or a host:port for NDFS.
+    </description>
+    <final>true</final>
+  </property>
+
+  <!-- Permissions configuration -->
+  <property>
+    <name>dfs.umaskmode</name>
+    <value>077</value>
+    <description>
+      The octal umask used when creating files and directories.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.block.access.token.enable</name>
+    <value>${SECURITY}</value>
+    <description>
+      Are access tokens are used as capabilities for accessing datanodes.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.namenode.kerberos.principal</name>
+    <value>nn/_HOST@${local.realm}</value>
+    <description>
+      Kerberos principal name for the NameNode
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.secondary.namenode.kerberos.principal</name>
+    <value>nn/_HOST@${local.realm}</value>
+    <description>
+        Kerberos principal name for the secondary NameNode.
+    </description>
+  </property>
+
+
+  <property>
+    <name>dfs.namenode.kerberos.https.principal</name>
+    <value>host/_HOST@${local.realm}</value>
+    <description>
+       The Kerberos principal for the host that the NameNode runs on.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.secondary.namenode.kerberos.https.principal</name>
+    <value>host/_HOST@${local.realm}</value>
+    <description>
+      The Kerberos principal for the hostthat the secondary NameNode runs on.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.secondary.https.port</name>
+    <value>50490</value>
+    <description>The https port where secondary-namenode binds</description>
+
+  </property>
+
+  <property>
+    <name>dfs.datanode.kerberos.principal</name>
+    <value>dn/_HOST@${local.realm}</value>
+    <description>
+      The Kerberos principal that the DataNode runs as. "_HOST" is replaced by 
+      the real host name.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.namenode.keytab.file</name>
+    <value>/etc/security/keytabs/nn.service.keytab</value>
+    <description>
+      Combined keytab file containing the namenode service and host principals.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.secondary.namenode.keytab.file</name>
+    <value>/etc/security/keytabs/nn.service.keytab</value>
+    <description>
+      Combined keytab file containing the namenode service and host principals.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.datanode.keytab.file</name>
+    <value>/etc/security/keytabs/dn.service.keytab</value>
+    <description>
+        The filename of the keytab file for the DataNode.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.https.port</name>
+    <value>50470</value>
+    <description>The https port where namenode binds</description>
+  </property>
+
+  <property>
+    <name>dfs.https.address</name>
+    <value>${HADOOP_NN_HOST}:50470</value>
+    <description>The https address where namenode binds</description>
+  </property>
+
+  <property>
+    <name>dfs.datanode.data.dir.perm</name>
+    <value>700</value>
+    <description>The permissions that should be there on dfs.data.dir
+      directories. The datanode will not come up if the permissions are
+      different on existing dfs.data.dir directories. If the directories
+      don't exist, they will be created with this permission.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.cluster.administrators</name>
+    <value>${HADOOP_HDFS_USER}</value>
+    <description>ACL for who all can view the default servlets in the HDFS</description>
+  </property>
+  
+  <property>
+    <name>dfs.permissions.superusergroup</name>
+    <value>${HADOOP_GROUP}</value>
+    <description>The name of the group of super-users.</description>
+  </property>
+  
+  <property>
+    <name>dfs.namenode.http-address</name>
+    <value>${HADOOP_NN_HOST}:50070</value>
+    <description>
+      The address and the base port where the dfs namenode web ui will listen on.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.namenode.https-address</name>
+    <value>${HADOOP_NN_HOST}:50470</value>
+  </property>
+
+  <property>
+    <name>dfs.secondary.http.address</name>
+    <value>${HADOOP_SNN_HOST}:50090</value>
+    <description>
+      The secondary namenode http server address and port.
+      If the port is 0 then the server will start on a free port.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.hosts</name>
+    <value>${HADOOP_CONF_DIR}/dfs.include</value>
+    <description>Names a file that contains a list of hosts that are
+      permitted to connect to the namenode. The full pathname of the file
+      must be specified.  If the value is empty, all hosts are
+      permitted.</description>
+  </property>
+
+  <property>
+    <name>dfs.hosts.exclude</name>
+    <value>${HADOOP_CONF_DIR}/dfs.exclude</value>
+    <description>Names a file that contains a list of hosts that are
+      not permitted to connect to the namenode.  The full pathname of the
+      file must be specified.  If the value is empty, no hosts are
+      excluded.
+    </description>
+  </property>
+</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-queue-acls.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+<name>mapred.queue.default.acl-submit-job</name>
+<value>*</value>
+</property>
+<property>
+<name>mapred.queue.default.acl-administer-jobs</name>
+<value>*</value>
+</property>
+</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml

@@ -0,0 +1,268 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ 
+  <property>
+    <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
+    <value>250</value>
+    <description>Normally, this is the amount of time before killing
+      processes, and the recommended-default is 5.000 seconds - a value of
+      5000 here.  In this case, we are using it solely to blast tasks before
+      killing them, and killing them very quickly (1/4 second) to guarantee
+      that we do not leave VMs around for later jobs.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.system.dir</name>
+    <value>/mapred/mapredsystem</value>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker</name>
+    <value>${HADOOP_JT_HOST}:9000</value>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker.http.address</name>
+    <value>${HADOOP_JT_HOST}:50030</value>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>mapred.local.dir</name>
+    <value>${HADOOP_MAPRED_DIR}</value>
+    <final>true</final>
+  </property>
+
+  <property>
+    <name>mapreduce.cluster.administrators</name>
+    <value>${HADOOP_MR_USER}</value>
+  </property>
+
+  <property>
+    <name>mapred.map.tasks.speculative.execution</name>
+    <value>false</value>
+    <description>If true, then multiple instances of some map tasks
+               may be executed in parallel.</description>
+  </property>
+
+  <property>
+    <name>mapred.reduce.tasks.speculative.execution</name>
+    <value>false</value>
+    <description>If true, then multiple instances of some reduce tasks
+               may be executed in parallel.</description>
+  </property>
+
+  <property>
+    <name>mapred.output.compression.type</name>
+    <value>BLOCK</value>
+    <description>If the job outputs are to compressed as SequenceFiles, how 
+       should they be compressed? Should be one of NONE, RECORD or BLOCK.
+    </description>
+  </property>
+
+  <property>
+    <name>jetty.connector</name>
+    <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
+  </property>
+
+  <property>
+    <name>mapred.task.tracker.task-controller</name>
+    <value>${TASK_CONTROLLER}</value>
+  </property>
+
+  <property>
+    <name>mapred.child.root.logger</name>
+    <value>INFO,TLA</value>
+  </property>
+
+  <property>
+    <name>stream.tmpdir</name>
+    <value>${mapred.temp.dir}</value>
+  </property>
+
+
+  <property>
+    <name>mapred.child.java.opts</name>
+    <value>-server -Xmx640m -Djava.net.preferIPv4Stack=true</value>
+  </property>
+
+  <property>
+    <name>mapred.child.ulimit</name>
+    <value>8388608</value>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker.persist.jobstatus.active</name>
+    <value>true</value>
+    <description>Indicates if persistency of job status information is
+      active or not.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker.persist.jobstatus.dir</name>
+    <value>file:///${HADOOP_LOG_DIR}/${HADOOP_MR_USER}/jobstatus</value>
+    <description>The directory where the job status information is persisted
+      in a file system to be available after it drops of the memory queue and
+      between jobtracker restarts.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.job.tracker.history.completed.location</name>
+    <value>/mapred/history/done</value>
+  </property>
+
+  <property>
+    <name>mapred.heartbeats.in.second</name>
+    <value>200</value>
+    <description>to enable HADOOP:5784</description>
+  </property>
+
+  <property>
+    <name>mapreduce.tasktracker.outofband.heartbeat</name>
+    <value>true</value>
+    <description>to enable MAPREDUCE:270</description>
+  </property>
+
+  <property>
+    <name>mapred.jobtracker.maxtasks.per.job</name>
+    <value>200000</value>
+    <final>true</final>
+    <description>The maximum number of tasks for a single job.
+      A value of -1 indicates that there is no maximum.  
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.kerberos.principal</name>
+    <value>jt/_HOST@${local.realm}</value>   
+    <description>
+       JT principal
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.tasktracker.kerberos.principal</name>
+    <value>tt/_HOST@${local.realm}</value>   
+    <description>       
+       TT principal.
+    </description>
+  </property>
+
+
+  <property>
+    <name>hadoop.job.history.user.location</name>
+    <value>none</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.keytab.file</name>
+    <value>/etc/security/keytabs/jt.service.keytab</value>
+    <description>
+        The keytab for the jobtracker principal.
+    </description>
+  </property>
+
+  <property>
+    <name>mapreduce.tasktracker.keytab.file</name>
+    <value>/etc/security/keytabs/tt.service.keytab</value>
+    <description>The filename of the keytab for the task tracker</description>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.staging.root.dir</name>
+    <value>/user</value>
+    <description>The Path prefix for where the staging directories should be 
+      placed. The next level is always the user's
+      name. It is a path in the default file system.
+    </description>
+  </property>
+
+
+  <property>
+    <name>mapreduce.job.acl-modify-job</name>
+    <value></value>
+  </property>
+
+  <property>
+    <name>mapreduce.job.acl-view-job</name>
+    <value>Dr.Who</value>
+  </property>
+
+  <property>
+    <name>mapreduce.tasktracker.group</name>
+    <value>${HADOOP_GROUP}</value>
+    <description>The group that the task controller uses for accessing the
+      task controller. The mapred user must be a member and users should *not*
+      be members.
+    </description> 
+  </property>
+
+  <property>
+    <name>mapred.acls.enabled</name>
+    <value>true</value>
+  </property>
+  
+  <property>
+    <name>mapred.jobtracker.taskScheduler</name>
+    <value>org.apache.hadoop.mapred.CapacityTaskScheduler</value>
+  </property>
+  <property>
+    <name>mapred.queue.names</name>
+    <value>default</value>
+  </property>
+
+  <!-- settings for the history server -->
+  <property>
+    <name>mapreduce.history.server.embedded</name>
+    <value>false</value>
+  </property>
+  <property>
+    <name>mapreduce.history.server.http.address</name>
+    <value>${HADOOP_JT_HOST}:51111</value>
+  </property>
+  <property>
+    <name>mapreduce.jobhistory.kerberos.principal</name>
+    <value>jt/_HOST@${local.realm}</value>   
+    <description>history server principal</description>
+  </property>
+  <property>
+    <name>mapreduce.jobhistory.keytab.file</name>
+    <value>/etc/security/keytabs/jt.service.keytab</value>
+    <description>
+        The keytab for the jobtracker principal.
+    </description>
+  </property>
+  
+  <property>
+    <name>mapred.hosts</name>
+    <value>${HADOOP_CONF_DIR}/mapred.include</value>
+    <description>Names a file that contains the list of nodes that may
+    connect to the jobtracker.  If the value is empty, all hosts are
+    permitted.</description>
+  </property>
+          
+  <property>
+    <name>mapred.hosts.exclude</name>
+    <value>${HADOOP_CONF_DIR}/mapred.exclude</value>
+    <description>Names a file that contains the list of hosts that
+      should be excluded by the jobtracker.  If the value is empty, no
+      hosts are excluded.</description>
+  </property>
+  <property>
+    <name>mapred.jobtracker.retirejob.check</name>
+    <value>10000</value>
+  </property>
+  <property>
+    <name>mapred.jobtracker.retirejob.interval</name>
+    <value>0</value>
+  </property>
+</configuration>

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/taskcontroller.cfg

@@ -0,0 +1,3 @@
+mapreduce.cluster.local.dir=${HADOOP_MAPRED_DIR}
+mapreduce.tasktracker.group=${HADOOP_GROUP}
+hadoop.log.dir=${HADOOP_LOG_DIR}/${HADOOP_MR_USER}

hadoop-common-project/hadoop-common/src/main/packages/update-hadoop-env.sh

@@ -134,7 +134,9 @@ if [ "${UNINSTALL}" -eq "1" ]; then
     rm -rf ${HADOOP_PREFIX}/etc/hadoop
   fi
   rm -f /etc/default/hadoop-env.sh
-  rm -f /etc/profile.d/hadoop-env.sh
+  if [ -d /etc/profile.d ]; then
+    rm -f /etc/profile.d/hadoop-env.sh
+  fi
 else
   # Create symlinks
   if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then
@@ -142,7 +144,9 @@ else
     ln -sf ${HADOOP_CONF_DIR} ${HADOOP_PREFIX}/etc/hadoop
   fi
   ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/default/hadoop-env.sh
-  ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh
+  if [ -d /etc/profile.d ]; then
+    ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh
+  fi
 
   mkdir -p ${HADOOP_LOG_DIR}
   chown root:hadoop ${HADOOP_LOG_DIR}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.serializer;
+
+import org.junit.Test;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+
+public class TestSerializationFactory {
+
+  @Test
+  public void testSerializerAvailability() {
+    Configuration conf = new Configuration();
+    SerializationFactory factory = new SerializationFactory(conf);
+    // Test that a valid serializer class is returned when its present
+    assertNotNull("A valid class must be returned for default Writable Serde",
+        factory.getSerializer(Writable.class));
+    assertNotNull("A valid class must be returned for default Writable serDe",
+        factory.getDeserializer(Writable.class));
+    // Test that a null is returned when none can be found.
+    assertNull("A null should be returned if there are no serializers found.",
+        factory.getSerializer(TestSerializationFactory.class));
+    assertNull("A null should be returned if there are no deserializers found",
+        factory.getDeserializer(TestSerializationFactory.class));
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -5,6 +5,11 @@ Trunk (unreleased changes)
     HDFS-395.  DFS Scalability: Incremental block reports. (Tomasz Nykiel
                via hairong)
 
+    HDFS-2284. Add a new FileSystem, webhdfs://, for supporting write Http
+    access to HDFS.  (szetszwo)
+
+    HDFS-2317. Support read access to HDFS in webhdfs.  (szetszwo)
+
   IMPROVEMENTS
 
     HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
@@ -1029,7 +1034,9 @@ Release 0.23.0 - Unreleased
     (todd)
 
     HDFS-2289. Ensure jsvc is bundled with the HDFS distribution artifact.
-    (Alejandro Abdelnur via acmurthy) 
+    (Alejandro Abdelnur via acmurthy)
+
+    HDFS-2323. start-dfs.sh script fails for tarball install (tomwhite)
 
   BREAKDOWN OF HDFS-1073 SUBTASKS
 

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh

@@ -51,7 +51,7 @@ NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
 
 echo "Starting namenodes on [$NAMENODES]"
 
-"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
   --config "$HADOOP_CONF_DIR" \
   --hostnames "$NAMENODES" \
   --script "$bin/hdfs" start namenode $nameStartOpt
@@ -64,7 +64,7 @@ if [ -n "$HADOOP_SECURE_DN_USER" ]; then
     "Attempting to start secure cluster, skipping datanodes. " \
     "Run start-secure-dns.sh as root to complete startup."
 else
-  "$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
     --config "$HADOOP_CONF_DIR" \
     --script "$bin/hdfs" start datanode $dataStartOpt
 fi
@@ -84,7 +84,7 @@ if [ "$SECONDARY_NAMENODES" = '0.0.0.0' ] ; then
 else
   echo "Starting secondary namenodes [$SECONDARY_NAMENODES]"
 
-  "$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
     --config "$HADOOP_CONF_DIR" \
     --hostnames "$SECONDARY_NAMENODES" \
     --script "$bin/hdfs" start secondarynamenode

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh

@@ -25,7 +25,7 @@ bin=`cd "$bin"; pwd`
 . "$bin"/../libexec/hdfs-config.sh
 
 if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
-  "$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
+  "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
 else
   echo $usage
 fi

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh

@@ -27,7 +27,7 @@ NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes)
 
 echo "Stopping namenodes on [$NAMENODES]"
 
-"$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
   --config "$HADOOP_CONF_DIR" \
   --hostnames "$NAMENODES" \
   --script "$bin/hdfs" stop namenode
@@ -40,7 +40,7 @@ if [ -n "$HADOOP_SECURE_DN_USER" ]; then
     "Attempting to stop secure cluster, skipping datanodes. " \
     "Run stop-secure-dns.sh as root to complete shutdown."
 else
-  "$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
     --config "$HADOOP_CONF_DIR" \
     --script "$bin/hdfs" stop datanode
 fi
@@ -60,7 +60,7 @@ if [ "$SECONDARY_NAMENODES" = '0.0.0.0' ] ; then
 else
   echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]"
 
-  "$HADOOP_PREFIX/bin/hadoop-daemons.sh" \
+  "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
     --config "$HADOOP_CONF_DIR" \
     --hostnames "$SECONDARY_NAMENODES" \
     --script "$bin/hdfs" stop secondarynamenode

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh

@@ -25,7 +25,7 @@ bin=`cd "$bin"; pwd`
 . "$bin"/../libexec/hdfs-config.sh
 
 if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
-  "$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
+  "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
 else
   echo $usage
 fi

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java

@@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.StreamFile;
  * is made on the successive read(). The normal input stream functions are 
  * connected to the currently active input stream. 
  */
-class ByteRangeInputStream extends FSInputStream {
+public class ByteRangeInputStream extends FSInputStream {
   
   /**
    * This class wraps a URL to allow easy mocking when testing. The URL class
@@ -71,7 +71,8 @@ class ByteRangeInputStream extends FSInputStream {
 
   StreamStatus status = StreamStatus.SEEK;
 
-  ByteRangeInputStream(final URL url) {
+  /** Create an input stream with the URL. */
+  public ByteRangeInputStream(final URL url) {
     this(new URLOpener(url), new URLOpener(null));
   }
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java

@@ -802,7 +802,7 @@ public class DFSClient implements java.io.Closeable {
    * 
    * @see ClientProtocol#append(String, String) 
    */
-  DFSOutputStream append(String src, int buffersize, Progressable progress) 
+  public DFSOutputStream append(String src, int buffersize, Progressable progress) 
       throws IOException {
     checkOpen();
     HdfsFileStatus stat = getFileInfo(src);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java

@@ -98,7 +98,7 @@ import org.apache.hadoop.util.PureJavaCrc32;
  * datanode from the original pipeline. The DataStreamer now
  * starts sending packets from the dataQueue.
 ****************************************************************/
-class DFSOutputStream extends FSOutputSummer implements Syncable {
+public class DFSOutputStream extends FSOutputSummer implements Syncable {
   private final DFSClient dfsClient;
   private static final int MAX_PACKETS = 80; // each packet 64K, total 5MB
   private Socket s;
@@ -1707,7 +1707,7 @@ class DFSOutputStream extends FSOutputSummer implements Syncable {
   /**
    * Returns the size of a file as it was when this stream was opened
    */
-  long getInitialLen() {
+  public long getInitialLen() {
     return initialFileSize;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -229,12 +229,11 @@ public class DistributedFileSystem extends FileSystem {
     return dfs.recoverLease(getPathName(f));
   }
 
-  @SuppressWarnings("deprecation")
   @Override
   public FSDataInputStream open(Path f, int bufferSize) throws IOException {
     statistics.incrementReadOps(1);
     return new DFSClient.DFSDataInputStream(
-          dfs.open(getPathName(f), bufferSize, verifyChecksum, statistics));
+          dfs.open(getPathName(f), bufferSize, verifyChecksum));
   }
 
   /** This optional operation is not yet supported. */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java

@@ -250,7 +250,7 @@ public class HftpFileSystem extends FileSystem {
    * @return namenode URL referring to the given path
    * @throws IOException on error constructing the URL
    */
-  URL getNamenodeURL(String path, String query) throws IOException {
+  protected URL getNamenodeURL(String path, String query) throws IOException {
     final URL url = new URL("http", nnAddr.getHostName(),
           nnAddr.getPort(), path + '?' + query);
     if (LOG.isTraceEnabled()) {
@@ -317,6 +317,7 @@ public class HftpFileSystem extends FileSystem {
 
   @Override
   public FSDataInputStream open(Path f, int buffersize) throws IOException {
+    f = f.makeQualified(getUri(), getWorkingDirectory());
     String path = "/data" + ServletUtil.encodePath(f.toUri().getPath());
     String query = addDelegationTokenParam("ugi=" + getEncodedUgiParameter());
     URL u = getNamenodeURL(path, query);    

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -18,7 +18,50 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 
-import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SIMULATEDDATASTORAGE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STORAGEID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SYNCONCLOSE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_TRANSFERTO_ALLOWED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_FEDERATION_NAMESERVICES;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.server.common.Util.now;
 
 import java.io.BufferedOutputStream;
@@ -93,9 +136,11 @@ import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.server.datanode.FSDataset.VolumeInfo;
 import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
+import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.FileChecksumServlets;
 import org.apache.hadoop.hdfs.server.namenode.StreamFile;
+import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
 import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
@@ -109,7 +154,8 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
 import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
-import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.Param;
 import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.ProtocolSignature;
@@ -502,6 +548,11 @@ public class DataNode extends Configured
     this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
     this.infoServer.addServlet(null, "/blockScannerReport", 
                                DataBlockScanner.Servlet.class);
+
+    infoServer.addJerseyResourcePackage(
+        DatanodeWebHdfsMethods.class.getPackage().getName()
+        + ";" + Param.class.getPackage().getName(),
+        "/" + WebHdfsFileSystem.PATH_PREFIX + "/*");
     this.infoServer.start();
   }
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java

@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.web.resources;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+
+import javax.servlet.ServletContext;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
+import org.apache.hadoop.hdfs.DFSOutputStream;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
+import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.LengthParam;
+import org.apache.hadoop.hdfs.web.resources.OffsetParam;
+import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
+import org.apache.hadoop.hdfs.web.resources.Param;
+import org.apache.hadoop.hdfs.web.resources.PermissionParam;
+import org.apache.hadoop.hdfs.web.resources.PostOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
+import org.apache.hadoop.io.IOUtils;
+
+/** Web-hdfs DataNode implementation. */
+@Path("")
+public class DatanodeWebHdfsMethods {
+  public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
+
+  private @Context ServletContext context;
+
+  /** Handle HTTP PUT request. */
+  @PUT
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Consumes({"*/*"})
+  @Produces({MediaType.APPLICATION_JSON})
+  public Response put(
+      final InputStream in,
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
+          final PutOpParam op,
+      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
+          final PermissionParam permission,
+      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
+          final OverwriteParam overwrite,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize,
+      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
+          final ReplicationParam replication,
+      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
+          final BlockSizeParam blockSize
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ": " + path
+            + Param.toSortedString(", ", permission, overwrite, bufferSize,
+                replication, blockSize));
+    }
+
+    final String fullpath = path.getAbsolutePath();
+    final DataNode datanode = (DataNode)context.getAttribute("datanode");
+
+    switch(op.getValue()) {
+    case CREATE:
+    {
+      final Configuration conf = new Configuration(datanode.getConf());
+      final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
+      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
+      final FSDataOutputStream out = new FSDataOutputStream(dfsclient.create(
+          fullpath, permission.getFsPermission(), 
+          overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
+              : EnumSet.of(CreateFlag.CREATE),
+          replication.getValue(), blockSize.getValue(), null,
+          bufferSize.getValue()), null);
+      try {
+        IOUtils.copyBytes(in, out, bufferSize.getValue());
+      } finally {
+        out.close();
+      }
+      final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
+      final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
+          nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null);
+      return Response.created(uri).type(MediaType.APPLICATION_JSON).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }
+  }
+
+  /** Handle HTTP POST request. */
+  @POST
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Consumes({"*/*"})
+  @Produces({MediaType.APPLICATION_JSON})
+  public Response post(
+      final InputStream in,
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
+          final PostOpParam op,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ": " + path
+            + Param.toSortedString(", ", bufferSize));
+    }
+
+    final String fullpath = path.getAbsolutePath();
+    final DataNode datanode = (DataNode)context.getAttribute("datanode");
+
+    switch(op.getValue()) {
+    case APPEND:
+    {
+      final Configuration conf = new Configuration(datanode.getConf());
+      final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
+      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
+      final DFSOutputStream dfsout = dfsclient.append(fullpath,
+          bufferSize.getValue(), null);
+      final FSDataOutputStream out = new FSDataOutputStream(dfsout, null,
+          dfsout.getInitialLen());
+      try {
+        IOUtils.copyBytes(in, out, bufferSize.getValue());
+      } finally {
+        out.close();
+      }
+      return Response.ok().type(MediaType.APPLICATION_JSON).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }
+  }
+
+  /** Handle HTTP GET request. */
+  @GET
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
+  public Response get(
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
+          final GetOpParam op,
+      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
+          final OffsetParam offset,
+      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
+          final LengthParam length,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ": " + path
+            + Param.toSortedString(", ", offset, length, bufferSize));
+    }
+
+    final String fullpath = path.getAbsolutePath();
+    final DataNode datanode = (DataNode)context.getAttribute("datanode");
+
+    switch(op.getValue()) {
+    case OPEN:
+    {
+      final Configuration conf = new Configuration(datanode.getConf());
+      final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
+      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
+      final DFSDataInputStream in = new DFSClient.DFSDataInputStream(
+          dfsclient.open(fullpath, bufferSize.getValue(), true));
+      in.seek(offset.getValue());
+
+      final StreamingOutput streaming = new StreamingOutput() {
+        @Override
+        public void write(final OutputStream out) throws IOException {
+          final Long n = length.getValue();
+          if (n == null) {
+            IOUtils.copyBytes(in, out, bufferSize.getValue());
+          } else {
+            IOUtils.copyBytes(in, out, n, false);
+          }
+        }
+      };
+      return Response.ok(streaming).type(MediaType.APPLICATION_OCTET_STREAM).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -335,6 +335,11 @@ public class NameNode {
   }
 
   protected InetSocketAddress getHttpServerAddress(Configuration conf) {
+    return getHttpAddress(conf);
+  }
+
+  /** @return the NameNode HTTP address set in the conf. */
+  public static InetSocketAddress getHttpAddress(Configuration conf) {
     return  NetUtils.createSocketAddr(
         conf.get(DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTP_ADDRESS_DEFAULT));
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java

@@ -24,18 +24,20 @@ import java.security.PrivilegedExceptionAction;
 import javax.servlet.ServletContext;
 
 import org.apache.commons.logging.Log;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.Param;
 import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-
 /**
  * Encapsulates the HTTP server started by the NameNode. 
  */
@@ -179,6 +181,11 @@ public class NameNodeHttpServer {
         FileChecksumServlets.RedirectServlet.class, false);
     httpServer.addInternalServlet("contentSummary", "/contentSummary/*",
         ContentSummaryServlet.class, false);
+
+    httpServer.addJerseyResourcePackage(
+        NamenodeWebHdfsMethods.class.getPackage().getName()
+        + ";" + Param.class.getPackage().getName(),
+        "/" + WebHdfsFileSystem.PATH_PREFIX + "/*");
   }
 
   public static FSImage getFsImageFromContext(ServletContext context) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java

@@ -0,0 +1,400 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.web.resources;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.EnumSet;
+
+import javax.servlet.ServletContext;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.hdfs.web.JsonUtil;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
+import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
+import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
+import org.apache.hadoop.hdfs.web.resources.DstPathParam;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.GroupParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.LengthParam;
+import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
+import org.apache.hadoop.hdfs.web.resources.OffsetParam;
+import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
+import org.apache.hadoop.hdfs.web.resources.OwnerParam;
+import org.apache.hadoop.hdfs.web.resources.Param;
+import org.apache.hadoop.hdfs.web.resources.PermissionParam;
+import org.apache.hadoop.hdfs.web.resources.PostOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
+import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
+import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
+import org.apache.hadoop.net.NodeBase;
+
+/** Web-hdfs NameNode implementation. */
+@Path("")
+public class NamenodeWebHdfsMethods {
+  private static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class);
+
+  private @Context ServletContext context;
+
+  private static DatanodeInfo chooseDatanode(final NameNode namenode,
+      final String path, final HttpOpParam.Op op, final long openOffset
+      ) throws IOException {
+    if (op == GetOpParam.Op.OPEN || op == PostOpParam.Op.APPEND) {
+      final NamenodeProtocols np = namenode.getRpcServer();
+      final HdfsFileStatus status = np.getFileInfo(path);
+      final long len = status.getLen();
+      if (op == GetOpParam.Op.OPEN && (openOffset < 0L || openOffset >= len)) {
+        throw new IOException("Offset=" + openOffset + " out of the range [0, "
+          + len + "); " + op + ", path=" + path);
+      }
+
+      if (len > 0) {
+        final long offset = op == GetOpParam.Op.OPEN? openOffset: len - 1;
+        final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
+        final int count = locations.locatedBlockCount();
+        if (count > 0) {
+          return JspHelper.bestNode(locations.get(0));
+        }
+      }
+    } 
+
+    return (DatanodeDescriptor)namenode.getNamesystem().getBlockManager(
+        ).getDatanodeManager().getNetworkTopology().chooseRandom(
+        NodeBase.ROOT);
+  }
+
+  private static URI redirectURI(final NameNode namenode,
+      final String path, final HttpOpParam.Op op, final long openOffset,
+      final Param<?, ?>... parameters) throws URISyntaxException, IOException {
+    final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset);
+    final String query = op.toQueryString() + Param.toSortedString("&", parameters);
+    final String uripath = "/" + WebHdfsFileSystem.PATH_PREFIX + path;
+
+    final URI uri = new URI("http", null, dn.getHostName(), dn.getInfoPort(),
+        uripath, query, null);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("redirectURI=" + uri);
+    }
+    return uri;
+  }
+
+  /** Handle HTTP PUT request. */
+  @PUT
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Consumes({"*/*"})
+  @Produces({MediaType.APPLICATION_JSON})
+  public Response put(
+      final InputStream in,
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
+          final PutOpParam op,
+      @QueryParam(DstPathParam.NAME) @DefaultValue(DstPathParam.DEFAULT)
+          final DstPathParam dstPath,
+      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
+          final OwnerParam owner,
+      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
+          final GroupParam group,
+      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
+          final PermissionParam permission,
+      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
+          final OverwriteParam overwrite,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize,
+      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
+          final ReplicationParam replication,
+      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
+          final BlockSizeParam blockSize,
+      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
+          final ModificationTimeParam modificationTime,
+      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
+          final AccessTimeParam accessTime,
+      @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
+          final RenameOptionSetParam renameOptions
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ": " + path
+          + Param.toSortedString(", ", dstPath, owner, group, permission,
+              overwrite, bufferSize, replication, blockSize,
+              modificationTime, accessTime, renameOptions));
+    }
+
+    final String fullpath = path.getAbsolutePath();
+    final NameNode namenode = (NameNode)context.getAttribute("name.node");
+    final NamenodeProtocols np = namenode.getRpcServer();
+
+    switch(op.getValue()) {
+    case CREATE:
+    {
+      final URI uri = redirectURI(namenode, fullpath, op.getValue(), -1L,
+          permission, overwrite, bufferSize, replication, blockSize);
+      return Response.temporaryRedirect(uri).build();
+    } 
+    case MKDIRS:
+    {
+      final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true);
+      final String js = JsonUtil.toJsonString(PutOpParam.Op.MKDIRS, b);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
+    case RENAME:
+    {
+      final EnumSet<Options.Rename> s = renameOptions.getValue();
+      if (s.isEmpty()) {
+        @SuppressWarnings("deprecation")
+        final boolean b = np.rename(fullpath, dstPath.getValue());
+        final String js = JsonUtil.toJsonString(PutOpParam.Op.RENAME, b);
+        return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+      } else {
+        np.rename(fullpath, dstPath.getValue(),
+            s.toArray(new Options.Rename[s.size()]));
+        return Response.ok().type(MediaType.APPLICATION_JSON).build();
+      }
+    }
+    case SETREPLICATION:
+    {
+      final boolean b = np.setReplication(fullpath, replication.getValue());
+      final String js = JsonUtil.toJsonString(PutOpParam.Op.SETREPLICATION, b);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
+    case SETOWNER:
+    {
+      np.setOwner(fullpath, owner.getValue(), group.getValue());
+      return Response.ok().type(MediaType.APPLICATION_JSON).build();
+    }
+    case SETPERMISSION:
+    {
+      np.setPermission(fullpath, permission.getFsPermission());
+      return Response.ok().type(MediaType.APPLICATION_JSON).build();
+    }
+    case SETTIMES:
+    {
+      np.setTimes(fullpath, modificationTime.getValue(), accessTime.getValue());
+      return Response.ok().type(MediaType.APPLICATION_JSON).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }
+  }
+
+  /** Handle HTTP POST request. */
+  @POST
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Consumes({"*/*"})
+  @Produces({MediaType.APPLICATION_JSON})
+  public Response post(
+      final InputStream in,
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
+          final PostOpParam op,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ": " + path
+            + Param.toSortedString(", ", bufferSize));
+    }
+
+    final String fullpath = path.getAbsolutePath();
+    final NameNode namenode = (NameNode)context.getAttribute("name.node");
+
+    switch(op.getValue()) {
+    case APPEND:
+    {
+      final URI uri = redirectURI(namenode, fullpath, op.getValue(), -1L,
+          bufferSize);
+      return Response.temporaryRedirect(uri).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }
+  }
+
+  private static final UriFsPathParam ROOT = new UriFsPathParam("");
+
+  /** Handle HTTP GET request for the root. */
+  @GET
+  @Path("/")
+  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
+  public Response root(
+      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
+          final GetOpParam op,
+      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
+          final OffsetParam offset,
+      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
+          final LengthParam length,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize
+      ) throws IOException, URISyntaxException {
+    return get(ROOT, op, offset, length, bufferSize);
+  }
+
+  /** Handle HTTP GET request. */
+  @GET
+  @Path("{" + UriFsPathParam.NAME + ":.*}")
+  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
+  public Response get(
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
+          final GetOpParam op,
+      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
+          final OffsetParam offset,
+      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
+          final LengthParam length,
+      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
+          final BufferSizeParam bufferSize
+      ) throws IOException, URISyntaxException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ", " + path
+          + Param.toSortedString(", ", offset, length, bufferSize));
+    }
+
+    final NameNode namenode = (NameNode)context.getAttribute("name.node");
+    final String fullpath = path.getAbsolutePath();
+    final NamenodeProtocols np = namenode.getRpcServer();
+
+    switch(op.getValue()) {
+    case OPEN:
+    {
+      final URI uri = redirectURI(namenode, fullpath, op.getValue(),
+          offset.getValue(), offset, length, bufferSize);
+      return Response.temporaryRedirect(uri).build();
+    }
+    case GETFILESTATUS:
+    {
+      final HdfsFileStatus status = np.getFileInfo(fullpath);
+      final String js = JsonUtil.toJsonString(status);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
+    case LISTSTATUS:
+    {
+      final StreamingOutput streaming = getListingStream(np, fullpath);
+      return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
+    }
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }    
+  }
+
+  private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
+      final String p, byte[] startAfter) throws IOException {
+    final DirectoryListing listing = np.getListing(p, startAfter, false);
+    if (listing == null) { // the directory does not exist
+      throw new FileNotFoundException("File " + p + " does not exist.");
+    }
+    return listing;
+  }
+  
+  private static StreamingOutput getListingStream(final NamenodeProtocols np, 
+      final String p) throws IOException {
+    final DirectoryListing first = getDirectoryListing(np, p,
+        HdfsFileStatus.EMPTY_NAME);
+
+    return new StreamingOutput() {
+      @Override
+      public void write(final OutputStream outstream) throws IOException {
+        final PrintStream out = new PrintStream(outstream);
+        out.print('[');
+
+        final HdfsFileStatus[] partial = first.getPartialListing();
+        if (partial.length > 0) {
+          out.print(JsonUtil.toJsonString(partial[0]));
+        }
+        for(int i = 1; i < partial.length; i++) {
+          out.println(',');
+          out.print(JsonUtil.toJsonString(partial[i]));
+        }
+
+        for(DirectoryListing curr = first; curr.hasMore(); ) { 
+          curr = getDirectoryListing(np, p, curr.getLastName());
+          for(HdfsFileStatus s : curr.getPartialListing()) {
+            out.println(',');
+            out.print(JsonUtil.toJsonString(s));
+          }
+        }
+        
+        out.println(']');
+      }
+    };
+  }
+
+  /** Handle HTTP DELETE request. */
+  @DELETE
+  @Path("{path:.*}")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Response delete(
+      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
+      @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
+          final DeleteOpParam op,
+      @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
+          final RecursiveParam recursive
+      ) throws IOException {
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace(op + ", " + path
+        + Param.toSortedString(", ", recursive));
+    }
+
+    switch(op.getValue()) {
+    case DELETE:
+      final NameNode namenode = (NameNode)context.getAttribute("name.node");
+      final String fullpath = path.getAbsolutePath();
+      final boolean b = namenode.getRpcServer().delete(fullpath, recursive.getValue());
+      final String js = JsonUtil.toJsonString(DeleteOpParam.Op.DELETE, b);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+
+    default:
+      throw new UnsupportedOperationException(op + " is not supported");
+    }    
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java

@@ -227,7 +227,10 @@ public class GetConf extends Configured implements Tool {
   void printList(List<InetSocketAddress> list) {
     StringBuilder buffer = new StringBuilder();
     for (InetSocketAddress address : list) {
-      buffer.append(address.getHostName()).append(" ");
+      if (buffer.length() > 0) {
+        buffer.append(" ");
+      }
+      buffer.append(address.getHostName());
     }
     printOut(buffer.toString());
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java

@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.ipc.RemoteException;
+import org.mortbay.util.ajax.JSON;
+
+/** JSON Utilities */
+public class JsonUtil {
+  private static final ThreadLocal<Map<String, Object>> jsonMap
+      = new ThreadLocal<Map<String, Object>>() {
+    @Override
+    protected Map<String, Object> initialValue() {
+      return new TreeMap<String, Object>();
+    }
+
+    @Override
+    public Map<String, Object> get() {
+      final Map<String, Object> m = super.get();
+      m.clear();
+      return m;
+    }
+  };
+
+  /** Convert an exception object to a Json string. */
+  public static String toJsonString(final Exception e) {
+    final Map<String, Object> m = jsonMap.get();
+    m.put("className", e.getClass().getName());
+    m.put("message", e.getMessage());
+    return JSON.toString(m);
+  }
+
+  /** Convert a Json map to a RemoteException. */
+  public static RemoteException toRemoteException(final Map<String, Object> m) {
+    final String className = (String)m.get("className");
+    final String message = (String)m.get("message");
+    return new RemoteException(className, message);
+  }
+
+  /** Convert a key-value pair to a Json string. */
+  public static String toJsonString(final Object key, final Object value) {
+    final Map<String, Object> m = jsonMap.get();
+    m.put(key instanceof String ? (String) key : key.toString(), value);
+    return JSON.toString(m);
+  }
+
+  /** Convert a FsPermission object to a string. */
+  public static String toString(final FsPermission permission) {
+    return String.format("%o", permission.toShort());
+  }
+
+  /** Convert a string to a FsPermission object. */
+  public static FsPermission toFsPermission(final String s) {
+    return new FsPermission(Short.parseShort(s, 8));
+  }
+
+  /** Convert a HdfsFileStatus object to a Json string. */
+  public static String toJsonString(final HdfsFileStatus status) {
+    final Map<String, Object> m = jsonMap.get();
+    if (status == null) {
+      m.put("isNull", true);
+    } else {
+      m.put("isNull", false);
+      m.put("localName", status.getLocalName());
+      m.put("isDir", status.isDir());
+      m.put("isSymlink", status.isSymlink());
+      if (status.isSymlink()) {
+        m.put("symlink", status.getSymlink());
+      }
+
+      m.put("len", status.getLen());
+      m.put("owner", status.getOwner());
+      m.put("group", status.getGroup());
+      m.put("permission", toString(status.getPermission()));
+      m.put("accessTime", status.getAccessTime());
+      m.put("modificationTime", status.getModificationTime());
+      m.put("blockSize", status.getBlockSize());
+      m.put("replication", status.getReplication());
+    }
+    return JSON.toString(m);
+  }
+
+  @SuppressWarnings("unchecked")
+  static Map<String, Object> parse(String jsonString) {
+    return (Map<String, Object>) JSON.parse(jsonString);
+  }
+
+  /** Convert a Json string to a HdfsFileStatus object. */
+  public static HdfsFileStatus toFileStatus(final Map<String, Object> m) {
+    if ((Boolean)m.get("isNull")) {
+      return null;
+    }
+
+    final String localName = (String) m.get("localName");
+    final boolean isDir = (Boolean) m.get("isDir");
+    final boolean isSymlink = (Boolean) m.get("isSymlink");
+    final byte[] symlink = isSymlink?
+        DFSUtil.string2Bytes((String)m.get("symlink")): null;
+
+    final long len = (Long) m.get("len");
+    final String owner = (String) m.get("owner");
+    final String group = (String) m.get("group");
+    final FsPermission permission = toFsPermission((String) m.get("permission"));
+    final long aTime = (Long) m.get("accessTime");
+    final long mTime = (Long) m.get("modificationTime");
+    final long blockSize = (Long) m.get("blockSize");
+    final short replication = (short) (long) (Long) m.get("replication");
+    return new HdfsFileStatus(len, isDir, replication, blockSize, mTime, aTime,
+        permission, owner, group,
+        symlink, DFSUtil.string2Bytes(localName));
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -0,0 +1,386 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import java.io.BufferedOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.ByteRangeInputStream;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HftpFileSystem;
+import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
+import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
+import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
+import org.apache.hadoop.hdfs.web.resources.DstPathParam;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.GroupParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
+import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
+import org.apache.hadoop.hdfs.web.resources.OwnerParam;
+import org.apache.hadoop.hdfs.web.resources.Param;
+import org.apache.hadoop.hdfs.web.resources.PermissionParam;
+import org.apache.hadoop.hdfs.web.resources.PostOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
+import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
+import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.mortbay.util.ajax.JSON;
+
+/** A FileSystem for HDFS over the web. */
+public class WebHdfsFileSystem extends HftpFileSystem {
+  /** File System URI: {SCHEME}://namenode:port/path/to/file */
+  public static final String SCHEME = "webhdfs";
+  /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
+  public static final String PATH_PREFIX = SCHEME;
+
+  private UserGroupInformation ugi;
+  protected Path workingDir;
+
+  @Override
+  public void initialize(URI uri, Configuration conf) throws IOException {
+    super.initialize(uri, conf);
+    setConf(conf);
+
+    ugi = UserGroupInformation.getCurrentUser();
+    this.workingDir = getHomeDirectory();
+  }
+
+  @Override
+  public URI getUri() {
+    try {
+      return new URI(SCHEME, null, nnAddr.getHostName(), nnAddr.getPort(),
+          null, null, null);
+    } catch (URISyntaxException e) {
+      return null;
+    }
+  }
+
+  @Override
+  public Path getHomeDirectory() {
+    return makeQualified(new Path("/user/" + ugi.getShortUserName()));
+  }
+
+  @Override
+  public synchronized Path getWorkingDirectory() {
+    return workingDir;
+  }
+
+  @Override
+  public synchronized void setWorkingDirectory(final Path dir) {
+    String result = makeAbsolute(dir).toUri().getPath();
+    if (!DFSUtil.isValidName(result)) {
+      throw new IllegalArgumentException("Invalid DFS directory name " + 
+                                         result);
+    }
+    workingDir = makeAbsolute(dir);
+  }
+
+  private Path makeAbsolute(Path f) {
+    return f.isAbsolute()? f: new Path(workingDir, f);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static <T> T jsonParse(final InputStream in) throws IOException {
+    if (in == null) {
+      throw new IOException("The input stream is null.");
+    }
+    return (T)JSON.parse(new InputStreamReader(in));
+  }
+
+  private static void validateResponse(final HttpOpParam.Op op,
+      final HttpURLConnection conn) throws IOException {
+    final int code = conn.getResponseCode();
+    if (code != op.getExpectedHttpResponseCode()) {
+      final Map<String, Object> m;
+      try {
+        m = jsonParse(conn.getErrorStream());
+      } catch(IOException e) {
+        throw new IOException("Unexpected HTTP response: code=" + code + " != "
+            + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
+            + ", message=" + conn.getResponseMessage(), e);
+      }
+
+      final RemoteException re = JsonUtil.toRemoteException(m);
+      throw re.unwrapRemoteException(AccessControlException.class,
+          DSQuotaExceededException.class,
+          FileAlreadyExistsException.class,
+          FileNotFoundException.class,
+          ParentNotDirectoryException.class,
+          SafeModeException.class,
+          NSQuotaExceededException.class,
+          UnresolvedPathException.class);
+    }
+  }
+
+  private URL toUrl(final HttpOpParam.Op op, final Path fspath,
+      final Param<?,?>... parameters) throws IOException {
+    //initialize URI path and query
+    final String path = "/" + PATH_PREFIX
+        + makeQualified(fspath).toUri().getPath();
+    final String query = op.toQueryString()
+        + Param.toSortedString("&", parameters);
+    final URL url = getNamenodeURL(path, query);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("url=" + url);
+    }
+    return url;
+  }
+
+  private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
+      final Param<?,?>... parameters) throws IOException {
+    final URL url = toUrl(op, fspath, parameters);
+
+    //connect and get response
+    final HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+    try {
+      conn.setRequestMethod(op.getType().toString());
+      conn.setDoOutput(op.getDoOutput());
+      if (op.getDoOutput()) {
+        conn.setRequestProperty("Expect", "100-Continue");
+        conn.setInstanceFollowRedirects(true);
+      }
+      conn.connect();
+      return conn;
+    } catch(IOException e) {
+      conn.disconnect();
+      throw e;
+    }
+  }
+
+  /**
+   * Run a http operation.
+   * Connect to the http server, validate response, and obtain the JSON output.
+   * 
+   * @param op http operation
+   * @param fspath file system path
+   * @param parameters parameters for the operation
+   * @return a JSON object, e.g. Object[], Map<String, Object>, etc.
+   * @throws IOException
+   */
+  private <T> T run(final HttpOpParam.Op op, final Path fspath,
+      final Param<?,?>... parameters) throws IOException {
+    final HttpURLConnection conn = httpConnect(op, fspath, parameters);
+    validateResponse(op, conn);
+    try {
+      return jsonParse(conn.getInputStream());
+    } finally {
+      conn.disconnect();
+    }
+  }
+
+  private FsPermission applyUMask(FsPermission permission) {
+    if (permission == null) {
+      permission = FsPermission.getDefault();
+    }
+    return permission.applyUMask(FsPermission.getUMask(getConf()));
+  }
+
+  private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
+    final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
+    final Map<String, Object> json = run(op, f);
+    final HdfsFileStatus status = JsonUtil.toFileStatus(json);
+    if (status == null) {
+      throw new FileNotFoundException("File does not exist: " + f);
+    }
+    return status;
+  }
+
+  @Override
+  public FileStatus getFileStatus(Path f) throws IOException {
+    statistics.incrementReadOps(1);
+    return makeQualified(getHdfsFileStatus(f), f);
+  }
+
+  private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
+    return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
+        f.getBlockSize(), f.getModificationTime(),
+        f.getAccessTime(),
+        f.getPermission(), f.getOwner(), f.getGroup(),
+        f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
+  }
+
+  @Override
+  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
+    final Map<String, Object> json = run(op, f,
+        new PermissionParam(applyUMask(permission)));
+    return (Boolean)json.get(op.toString());
+  }
+
+  @Override
+  public boolean rename(final Path src, final Path dst) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.RENAME;
+    final Map<String, Object> json = run(op, src,
+        new DstPathParam(makeQualified(dst).toUri().getPath()));
+    return (Boolean)json.get(op.toString());
+  }
+
+  @SuppressWarnings("deprecation")
+  @Override
+  public void rename(final Path src, final Path dst,
+      final Options.Rename... options) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.RENAME;
+    run(op, src, new DstPathParam(makeQualified(dst).toUri().getPath()),
+        new RenameOptionSetParam(options));
+  }
+
+  @Override
+  public void setOwner(final Path p, final String owner, final String group
+      ) throws IOException {
+    if (owner == null && group == null) {
+      throw new IOException("owner == null && group == null");
+    }
+
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
+    run(op, p, new OwnerParam(owner), new GroupParam(group));
+  }
+
+  @Override
+  public void setPermission(final Path p, final FsPermission permission
+      ) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
+    run(op, p, new PermissionParam(permission));
+  }
+
+  @Override
+  public boolean setReplication(final Path p, final short replication
+     ) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
+    final Map<String, Object> json = run(op, p,
+        new ReplicationParam(replication));
+    return (Boolean)json.get(op.toString());
+  }
+
+  @Override
+  public void setTimes(final Path p, final long mtime, final long atime
+      ) throws IOException {
+    statistics.incrementWriteOps(1);
+    final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
+    run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
+  }
+
+  private FSDataOutputStream write(final HttpOpParam.Op op,
+      final HttpURLConnection conn, final int bufferSize) throws IOException {
+    return new FSDataOutputStream(new BufferedOutputStream(
+        conn.getOutputStream(), bufferSize), statistics) {
+      @Override
+      public void close() throws IOException {
+        try {
+          super.close();
+        } finally {
+          validateResponse(op, conn);
+        }
+      }
+    };
+  }
+
+  @Override
+  public FSDataOutputStream create(final Path f, final FsPermission permission,
+      final boolean overwrite, final int bufferSize, final short replication,
+      final long blockSize, final Progressable progress) throws IOException {
+    statistics.incrementWriteOps(1);
+
+    final HttpOpParam.Op op = PutOpParam.Op.CREATE;
+    final HttpURLConnection conn = httpConnect(op, f, 
+        new PermissionParam(applyUMask(permission)),
+        new OverwriteParam(overwrite),
+        new BufferSizeParam(bufferSize),
+        new ReplicationParam(replication),
+        new BlockSizeParam(blockSize));
+    return write(op, conn, bufferSize);
+  }
+
+  @Override
+  public FSDataOutputStream append(final Path f, final int bufferSize,
+      final Progressable progress) throws IOException {
+    statistics.incrementWriteOps(1);
+
+    final HttpOpParam.Op op = PostOpParam.Op.APPEND;
+    final HttpURLConnection conn = httpConnect(op, f, 
+        new BufferSizeParam(bufferSize));
+    return write(op, conn, bufferSize);
+  }
+
+  @Override
+  public boolean delete(Path f, boolean recursive) throws IOException {
+    final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
+    final Map<String, Object> json = run(op, f, new RecursiveParam(recursive));
+    return (Boolean)json.get(op.toString());
+  }
+
+  @Override
+  public FSDataInputStream open(final Path f, final int buffersize
+      ) throws IOException {
+    statistics.incrementReadOps(1);
+    final HttpOpParam.Op op = GetOpParam.Op.OPEN;
+    final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
+    return new FSDataInputStream(new ByteRangeInputStream(url));
+  }
+
+  @Override
+  public FileStatus[] listStatus(final Path f) throws IOException {
+    statistics.incrementReadOps(1);
+
+    final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
+    final Object[] array = run(op, f);
+
+    //convert FileStatus
+    final FileStatus[] statuses = new FileStatus[array.length];
+    for(int i = 0; i < array.length; i++) {
+      @SuppressWarnings("unchecked")
+      final Map<String, Object> m = (Map<String, Object>)array[i];
+      statuses[i] = makeQualified(JsonUtil.toFileStatus(m), f);
+    }
+    return statuses;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/AccessTimeParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Access time parameter. */
+public class AccessTimeParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "accessTime";
+  /** Default parameter value. */
+  public static final String DEFAULT = "-1";
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public AccessTimeParam(final Long value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public AccessTimeParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BlockSizeParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Block size parameter. */
+public class BlockSizeParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "blockSize";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public BlockSizeParam(final Long value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public BlockSizeParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BooleanParam.java

@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Boolean parameter. */
+abstract class BooleanParam extends Param<Boolean, BooleanParam.Domain> {
+  static final String TRUE = "true";
+  static final String FALSE = "false";
+
+  BooleanParam(final Domain domain, final Boolean value) {
+    super(domain, value);
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain extends Param.Domain<Boolean> {
+    Domain(final String paramName) {
+      super(paramName);
+    }
+
+    @Override
+    public String getDomain() {
+      return "<" + NULL + " | boolean>";
+    }
+
+    @Override
+    Boolean parse(final String str) {
+      if (TRUE.equalsIgnoreCase(str)) {
+        return true;
+      } else if (FALSE.equalsIgnoreCase(str)) {
+        return false;
+      }
+      throw new IllegalArgumentException("Failed to parse \"" + str
+          + "\" to Boolean.");
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/BufferSizeParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Buffer size parameter. */
+public class BufferSizeParam extends IntegerParam {
+  /** Parameter name. */
+  public static final String NAME = "bufferSize";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public BufferSizeParam(final Integer value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public BufferSizeParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.net.HttpURLConnection;
+
+/** Http DELETE operation parameter. */
+public class DeleteOpParam extends HttpOpParam<DeleteOpParam.Op> {
+  /** Parameter name. */
+  public static final String NAME = "deleteOp";
+
+  /** Delete operations. */
+  public static enum Op implements HttpOpParam.Op {
+    DELETE(HttpURLConnection.HTTP_OK),
+
+    NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
+
+    final int expectedHttpResponseCode;
+
+    Op(final int expectedHttpResponseCode) {
+      this.expectedHttpResponseCode = expectedHttpResponseCode;
+    }
+
+    @Override
+    public HttpOpParam.Type getType() {
+      return HttpOpParam.Type.DELETE;
+    }
+
+    @Override
+    public boolean getDoOutput() {
+      return false;
+    }
+
+    @Override
+    public int getExpectedHttpResponseCode() {
+      return expectedHttpResponseCode;
+    }
+
+    @Override
+    public String toQueryString() {
+      return NAME + "=" + this;
+    }
+  }
+
+  private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public DeleteOpParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DstPathParam.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.Path;
+
+/** Destination path parameter. */
+public class DstPathParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "dstPath";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public DstPathParam(final String str) {
+    super(DOMAIN, str == null || str.equals(DEFAULT)? null: new Path(str).toUri().getPath());
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/EnumParam.java

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.util.Arrays;
+
+abstract class EnumParam<E extends Enum<E>> extends Param<E, EnumParam.Domain<E>> {
+  EnumParam(final Domain<E> domain, final E value) {
+    super(domain, value);
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain<E extends Enum<E>> extends Param.Domain<E> {
+    private final Class<E> enumClass;
+
+    Domain(String name, final Class<E> enumClass) {
+      super(name);
+      this.enumClass = enumClass;
+    }
+
+    @Override
+    public final String getDomain() {
+      return Arrays.asList(enumClass.getEnumConstants()).toString();
+    }
+
+    @Override
+    final E parse(final String str) {
+      return Enum.valueOf(enumClass, str.toUpperCase());
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/EnumSetParam.java

@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.Iterator;
+
+abstract class EnumSetParam<E extends Enum<E>> extends Param<EnumSet<E>, EnumSetParam.Domain<E>> {
+  /** Convert an EnumSet to a string of comma separated values. */
+  static <E extends Enum<E>> String toString(EnumSet<E> set) {
+    if (set == null || set.isEmpty()) {
+      return "";
+    } else {
+      final StringBuilder b = new StringBuilder();
+      final Iterator<E> i = set.iterator();
+      b.append(i.next());
+      for(; i.hasNext(); ) {
+        b.append(',').append(i.next());
+      }
+      return b.toString();
+    }
+  }
+
+  static <E extends Enum<E>> EnumSet<E> toEnumSet(final Class<E> clazz,
+      final E... values) {
+    final EnumSet<E> set = EnumSet.noneOf(clazz);
+    set.addAll(Arrays.asList(values));
+    return set;
+  }
+
+  EnumSetParam(final Domain<E> domain, final EnumSet<E> value) {
+    super(domain, value);
+  }
+
+  @Override
+  public String toString() {
+    return getName() + "=" + toString(value);
+  }
+
+  
+  /** The domain of the parameter. */
+  static final class Domain<E extends Enum<E>> extends Param.Domain<EnumSet<E>> {
+    private final Class<E> enumClass;
+
+    Domain(String name, final Class<E> enumClass) {
+      super(name);
+      this.enumClass = enumClass;
+    }
+
+    @Override
+    public final String getDomain() {
+      return Arrays.asList(enumClass.getEnumConstants()).toString();
+    }
+
+    /** The string contains a comma separated values. */
+    @Override
+    final EnumSet<E> parse(final String str) {
+      final EnumSet<E> set = EnumSet.noneOf(enumClass);
+      if (!str.isEmpty()) {
+        for(int i, j = 0; j >= 0; ) {
+          i = j;
+          j = str.indexOf(',', i+1);
+          final String sub = j >= 0? str.substring(i, j): str.substring(i);
+          set.add(Enum.valueOf(enumClass, sub.trim().toUpperCase()));
+        }
+      }
+      return set;
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ExceptionHandler.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.web.JsonUtil;
+
+/** Handle exceptions. */
+@Provider
+public class ExceptionHandler implements ExceptionMapper<Exception> {
+  public static final Log LOG = LogFactory.getLog(ExceptionHandler.class);
+
+  @Override
+  public Response toResponse(final Exception e) {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("GOT EXCEPITION", e);
+    }
+
+    final Response.Status s;
+    if (e instanceof SecurityException) {
+      s = Response.Status.UNAUTHORIZED;
+    } else if (e instanceof FileNotFoundException) {
+      s = Response.Status.NOT_FOUND;
+    } else if (e instanceof IOException) {
+      s = Response.Status.FORBIDDEN;
+    } else if (e instanceof UnsupportedOperationException) {
+      s = Response.Status.BAD_REQUEST;
+    } else {
+      s = Response.Status.INTERNAL_SERVER_ERROR;
+    }
+ 
+    final String js = JsonUtil.toJsonString(e);
+    return Response.status(s).type(MediaType.APPLICATION_JSON).entity(js).build();
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.net.HttpURLConnection;
+
+/** Http GET operation parameter. */
+public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
+  /** Parameter name. */
+  public static final String NAME = "getOp";
+
+  /** Get operations. */
+  public static enum Op implements HttpOpParam.Op {
+    OPEN(HttpURLConnection.HTTP_OK),
+
+    GETFILESTATUS(HttpURLConnection.HTTP_OK),
+    LISTSTATUS(HttpURLConnection.HTTP_OK),
+
+    NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
+
+    final int expectedHttpResponseCode;
+
+    Op(final int expectedHttpResponseCode) {
+      this.expectedHttpResponseCode = expectedHttpResponseCode;
+    }
+
+    @Override
+    public HttpOpParam.Type getType() {
+      return HttpOpParam.Type.GET;
+    }
+
+    @Override
+    public boolean getDoOutput() {
+      return false;
+    }
+
+    @Override
+    public int getExpectedHttpResponseCode() {
+      return expectedHttpResponseCode;
+    }
+
+    @Override
+    public String toQueryString() {
+      return NAME + "=" + this;
+    }
+  }
+
+  private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public GetOpParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GroupParam.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Group parameter. */
+public class GroupParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "group";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public GroupParam(final String str) {
+    super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Http operation parameter. */
+public abstract class HttpOpParam<E extends Enum<E> & HttpOpParam.Op> extends EnumParam<E> {
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  /** Http operation types */
+  public static enum Type {
+    GET, PUT, POST, DELETE;
+  }
+
+  /** Http operation interface. */
+  public static interface Op {
+    /** @return the Http operation type. */
+    public Type getType();
+
+    /** @return true if the operation has output. */
+    public boolean getDoOutput();
+
+    /** @return true if the operation has output. */
+    public int getExpectedHttpResponseCode();
+
+    /** @return a URI query string. */
+    public String toQueryString();
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  HttpOpParam(final Domain<E> domain, final E value) {
+    super(domain, value);
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/IntegerParam.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Integer parameter. */
+abstract class IntegerParam extends Param<Integer, IntegerParam.Domain> {
+  IntegerParam(final Domain domain, final Integer value) {
+    super(domain, value);
+  }
+  
+  @Override
+  public String toString() {
+    return getName() + "=" + domain.toString(getValue());
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain extends Param.Domain<Integer> {
+    /** The radix of the number. */
+    final int radix;
+
+    Domain(final String paramName) {
+      this(paramName, 10);
+    }
+
+    Domain(final String paramName, final int radix) {
+      super(paramName);
+      this.radix = radix;
+    }
+
+    @Override
+    public String getDomain() {
+      return "<" + NULL + " | int in radix " + radix + ">";
+    }
+
+    @Override
+    Integer parse(final String str) {
+      return NULL.equals(str)? null: Integer.parseInt(str, radix);
+    }
+
+    /** Convert an Integer to a String. */ 
+    String toString(final Integer n) {
+      return n == null? NULL: Integer.toString(n, radix);
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LengthParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Length parameter. */
+public class LengthParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "length";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public LengthParam(final Long value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public LengthParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/LongParam.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Long parameter. */
+abstract class LongParam extends Param<Long, LongParam.Domain> {
+  LongParam(final Domain domain, final Long value) {
+    super(domain, value);
+  }
+  
+  @Override
+  public String toString() {
+    return getName() + "=" + domain.toString(getValue());
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain extends Param.Domain<Long> {
+    /** The radix of the number. */
+    final int radix;
+
+    Domain(final String paramName) {
+      this(paramName, 10);
+    }
+
+    Domain(final String paramName, final int radix) {
+      super(paramName);
+      this.radix = radix;
+    }
+
+    @Override
+    public String getDomain() {
+      return "<" + NULL + " | short in radix " + radix + ">";
+    }
+
+    @Override
+    Long parse(final String str) {
+      return NULL.equals(str)? null: Long.parseLong(str, radix);
+    }
+
+    /** Convert a Short to a String. */ 
+    String toString(final Long n) {
+      return n == null? NULL: Long.toString(n, radix);
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ModificationTimeParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Modification time parameter. */
+public class ModificationTimeParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "modificationTime";
+  /** Default parameter value. */
+  public static final String DEFAULT = "-1";
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public ModificationTimeParam(final Long value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public ModificationTimeParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OffsetParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Offset parameter. */
+public class OffsetParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "offset";
+  /** Default parameter value. */
+  public static final String DEFAULT = "0";
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public OffsetParam(final Long value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public OffsetParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Recursive parameter. */
+public class OverwriteParam extends BooleanParam {
+  /** Parameter name. */
+  public static final String NAME = "overwrite";
+  /** Default parameter value. */
+  public static final String DEFAULT = FALSE;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public OverwriteParam(final Boolean value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public OverwriteParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/OwnerParam.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Owner parameter. */
+public class OwnerParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "owner";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public OwnerParam(final String str) {
+    super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/Param.java

@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.util.Arrays;
+import java.util.Comparator;
+
+
+/** Base class of parameters. */
+public abstract class Param<T, D extends Param.Domain<T>> {
+  static final String NULL = "null";
+  
+  static final Comparator<Param<?,?>> NAME_CMP = new Comparator<Param<?,?>>() {
+    @Override
+    public int compare(Param<?, ?> left, Param<?, ?> right) {
+      return left.getName().compareTo(right.getName());
+    }
+  };
+
+  /** Convert the parameters to a sorted String. */
+  public static String toSortedString(final String separator,
+      final Param<?, ?>... parameters) {
+    Arrays.sort(parameters, NAME_CMP);
+    final StringBuilder b = new StringBuilder();
+    for(Param<?, ?> p : parameters) {
+      if (p.getValue() != null) {
+        b.append(separator).append(p);
+      }
+    }
+    return b.toString();
+  }
+
+  /** The domain of the parameter. */
+  final D domain;
+  /** The actual parameter value. */
+  final T value;
+
+  Param(final D domain, final T value) {
+    this.domain = domain;
+    this.value = value;
+  }
+
+  /** @return the parameter value. */
+  public final T getValue() {
+    return value;
+  }
+
+  /** @return the parameter name. */
+  public abstract String getName();
+
+  @Override
+  public String toString() {
+    return getName() + "=" + value;
+  }
+
+  /** Base class of parameter domains. */
+  static abstract class Domain<T> {
+    /** Parameter name. */
+    final String paramName;
+    
+    Domain(final String paramName) {
+      this.paramName = paramName;
+    }
+ 
+    /** @return the parameter name. */
+    public final String getParamName() {
+      return paramName;
+    }
+
+    /** @return a string description of the domain of the parameter. */
+    public abstract String getDomain();
+
+    /** @return the parameter value represented by the string. */
+    abstract T parse(String str);
+
+    /** Parse the given string.
+     * @return the parameter value represented by the string.
+     */
+    public final T parse(final String varName, final String str) {
+      try {
+        return str != null && str.trim().length() > 0 ? parse(str) : null;
+      } catch(Exception e) {
+        throw new IllegalArgumentException("Failed to parse \"" + str
+            + "\" for the parameter " + varName
+            + ".  The value must be in the domain " + getDomain(), e);
+      }
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/** Permission parameter, use a Short to represent a FsPermission. */
+public class PermissionParam extends ShortParam {
+  /** Parameter name. */
+  public static final String NAME = "permission";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME, 8);
+  
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public PermissionParam(final FsPermission value) {
+    super(DOMAIN, value == null? null: value.toShort());
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public PermissionParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  /** @return the represented FsPermission. */
+  public FsPermission getFsPermission() {
+    final Short mode = getValue();
+    return mode == null? FsPermission.getDefault(): new FsPermission(mode);
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.net.HttpURLConnection;
+
+/** Http POST operation parameter. */
+public class PostOpParam extends HttpOpParam<PostOpParam.Op> {
+  /** Parameter name. */
+  public static final String NAME = "postOp";
+
+  /** Post operations. */
+  public static enum Op implements HttpOpParam.Op {
+    APPEND(HttpURLConnection.HTTP_OK),
+
+    NULL(HttpURLConnection.HTTP_NOT_IMPLEMENTED);
+
+    final int expectedHttpResponseCode;
+
+    Op(final int expectedHttpResponseCode) {
+      this.expectedHttpResponseCode = expectedHttpResponseCode;
+    }
+
+    @Override
+    public Type getType() {
+      return Type.POST;
+    }
+
+    @Override
+    public boolean getDoOutput() {
+      return true;
+    }
+
+    @Override
+    public int getExpectedHttpResponseCode() {
+      return expectedHttpResponseCode;
+    }
+
+    /** @return a URI query string. */
+    public String toQueryString() {
+      return NAME + "=" + this;
+    }
+  }
+
+  private static final Domain<Op> DOMAIN = new Domain<PostOpParam.Op>(NAME, Op.class);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public PostOpParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.net.HttpURLConnection;
+
+/** Http POST operation parameter. */
+public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
+  /** Parameter name. */
+  public static final String NAME = "putOp";
+
+  /** Put operations. */
+  public static enum Op implements HttpOpParam.Op {
+    CREATE(true, HttpURLConnection.HTTP_CREATED),
+
+    MKDIRS(false, HttpURLConnection.HTTP_OK),
+    RENAME(false, HttpURLConnection.HTTP_OK),
+    SETREPLICATION(false, HttpURLConnection.HTTP_OK),
+
+    SETOWNER(false, HttpURLConnection.HTTP_OK),
+    SETPERMISSION(false, HttpURLConnection.HTTP_OK),
+    SETTIMES(false, HttpURLConnection.HTTP_OK),
+    
+    NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED);
+
+    final boolean doOutput;
+    final int expectedHttpResponseCode;
+
+    Op(final boolean doOutput, final int expectedHttpResponseCode) {
+      this.doOutput = doOutput;
+      this.expectedHttpResponseCode = expectedHttpResponseCode;
+    }
+
+    @Override
+    public HttpOpParam.Type getType() {
+      return HttpOpParam.Type.PUT;
+    }
+
+    @Override
+    public boolean getDoOutput() {
+      return doOutput;
+    }
+
+    @Override
+    public int getExpectedHttpResponseCode() {
+      return expectedHttpResponseCode;
+    }
+
+    @Override
+    public String toQueryString() {
+      return NAME + "=" + this;
+    }
+  }
+
+  private static final Domain<Op> DOMAIN = new Domain<Op>(NAME, Op.class);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public PutOpParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/RecursiveParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Recursive parameter. */
+public class RecursiveParam extends BooleanParam {
+  /** Parameter name. */
+  public static final String NAME = "recursive";
+  /** Default parameter value. */
+  public static final String DEFAULT = FALSE;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public RecursiveParam(final Boolean value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public RecursiveParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/RenameOptionSetParam.java

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.Options;
+
+/** Rename option set parameter. */
+public class RenameOptionSetParam extends EnumSetParam<Options.Rename> {
+  /** Parameter name. */
+  public static final String NAME = "renameOptions";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain<Options.Rename> DOMAIN = new Domain<Options.Rename>(
+      NAME, Options.Rename.class);
+
+  /**
+   * Constructor.
+   * @param options rename options.
+   */
+  public RenameOptionSetParam(final Options.Rename... options) {
+    super(DOMAIN, toEnumSet(Options.Rename.class, options));
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public RenameOptionSetParam(final String str) {
+    super(DOMAIN, DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ReplicationParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Replication parameter. */
+public class ReplicationParam extends ShortParam {
+  /** Parameter name. */
+  public static final String NAME = "replication";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public ReplicationParam(final Short value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public ReplicationParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ShortParam.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** Short parameter. */
+abstract class ShortParam extends Param<Short, ShortParam.Domain> {
+  ShortParam(final Domain domain, final Short value) {
+    super(domain, value);
+  }
+  
+  @Override
+  public String toString() {
+    return getName() + "=" + domain.toString(getValue());
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain extends Param.Domain<Short> {
+    /** The radix of the number. */
+    final int radix;
+
+    Domain(final String paramName) {
+      this(paramName, 10);
+    }
+
+    Domain(final String paramName, final int radix) {
+      super(paramName);
+      this.radix = radix;
+    }
+
+    @Override
+    public String getDomain() {
+      return "<" + NULL + " | short in radix " + radix + ">";
+    }
+
+    @Override
+    Short parse(final String str) {
+      return NULL.equals(str)? null: Short.parseShort(str, radix);
+    }
+
+    /** Convert a Short to a String. */ 
+    String toString(final Short n) {
+      return n == null? NULL: Integer.toString(n, radix);
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/StringParam.java

@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.util.regex.Pattern;
+
+/** String parameter. */
+abstract class StringParam extends Param<String, StringParam.Domain> {
+  StringParam(final Domain domain, String str) {
+    super(domain, domain.parse(str));
+  }
+
+  /** The domain of the parameter. */
+  static final class Domain extends Param.Domain<String> {
+    /** The pattern defining the domain; null . */
+    private final Pattern pattern;
+
+    Domain(final String paramName, final Pattern pattern) {
+      super(paramName);
+      this.pattern = pattern;
+    }
+
+    @Override
+    public final String getDomain() {
+      return pattern == null ? "<String>" : pattern.pattern();
+    }
+
+    @Override
+    final String parse(final String str) {
+      if (pattern != null) {
+        if (!pattern.matcher(str).matches()) {
+          throw new IllegalArgumentException("Invalid value: \"" + str
+              + "\" does not belong to the domain " + getDomain());
+        }
+      }
+      return str;
+    }
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UriFsPathParam.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** The FileSystem path parameter. */
+public class UriFsPathParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "path";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public UriFsPathParam(String str) {
+    super(DOMAIN, str);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  /** @return the absolute path. */
+  public final String getAbsolutePath() {
+    final String path = getValue(); //The first / has been stripped out.
+    return path == null? null: "/" + path;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** User parameter. */
+public class UserParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "user.name";
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public UserParam(final String str) {
+    super(DOMAIN, str == null || str.equals(DEFAULT)? null: str);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserProvider.java

@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import java.lang.reflect.Type;
+import java.security.Principal;
+
+import javax.ws.rs.core.Context;
+import javax.ws.rs.ext.Provider;
+
+import com.sun.jersey.api.core.HttpContext;
+import com.sun.jersey.core.spi.component.ComponentContext;
+import com.sun.jersey.core.spi.component.ComponentScope;
+import com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable;
+import com.sun.jersey.spi.inject.Injectable;
+import com.sun.jersey.spi.inject.InjectableProvider;
+
+@Provider
+public class UserProvider extends AbstractHttpContextInjectable<Principal>
+    implements InjectableProvider<Context, Type> {
+
+  @Override
+  public Principal getValue(final HttpContext context) {
+    //get principal from the request
+    final Principal principal = context.getRequest().getUserPrincipal();
+    if (principal != null) {
+      return principal;
+    }
+
+    //get username from the parameter
+    final String username = context.getRequest().getQueryParameters().getFirst(
+        UserParam.NAME);
+    if (username != null) {
+      final UserParam userparam = new UserParam(username);
+      return new Principal() {
+        @Override
+        public String getName() {
+          return userparam.getValue();
+        }
+      };
+    }
+
+    //user not found
+    return null;
+  }
+
+  @Override
+  public ComponentScope getScope() {
+    return ComponentScope.PerRequest;
+  }
+
+  @Override
+  public Injectable<Principal> getInjectable(
+      final ComponentContext componentContext, final Context context,
+      final Type type) {
+    return type.equals(Principal.class)? this : null;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java

@@ -815,6 +815,8 @@ public class MiniDFSCluster {
                              long[] simulatedCapacities,
                              boolean setupHostsFile,
                              boolean checkDataNodeAddrConfig) throws IOException {
+    conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1");
+
     int curDatanodesNum = dataNodes.size();
     // for mincluster's the default initialDelay for BRs is 0
     if (conf.get(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY) == null) {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java

@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import java.net.URI;
+
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSMainOperationsBaseTest;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods;
+import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
+import org.apache.log4j.Level;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
+  {
+    ((Log4JLogger)ExceptionHandler.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger)DatanodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);
+  }
+
+  private static MiniDFSCluster cluster = null;
+  private static Path defaultWorkingDirectory;
+
+  @BeforeClass
+  public static void setupCluster() {
+    Configuration conf = new Configuration();
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+      cluster.waitActive();
+
+      final String uri = WebHdfsFileSystem.SCHEME  + "://"
+          + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+      fSys = FileSystem.get(new URI(uri), conf); 
+      defaultWorkingDirectory = fSys.getWorkingDirectory();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @AfterClass
+  public static void shutdownCluster() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Override
+  protected Path getDefaultWorkingDirectory() {
+    return defaultWorkingDirectory;
+  }
+  
+  //The following test failed since WebHdfsFileSystem did not support
+  //authentication.
+  //Disable it.
+  @Test
+  public void testListStatusThrowsExceptionForUnreadableDir() {}
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.web.JsonUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestJsonUtil {
+  static FileStatus toFileStatus(HdfsFileStatus f, String parent) {
+    return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
+        f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
+        f.getPermission(), f.getOwner(), f.getGroup(),
+        new Path(f.getFullName(parent)));
+  }
+
+  @Test
+  public void testHdfsFileStatus() {
+    final long now = System.currentTimeMillis();
+    final String parent = "/dir";
+    final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L<<26,
+        now, now + 10, new FsPermission((short)0644), "user", "group",
+        DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"));
+    final FileStatus fstatus = toFileStatus(status, parent);
+    System.out.println("status  = " + status);
+    System.out.println("fstatus = " + fstatus);
+    final String json = JsonUtil.toJsonString(status);
+    System.out.println("json    = " + json.replace(",", ",\n  "));
+    final HdfsFileStatus s2 = JsonUtil.toFileStatus(JsonUtil.parse(json));
+    final FileStatus fs2 = toFileStatus(s2, parent);
+    System.out.println("s2      = " + s2);
+    System.out.println("fs2     = " + fs2);
+    Assert.assertEquals(fstatus, fs2);
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.web;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
+  private static final Configuration conf = new Configuration();
+  private static final MiniDFSCluster cluster;
+  private String defaultWorkingDirectory;
+
+  static {
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+      cluster.waitActive();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  protected void setUp() throws Exception {
+    final String uri = WebHdfsFileSystem.SCHEME  + "://"
+        + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+    fs = FileSystem.get(new URI(uri), conf); 
+    defaultWorkingDirectory = fs.getWorkingDirectory().toUri().getPath();
+  }
+
+  @Override
+  protected String getDefaultWorkingDirectory() {
+    return defaultWorkingDirectory;
+  }
+}

hadoop-mapreduce-project/CHANGES.txt

@@ -2,9 +2,29 @@ Hadoop MapReduce Change Log
 
 Trunk (unreleased changes)
 
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+    MAPREDUCE-2669. Add new examples for Mean, Median, and Standard Deviation.
+    (Plamen Jeliazkov via shv)
+
   IMPROVEMENTS
 
-    MAPREDUCE-2887 due to HADOOP-7524 Change RPC to allow multiple protocols including multuple versions of the same protocol (sanjay Radia)
+    MAPREDUCE-2887 due to HADOOP-7524 Change RPC to allow multiple protocols
+                   including multuple versions of the same protocol (sanjay Radia)
+
+    MAPREDUCE-2934. MR portion of HADOOP-7607 - Simplify the RPC proxy cleanup
+                    process (atm)
+
+  BUG FIXES
+
+    MAPREDUCE-2784. [Gridmix] Bug fixes in ExecutionSummarizer and 
+    ResourceUsageMatcher. (amarrk)
+
+    MAPREDUCE-2978. Fixed test-patch to make Jenkins report correct number of
+    findBugs, correct links to findBugs artifacts and no links to the
+    artifacts when there are no warnings. (Tom White via vinodkv).
 
 Release 0.23.0 - Unreleased
 
@@ -248,6 +268,24 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2655. Add audit logs to ResourceManager and NodeManager. (Thomas
     Graves via acmurthy)
 
+    MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
+    Evans via acmurthy) 
+
+    MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy) 
+
+    MAPREDUCE-2711. Update TestBlockPlacementPolicyRaid for the new namesystem
+    and block management APIs.  (szetszwo)
+
+    MAPREDUCE-2933. Change allocate call to return ContainerStatus for
+    completed containers rather than Container. (acmurthy) 
+
+    MAPREDUCE-2675. Reformat JobHistory Server main page to be more
+    useful. (Robert Joseph Evans via vinodkv).
+
+    MAPREDUCE-2896. Simplify all apis to in
+    org.apache.hadoop.yarn.api.records.* to be get/set only. Added javadocs to
+    all public records. (acmurthy)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and
@@ -1221,6 +1259,59 @@ Release 0.23.0 - Unreleased
 
    MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev)
 
+   MAPREDUCE-2908. Fix all findbugs warnings. (vinodkv via acmurthy) 
+
+   MAPREDUCE-2942. TestNMAuditLogger.testNMAuditLoggerWithIP failing (Thomas Graves 
+   via mahadev)
+
+   MAPREDUCE-2947. Fixed race condition in AuxiliaryServices. (vinodkv via
+   acmurthy) 
+
+   MAPREDUCE-2844. Fixed display of nodes in UI. (Ravi Teja Ch N V via
+   acmurthy) 
+
+   MAPREDUCE-2677. Fixed 404 for some links from HistoryServer. (Robert Evans
+   via acmurthy) 
+
+   MAPREDUCE-2937. Ensure reason for application failure is displayed to the
+   user. (mahadev via acmurthy) 
+
+   MAPREDUCE-2953. Fix a race condition on submission which caused client to 
+   incorrectly assume application was gone by making submission synchronous
+   for RMAppManager. (Thomas Graves via acmurthy) 
+
+   MAPREDUCE-2963. Fix hang in TestMRJobs. (Siddharth Seth via acmurthy) 
+
+   MAPREDUCE-2954. Fixed a deadlock in NM caused due to wrong synchronization
+   in protocol buffer records. (Siddharth Seth via vinodkv)
+
+   MAPREDUCE-2975. Fixed YARNRunner to use YarnConfiguration rather than
+   Configuration. (mahadev via acmurthy) 
+ 
+   MAPREDUCE-2971. ant build mapreduce fails protected access jc.displayJobList
+   (jobs) (Thomas Graves via mahadev)
+
+   MAPREDUCE-2691. Finishing up the cleanup of distributed cache file resources
+   and related tests. (Siddharth Seth via vinodkv)
+
+   MAPREDUCE-2749. Ensure NM registers with RM after starting all its services
+   correctly. (Thomas Graves via acmurthy)
+
+   MAPREDUCE-2979. Removed the needless ClientProtocolProvider configuration
+   from the hadoop-mapreduce-client-core module. (Siddharth Seth via vinodkv)
+
+   MAPREDUCE-2985. Fixed findbugs warnings in ResourceLocalizationService.
+   (Thomas Graves via acmurthy)
+
+   MAPREDUCE-2874. Fix formatting of ApplicationId in web-ui. (Eric Payne via
+   acmurthy)
+
+   MAPREDUCE-2995. Better handling of expired containers in MapReduce
+   ApplicationMaster. (vinodkv via acmurthy) 
+
+   MAPREDUCE-2995. Fixed race condition in ContainerLauncher. (vinodkv via 
+   acmurthy) 
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/INSTALL

@@ -56,12 +56,12 @@ export YARN_CONF_DIR=$HADOOP_CONF_DIR
 
 Step 9) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
     <property>
-      <name>nodemanager.auxiluary.services</name>
+      <name>yarn.nodemanager.aux-services</name>
       <value>mapreduce.shuffle</value>
     </property>
 
     <property>
-      <name>nodemanager.aux.service.mapreduce.shuffle.class</name>
+      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
       <value>org.apache.hadoop.mapred.ShuffleHandler</value>
     </property>
 

hadoop-mapreduce-project/dev-support/findbugs-exclude.xml

@@ -138,6 +138,11 @@
        <Method name="run" />
        <Bug pattern="DM_EXIT" />
      </Match>
+     <Match>
+       <Class name="org.apache.hadoop.mapreduce.security.token.DelegationTokenRenewal$DelegationTokenCancelThread" />
+       <Method name="run" />
+       <Bug pattern="DM_EXIT" />
+    </Match>
      <!--
        We need to cast objects between old and new api objects
      -->
@@ -155,7 +160,8 @@
      </Match>
      <Match>
        <Class name="org.apache.hadoop.mapred.FileOutputCommitter" />
-       <Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
+       <Method name="commitJob" />
+       <Bug pattern="NM_WRONG_PACKAGE" />
      </Match>
      <Match>
        <Class name="org.apache.hadoop.mapred.OutputCommitter" />
@@ -166,6 +172,14 @@
        </Or>
        <Bug pattern="NM_WRONG_PACKAGE_INTENTIONAL" />
      </Match>
+     <Match>
+       <Class name="org.apache.hadoop.mapred.TaskCompletionEvent" />
+       <Or>
+       <Method name="setTaskStatus" />
+       <Method name="setTaskAttemptId" />
+       </Or>
+       <Bug pattern="NM_WRONG_PACKAGE" />
+     </Match>
      <Match>
        <Class name="org.apache.hadoop.mapred.lib.db.DBInputFormat$DBRecordReader" />
        <Method name="next" />

hadoop-mapreduce-project/dev-support/test-patch.properties

@@ -14,5 +14,5 @@
 # limitations under the License.
 
 OK_RELEASEAUDIT_WARNINGS=2
-OK_FINDBUGS_WARNINGS=13
+OK_FINDBUGS_WARNINGS=0
 OK_JAVADOC_WARNINGS=0

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java

@@ -42,7 +42,6 @@ import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
 import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java

@@ -34,10 +34,10 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RPC.Server;
 import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.mapred.SortedRanges.Range;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
 import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
@@ -104,8 +104,8 @@ public class TaskAttemptListenerImpl extends CompositeService
     try {
       server =
           RPC.getServer(TaskUmbilicalProtocol.class, this, "0.0.0.0", 0, 
-              conf.getInt(AMConstants.AM_TASK_LISTENER_THREADS, 
-                  AMConstants.DEFAULT_AM_TASK_LISTENER_THREADS),
+              conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, 
+                  MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT),
               false, conf, jobTokenSecretManager);
       server.start();
       InetSocketAddress listenerAddress = server.getListenerAddress();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedProgressSplitsBlock.java

@@ -20,9 +20,6 @@ package org.apache.hadoop.mapred;
 
 // Workaround for ProgressSplitBlock being package access
 public class WrappedProgressSplitsBlock extends ProgressSplitsBlock {
-
-  public static final int DEFAULT_NUMBER_PROGRESS_SPLITS = 12;
-
   private WrappedPeriodicStatsAccumulator wrappedProgressWallclockTime;
   private WrappedPeriodicStatsAccumulator wrappedProgressCPUTime;
   private WrappedPeriodicStatsAccumulator wrappedProgressVirtualMemoryKbytes;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -37,12 +37,12 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapreduce.JobCounter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.Counter;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.JobState;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -140,7 +140,7 @@ public class JobHistoryEventHandler extends AbstractService
         LOG.info("Creating intermediate history logDir: ["
             + doneDirPath
             + "] + based on conf. Should ideally be created by the JobHistoryServer: "
-            + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY);
+            + MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR);
           mkdir(
               doneDirFS,
               doneDirPath,
@@ -154,7 +154,7 @@ public class JobHistoryEventHandler extends AbstractService
           String message = "Not creating intermediate history logDir: ["
                 + doneDirPath
                 + "] based on conf: "
-                + JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY
+                + MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR
                 + ". Either set to true or pre-create this directory with appropriate permissions";
         LOG.error(message);
         throw new YarnException(message);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/AMConstants.java

@@ -1,74 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.mapreduce.v2.app;
-
-import org.apache.hadoop.mapreduce.v2.MRConstants;
-
-public interface AMConstants {
-
-  public static final String CONTAINERLAUNCHER_THREADPOOL_SIZE =
-      "yarn.mapreduce.containerlauncher.threadpool-size";
-
-  public static final String AM_RM_SCHEDULE_INTERVAL =
-      "yarn.appMaster.scheduler.interval";
-
-  public static final int DEFAULT_AM_RM_SCHEDULE_INTERVAL = 2000;
-
-  public static final String AM_TASK_LISTENER_THREADS =
-      MRConstants.YARN_MR_PREFIX + "task.listener.threads";
-
-  public static final int DEFAULT_AM_TASK_LISTENER_THREADS = 10;
-
-  public static final String AM_JOB_CLIENT_THREADS =
-      MRConstants.YARN_MR_PREFIX + "job.client.threads";
-
-  public static final int DEFAULT_AM_JOB_CLIENT_THREADS = 1;
-
-  public static final String SPECULATOR_CLASS =
-      MRConstants.YARN_MR_PREFIX + "speculator.class";
-
-  public static final String TASK_RUNTIME_ESTIMATOR_CLASS =
-      MRConstants.YARN_MR_PREFIX + "task.runtime.estimator.class";
-
-  public static final String TASK_ATTEMPT_PROGRESS_RUNTIME_LINEARIZER_CLASS =
-      MRConstants.YARN_MR_PREFIX + "task.runtime.linearizer.class";
-
-  public static final String EXPONENTIAL_SMOOTHING_LAMBDA_MILLISECONDS =
-      MRConstants.YARN_MR_PREFIX
-          + "task.runtime.estimator.exponential.smooth.lambda";
-
-  public static final String EXPONENTIAL_SMOOTHING_SMOOTH_RATE =
-      MRConstants.YARN_MR_PREFIX
-          + "task.runtime.estimator.exponential.smooth.smoothsrate";
-
-  public static final String RECOVERY_ENABLE = MRConstants.YARN_MR_PREFIX
-      + "recovery.enable";
-  
-  public static final float DEFAULT_REDUCE_RAMP_UP_LIMIT = 0.5f;
-  public static final String REDUCE_RAMPUP_UP_LIMIT = MRConstants.YARN_MR_PREFIX
-  + "reduce.rampup.limit";
-  
-  public static final float DEFAULT_REDUCE_PREEMPTION_LIMIT = 0.5f;
-  public static final String REDUCE_PREEMPTION_LIMIT = MRConstants.YARN_MR_PREFIX
-  + "reduce.preemption.limit";
-
-  public static final String NODE_BLACKLISTING_ENABLE = MRConstants.YARN_MR_PREFIX
-  + "node.blacklisting.enable";
-  
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java

@@ -154,7 +154,7 @@ public class MRAppMaster extends CompositeService {
     // for an app later
     appName = conf.get(MRJobConfig.JOB_NAME, "<missing app name>");
 
-    if (conf.getBoolean(AMConstants.RECOVERY_ENABLE, false)
+    if (conf.getBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, false)
          && startCount > 1) {
       LOG.info("Recovery is enabled. Will try to recover from previous life.");
       Recovery recoveryServ = new RecoveryService(appID, clock, startCount);
@@ -349,7 +349,7 @@ public class MRAppMaster extends CompositeService {
     try {
       speculatorClass
           // "yarn.mapreduce.job.speculator.class"
-          = conf.getClass(AMConstants.SPECULATOR_CLASS,
+          = conf.getClass(MRJobConfig.MR_AM_JOB_SPECULATOR,
                           DefaultSpeculator.class,
                           Speculator.class);
       Constructor<? extends Speculator> speculatorConstructor
@@ -360,19 +360,19 @@ public class MRAppMaster extends CompositeService {
       return result;
     } catch (InstantiationException ex) {
       LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
+          + MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
       throw new YarnException(ex);
     } catch (IllegalAccessException ex) {
       LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
+          + MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
       throw new YarnException(ex);
     } catch (InvocationTargetException ex) {
       LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
+          + MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
       throw new YarnException(ex);
     } catch (NoSuchMethodException ex) {
       LOG.error("Can't make a speculator -- check "
-          + AMConstants.SPECULATOR_CLASS + " " + ex);
+          + MRJobConfig.MR_AM_JOB_SPECULATOR, ex);
       throw new YarnException(ex);
     }
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
 import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
@@ -59,7 +60,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.app.job.Task;
@@ -141,8 +141,8 @@ public class MRClientService extends AbstractService
     server =
         rpc.getServer(MRClientProtocol.class, protocolHandler, address,
             conf, secretManager,
-            conf.getInt(AMConstants.AM_JOB_CLIENT_THREADS, 
-                AMConstants.DEFAULT_AM_JOB_CLIENT_THREADS));
+            conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
+                MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT));
     server.start();
     this.bindAddress =
         NetUtils.createSocketAddr(hostNameResolved.getHostAddress()

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -94,7 +94,6 @@ import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
 import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
 import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
-import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.Credentials;
@@ -138,6 +137,7 @@ public abstract class TaskAttemptImpl implements
   protected final Configuration conf;
   protected final Path jobFile;
   protected final int partition;
+  @SuppressWarnings("rawtypes")
   protected final EventHandler eventHandler;
   private final TaskAttemptId attemptId;
   private final Clock clock;
@@ -204,6 +204,11 @@ public abstract class TaskAttemptImpl implements
      .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED,
          TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED,
          new DeallocateContainerTransition(TaskAttemptState.FAILED, false))
+     .addTransition(TaskAttemptState.ASSIGNED,
+         TaskAttemptState.FAIL_CONTAINER_CLEANUP,
+         TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+         CLEANUP_CONTAINER_TRANSITION)
+      // ^ If RM kills the container due to expiry, preemption etc. 
      .addTransition(TaskAttemptState.ASSIGNED, 
          TaskAttemptState.KILL_CONTAINER_CLEANUP,
          TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
@@ -432,7 +437,8 @@ public abstract class TaskAttemptImpl implements
   //this is the last status reported by the REMOTE running attempt
   private TaskAttemptStatus reportedStatus;
 
-  public TaskAttemptImpl(TaskId taskId, int i, EventHandler eventHandler,
+  public TaskAttemptImpl(TaskId taskId, int i, 
+      @SuppressWarnings("rawtypes") EventHandler eventHandler,
       TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
       Configuration conf, String[] dataLocalHosts, OutputCommitter committer,
       Token<JobTokenIdentifier> jobToken,
@@ -528,6 +534,13 @@ public abstract class TaskAttemptImpl implements
     ContainerLaunchContext container =
         recordFactory.newRecordInstance(ContainerLaunchContext.class);
 
+    // Application resources
+    Map<String, LocalResource> localResources = 
+        new HashMap<String, LocalResource>();
+    
+    // Application environment
+    Map<String, String> environment = new HashMap<String, String>();
+    
     try {
       FileSystem remoteFS = FileSystem.get(conf);
 
@@ -536,7 +549,7 @@ public abstract class TaskAttemptImpl implements
         Path remoteJobJar = (new Path(remoteTask.getConf().get(
               MRJobConfig.JAR))).makeQualified(remoteFS.getUri(), 
                                                remoteFS.getWorkingDirectory());
-        container.setLocalResource(
+        localResources.put(
             MRConstants.JOB_JAR,
             createLocalResource(remoteFS, recordFactory, remoteJobJar,
                 LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
@@ -558,7 +571,7 @@ public abstract class TaskAttemptImpl implements
           new Path(path, oldJobId.toString());
       Path remoteJobConfPath = 
           new Path(remoteJobSubmitDir, MRConstants.JOB_CONF_FILE);
-      container.setLocalResource(
+      localResources.put(
           MRConstants.JOB_CONF_FILE,
           createLocalResource(remoteFS, recordFactory, remoteJobConfPath,
               LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
@@ -566,9 +579,14 @@ public abstract class TaskAttemptImpl implements
           + remoteJobConfPath.toUri().toASCIIString());
       // //////////// End of JobConf setup
 
+      
       // Setup DistributedCache
-      setupDistributedCache(remoteFS, conf, container);
+      setupDistributedCache(remoteFS, conf, localResources, environment);
 
+      // Set local-resources and environment
+      container.setLocalResources(localResources);
+      container.setEnv(environment);
+      
       // Setup up tokens
       Credentials taskCredentials = new Credentials();
 
@@ -595,12 +613,12 @@ public abstract class TaskAttemptImpl implements
 
       // Add shuffle token
       LOG.info("Putting shuffle token in serviceData");
-      container
-          .setServiceData(
-              ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
-              ShuffleHandler.serializeServiceData(jobToken));
+      Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
+      serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
+          ShuffleHandler.serializeServiceData(jobToken));
+      container.setServiceData(serviceData);
 
-      MRApps.addToClassPath(container.getAllEnv(), getInitialClasspath());
+      MRApps.addToClassPath(container.getEnv(), getInitialClasspath());
     } catch (IOException e) {
       throw new YarnException(e);
     }
@@ -623,11 +641,11 @@ public abstract class TaskAttemptImpl implements
     classPaths.add(workDir.toString()); // TODO
 
     // Construct the actual Container
-    container.addAllCommands(MapReduceChildJVM.getVMCommand(
+    container.setCommands(MapReduceChildJVM.getVMCommand(
         taskAttemptListener.getAddress(), remoteTask, javaHome,
         workDir.toString(), containerLogDir, childTmpDir, jvmID));
 
-    MapReduceChildJVM.setVMEnv(container.getAllEnv(), classPaths,
+    MapReduceChildJVM.setVMEnv(container.getEnv(), classPaths,
         workDir.toString(), containerLogDir, nmLdLibraryPath, remoteTask,
         localizedApplicationTokensFile);
 
@@ -649,11 +667,15 @@ public abstract class TaskAttemptImpl implements
     return result;
   }
 
-  private void setupDistributedCache(FileSystem remoteFS, Configuration conf, 
-      ContainerLaunchContext container) throws IOException {
+  private void setupDistributedCache(FileSystem remoteFS, 
+      Configuration conf, 
+      Map<String, LocalResource> localResources,
+      Map<String, String> env) 
+  throws IOException {
     
     // Cache archives
-    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE, 
+    parseDistributedCacheArtifacts(remoteFS, localResources, env, 
+        LocalResourceType.ARCHIVE, 
         DistributedCache.getCacheArchives(conf), 
         parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)), 
         getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
@@ -661,7 +683,9 @@ public abstract class TaskAttemptImpl implements
         DistributedCache.getArchiveClassPaths(conf));
     
     // Cache files
-    parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE, 
+    parseDistributedCacheArtifacts(remoteFS, 
+        localResources, env, 
+        LocalResourceType.FILE, 
         DistributedCache.getCacheFiles(conf),
         parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
         getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
@@ -673,7 +697,10 @@ public abstract class TaskAttemptImpl implements
   // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], 
   // long[], boolean[], Path[], FileType)
   private void parseDistributedCacheArtifacts(
-      FileSystem remoteFS, ContainerLaunchContext container, LocalResourceType type,
+      FileSystem remoteFS, 
+      Map<String, LocalResource> localResources,
+      Map<String, String> env,
+      LocalResourceType type,
       URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[], 
       Path[] pathsToPutOnClasspath) throws IOException {
 
@@ -710,9 +737,9 @@ public abstract class TaskAttemptImpl implements
           throw new IllegalArgumentException("Resource name must be relative");
         }
         String linkName = name.toUri().getPath();
-        container.setLocalResource(
+        localResources.put(
             linkName,
-            BuilderUtils.newLocalResource(recordFactory,
+            BuilderUtils.newLocalResource(
                 p.toUri(), type, 
                 visibilities[i]
                   ? LocalResourceVisibility.PUBLIC
@@ -720,8 +747,7 @@ public abstract class TaskAttemptImpl implements
                 sizes[i], timestamps[i])
         );
         if (classPaths.containsKey(u.getPath())) {
-          Map<String, String> environment = container.getAllEnv();
-          MRApps.addToClassPath(environment, linkName);
+          MRApps.addToClassPath(env, linkName);
         }
       }
     }
@@ -893,6 +919,7 @@ public abstract class TaskAttemptImpl implements
     }
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public void handle(TaskAttemptEvent event) {
     LOG.info("Processing " + event.getTaskAttemptID() +
@@ -903,7 +930,8 @@ public abstract class TaskAttemptImpl implements
       try {
         stateMachine.doTransition(event.getType(), event);
       } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
+        LOG.error("Can't handle this event at current state for "
+            + this.attemptId, e);
         eventHandler.handle(new JobDiagnosticsUpdateEvent(
             this.attemptId.getTaskId().getJobId(), "Invalid event " + event.getType() + 
             " on TaskAttempt " + this.attemptId));
@@ -981,8 +1009,8 @@ public abstract class TaskAttemptImpl implements
     try {
       if (progressSplitBlock == null) {
         progressSplitBlock = new WrappedProgressSplitsBlock(conf.getInt(
-            JHConfig.JOBHISTORY_TASKPROGRESS_NUMBER_SPLITS_KEY,
-            WrappedProgressSplitsBlock.DEFAULT_NUMBER_PROGRESS_SPLITS));
+            MRJobConfig.MR_AM_NUM_PROGRESS_SPLITS,
+            MRJobConfig.DEFAULT_MR_AM_NUM_PROGRESS_SPLITS));
       }
       return progressSplitBlock;
     } finally {
@@ -1035,6 +1063,7 @@ public abstract class TaskAttemptImpl implements
     public RequestContainerTransition(boolean rescheduled) {
       this.rescheduled = rescheduled;
     }
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1063,6 +1092,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class ContainerAssignedTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(final TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1112,6 +1142,7 @@ public abstract class TaskAttemptImpl implements
       this.finalState = finalState;
       this.withdrawsContainerRequest = withdrawsContainerRequest;
     }
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1158,6 +1189,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class LaunchedContainerTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent evnt) {
@@ -1208,6 +1240,7 @@ public abstract class TaskAttemptImpl implements
    
   private static class CommitPendingTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1219,6 +1252,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class TaskCleanupTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1234,6 +1268,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class SucceededTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1263,6 +1298,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class FailedTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
       // set the finish time
@@ -1287,6 +1323,7 @@ public abstract class TaskAttemptImpl implements
     }
   }
 
+  @SuppressWarnings({ "unchecked" })
   private void logAttemptFinishedEvent(TaskAttemptState state) {
     //Log finished events only if an attempt started.
     if (getLaunchTime() == 0) return; 
@@ -1320,6 +1357,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class TooManyFetchFailureTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
       //add to diagnostic
@@ -1347,6 +1385,7 @@ public abstract class TaskAttemptImpl implements
   private static class KilledTransition implements
       SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
 
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt,
         TaskAttemptEvent event) {
@@ -1373,6 +1412,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class CleanupContainerTransition implements
        SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {
@@ -1399,6 +1439,7 @@ public abstract class TaskAttemptImpl implements
 
   private static class StatusUpdater 
        implements SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+    @SuppressWarnings("unchecked")
     @Override
     public void transition(TaskAttemptImpl taskAttempt, 
         TaskAttemptEvent event) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java

@@ -528,7 +528,8 @@ public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
       try {
         stateMachine.doTransition(event.getType(), event);
       } catch (InvalidStateTransitonException e) {
-        LOG.error("Can't handle this event at current state", e);
+        LOG.error("Can't handle this event at current state for "
+            + this.taskId, e);
         internalError(event.getType());
       }
       if (oldState != getState()) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java

@@ -21,9 +21,9 @@ package org.apache.hadoop.mapreduce.v2.app.launcher;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedAction;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -33,8 +33,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.app.AMConstants;
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
@@ -79,8 +79,8 @@ public class ContainerLauncherImpl extends AbstractService implements
   private RecordFactory recordFactory;
   //have a cache/map of UGIs so as to avoid creating too many RPC
   //client connection objects to the same NodeManager
-  private Map<String, UserGroupInformation> ugiMap = 
-    new HashMap<String, UserGroupInformation>();
+  private ConcurrentMap<String, UserGroupInformation> ugiMap = 
+    new ConcurrentHashMap<String, UserGroupInformation>();
 
   public ContainerLauncherImpl(AppContext context) {
     super(ContainerLauncherImpl.class.getName());
@@ -102,7 +102,7 @@ public class ContainerLauncherImpl extends AbstractService implements
   public void start() {
     launcherPool =
         new ThreadPoolExecutor(getConfig().getInt(
-            AMConstants.CONTAINERLAUNCHER_THREADPOOL_SIZE, 10),
+            MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT, 10),
             Integer.MAX_VALUE, 1, TimeUnit.HOURS,
             new LinkedBlockingQueue<Runnable>());
     launcherPool.prestartAllCoreThreads(); // Wait for work.
@@ -142,22 +142,19 @@ public class ContainerLauncherImpl extends AbstractService implements
 
     UserGroupInformation user = UserGroupInformation.getCurrentUser();
 
-    // TODO: Synchronization problems!!
     if (UserGroupInformation.isSecurityEnabled()) {
-      if(!ugiMap.containsKey(containerManagerBindAddr)) {
-        Token<ContainerTokenIdentifier> token =
-          new Token<ContainerTokenIdentifier>(
-              containerToken.getIdentifier().array(),
-              containerToken.getPassword().array(), new Text(
-                  containerToken.getKind()), new Text(
-                      containerToken.getService()));
-        //the user in createRemoteUser in this context is not important
-        user = UserGroupInformation.createRemoteUser(containerManagerBindAddr);
-        user.addToken(token);
-        ugiMap.put(containerManagerBindAddr, user);
-      } else {
-        user = ugiMap.get(containerManagerBindAddr);    
-      }
+
+      Token<ContainerTokenIdentifier> token = new Token<ContainerTokenIdentifier>(
+          containerToken.getIdentifier().array(), containerToken
+              .getPassword().array(), new Text(containerToken.getKind()),
+          new Text(containerToken.getService()));
+      // the user in createRemoteUser in this context is not important
+      UserGroupInformation ugi = UserGroupInformation
+          .createRemoteUser(containerManagerBindAddr);
+      ugi.addToken(token);
+      ugiMap.putIfAbsent(containerManagerBindAddr, ugi);
+
+      user = ugiMap.get(containerManagerBindAddr);    
     }
     ContainerManager proxy =
         user.doAs(new PrivilegedAction<ContainerManager>() {