Merge branch 'trunk' into HDFS-7240

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -50,6 +50,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
 import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
@@ -950,6 +951,51 @@ public abstract class FileSystem extends Configured implements Closeable {
         IO_FILE_BUFFER_SIZE_DEFAULT));
   }
 
+  /**
+   * Open an FSDataInputStream matching the PathHandle instance. The
+   * implementation may encode metadata in PathHandle to address the
+   * resource directly and verify that the resource referenced
+   * satisfies constraints specified at its construciton.
+   * @param fd PathHandle object returned by the FS authority.
+   * @param bufferSize the size of the buffer to use
+   * @throws IOException IO failure
+   * @throws UnsupportedOperationException If not overridden by subclass
+   */
+  public FSDataInputStream open(PathHandle fd, int bufferSize)
+      throws IOException {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Create a durable, serializable handle to the referent of the given
+   * entity.
+   * @param stat Referent in the target FileSystem
+   * @param opt If absent, assume {@link HandleOpt#path()}.
+   * @throws IllegalArgumentException If the FileStatus does not belong to
+   *         this FileSystem
+   * @throws UnsupportedOperationException If
+   *         {@link #createPathHandle(FileStatus, HandleOpt[])}
+   *         not overridden by subclass.
+   * @throws UnsupportedOperationException If this FileSystem cannot enforce
+   *         the specified constraints.
+   */
+  public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
+    if (null == opt || 0 == opt.length) {
+      return createPathHandle(stat, HandleOpt.path());
+    }
+    return createPathHandle(stat, opt);
+  }
+
+  /**
+   * Hook to implement support for {@link PathHandle} operations.
+   * @param stat Referent in the target FileSystem
+   * @param opt Constraints that determine the validity of the
+   *            {@link PathHandle} reference.
+   */
+  protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * Create an FSDataOutputStream at the indicated Path.
    * Files are overwritten by default.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.Progressable;
@@ -163,6 +164,17 @@ public class FilterFileSystem extends FileSystem {
     return fs.open(f, bufferSize);
   }
 
+  @Override
+  public FSDataInputStream open(PathHandle fd, int bufferSize)
+      throws IOException {
+    return fs.open(fd, bufferSize);
+  }
+
+  @Override
+  protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opts) {
+    return fs.getPathHandle(stat, opts);
+  }
+
   @Override
   public FSDataOutputStream append(Path f, int bufferSize,
       Progressable progress) throws IOException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.fs;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.HandleOpt;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.util.LineReader;
@@ -699,6 +700,19 @@ public class HarFileSystem extends FileSystem {
         hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
   }
 
+  @Override
+  protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opts) {
+    // har consistency managed through metadata cache
+    // could extend HarMetaData to track more explicitly
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public FSDataInputStream open(PathHandle fd, int bufferSize)
+      throws IOException {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * Used for delegation token related functionality. Must delegate to
    * underlying file system.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java

@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.fs;
 
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.BiFunction;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -325,4 +329,180 @@ public final class Options {
       return processChecksumOpt(defaultOpt, userOpt, -1);
     }
   }
+
+  /**
+   * Options for creating {@link PathHandle} references.
+   */
+  public static class HandleOpt {
+    protected HandleOpt() {
+    }
+
+    /**
+     * Utility function for mapping
+     * {@link FileSystem#getPathHandle(FileStatus, HandleOpt[])} to a
+     * fixed set of handle options.
+     * @param fs Target filesystem
+     * @param opt Options to bind in partially evaluated function
+     * @return Function reference with options fixed
+     */
+    public static Function<FileStatus, PathHandle> resolve(
+        FileSystem fs, HandleOpt... opt) {
+      return resolve(fs::getPathHandle, opt);
+    }
+
+    /**
+     * Utility function for partial evaluation of {@link FileStatus}
+     * instances to a fixed set of handle options.
+     * @param fsr Function reference
+     * @param opt Options to associate with {@link FileStatus} instances to
+     *            produce {@link PathHandle} instances.
+     * @return Function reference with options fixed
+     */
+    public static Function<FileStatus, PathHandle> resolve(
+        BiFunction<FileStatus, HandleOpt[], PathHandle> fsr,
+        HandleOpt... opt) {
+      return (stat) -> fsr.apply(stat, opt);
+    }
+
+    /**
+     * Handle is valid iff the referent is neither moved nor changed.
+     * Equivalent to changed(false), moved(false).
+     * @return Options requiring that the content and location of the entity
+     * be unchanged between calls.
+     */
+    public static HandleOpt[] exact() {
+      return new HandleOpt[] {changed(false), moved(false) };
+    }
+
+    /**
+     * Handle is valid iff the content of the referent is the same.
+     * Equivalent to changed(false), moved(true).
+     * @return Options requiring that the content of the entity is unchanged,
+     * but it may be at a different location.
+     */
+    public static HandleOpt[] content() {
+      return new HandleOpt[] {changed(false), moved(true)  };
+    }
+
+    /**
+     * Handle is valid iff the referent is unmoved in the namespace.
+     * Equivalent to changed(true), moved(false).
+     * @return Options requiring that the referent exist in the same location,
+     * but its content may have changed.
+     */
+    public static HandleOpt[] path() {
+      return new HandleOpt[] {changed(true),  moved(false) };
+    }
+
+    /**
+     * Handle is valid iff the referent exists in the namespace.
+     * Equivalent to changed(true), moved(true).
+     * @return Options requiring that the implementation resolve a reference
+     * to this entity regardless of changes to content or location.
+     */
+    public static HandleOpt[] reference() {
+      return new HandleOpt[] {changed(true),  moved(true)  };
+    }
+
+    /**
+     * @param allow If true, resolve references to this entity even if it has
+     *             been modified.
+     * @return Handle option encoding parameter.
+     */
+    public static Data changed(boolean allow) {
+      return new Data(allow);
+    }
+
+    /**
+     * @param allow If true, resolve references to this entity anywhere in
+     *              the namespace.
+     * @return Handle option encoding parameter.
+     */
+    public static Location moved(boolean allow) {
+      return new Location(allow);
+    }
+
+    /**
+     * Utility method to extract a HandleOpt from the set provided.
+     * @param c Target class
+     * @param opt List of options
+     * @param <T> Type constraint for exact match
+     * @throws IllegalArgumentException If more than one matching type is found.
+     * @return An option assignable from the specified type or null if either
+     * opt is null or a suitable match is not found.
+     */
+    public static <T extends HandleOpt> Optional<T> getOpt(
+        Class<T> c, HandleOpt... opt) {
+      if (null == opt) {
+        return Optional.empty();
+      }
+      T ret = null;
+      for (HandleOpt o : opt) {
+        if (c.isAssignableFrom(o.getClass())) {
+          if (ret != null) {
+            throw new IllegalArgumentException("Duplicate option "
+                + c.getSimpleName());
+          }
+
+          @SuppressWarnings("unchecked")
+          T tmp = (T) o;
+          ret = tmp;
+        }
+      }
+      return Optional.ofNullable(ret);
+    }
+
+    /**
+     * Option storing standard constraints on data.
+     */
+    public static class Data extends HandleOpt {
+      private final boolean allowChanged;
+      Data(boolean allowChanged) {
+        this.allowChanged = allowChanged;
+      }
+
+      /**
+       * Tracks whether any changes to file content are permitted.
+       * @return True if content changes are allowed, false otherwise.
+       */
+      public boolean allowChange() {
+        return allowChanged;
+      }
+      @Override
+      public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("data(allowChange=")
+          .append(allowChanged).append(")");
+        return sb.toString();
+      }
+    }
+
+    /**
+     * Option storing standard constraints on location.
+     */
+    public static class Location extends HandleOpt {
+      private final boolean allowChanged;
+      Location(boolean allowChanged) {
+        this.allowChanged = allowChanged;
+      }
+
+      /**
+       * Tracks whether any changes to file location are permitted.
+       * @return True if relocation in the namespace is allowed, false
+       * otherwise.
+       */
+      public boolean allowChange() {
+        return allowChanged;
+      }
+      @Override
+      public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("loc(allowChange=")
+            .append(allowChanged).append(")");
+        return sb.toString();
+      }
+    }
+
+  }
+
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathHandle.java

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Opaque, serializable reference to an entity in the FileSystem. May contain
+ * metadata sufficient to resolve or verify subsequent accesses indepedent of
+ * other modifications to the FileSystem.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface PathHandle extends Serializable {
+
+  /**
+   * @return Serialized from in bytes.
+   */
+  default byte[] toByteArray() {
+    ByteBuffer bb = bytes();
+    byte[] ret = new byte[bb.remaining()];
+    bb.get(ret);
+    return ret;
+  }
+
+  ByteBuffer bytes();
+
+  @Override
+  boolean equals(Object other);
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawPathHandle.java

@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamException;
+import java.nio.ByteBuffer;
+
+/**
+ * Generic format of FileStatus objects. When the origin is unknown, the
+ * attributes of the handle are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class RawPathHandle implements PathHandle {
+
+  private static final long serialVersionUID = 0x12ba4689510L;
+
+  public static final int MAX_SIZE = 1 << 20;
+
+  private transient ByteBuffer fd;
+
+  /**
+   * Store a reference to the given bytes as the serialized form.
+   * @param fd serialized bytes
+   */
+  public RawPathHandle(ByteBuffer fd) {
+    this.fd = null == fd
+        ? ByteBuffer.allocate(0)
+        : fd.asReadOnlyBuffer();
+  }
+
+  /**
+   * Initialize using a copy of bytes from the serialized handle.
+   * @param handle PathHandle to preserve in serialized form.
+   */
+  public RawPathHandle(PathHandle handle) {
+    ByteBuffer hb = null == handle
+        ? ByteBuffer.allocate(0)
+        : handle.bytes();
+    fd = ByteBuffer.allocate(hb.remaining());
+    fd.put(hb);
+    fd.flip();
+  }
+
+  @Override
+  public ByteBuffer bytes() {
+    return fd.asReadOnlyBuffer();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (!(other instanceof PathHandle)) {
+      return false;
+    }
+    PathHandle o = (PathHandle) other;
+    return bytes().equals(o.bytes());
+  }
+
+  @Override
+  public int hashCode() {
+    return bytes().hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return bytes().toString();
+  }
+
+  private void writeObject(ObjectOutputStream out) throws IOException {
+    out.defaultWriteObject();
+    out.writeInt(fd.remaining());
+    if (fd.hasArray()) {
+      out.write(fd.array(), fd.position(), fd.remaining());
+    } else {
+      byte[] x = new byte[fd.remaining()];
+      fd.slice().get(x);
+      out.write(x);
+    }
+  }
+
+  private void readObject(ObjectInputStream in)
+      throws IOException, ClassNotFoundException {
+    in.defaultReadObject();
+    int len = in.readInt();
+    if (len < 0 || len > MAX_SIZE) {
+      throw new IOException("Illegal buffer length " + len);
+    }
+    byte[] x = new byte[len];
+    in.readFully(x);
+    fd = ByteBuffer.wrap(x);
+  }
+
+  private void readObjectNoData() throws ObjectStreamException {
+    throw new InvalidObjectException("Stream data required");
+  }
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java

@@ -64,6 +64,7 @@ abstract public class FsCommand extends Command {
     factory.registerCommands(SetReplication.class);
     factory.registerCommands(Stat.class);
     factory.registerCommands(Tail.class);
+    factory.registerCommands(Head.class);
     factory.registerCommands(Test.class);
     factory.registerCommands(Touch.class);
     factory.registerCommands(Truncate.class);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java

@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.io.IOUtils;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Show the first 1KB of the file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+class Head extends FsCommand {
+  public static void registerCommands(CommandFactory factory) {
+    factory.addClass(Head.class, "-head");
+  }
+  public static final String NAME = "head";
+  public static final String USAGE = "<file>";
+  public static final String DESCRIPTION =
+      "Show the first 1KB of the file.\n";
+
+  private long endingOffset = 1024;
+
+  @Override
+  protected void processOptions(LinkedList<String> args) throws IOException {
+    CommandFormat cf = new CommandFormat(1, 1);
+    cf.parse(args);
+  }
+
+  @Override
+  protected List<PathData> expandArgument(String arg) throws IOException {
+    List<PathData> items = new LinkedList<PathData>();
+    items.add(new PathData(arg, getConf()));
+    return items;
+  }
+
+  @Override
+  protected void processPath(PathData item) throws IOException {
+    if (item.stat.isDirectory()) {
+      throw new PathIsDirectoryException(item.toString());
+    }
+
+    dumpToOffset(item);
+  }
+
+  private void dumpToOffset(PathData item) throws IOException {
+    FSDataInputStream in = item.fs.open(item.path);
+    try {
+      IOUtils.copyBytes(in, System.out, endingOffset, false);
+    } finally {
+      in.close();
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java

@@ -204,43 +204,8 @@ public class BZip2Codec implements Configurable, SplittableCompressionCodec {
           Seekable.class.getName());
     }
 
-    //find the position of first BZip2 start up marker
-    ((Seekable)seekableIn).seek(0);
-
-    // BZip2 start of block markers are of 6 bytes.  But the very first block
-    // also has "BZh9", making it 10 bytes.  This is the common case.  But at
-    // time stream might start without a leading BZ.
-    final long FIRST_BZIP2_BLOCK_MARKER_POSITION =
-      CBZip2InputStream.numberOfBytesTillNextMarker(seekableIn);
-    long adjStart = 0L;
-    if (start != 0) {
-      // Other than the first of file, the marker size is 6 bytes.
-      adjStart = Math.max(0L, start - (FIRST_BZIP2_BLOCK_MARKER_POSITION
-          - (HEADER_LEN + SUB_HEADER_LEN)));
-    }
-
-    ((Seekable)seekableIn).seek(adjStart);
-    SplitCompressionInputStream in =
-      new BZip2CompressionInputStream(seekableIn, adjStart, end, readMode);
-
-
-    // The following if clause handles the following case:
-    // Assume the following scenario in BZip2 compressed stream where
-    // . represent compressed data.
-    // .....[48 bit Block].....[48 bit   Block].....[48 bit Block]...
-    // ........................[47 bits][1 bit].....[48 bit Block]...
-    // ................................^[Assume a Byte alignment here]
-    // ........................................^^[current position of stream]
-    // .....................^^[We go back 10 Bytes in stream and find a Block marker]
-    // ........................................^^[We align at wrong position!]
-    // ...........................................................^^[While this pos is correct]
-
-    if (in.getPos() < start) {
-      ((Seekable)seekableIn).seek(start);
-      in = new BZip2CompressionInputStream(seekableIn, start, end, readMode);
-    }
-
-    return in;
+    ((Seekable)seekableIn).seek(start);
+    return new BZip2CompressionInputStream(seekableIn, start, end, readMode);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java

@@ -52,20 +52,20 @@ import org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE;
  * This Ant code was enhanced so that it can de-compress blocks of bzip2 data.
  * Current position in the stream is an important statistic for Hadoop. For
  * example in LineRecordReader, we solely depend on the current position in the
- * stream to know about the progess. The notion of position becomes complicated
+ * stream to know about the progress. The notion of position becomes complicated
  * for compressed files. The Hadoop splitting is done in terms of compressed
  * file. But a compressed file deflates to a large amount of data. So we have
  * handled this problem in the following way.
  *
  * On object creation time, we find the next block start delimiter. Once such a
  * marker is found, the stream stops there (we discard any read compressed data
- * in this process) and the position is updated (i.e. the caller of this class
- * will find out the stream location). At this point we are ready for actual
- * reading (i.e. decompression) of data.
+ * in this process) and the position is reported as the beginning of the block
+ * start delimiter. At this point we are ready for actual reading
+ * (i.e. decompression) of data.
  *
  * The subsequent read calls give out data. The position is updated when the
  * caller of this class has read off the current block + 1 bytes. In between the
- * block reading, position is not updated. (We can only update the postion on
+ * block reading, position is not updated. (We can only update the position on
  * block boundaries).
  * </p>
  *
@@ -204,11 +204,12 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
   * in the stream.  It can find bit patterns of length <= 63 bits.  Specifically
   * this method is used in CBZip2InputStream to find the end of block (EOB)
   * delimiter in the stream, starting from the current position of the stream.
-  * If marker is found, the stream position will be right after marker at the
-  * end of this call.
+  * If marker is found, the stream position will be at the byte containing
+  * the starting bit of the marker.
   *
   * @param marker  The bit pattern to be found in the stream
   * @param markerBitLength  No of bits in the marker
+  * @return true if the marker was found otherwise false
   *
   * @throws IOException
   * @throws IllegalArgumentException  if marketBitLength is greater than 63
@@ -224,23 +225,33 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
       long bytes = 0;
       bytes = this.bsR(markerBitLength);
       if (bytes == -1) {
+        this.reportedBytesReadFromCompressedStream =
+            this.bytesReadFromCompressedStream;
         return false;
       }
       while (true) {
         if (bytes == marker) {
+          // Report the byte position where the marker starts
+          long markerBytesRead = (markerBitLength + this.bsLive + 7) / 8;
+          this.reportedBytesReadFromCompressedStream =
+              this.bytesReadFromCompressedStream - markerBytesRead;
           return true;
-
         } else {
           bytes = bytes << 1;
           bytes = bytes & ((1L << markerBitLength) - 1);
           int oneBit = (int) this.bsR(1);
           if (oneBit != -1) {
             bytes = bytes | oneBit;
-          } else
+          } else {
+            this.reportedBytesReadFromCompressedStream =
+                this.bytesReadFromCompressedStream;
             return false;
+          }
         }
       }
     } catch (IOException ex) {
+      this.reportedBytesReadFromCompressedStream =
+          this.bytesReadFromCompressedStream;
       return false;
     }
   }
@@ -302,7 +313,6 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
     } else if (readMode == READ_MODE.BYBLOCK) {
       this.currentState = STATE.NO_PROCESS_STATE;
       skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER,DELIMITER_BIT_LENGTH);
-      this.reportedBytesReadFromCompressedStream = this.bytesReadFromCompressedStream;
       if(!skipDecompression){
         changeStateToProcessABlock();
       }
@@ -419,8 +429,6 @@ public class CBZip2InputStream extends InputStream implements BZip2Constants {
       result = b;
 
       skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH);
-      //Exactly when we are about to start a new block, we advertise the stream position.
-      this.reportedBytesReadFromCompressedStream = this.bytesReadFromCompressedStream;
 
       changeStateToProcessABlock();
     }

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -400,6 +400,19 @@ Exit Code:
 
 Returns 0 on success and non-zero on error.
 
+head
+----
+
+Usage: `hadoop fs -head URI`
+
+Displays first kilobyte of the file to stdout.
+
+Example:
+
+* `hadoop fs -head pathname`
+
+Exit Code: Returns 0 on success and -1 on error.
+
 help
 ----
 

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -702,6 +702,121 @@ symbolic links
 exists in the metadata, but no copies of any its blocks can be located;
 -`FileNotFoundException` would seem more accurate and useful.
 
+### `PathHandle getPathHandle(FileStatus stat, HandleOpt... options)`
+
+Implementaions without a compliant call MUST throw `UnsupportedOperationException`
+
+#### Preconditions
+
+    let stat = getFileStatus(Path p)
+    let FS' where:
+      (FS.Directories', FS.Files', FS.Symlinks')
+      p' in paths(FS') where:
+        exists(FS, stat.path) implies exists(FS', p')
+
+The referent of a `FileStatus` instance, at the time it was resolved, is the
+same referent as the result of `getPathHandle(FileStatus)`. The `PathHandle`
+may be used in subsequent operations to ensure invariants hold between
+calls.
+
+The `options` parameter specifies whether a subsequent call e.g.,
+`open(PathHandle)` will succeed if the referent data or location changed. By
+default, any modification results in an error. The caller MAY specify
+relaxations that allow operations to succeed even if the referent exists at
+a different path and/or its data are changed.
+
+An implementation MUST throw `UnsupportedOperationException` if it cannot
+support the semantics specified by the caller. The default set of options
+are as follows.
+
+|            | Unmoved  | Moved     |
+|-----------:|:--------:|:---------:|
+| Unchanged  | EXACT    | CONTENT   |
+| Changed    | PATH     | REFERENCE |
+
+Changes to ownership, extended attributes, and other metadata are not
+required to match the `PathHandle`. Implementations can extend the set of
+`HandleOpt` parameters with custom constraints.
+
+##### Examples
+
+A client specifies that the `PathHandle` should track the entity across
+renames using `REFERENCE`. The implementation MUST throw an
+`UnsupportedOperationException` when creating the `PathHandle` unless
+failure to resolve the reference implies the entity no longer exists.
+
+A client specifies that the `PathHandle` should resolve only if the entity
+is unchanged using `PATH`. The implementation MUST throw an
+`UnsupportedOperationException` when creating the `PathHandle` unless it can
+distinguish between an identical entity located subsequently at the same
+path.
+
+#### Postconditions
+
+    result = PathHandle(p')
+
+#### Implementation notes
+
+The referent of a `PathHandle` is the namespace when the `FileStatus`
+instance was created, _not_ its state when the `PathHandle` is created. An
+implementation MAY reject attempts to create or resolve `PathHandle`
+instances that are valid, but expensive to service.
+
+Object stores that implement rename by copying objects MUST NOT claim to
+support `CONTENT` and `REFERENCE` unless the lineage of the object is
+resolved.
+
+It MUST be possible to serialize a `PathHandle` instance and reinstantiate
+it in one or more processes, on another machine, and arbitrarily far into
+the future without changing its semantics. The implementation MUST refuse to
+resolve instances if it can no longer guarantee its invariants.
+
+#### HDFS implementation notes
+
+HDFS does not support `PathHandle` references to directories or symlinks.
+Support for `CONTENT` and `REFERENCE` looks up files by INode. INodes are
+not unique across NameNodes, so federated clusters SHOULD include enough
+metadata in the `PathHandle` to detect references from other namespaces.
+
+### `FSDataInputStream open(PathHandle handle, int bufferSize)`
+
+Implementaions without a compliant call MUST throw `UnsupportedOperationException`
+
+#### Preconditions
+
+    let fd = getPathHandle(FileStatus stat)
+    if stat.isdir : raise IOException
+    let FS' where:
+      (FS.Directories', FS.Files', FS.Symlinks')
+      p' in FS.Files' where:
+        FS.Files'[p'] = fd
+    if not exists(FS', p') : raise FileNotFoundException
+
+The implementation MUST resolve the referent of the `PathHandle` following
+the constraints specified at its creation by `getPathHandle(FileStatus)`.
+
+Metadata necessary for the `FileSystem` to satisfy this contract MAY be
+encoded in the `PathHandle`.
+
+#### Postconditions
+
+    result = FSDataInputStream(0, FS.Files'[p'])
+
+The stream returned is subject to the constraints of a stream returned by
+`open(Path)`. Constraints checked on open MAY hold to hold for the stream, but
+this is not guaranteed.
+
+For example, a `PathHandle` created with `CONTENT` constraints MAY return a
+stream that ignores updates to the file after it is opened, if it was
+unmodified when `open(PathHandle)` was resolved.
+
+#### Implementation notes
+
+An implementation MAY check invariants either at the server or before
+returning the stream to the client. For example, an implementation may open
+the file, then verify the invariants in the `PathHandle` using
+`getFileStatus(Path)` to implement `CONTENT`. This could yield false
+positives and it requires additional RPC traffic.
 
 ### `boolean delete(Path p, boolean recursive)`
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java

@@ -15,29 +15,38 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-
 package org.apache.hadoop.fs.contract;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Options.HandleOpt;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.fs.RawPathHandle;
 import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
 
+import static org.apache.hadoop.fs.contract.ContractTestUtils.appendFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyRead;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyFileContents;
+
+import org.junit.Test;
 
 /**
  * Test Seek operations
  */
-public abstract class AbstractContractOpenTest extends AbstractFSContractTestBase {
+public abstract class AbstractContractOpenTest
+    extends AbstractFSContractTestBase {
 
   private FSDataInputStream instream;
 
@@ -163,5 +172,229 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
     instream.close();
   }
 
+  /**
+   * Skip a test case if the FS doesn't support file references.
+   * The feature is assumed to be unsupported unless stated otherwise.
+   */
+  protected void assumeSupportsFileReference() throws IOException {
+    if (getContract().isSupported(SUPPORTS_FILE_REFERENCE, false)) {
+      return;
+    }
+    skip("Skipping as unsupported feature: " + SUPPORTS_FILE_REFERENCE);
+  }
+
+  /**
+   * Skip a test case if the FS doesn't support content validation.
+   * The feature is assumed to be unsupported unless stated otherwise.
+   */
+  protected void assumeSupportsContentCheck() throws IOException {
+    if (getContract().isSupported(SUPPORTS_CONTENT_CHECK, false)) {
+      return;
+    }
+    skip("Skipping as unsupported feature: " + SUPPORTS_CONTENT_CHECK);
+  }
+
+  private PathHandle getHandleOrSkip(FileStatus stat, HandleOpt... opts) {
+    try {
+      return getFileSystem().getPathHandle(stat, opts);
+    } catch (UnsupportedOperationException e) {
+      skip("FileSystem does not support " + Arrays.toString(opts));
+    }
+    // unreachable
+    return null;
+  }
+
+  /**
+   * Verify {@link HandleOpt#exact()} handle semantics.
+   * @throws Throwable on error
+   */
+  @Test
+  public void testOpenFileByExact() throws Throwable {
+    describe("verify open(getPathHandle(FileStatus, exact())) operations" +
+        "detect changes");
+    assumeSupportsContentCheck();
+    assumeSupportsFileReference();
+    Path path1 = path("testopenfilebyexact1");
+    Path path2 = path("testopenfilebyexact2");
+    byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+    createFile(getFileSystem(), path1, false, file1);
+    FileStatus stat = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat);
+    assertEquals(path1, stat.getPath());
+    ContractTestUtils.rename(getFileSystem(), path1, path2);
+    // create identical file at same location, orig still exists at path2
+    createFile(getFileSystem(), path1, false, file1);
+
+    PathHandle fd = getHandleOrSkip(stat, HandleOpt.exact());
+
+    // verify path1, path2 contents identical
+    verifyFileContents(getFileSystem(), path1, file1);
+    verifyFileContents(getFileSystem(), path2, file1);
+    try {
+      // the PathHandle will not resolve, even though
+      // the original entity exists, it has not been modified, and an
+      // identical file exists at the old path. The handle would also
+      // fail to resolve if path1 had been modified
+      instream = getFileSystem().open(fd, 1 << 15);
+      fail("Expected an exception");
+    } catch (IOException e) {
+      // expected
+    }
+  }
+
+  /**
+   * Verify {@link HandleOpt#content()} handle semantics.
+   * @throws Throwable on error
+   */
+  @Test
+  public void testOpenFileByContent() throws Throwable {
+    describe("verify open(getPathHandle(FileStatus, content())) operations" +
+        "follow relocation");
+    assumeSupportsContentCheck();
+    assumeSupportsFileReference();
+    Path path1 = path("testopenfilebycontent1");
+    Path path2 = path("testopenfilebycontent2");
+    byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+    createFile(getFileSystem(), path1, false, file1);
+    FileStatus stat = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat);
+    assertEquals(path1, stat.getPath());
+    // rename the file after obtaining FileStatus
+    ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+    // obtain handle to entity from #getFileStatus call
+    PathHandle fd = getHandleOrSkip(stat, HandleOpt.content());
+
+    try (FSDataInputStream in = getFileSystem().open(fd, 1 << 15)) {
+      // verify read of consistent content at new location
+      verifyRead(in, file1, 0, TEST_FILE_LEN);
+    }
+
+    // modify the file at its new location by appending data
+    byte[] file1a = dataset(TEST_FILE_LEN, 44, 255);
+    appendFile(getFileSystem(), path2, file1a);
+    byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
+    System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
+    // verify fd entity contains contents of file1 + appended bytes
+    verifyFileContents(getFileSystem(), path2, file1x);
+
+    try {
+      // handle should not resolve when content changed
+      instream = getFileSystem().open(fd, 1 << 15);
+      fail("Failed to detect change to content");
+    } catch (IOException e) {
+      // expected
+    }
+  }
+
+
+  /**
+   * Verify {@link HandleOpt#path()} handle semantics.
+   * @throws Throwable on error
+   */
+  @Test
+  public void testOpenFileByPath() throws Throwable {
+    describe("verify open(getPathHandle(FileStatus, path())) operations" +
+        "detect changes");
+    assumeSupportsContentCheck();
+    Path path1 = path("testopenfilebypath1");
+    Path path2 = path("testopenfilebypath2");
+
+    byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+    createFile(getFileSystem(), path1, false, file1);
+    FileStatus stat = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat);
+    assertEquals(path1, stat.getPath());
+    ContractTestUtils.rename(getFileSystem(), path1, path2);
+    // create identical file at same location, orig still exists at path2
+    createFile(getFileSystem(), path1, false, file1);
+
+    PathHandle fd = getHandleOrSkip(stat, HandleOpt.path());
+
+    // verify path1, path2 contents identical
+    verifyFileContents(getFileSystem(), path1, file1);
+    verifyFileContents(getFileSystem(), path2, file1);
+    try {
+      // verify attempt to resolve the handle fails
+      instream = getFileSystem().open(fd, 1 << 15);
+      fail("Expected an exception");
+    } catch (IOException e) {
+      // expected
+    }
+  }
+
+  /**
+   * Verify {@link HandleOpt#reference()} handle semantics.
+   * @throws Throwable on error
+   */
+  @Test
+  public void testOpenFileByReference() throws Throwable {
+    describe("verify open(getPathHandle(FileStatus, reference())) operations" +
+        " are independent of rename");
+    assumeSupportsFileReference();
+    Path path1 = path("testopenfilebyref1");
+    Path path2 = path("testopenfilebyref2");
+
+    byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+    createFile(getFileSystem(), path1, false, file1);
+    FileStatus stat = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat);
+    assertEquals(path1, stat.getPath());
+    ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+    byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
+    createFile(getFileSystem(), path1, false, file2);
+    byte[] file1a = dataset(TEST_FILE_LEN, 42, 255);
+    appendFile(getFileSystem(), path2, file1a);
+    byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
+    System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
+
+    PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
+
+    // verify path2 contains contents of file1 + appended bytes
+    verifyFileContents(getFileSystem(), path2, file1x);
+    // verify path1 contents contents of file2
+    verifyFileContents(getFileSystem(), path1, file2);
+
+    // verify fd contains contents of file1 + appended bytes
+    instream = getFileSystem().open(fd, 1 << 15);
+    verifyRead(instream, file1x, 0, TEST_FILE_LEN);
+  }
+
+  /**
+   * Verify {@link PathHandle} may be serialized and restored.
+   * @throws Throwable on error
+   */
+  @Test
+  public void testOpenFileBySerializedReference() throws Throwable {
+    describe("verify PathHandle supports generic serialization");
+    assumeSupportsFileReference();
+    Path path1 = path("testopenfilebyref1");
+    Path path2 = path("testopenfilebyref2");
+
+    byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+    createFile(getFileSystem(), path1, false, file1);
+    FileStatus stat = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat);
+    assertEquals(path1, stat.getPath());
+    ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+    byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
+    createFile(getFileSystem(), path1, false, file2);
+
+    PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
+
+    // serialize PathHandle
+    ByteBuffer sb = fd.bytes();
+    PathHandle fdb = new RawPathHandle(sb);
+
+    instream = getFileSystem().open(fdb, 1 << 15);
+    // verify stat contains contents of file1
+    verifyRead(instream, file1, 0, TEST_FILE_LEN);
+    // verify path2 contains contents of file1
+    verifyFileContents(getFileSystem(), path2, file1);
+    // verify path1 contents contents of file2
+    verifyFileContents(getFileSystem(), path1, file2);
+  }
 
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java

@@ -191,6 +191,16 @@ public interface ContractOptions {
    */
   String SUPPORTS_POSITIONED_READABLE = "supports-positioned-readable";
 
+  /**
+   * Indicates that FS exposes durable references to files.
+   */
+  String SUPPORTS_FILE_REFERENCE = "supports-file-reference";
+
+  /**
+   * Indicates that FS supports content checks on open.
+   */
+  String SUPPORTS_CONTENT_CHECK = "supports-content-check";
+
   /**
    * Maximum path length
    * {@value}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java

@@ -402,6 +402,21 @@ public class ContractTestUtils extends Assert {
 
   }
 
+  /**
+   * Rename operation. Safety check for attempts to rename the root directory.
+   * Verifies that src no longer exists after rename.
+   * @param fileSystem filesystem to work with
+   * @param src source path
+   * @param dst destination path
+   * @throws IOException If rename fails or src is the root directory.
+   */
+  public static void rename(FileSystem fileSystem, Path src, Path dst)
+      throws IOException {
+    rejectRootOperation(src, false);
+    assertTrue(fileSystem.rename(src, dst));
+    assertPathDoesNotExist(fileSystem, "renamed", src);
+  }
+
   /**
    * Block any operation on the root path. This is a safety check
    * @param path path in the filesystem
@@ -622,6 +637,23 @@ public class ContractTestUtils extends Assert {
     }
   }
 
+  /**
+   * Append to an existing file.
+   * @param fs filesystem
+   * @param path path to file
+   * @param data data to append. Can be null
+   * @throws IOException On any error
+   */
+  public static void appendFile(FileSystem fs,
+                                Path path,
+                                byte[] data) throws IOException {
+    try (FSDataOutputStream stream = fs.appendFile(path).build()) {
+      if (data != null && data.length > 0) {
+        stream.write(data);
+      }
+    }
+  }
+
   /**
    * Touch a file.
    * @param fs filesystem

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java

@@ -862,4 +862,25 @@ public class DFSUtilClient {
     }
     return threadPoolExecutor;
   }
+
+  private static final int INODE_PATH_MAX_LENGTH = 3 * Path.SEPARATOR.length()
+      + HdfsConstants.DOT_RESERVED_STRING.length()
+      + HdfsConstants.DOT_INODES_STRING.length()
+      + (int)Math.ceil(Math.log10(Long.MAX_VALUE)) + 1;
+
+  /**
+   * Create the internal unique file path from HDFS file ID (inode ID). Unlike
+   * a regular file path, this one is guaranteed to refer to the same file at
+   * all times, across overwrites, etc.
+   * @param fileId File ID.
+   * @return The internal ID-based path.
+   */
+  public static Path makePathFromFileId(long fileId) {
+    StringBuilder sb = new StringBuilder(INODE_PATH_MAX_LENGTH);
+    sb.append(Path.SEPARATOR).append(HdfsConstants.DOT_RESERVED_STRING)
+      .append(Path.SEPARATOR).append(HdfsConstants.DOT_INODES_STRING)
+      .append(Path.SEPARATOR).append(fileId);
+    return new Path(sb.toString());
+  }
+
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -44,9 +44,11 @@ import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.FsStatus;
 import org.apache.hadoop.fs.GlobalStorageStatistics;
 import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
+import org.apache.hadoop.fs.PathHandle;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.QuotaUsage;
@@ -81,6 +83,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
 import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
 import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
@@ -105,6 +108,7 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /****************************************************************
  * Implementation of the abstract FileSystem for the DFS system.
@@ -319,6 +323,56 @@ public class DistributedFileSystem extends FileSystem {
     }.resolve(this, absF);
   }
 
+  /**
+   * Opens an FSDataInputStream with the indicated file ID extracted from
+   * the {@link PathHandle}.
+   * @param fd Reference to entity in this FileSystem.
+   * @param bufferSize the size of the buffer to be used.
+   */
+  @Override
+  public FSDataInputStream open(PathHandle fd, int bufferSize)
+      throws IOException {
+    if (!(fd instanceof HdfsPathHandle)) {
+      fd = new HdfsPathHandle(fd.bytes());
+    }
+    HdfsPathHandle id = (HdfsPathHandle) fd;
+    return open(DFSUtilClient.makePathFromFileId(id.getInodeId()), bufferSize);
+  }
+
+  /**
+   * Create a handle to an HDFS file.
+   * @param st HdfsFileStatus instance from NameNode
+   * @param opts Standard handle arguments
+   * @throws IllegalArgumentException If the FileStatus instance refers to a
+   * directory, symlink, or another namesystem.
+   * @throws UnsupportedOperationException If opts are not specified or both
+   * data and location are not allowed to change.
+   * @return A handle to the file.
+   */
+  @Override
+  protected PathHandle createPathHandle(FileStatus st, HandleOpt... opts) {
+    if (!(st instanceof HdfsFileStatus)) {
+      throw new IllegalArgumentException("Invalid FileStatus "
+          + st.getClass().getSimpleName());
+    }
+    if (st.isDirectory() || st.isSymlink()) {
+      throw new IllegalArgumentException("PathHandle only available for files");
+    }
+    if (!getUri().getAuthority().equals(st.getPath().toUri().getAuthority())) {
+      throw new IllegalArgumentException("Wrong FileSystem: " + st.getPath());
+    }
+    HandleOpt.Data data = HandleOpt.getOpt(HandleOpt.Data.class, opts)
+        .orElse(HandleOpt.changed(false));
+    HandleOpt.Location loc = HandleOpt.getOpt(HandleOpt.Location.class, opts)
+        .orElse(HandleOpt.moved(false));
+    if (!data.allowChange() || !loc.allowChange()) {
+      throw new UnsupportedOperationException("Unsupported opts "
+          + Arrays.stream(opts)
+                  .map(HandleOpt::toString).collect(Collectors.joining(",")));
+    }
+    return new HdfsPathHandle((HdfsFileStatus)st);
+  }
+
   @Override
   public FSDataOutputStream append(Path f, final int bufferSize,
       final Progressable progress) throws IOException {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java

@@ -646,7 +646,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
     default:
       LOG.warn(this + ": unknown response code " + resp.getStatus() +
           " while attempting to set up short-circuit access. " +
-          resp.getMessage());
+          resp.getMessage() + ". Disabling short-circuit read for DataNode "
+          + datanode + " temporarily.");
       clientContext.getDomainSocketFactory()
           .disableShortCircuitForPath(pathInfo.getPath());
       return null;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java

@@ -58,6 +58,10 @@ public final class HdfsConstants {
           = Path.SEPARATOR + DOT_SNAPSHOT_DIR;
   public static final String SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR
       = Path.SEPARATOR + DOT_SNAPSHOT_DIR + Path.SEPARATOR;
+  public final static String DOT_RESERVED_STRING = ".reserved";
+  public final static String DOT_RESERVED_PATH_PREFIX = Path.SEPARATOR
+      + DOT_RESERVED_STRING;
+  public final static String DOT_INODES_STRING = ".inodes";
 
   /**
    * Generation stamp of blocks that pre-date the introduction

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocol;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.Arrays;
 import java.util.EnumSet;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -63,12 +64,12 @@ public class HdfsFileStatus extends FileStatus {
 
   /**
    * Constructor.
-   * @param length            the number of bytes the file has
-   * @param isdir             if the path is a directory
-   * @param block_replication the replication factor
-   * @param blocksize         the block size
-   * @param modification_time modification time
-   * @param access_time access time
+   * @param length the number of bytes the file has
+   * @param isdir if the path is a directory
+   * @param replication the replication factor
+   * @param blocksize the block size
+   * @param mtime modification time
+   * @param atime access time
    * @param permission permission
    * @param owner the owner of the path
    * @param group the group of the path
@@ -80,16 +81,18 @@ public class HdfsFileStatus extends FileStatus {
    * @param storagePolicy ID which specifies storage policy
    * @param ecPolicy the erasure coding policy
    */
-  public HdfsFileStatus(long length, boolean isdir, int block_replication,
-                        long blocksize, long modification_time,
-                        long access_time, FsPermission permission,
-                        EnumSet<Flags> flags, String owner, String group,
-                        byte[] symlink, byte[] path, long fileId,
-                        int childrenNum, FileEncryptionInfo feInfo,
-                        byte storagePolicy, ErasureCodingPolicy ecPolicy) {
-    super(length, isdir, block_replication, blocksize, modification_time,
-        access_time, convert(isdir, symlink != null, permission, flags),
-        owner, group, null, null);
+  protected HdfsFileStatus(long length, boolean isdir, int replication,
+                         long blocksize, long mtime, long atime,
+                         FsPermission permission, EnumSet<Flags> flags,
+                         String owner, String group,
+                         byte[] symlink, byte[] path, long fileId,
+                         int childrenNum, FileEncryptionInfo feInfo,
+                         byte storagePolicy, ErasureCodingPolicy ecPolicy) {
+    super(length, isdir, replication, blocksize, mtime,
+        atime, convert(isdir, symlink != null, permission, flags),
+        owner, group, null, null,
+        flags.contains(Flags.HAS_ACL), flags.contains(Flags.HAS_CRYPT),
+        flags.contains(Flags.HAS_EC));
     this.flags = flags;
     this.uSymlink = symlink;
     this.uPath = path;
@@ -278,6 +281,222 @@ public class HdfsFileStatus extends FileStatus {
     // fully-qualify path
     setPath(getFullPath(parent).makeQualified(defaultUri, null));
     return this; // API compatibility
+
+  }
+
+  /**
+   * Builder class for HdfsFileStatus instances. Note default values for
+   * parameters.
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public static class Builder {
+    // Changing default values will affect cases where values are not
+    // specified. Be careful!
+    private long length                    = 0L;
+    private boolean isdir                  = false;
+    private int replication                = 0;
+    private long blocksize                 = 0L;
+    private long mtime                     = 0L;
+    private long atime                     = 0L;
+    private FsPermission permission        = null;
+    private EnumSet<Flags> flags           = EnumSet.noneOf(Flags.class);
+    private String owner                   = null;
+    private String group                   = null;
+    private byte[] symlink                 = null;
+    private byte[] path                    = EMPTY_NAME;
+    private long fileId                    = -1L;
+    private int childrenNum                = 0;
+    private FileEncryptionInfo feInfo      = null;
+    private byte storagePolicy             =
+        HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
+    private ErasureCodingPolicy ecPolicy   = null;
+
+    /**
+     * Set the length of the entity (default = 0).
+     * @param length Entity length
+     * @return This Builder instance
+     */
+    public Builder length(long length) {
+      this.length = length;
+      return this;
+    }
+
+    /**
+     * Set the isDir flag for the entity (default = false).
+     * @param isdir True if the referent is a directory.
+     * @return This Builder instance
+     */
+    public Builder isdir(boolean isdir) {
+      this.isdir = isdir;
+      return this;
+    }
+
+    /**
+     * Set the replication of this entity (default = 0).
+     * @param replication Number of replicas
+     * @return This Builder instance
+     */
+    public Builder replication(int replication) {
+      this.replication = replication;
+      return this;
+    }
+
+    /**
+     * Set the blocksize of this entity (default = 0).
+     * @param blocksize Target, default blocksize
+     * @return This Builder instance
+     */
+    public Builder blocksize(long blocksize) {
+      this.blocksize = blocksize;
+      return this;
+    }
+
+    /**
+     * Set the modification time of this entity (default = 0).
+     * @param mtime Last modified time
+     * @return This Builder instance
+     */
+    public Builder mtime(long mtime) {
+      this.mtime = mtime;
+      return this;
+    }
+
+    /**
+     * Set the access time of this entity (default = 0).
+     * @param atime Last accessed time
+     * @return This Builder instance
+     */
+    public Builder atime(long atime) {
+      this.atime = atime;
+      return this;
+    }
+
+    /**
+     * Set the permission mask of this entity (default = null).
+     * @param permission Permission bitmask
+     * @return This Builder instance
+     */
+    public Builder perm(FsPermission permission) {
+      this.permission = permission;
+      return this;
+    }
+
+    /**
+     * Set {@link Flags} for this entity
+     * (default = {@link EnumSet#noneOf(Class)}).
+     * @param flags Flags
+     * @return This builder instance
+     */
+    public Builder flags(EnumSet<Flags> flags) {
+      this.flags = flags;
+      return this;
+    }
+
+    /**
+     * Set the owner for this entity (default = null).
+     * @param owner Owner
+     * @return This Builder instance
+     */
+    public Builder owner(String owner) {
+      this.owner = owner;
+      return this;
+    }
+
+    /**
+     * Set the group for this entity (default = null).
+     * @param group Group
+     * @return This Builder instance
+     */
+    public Builder group(String group) {
+      this.group = group;
+      return this;
+    }
+
+    /**
+     * Set symlink bytes for this entity (default = null).
+     * @param symlink Symlink bytes (see
+     *                {@link DFSUtilClient#bytes2String(byte[])})
+     * @return This Builder instance
+     */
+    public Builder symlink(byte[] symlink) {
+      this.symlink = null == symlink
+          ? null
+          : Arrays.copyOf(symlink, symlink.length);
+      return this;
+    }
+
+    /**
+     * Set path bytes for this entity (default = {@link #EMPTY_NAME}).
+     * @param path Path bytes (see {@link #makeQualified(URI, Path)}).
+     * @return This Builder instance
+     */
+    public Builder path(byte[] path) {
+      this.path = null == path
+          ? null
+          : Arrays.copyOf(path, path.length);
+      return this;
+    }
+
+    /**
+     * Set the fileId for this entity (default = -1).
+     * @param fileId FileId
+     * @return This Builder instance
+     */
+    public Builder fileId(long fileId) {
+      this.fileId = fileId;
+      return this;
+    }
+
+    /**
+     * Set the number of children for this entity (default = 0).
+     * @param childrenNum Number of children
+     * @return This Builder instance
+     */
+    public Builder children(int childrenNum) {
+      this.childrenNum = childrenNum;
+      return this;
+    }
+
+    /**
+     * Set the encryption info for this entity (default = null).
+     * @param feInfo Encryption info
+     * @return This Builder instance
+     */
+    public Builder feInfo(FileEncryptionInfo feInfo) {
+      this.feInfo = feInfo;
+      return this;
+    }
+
+    /**
+     * Set the storage policy for this entity
+     * (default = {@link HdfsConstants#BLOCK_STORAGE_POLICY_ID_UNSPECIFIED}).
+     * @param storagePolicy Storage policy
+     * @return This Builder instance
+     */
+    public Builder storagePolicy(byte storagePolicy) {
+      this.storagePolicy = storagePolicy;
+      return this;
+    }
+
+    /**
+     * Set the erasure coding policy for this entity (default = null).
+     * @param ecPolicy Erasure coding policy
+     * @return This Builder instance
+     */
+    public Builder ecPolicy(ErasureCodingPolicy ecPolicy) {
+      this.ecPolicy = ecPolicy;
+      return this;
+    }
+
+    /**
+     * @return An {@link HdfsFileStatus} instance from these parameters.
+     */
+    public HdfsFileStatus build() {
+      return new HdfsFileStatus(length, isdir, replication, blocksize,
+          mtime, atime, permission, flags, owner, group, symlink, path, fileId,
+          childrenNum, feInfo, storagePolicy, ecPolicy);
+    }
   }
 
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsPathHandleProto;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Opaque handle to an entity in HDFS.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class HdfsPathHandle implements PathHandle {
+
+  private static final long serialVersionUID = 0xc5308795428L;
+
+  private final long inodeId;
+
+  public HdfsPathHandle(HdfsFileStatus hstat) {
+    this(hstat.getFileId());
+  }
+
+  public HdfsPathHandle(long inodeId) {
+    this.inodeId = inodeId;
+  }
+
+  public HdfsPathHandle(ByteBuffer bytes) throws IOException {
+    if (null == bytes) {
+      throw new IOException("Missing PathHandle");
+    }
+    HdfsPathHandleProto p =
+        HdfsPathHandleProto.parseFrom(ByteString.copyFrom(bytes));
+    inodeId = p.getInodeId();
+  }
+
+  public long getInodeId() {
+    return inodeId;
+  }
+
+  @Override
+  public ByteBuffer bytes() {
+    return HdfsPathHandleProto.newBuilder()
+      .setInodeId(getInodeId())
+      .build()
+      .toByteString()
+      .asReadOnlyByteBuffer();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (null == other) {
+      return false;
+    }
+    if (!HdfsPathHandle.class.equals(other.getClass())) {
+      // require exact match
+      return false;
+    }
+    HdfsPathHandle o = (HdfsPathHandle)other;
+    return getInodeId() == o.getInodeId();
+  }
+
+  @Override
+  public int hashCode() {
+    return Long.hashCode(inodeId);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("{ ");
+    sb.append("inodeId : ").append(Long.toString(getInodeId()));
+    sb.append(" }");
+    return sb.toString();
+  }
+
+}

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java

@@ -62,10 +62,18 @@ public class SnapshottableDirectoryStatus {
       String owner, String group, byte[] localName, long inodeId,
       int childrenNum, int snapshotNumber, int snapshotQuota,
       byte[] parentFullPath) {
-    this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
-        access_time, permission, flags, owner, group, null, localName, inodeId,
-        childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
-        null);
+    this.dirStatus = new HdfsFileStatus.Builder()
+      .isdir(true)
+      .mtime(modification_time)
+      .atime(access_time)
+      .perm(permission)
+      .flags(flags)
+      .owner(owner)
+      .group(group)
+      .path(localName)
+      .fileId(inodeId)
+      .children(childrenNum)
+      .build();
     this.snapshotNumber = snapshotNumber;
     this.snapshotQuota = snapshotQuota;
     this.parentFullPath = parentFullPath;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -82,13 +82,14 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -159,6 +160,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsPathHandleProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
@@ -1561,6 +1563,19 @@ public class PBHelperClient {
     return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
   }
 
+  public static HdfsPathHandle convert(HdfsPathHandleProto fd) {
+    if (null == fd) {
+      return null;
+    }
+    return new HdfsPathHandle(fd.getInodeId());
+  }
+
+  public static HdfsPathHandleProto convert(HdfsPathHandle fd) {
+    return HdfsPathHandleProto.newBuilder()
+        .setInodeId(fd.getInodeId())
+        .build();
+  }
+
   public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
     if (fs == null) {
       return null;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java

@@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.ContentSummary.Builder;
 import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FsServerDefaults;
@@ -152,11 +151,23 @@ class JsonUtilClient {
     final byte storagePolicy = m.containsKey("storagePolicy") ?
         (byte) ((Number) m.get("storagePolicy")).longValue() :
         HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
-    return new HdfsFileStatus(len,
-        type == WebHdfsConstants.PathType.DIRECTORY, replication, blockSize,
-        mTime, aTime, permission, f, owner, group, symlink,
-        DFSUtilClient.string2Bytes(localName), fileId, childrenNum,
-        null, storagePolicy, null);
+    return new HdfsFileStatus.Builder()
+      .length(len)
+      .isdir(type == WebHdfsConstants.PathType.DIRECTORY)
+      .replication(replication)
+      .blocksize(blockSize)
+      .mtime(mTime)
+      .atime(aTime)
+      .perm(permission)
+      .flags(f)
+      .owner(owner)
+      .group(group)
+      .symlink(symlink)
+      .path(DFSUtilClient.string2Bytes(localName))
+      .fileId(fileId)
+      .children(childrenNum)
+      .storagePolicy(storagePolicy)
+      .build();
   }
 
   static HdfsFileStatus[] toHdfsFileStatusArray(final Map<?, ?> json) {
@@ -399,9 +410,9 @@ class JsonUtilClient {
     final long spaceQuota = ((Number) m.get("spaceQuota")).longValue();
     final Map<?, ?> typem = (Map<?, ?>) m.get("typeQuota");
 
-    Builder contentSummaryBuilder = new ContentSummary.Builder().length(length)
-        .fileCount(fileCount).directoryCount(directoryCount).quota(quota)
-        .spaceConsumed(spaceConsumed).spaceQuota(spaceQuota);
+    ContentSummary.Builder contentSummaryBuilder =new ContentSummary.Builder()
+        .length(length).fileCount(fileCount).directoryCount(directoryCount)
+        .quota(quota).spaceConsumed(spaceConsumed).spaceQuota(spaceQuota);
     if (typem != null) {
       for (StorageType t : StorageType.getTypesSupportingQuota()) {
         Map<?, ?> type = (Map<?, ?>) typem.get(t.toString());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto

@@ -398,6 +398,13 @@ message AddErasureCodingPolicyResponseProto {
   optional string errorMsg = 3;
 }
 
+/**
+ * Placeholder type for consistent HDFS operations.
+ */
+message HdfsPathHandleProto {
+  optional uint64 inodeId = 1;
+}
+
 /**
  * Status of a file, directory or symlink
  * Optionally includes a file's block locations if requested by client on the rpc call.

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c

@@ -2688,7 +2688,7 @@ static int translateZCRException(JNIEnv *env, jthrowable exc)
         ret = EPROTONOSUPPORT;
         goto done;
     }
-    ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+    ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL,
             "hadoopZeroCopyRead: ZeroCopyCursor#read failed");
 done:
     free(className);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -596,6 +596,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s
   public static final String  DFS_DN_EC_RECONSTRUCTION_THREADS_KEY = "dfs.datanode.ec.reconstruction.threads";
   public static final int     DFS_DN_EC_RECONSTRUCTION_THREADS_DEFAULT = 8;
+  public static final String  DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_KEY =
+      "dfs.datanode.ec.reconstruction.xmits.weight";
+  public static final float   DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_DEFAULT =
+      0.5f;
 
   public static final String
       DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY =

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java

@@ -89,4 +89,10 @@ public class DataNodeFaultInjector {
 
   public void throwTooManyOpenFiles() throws FileNotFoundException {
   }
+
+  /**
+   * Used as a hook to inject failure in erasure coding reconstruction
+   * process.
+   */
+  public void stripedBlockReconstruction() throws IOException {}
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java

@@ -387,6 +387,8 @@ class DataXceiver extends Receiver implements Runnable {
       } catch (IOException e) {
         bld.setStatus(ERROR);
         bld.setMessage(e.getMessage());
+        LOG.error("Request short-circuit read file descriptor" +
+            " failed with unknown error.", e);
       }
       bld.build().writeDelimitedTo(socketOut);
       if (fis != null) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.erasurecode;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -47,6 +48,7 @@ public final class ErasureCodingWorker {
 
   private final DataNode datanode;
   private final Configuration conf;
+  private final float xmitWeight;
 
   private ThreadPoolExecutor stripedReconstructionPool;
   private ThreadPoolExecutor stripedReadPool;
@@ -54,6 +56,14 @@ public final class ErasureCodingWorker {
   public ErasureCodingWorker(Configuration conf, DataNode datanode) {
     this.datanode = datanode;
     this.conf = conf;
+    this.xmitWeight = conf.getFloat(
+        DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_KEY,
+        DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_DEFAULT
+    );
+    Preconditions.checkArgument(this.xmitWeight >= 0,
+        "Invalid value configured for " +
+            DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_KEY +
+            ", it can not be negative value (" + this.xmitWeight + ").");
 
     initializeStripedReadThreadPool();
     initializeStripedBlkReconstructionThreadPool(conf.getInt(
@@ -128,7 +138,7 @@ public final class ErasureCodingWorker {
           //   1) NN will not send more tasks than what DN can execute and
           //   2) DN will not throw away reconstruction tasks, and instead keeps
           //      an unbounded number of tasks in the executor's task queue.
-          xmitsSubmitted = task.getXmits();
+          xmitsSubmitted = Math.max((int)(task.getXmits() * xmitWeight), 1);
           getDatanode().incrementXmitsInProcess(xmitsSubmitted);
           stripedReconstructionPool.submit(task);
         } else {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 import org.apache.hadoop.util.Time;
 
@@ -80,6 +81,7 @@ class StripedBlockReconstructor extends StripedReconstructor
 
   void reconstruct() throws IOException {
     while (getPositionInBlock() < getMaxTargetLength()) {
+      DataNodeFaultInjector.get().stripedBlockReconstruction();
       long remaining = getMaxTargetLength() - getPositionInBlock();
       final int toReconstructLen =
           (int) Math.min(getStripedReader().getBufferSize(), remaining);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java

@@ -2016,10 +2016,18 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
       LOG.error("Cannot get the remote user: {}", e.getMessage());
     }
     long inodeId = 0;
-    return new HdfsFileStatus(0, true, 0, 0, modTime, accessTime, permission,
-        EnumSet.noneOf(HdfsFileStatus.Flags.class),
-        owner, group, new byte[0], DFSUtil.string2Bytes(name), inodeId,
-        childrenNum, null, (byte) 0, null);
+    return new HdfsFileStatus.Builder()
+      .isdir(true)
+      .mtime(modTime)
+      .atime(accessTime)
+      .perm(permission)
+      .owner(owner)
+      .group(group)
+      .symlink(new byte[0])
+      .path(DFSUtil.string2Bytes(name))
+      .fileId(inodeId)
+      .children(childrenNum)
+      .build();
   }
 
   /**
@@ -2043,4 +2051,4 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
     UserGroupInformation ugi = Server.getRemoteUser();
     return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
   }
-}
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java

@@ -487,9 +487,25 @@ class FSDirStatAndListingOp {
       int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
       ErasureCodingPolicy ecPolicy, LocatedBlocks locations) {
     if (locations == null) {
-      return new HdfsFileStatus(length, isdir, replication, blocksize,
-          mtime, atime, permission, flags, owner, group, symlink, path,
-          fileId, childrenNum, feInfo, storagePolicy, ecPolicy);
+      return new HdfsFileStatus.Builder()
+          .length(length)
+          .isdir(isdir)
+          .replication(replication)
+          .blocksize(blocksize)
+          .mtime(mtime)
+          .atime(atime)
+          .perm(permission)
+          .flags(flags)
+          .owner(owner)
+          .group(group)
+          .symlink(symlink)
+          .path(path)
+          .fileId(fileId)
+          .children(childrenNum)
+          .feInfo(feInfo)
+          .storagePolicy(storagePolicy)
+          .ecPolicy(ecPolicy)
+          .build();
     } else {
       return new HdfsLocatedFileStatus(length, isdir, replication, blocksize,
           mtime, atime, permission, flags, owner, group, symlink, path,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java

@@ -73,7 +73,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -123,30 +122,31 @@ public class FSDirectory implements Closeable {
 
   @VisibleForTesting
   static boolean CHECK_RESERVED_FILE_NAMES = true;
-  public final static String DOT_RESERVED_STRING = ".reserved";
-  public final static String DOT_RESERVED_PATH_PREFIX = Path.SEPARATOR
-      + DOT_RESERVED_STRING;
+  public final static String DOT_RESERVED_STRING =
+      HdfsConstants.DOT_RESERVED_STRING;
+  public final static String DOT_RESERVED_PATH_PREFIX =
+      HdfsConstants.DOT_RESERVED_PATH_PREFIX;
   public final static byte[] DOT_RESERVED = 
       DFSUtil.string2Bytes(DOT_RESERVED_STRING);
   private final static String RAW_STRING = "raw";
   private final static byte[] RAW = DFSUtil.string2Bytes(RAW_STRING);
-  public final static String DOT_INODES_STRING = ".inodes";
+  public final static String DOT_INODES_STRING =
+      HdfsConstants.DOT_INODES_STRING;
   public final static byte[] DOT_INODES = 
       DFSUtil.string2Bytes(DOT_INODES_STRING);
   private final static byte[] DOT_DOT =
       DFSUtil.string2Bytes("..");
 
   public final static HdfsFileStatus DOT_RESERVED_STATUS =
-      new HdfsFileStatus(0, true, 0, 0, 0, 0, new FsPermission((short) 01770),
-          EnumSet.noneOf(HdfsFileStatus.Flags.class), null, null, null,
-          HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
-          HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+      new HdfsFileStatus.Builder()
+        .isdir(true)
+        .perm(new FsPermission((short) 01770))
+        .build();
 
   public final static HdfsFileStatus DOT_SNAPSHOT_DIR_STATUS =
-      new HdfsFileStatus(0, true, 0, 0, 0, 0, null,
-          EnumSet.noneOf(HdfsFileStatus.Flags.class), null, null, null,
-          HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
-          HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+      new HdfsFileStatus.Builder()
+        .isdir(true)
+        .build();
 
   INodeDirectory rootDir;
   private final FSNamesystem namesystem;
@@ -434,16 +434,22 @@ public class FSDirectory implements Closeable {
    * @return Array of HdfsFileStatus
    */
   void createReservedStatuses(long cTime) {
-    HdfsFileStatus inodes = new HdfsFileStatus(0, true, 0, 0, cTime, cTime,
-        new FsPermission((short) 0770),
-        EnumSet.noneOf(HdfsFileStatus.Flags.class), null, supergroup, null,
-        DOT_INODES, -1L, 0, null,
-        HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
-    HdfsFileStatus raw = new HdfsFileStatus(0, true, 0, 0, cTime, cTime,
-        new FsPermission((short) 0770),
-        EnumSet.noneOf(HdfsFileStatus.Flags.class), null, supergroup, null,
-        RAW, -1L, 0, null,
-        HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+    HdfsFileStatus inodes = new HdfsFileStatus.Builder()
+        .isdir(true)
+        .mtime(cTime)
+        .atime(cTime)
+        .perm(new FsPermission((short) 0770))
+        .group(supergroup)
+        .path(DOT_INODES)
+        .build();
+    HdfsFileStatus raw = new HdfsFileStatus.Builder()
+        .isdir(true)
+        .mtime(cTime)
+        .atime(cTime)
+        .perm(new FsPermission((short) 0770))
+        .group(supergroup)
+        .path(RAW)
+        .build();
     reservedStatuses = new HdfsFileStatus[] {inodes, raw};
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -4606,9 +4606,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     }
   }
 
-  /**
-   * Get the total number of objects in the system. 
-   */
   @Override // FSNamesystemMBean
   public long getMaxObjects() {
     return maxFsObjects;

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -3072,6 +3072,19 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.ec.reconstruction.xmits.weight</name>
+  <value>0.5</value>
+  <description>
+    Datanode uses xmits weight to calculate the relative cost of EC recovery
+    tasks comparing to replicated block recovery, of which xmits is always 1.
+    Namenode then uses xmits reported from datanode to throttle recovery tasks
+    for EC and replicated blocks.
+    The xmits of an erasure coding recovery task is calculated as the maximum
+    value between the number of read streams and the number of write streams.
+  </description>
+</property>
+
 <property>
   <name>dfs.namenode.quota.init-threads</name>
   <value>4</value>

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md

@@ -136,6 +136,12 @@ Deployment
   1. `dfs.datanode.ec.reconstruction.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms.
   1. `dfs.datanode.ec.reconstruction.stripedread.buffer.size` - Buffer size for reader service. Default value is 64KB.
   1. `dfs.datanode.ec.reconstruction.threads` - Number of threads used by the Datanode for background reconstruction work. Default value is 8 threads.
+  1. `dfs.datanode.ec.reconstruction.xmits.weight` - Relative weight of xmits used by EC background recovery task comparing to replicated block recovery. Default value is 0.5.
+  It sets to `0` to disable calculate weights for EC recovery tasks, that is, EC task always has `1` xmits.
+  The xmits of an erasure coding recovery task is calculated as the maximum value between the number of read streams and the number of write streams. For example, if an EC recovery
+  task need to read from 6 nodes and write to 2 nodes, it has xmits of `max(6, 2) * 0.5 = 3`. Recovery task for replicated file always counts
+  as `1` xmit. NameNode utilizes `dfs.namenode.replication.max-streams` minus the total `xmitsInProgress` on the DataNode that combines of the xmits from
+  replicated file and EC files, to schedule recovery tasks to this DataNode.
 
 ### Enable Intel ISA-L
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java

@@ -257,22 +257,32 @@ public class TestDFSClientRetries {
                          Matchers.<EnumSet<AddBlockFlag>>any()))
         .thenAnswer(answer);
     
-    Mockito.doReturn(
-            new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
-                (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
-                "owner", "group", new byte[0], new byte[0],
-                1010, 0, null, (byte) 0, null)).when(mockNN).getFileInfo(anyString());
+    Mockito.doReturn(new HdfsFileStatus.Builder()
+          .replication(1)
+          .blocksize(1024)
+          .perm(new FsPermission((short) 777))
+          .owner("owner")
+          .group("group")
+          .symlink(new byte[0])
+          .fileId(1010)
+          .build())
+      .when(mockNN)
+      .getFileInfo(anyString());
     
-    Mockito.doReturn(
-            new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
-                (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
-                "owner", "group", new byte[0], new byte[0],
-                1010, 0, null, (byte) 0, null))
+    Mockito.doReturn(new HdfsFileStatus.Builder()
+          .replication(1)
+          .blocksize(1024)
+          .perm(new FsPermission((short) 777))
+          .owner("owner")
+          .group("group")
+          .symlink(new byte[0])
+          .fileId(1010)
+          .build())
         .when(mockNN)
         .create(anyString(), (FsPermission) anyObject(), anyString(),
-            (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
-            anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
-            anyObject());
+          (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
+          anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
+          anyObject());
 
     final DFSClient client = new DFSClient(null, mockNN, conf, null);
     OutputStream os = client.create("testfile", true);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java

@@ -892,6 +892,33 @@ public class TestDFSShell {
     }
   }
 
+  /**
+   * Test that -head displays first kilobyte of the file to stdout.
+   */
+  @Test (timeout = 30000)
+  public void testHead() throws Exception {
+    final int fileLen = 5 * BLOCK_SIZE;
+
+    // create a text file with multiple KB bytes (and multiple blocks)
+    final Path testFile = new Path("testHead", "file1");
+    final String text = RandomStringUtils.randomAscii(fileLen);
+    try (OutputStream pout = dfs.create(testFile)) {
+      pout.write(text.getBytes());
+    }
+    final ByteArrayOutputStream out = new ByteArrayOutputStream();
+    System.setOut(new PrintStream(out));
+    final String[] argv = new String[]{"-head", testFile.toString()};
+    final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv);
+
+    assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret);
+    assertEquals("-head returned " + out.size() + " bytes data, expected 1KB",
+            1024, out.size());
+    // tailed out last 1KB of the file content
+    assertArrayEquals("Head output doesn't match input",
+            text.substring(0, 1024).getBytes(), out.toByteArray());
+    out.reset();
+  }
+
   /**
    * Test that -tail displays last kilobyte of the file to stdout.
    */

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java

@@ -54,6 +54,7 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
@@ -1049,4 +1050,19 @@ public class TestDFSUtil {
         DFSUtilClient.isHDFSEncryptionEnabled(conf));
 
   }
+
+  @Test
+  public void testFileIdPath() throws Throwable {
+    // /.reserved/.inodes/
+    String prefix = Path.SEPARATOR + HdfsConstants.DOT_RESERVED_STRING +
+                    Path.SEPARATOR + HdfsConstants.DOT_INODES_STRING +
+                    Path.SEPARATOR;
+    Random r = new Random();
+    for (int i = 0; i < 100; ++i) {
+      long inode = r.nextLong() & Long.MAX_VALUE;
+      assertEquals(new Path(prefix + inode),
+          DFSUtilClient.makePathFromFileId(inode));
+    }
+  }
+
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java

@@ -889,20 +889,25 @@ public class TestEncryptionZones {
   @SuppressWarnings("unchecked")
   private static void mockCreate(ClientProtocol mcp,
       CipherSuite suite, CryptoProtocolVersion version) throws Exception {
-    Mockito.doReturn(
-        new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
-            (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
-            "owner", "group", new byte[0], new byte[0],
-            1010, 0, new FileEncryptionInfo(suite,
-            version, new byte[suite.getAlgorithmBlockSize()],
-            new byte[suite.getAlgorithmBlockSize()],
-            "fakeKey", "fakeVersion"),
-            (byte) 0, null))
+    Mockito.doReturn(new HdfsFileStatus.Builder()
+          .replication(1)
+          .blocksize(1024)
+          .perm(new FsPermission((short) 777))
+          .owner("owner")
+          .group("group")
+          .symlink(new byte[0])
+          .path(new byte[0])
+          .fileId(1010)
+          .feInfo(new FileEncryptionInfo(suite, version,
+              new byte[suite.getAlgorithmBlockSize()],
+              new byte[suite.getAlgorithmBlockSize()],
+              "fakeKey", "fakeVersion"))
+          .build())
         .when(mcp)
         .create(anyString(), (FsPermission) anyObject(), anyString(),
-            (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
-            anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
-            anyObject());
+          (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
+          anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
+          anyObject());
   }
 
   // This test only uses mocks. Called from the end of an existing test to
@@ -1248,6 +1253,7 @@ public class TestEncryptionZones {
     Mockito.when(keyProvider.getConf()).thenReturn(conf);
     byte[] testIdentifier = "Test identifier for delegation token".getBytes();
 
+    @SuppressWarnings("rawtypes")
     Token<?> testToken = new Token(testIdentifier, new byte[0],
         new Text(), new Text());
     Mockito.when(((DelegationTokenExtension)keyProvider).

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java

@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.hdfs;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.net.URI;
 
 import org.apache.hadoop.fs.FSProtos.FileStatusProto;
@@ -33,7 +37,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
 import com.google.protobuf.ByteString;
 
 import org.junit.Test;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 /**
  * Verify compatible FileStatus/HdfsFileStatus serialization.
@@ -53,6 +57,27 @@ public class TestFileStatusSerialization {
     assertEquals(expected.getBlockSize(), actual.getBlockSize());
   }
 
+  private static final URI  BASEURI  = new Path("hdfs://foobar").toUri();
+  private static final Path BASEPATH = new Path("/dingos");
+  private static final String FILE   = "zot";
+  private static final Path FULLPATH = new Path("hdfs://foobar/dingos/zot");
+  private static HdfsFileStatusProto.Builder baseStatus() {
+    FsPermission perm = FsPermission.getFileDefault();
+    HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
+        .setFileType(FileType.IS_FILE)
+        .setPath(ByteString.copyFromUtf8("zot"))
+        .setLength(4344)
+        .setPermission(PBHelperClient.convert(perm))
+        .setOwner("hadoop")
+        .setGroup("unqbbc")
+        .setModificationTime(12345678L)
+        .setAccessTime(87654321L)
+        .setBlockReplication(10)
+        .setBlocksize(1L << 33)
+        .setFlags(0);
+    return hspb;
+  }
+
   /**
    * Test API backwards-compatibility with 2.x applications w.r.t. FsPermission.
    */
@@ -65,21 +90,12 @@ public class TestFileStatusSerialization {
     // test verifies.
     for (int i = 0; i < flagmask; ++i) {
       FsPermission perm = FsPermission.createImmutable((short) 0013);
-      HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
-          .setFileType(FileType.IS_FILE)
-          .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos/zot"))
-          .setLength(4344)
+      HdfsFileStatusProto.Builder hspb = baseStatus()
           .setPermission(PBHelperClient.convert(perm))
-          .setOwner("hadoop")
-          .setGroup("unqbbc")
-          .setModificationTime(12345678L)
-          .setAccessTime(87654321L)
-          .setBlockReplication(10)
-          .setBlocksize(1L << 33)
           .setFlags(i);
       HdfsFileStatus stat = PBHelperClient.convert(hspb.build());
-      stat.makeQualified(new URI("hdfs://foobar"), new Path("/dingos"));
-      assertEquals(new Path("hdfs://foobar/dingos/zot"), stat.getPath());
+      stat.makeQualified(BASEURI, BASEPATH);
+      assertEquals(FULLPATH, stat.getPath());
 
       // verify deprecated FsPermissionExtension methods
       FsPermission sp = stat.getPermission();
@@ -103,23 +119,29 @@ public class TestFileStatusSerialization {
       assertEquals(sp.getErasureCodedBit(), fstat.isErasureCoded());
     }
   }
-  // param for LocatedFileStatus, HttpFileStatus
+
+  @Test
+  public void testJavaSerialization() throws Exception {
+    HdfsFileStatusProto hsp = baseStatus().build();
+    HdfsFileStatus hs = PBHelperClient.convert(hsp);
+    hs.makeQualified(BASEURI, BASEPATH);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
+    try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+      oos.writeObject(hs);
+    }
+    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+    try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+      FileStatus deser = (FileStatus) ois.readObject();
+      assertEquals(hs, deser);
+      checkFields(hs, deser);
+    }
+  }
 
   @Test
   public void testCrossSerializationProto() throws Exception {
-    FsPermission perm = FsPermission.getFileDefault();
     for (FileType t : FileType.values()) {
-      HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
-          .setFileType(t)
-          .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos"))
-          .setLength(4344)
-          .setPermission(PBHelperClient.convert(perm))
-          .setOwner("hadoop")
-          .setGroup("unqbbc")
-          .setModificationTime(12345678L)
-          .setAccessTime(87654321L)
-          .setBlockReplication(10)
-          .setBlocksize(1L << 33);
+      HdfsFileStatusProto.Builder hspb = baseStatus()
+          .setFileType(t);
       if (FileType.IS_SYMLINK.equals(t)) {
         hspb.setSymlink(ByteString.copyFromUtf8("hdfs://yaks/dingos"));
       }
@@ -146,7 +168,9 @@ public class TestFileStatusSerialization {
       byte[] dst = fsp.toByteArray();
       HdfsFileStatusProto hsp2 = HdfsFileStatusProto.parseFrom(dst);
       assertEquals(hsp, hsp2);
-      checkFields(PBHelperClient.convert(hsp), PBHelperClient.convert(hsp2));
+      FileStatus hstat  = PBHelperClient.convert(hsp);
+      FileStatus hstat2 = PBHelperClient.convert(hsp2);
+      checkFields(hstat, hstat2);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java

@@ -30,7 +30,6 @@ import java.io.DataOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
-import java.util.EnumSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -352,22 +351,33 @@ public class TestLease {
       ugi[i] = UserGroupInformation.createUserForTesting("user" + i, groups);
     }
 
-    Mockito.doReturn(
-        new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
-            (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
-            "owner", "group", new byte[0], new byte[0],
-            1010, 0, null, (byte) 0, null)).when(mcp).getFileInfo(anyString());
-    Mockito
-        .doReturn(
-            new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
-                (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
-                "owner", "group", new byte[0], new byte[0],
-                1010, 0, null, (byte) 0, null))
+    Mockito.doReturn(new HdfsFileStatus.Builder()
+          .replication(1)
+          .blocksize(1024)
+          .perm(new FsPermission((short) 777))
+          .owner("owner")
+          .group("group")
+          .symlink(new byte[0])
+          .path(new byte[0])
+          .fileId(1010)
+          .build())
+        .when(mcp)
+        .getFileInfo(anyString());
+    Mockito.doReturn(new HdfsFileStatus.Builder()
+          .replication(1)
+          .blocksize(1024)
+          .perm(new FsPermission((short) 777))
+          .owner("owner")
+          .group("group")
+          .symlink(new byte[0])
+          .path(new byte[0])
+          .fileId(1010)
+          .build())
         .when(mcp)
         .create(anyString(), (FsPermission) anyObject(), anyString(),
-            (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
-            anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
-            anyObject());
+          (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
+          anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject(),
+          anyObject());
 
     final Configuration conf = new Configuration();
     final DFSClient c1 = createDFSClientAs(ugi[0], conf);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java

@@ -30,6 +30,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -45,6 +47,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo;
@@ -53,6 +56,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
 import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.Assert;
@@ -488,4 +492,64 @@ public class TestReconstructStripedFile {
         500, 30000
     );
   }
+
+  @Test(timeout = 180000)
+  public void testErasureCodingWorkerXmitsWeight() throws Exception {
+    testErasureCodingWorkerXmitsWeight(1f, ecPolicy.getNumDataUnits());
+    testErasureCodingWorkerXmitsWeight(0f, 1);
+    testErasureCodingWorkerXmitsWeight(10f, 10 * ecPolicy.getNumDataUnits());
+  }
+
+  private void testErasureCodingWorkerXmitsWeight(
+      float weight, int expectedWeight)
+      throws Exception {
+
+    // Reset cluster with customized xmits weight.
+    conf.setFloat(DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_XMITS_WEIGHT_KEY,
+        weight);
+    cluster.shutdown();
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
+    cluster.waitActive();
+    fs = cluster.getFileSystem();
+    fs.enableErasureCodingPolicy(
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+    fs.getClient().setErasureCodingPolicy("/",
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+
+    final int fileLen = cellSize * ecPolicy.getNumDataUnits() * 2;
+    writeFile(fs, "/ec-xmits-weight", fileLen);
+
+    DataNode dn = cluster.getDataNodes().get(0);
+    int corruptBlocks = dn.getFSDataset().getFinalizedBlocks(
+        cluster.getNameNode().getNamesystem().getBlockPoolId()).size();
+    int expectedXmits = corruptBlocks * expectedWeight;
+
+    final CyclicBarrier barrier = new CyclicBarrier(corruptBlocks + 1);
+    DataNodeFaultInjector oldInjector = DataNodeFaultInjector.get();
+    DataNodeFaultInjector delayInjector = new DataNodeFaultInjector() {
+      public void stripedBlockReconstruction() throws IOException {
+        try {
+          barrier.await();
+        } catch (InterruptedException | BrokenBarrierException e) {
+          throw new IOException(e);
+        }
+      }
+    };
+    DataNodeFaultInjector.set(delayInjector);
+
+    try {
+      shutdownDataNode(dn);
+      LambdaTestUtils.await(30 * 1000, 500,
+          () -> {
+            int totalXmits = cluster.getDataNodes().stream()
+                  .mapToInt(DataNode::getXmitsInProgress).sum();
+            return totalXmits == expectedXmits;
+          }
+      );
+    } finally {
+      barrier.await();
+      DataNodeFaultInjector.set(oldInjector);
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestMountTable.java

@@ -22,7 +22,6 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -113,7 +112,7 @@ public class TestMountTable {
   @Test
   public void testReadOnly() throws IOException {
 
-    Map<String, String> dest = new HashMap<>();
+    Map<String, String> dest = new LinkedHashMap<>();
     dest.put(DST_NS_0, DST_PATH_0);
     dest.put(DST_NS_1, DST_PATH_1);
     MountTable record1 = MountTable.newInstance(SRC, dest);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java

@@ -48,7 +48,6 @@ import java.nio.channels.FileChannel;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -92,6 +91,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -1340,25 +1340,20 @@ public class TestFsck {
 
     String pathString = "/tmp/testFile";
 
-    long length = 123L;
-    boolean isDir = false;
-    int blockReplication = 1;
-    long blockSize = 128 *1024L;
-    long modTime = 123123123L;
-    long accessTime = 123123120L;
-    FsPermission perms = FsPermission.getDefault();
-    String owner = "foo";
-    String group = "bar";
-    byte[] symlink = null;
-    byte[] path = DFSUtil.string2Bytes(pathString);
-    long fileId = 312321L;
-    int numChildren = 1;
-    byte storagePolicy = 0;
-
-    HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication,
-        blockSize, modTime, accessTime, perms,
-        EnumSet.noneOf(HdfsFileStatus.Flags.class), owner, group, symlink,
-        path, fileId, numChildren, null, storagePolicy, null);
+    HdfsFileStatus file = new HdfsFileStatus.Builder()
+        .length(123L)
+        .replication(1)
+        .blocksize(128 * 1024L)
+        .mtime(123123123L)
+        .atime(123123120L)
+        .perm(FsPermission.getDefault())
+        .owner("foo")
+        .group("bar")
+        .path(DFSUtil.string2Bytes(pathString))
+        .fileId(312321L)
+        .children(1)
+        .storagePolicy(HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED)
+        .build();
     Result replRes = new ReplicationResult(conf);
     Result ecRes = new ErasureCodingResult(conf);
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java

@@ -23,7 +23,6 @@ import static org.apache.hadoop.fs.permission.FsAction.*;
 import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
 
 import java.io.IOException;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -70,11 +69,19 @@ public class TestJsonUtil {
   public void testHdfsFileStatus() throws IOException {
     final long now = Time.now();
     final String parent = "/dir";
-    final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26,
-        now, now + 10, new FsPermission((short) 0644),
-        EnumSet.noneOf(HdfsFileStatus.Flags.class), "user", "group",
-        DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"),
-        HdfsConstants.GRANDFATHER_INODE_ID, 0, null, (byte) 0, null);
+    final HdfsFileStatus status = new HdfsFileStatus.Builder()
+        .length(1001L)
+        .replication(3)
+        .blocksize(1L << 26)
+        .mtime(now)
+        .atime(now + 10)
+        .perm(new FsPermission((short) 0644))
+        .owner("user")
+        .group("group")
+        .symlink(DFSUtil.string2Bytes("bar"))
+        .path(DFSUtil.string2Bytes("foo"))
+        .fileId(HdfsConstants.GRANDFATHER_INODE_ID)
+        .build();
     final FileStatus fstatus = toFileStatus(status, parent);
     System.out.println("status  = " + status);
     System.out.println("fstatus = " + fstatus);

hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml

@@ -101,4 +101,14 @@
     <value>true</value>
   </property>
 
-</configuration>
+  <property>
+    <name>fs.contract.supports-file-reference</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-content-check</name>
+    <value>false</value>
+  </property>
+
+</configuration>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.mapreduce.v2.app.job.impl;
 
+import static org.apache.commons.lang.StringUtils.isEmpty;
+
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -123,6 +125,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -136,6 +139,8 @@ import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.RackResolver;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -667,12 +672,8 @@ public abstract class TaskAttemptImpl implements
     this.jobFile = jobFile;
     this.partition = partition;
 
-    //TODO:create the resource reqt for this Task attempt
     this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
-    this.resourceCapability.setMemorySize(
-        getMemoryRequired(conf, taskId.getTaskType()));
-    this.resourceCapability.setVirtualCores(
-        getCpuRequired(conf, taskId.getTaskType()));
+    populateResourceCapability(taskId.getTaskType());
 
     this.dataLocalHosts = resolveHosts(dataLocalHosts);
     RackResolver.init(conf);
@@ -689,25 +690,137 @@ public abstract class TaskAttemptImpl implements
     stateMachine = stateMachineFactory.make(this);
   }
 
+  private void populateResourceCapability(TaskType taskType) {
+    String resourceTypePrefix =
+        getResourceTypePrefix(taskType);
+    boolean memorySet = false;
+    boolean cpuVcoresSet = false;
+    if (resourceTypePrefix != null) {
+      List<ResourceInformation> resourceRequests =
+          ResourceUtils.getRequestedResourcesFromConfig(conf,
+              resourceTypePrefix);
+      for (ResourceInformation resourceRequest : resourceRequests) {
+        String resourceName = resourceRequest.getName();
+        if (MRJobConfig.RESOURCE_TYPE_NAME_MEMORY.equals(resourceName) ||
+            MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY.equals(
+                resourceName)) {
+          if (memorySet) {
+            throw new IllegalArgumentException(
+                "Only one of the following keys " +
+                    "can be specified for a single job: " +
+                    MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY + ", " +
+                    MRJobConfig.RESOURCE_TYPE_NAME_MEMORY);
+          }
+          String units = isEmpty(resourceRequest.getUnits()) ?
+              ResourceUtils.getDefaultUnit(ResourceInformation.MEMORY_URI) :
+                resourceRequest.getUnits();
+          this.resourceCapability.setMemorySize(
+              UnitsConversionUtil.convert(units, "Mi",
+                  resourceRequest.getValue()));
+          memorySet = true;
+          String memoryKey = getMemoryKey(taskType);
+          if (memoryKey != null && conf.get(memoryKey) != null) {
+            LOG.warn("Configuration " + resourceTypePrefix + resourceName +
+                "=" + resourceRequest.getValue() + resourceRequest.getUnits() +
+                " is overriding the " + memoryKey + "=" + conf.get(memoryKey) +
+                " configuration");
+          }
+        } else if (MRJobConfig.RESOURCE_TYPE_NAME_VCORE.equals(
+            resourceName)) {
+          this.resourceCapability.setVirtualCores(
+              (int) UnitsConversionUtil.convert(resourceRequest.getUnits(), "",
+                  resourceRequest.getValue()));
+          cpuVcoresSet = true;
+          String cpuKey = getCpuVcoresKey(taskType);
+          if (cpuKey != null && conf.get(cpuKey) != null) {
+            LOG.warn("Configuration " + resourceTypePrefix +
+                MRJobConfig.RESOURCE_TYPE_NAME_VCORE + "=" +
+                resourceRequest.getValue() + resourceRequest.getUnits() +
+                " is overriding the " + cpuKey + "=" +
+                conf.get(cpuKey) + " configuration");
+          }
+        } else {
+          ResourceInformation resourceInformation =
+              this.resourceCapability.getResourceInformation(resourceName);
+          resourceInformation.setUnits(resourceRequest.getUnits());
+          resourceInformation.setValue(resourceRequest.getValue());
+          this.resourceCapability.setResourceInformation(resourceName,
+              resourceInformation);
+        }
+      }
+    }
+    if (!memorySet) {
+      this.resourceCapability.setMemorySize(getMemoryRequired(conf, taskType));
+    }
+    if (!cpuVcoresSet) {
+      this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType));
+    }
+  }
+
+  private String getCpuVcoresKey(TaskType taskType) {
+    switch (taskType) {
+    case MAP:
+      return MRJobConfig.MAP_CPU_VCORES;
+    case REDUCE:
+      return MRJobConfig.REDUCE_CPU_VCORES;
+    default:
+      return null;
+    }
+  }
+
+  private String getMemoryKey(TaskType taskType) {
+    switch (taskType) {
+    case MAP:
+      return MRJobConfig.MAP_MEMORY_MB;
+    case REDUCE:
+      return MRJobConfig.REDUCE_MEMORY_MB;
+    default:
+      return null;
+    }
+  }
+
+  private Integer getCpuVcoreDefault(TaskType taskType) {
+    switch (taskType) {
+    case MAP:
+      return MRJobConfig.DEFAULT_MAP_CPU_VCORES;
+    case REDUCE:
+      return MRJobConfig.DEFAULT_REDUCE_CPU_VCORES;
+    default:
+      return null;
+    }
+  }
+
   private int getMemoryRequired(JobConf conf, TaskType taskType) {
     return conf.getMemoryRequired(TypeConverter.fromYarn(taskType));
   }
 
   private int getCpuRequired(Configuration conf, TaskType taskType) {
     int vcores = 1;
-    if (taskType == TaskType.MAP)  {
-      vcores =
-          conf.getInt(MRJobConfig.MAP_CPU_VCORES,
-              MRJobConfig.DEFAULT_MAP_CPU_VCORES);
-    } else if (taskType == TaskType.REDUCE) {
-      vcores =
-          conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
-              MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
+    String cpuVcoreKey = getCpuVcoresKey(taskType);
+    if (cpuVcoreKey != null) {
+      Integer defaultCpuVcores = getCpuVcoreDefault(taskType);
+      if (null == defaultCpuVcores) {
+        defaultCpuVcores = vcores;
+      }
+      vcores = conf.getInt(cpuVcoreKey, defaultCpuVcores);
     }
-    
     return vcores;
   }
 
+  private String getResourceTypePrefix(TaskType taskType) {
+    switch (taskType) {
+    case MAP:
+      return MRJobConfig.MAP_RESOURCE_TYPE_PREFIX;
+    case REDUCE:
+      return MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX;
+    default:
+      LOG.info("TaskType " + taskType +
+          " does not support custom resource types - this support can be " +
+          "added in " + getClass().getSimpleName());
+      return null;
+    }
+  }
+
   /**
    * Create a {@link LocalResource} record with all the given parameters.
    * The NM that hosts AM container will upload resources to shared cache.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java

@@ -71,6 +71,17 @@ public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
             .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY);
     configurationPropsToSkipCompare
             .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+
+    // Resource type related properties are only prefixes,
+    // they need to be postfixed with the resource name
+    // in order to take effect.
+    // There is nothing to be added to mapred-default.xml
+    configurationPropsToSkipCompare.add(
+        MRJobConfig.MR_AM_RESOURCE_PREFIX);
+    configurationPropsToSkipCompare.add(
+        MRJobConfig.MAP_RESOURCE_TYPE_PREFIX);
+    configurationPropsToSkipCompare.add(
+        MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX);
   }
 
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java

@@ -28,13 +28,20 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -42,6 +49,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MapTaskAttemptImpl;
+import org.apache.hadoop.mapred.ReduceTaskAttemptImpl;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.JobCounter;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -82,24 +90,36 @@ import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ControlledClock;
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
+import com.google.common.collect.ImmutableList;
+
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class TestTaskAttempt{
-	
+
+  private static final String CUSTOM_RESOURCE_NAME = "a-custom-resource";
+
   static public class StubbedFS extends RawLocalFileSystem {
     @Override
     public FileStatus getFileStatus(Path f) throws IOException {
@@ -107,6 +127,63 @@ public class TestTaskAttempt{
     }
   }
 
+  private static class CustomResourceTypesConfigurationProvider
+      extends LocalConfigurationProvider {
+
+    @Override
+    public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+        String name) throws YarnException, IOException {
+      if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+        return new ByteArrayInputStream(
+            ("<configuration>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types</name>\n" +
+            "   <value>a-custom-resource</value>\n" +
+            " </property>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types.a-custom-resource.units</name>\n" +
+            "   <value>G</value>\n" +
+            " </property>\n" +
+            "</configuration>\n").getBytes());
+      } else {
+        return super.getConfigurationInputStream(bootstrapConf, name);
+      }
+    }
+  }
+
+  private static class TestAppender extends AppenderSkeleton {
+
+    private final List<LoggingEvent> logEvents = new CopyOnWriteArrayList<>();
+
+    @Override
+    public boolean requiresLayout() {
+      return false;
+    }
+
+    @Override
+    public void close() {
+    }
+
+    @Override
+    protected void append(LoggingEvent arg0) {
+      logEvents.add(arg0);
+    }
+
+    private List<LoggingEvent> getLogEvents() {
+      return logEvents;
+    }
+  }
+
+  @BeforeClass
+  public static void setupBeforeClass() {
+    ResourceUtils.resetResourceTypes(new Configuration());
+  }
+
+  @After
+  public void tearDown() {
+    ResourceUtils.resetResourceTypes(new Configuration());
+  }
+
   @Test
   public void testMRAppHistoryForMap() throws Exception {
     MRApp app = new FailingAttemptsMRApp(1, 0);
@@ -328,17 +405,18 @@ public class TestTaskAttempt{
   private TaskAttemptImpl createMapTaskAttemptImplForTest(
       EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo) {
     Clock clock = SystemClock.getInstance();
-    return createMapTaskAttemptImplForTest(eventHandler, taskSplitMetaInfo, clock);
+    return createMapTaskAttemptImplForTest(eventHandler, taskSplitMetaInfo,
+        clock, new JobConf());
   }
 
   private TaskAttemptImpl createMapTaskAttemptImplForTest(
-      EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
+      EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo,
+      Clock clock, JobConf jobConf) {
     ApplicationId appId = ApplicationId.newInstance(1, 1);
     JobId jobId = MRBuilderUtils.newJobId(appId, 1);
     TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
     TaskAttemptListener taListener = mock(TaskAttemptListener.class);
     Path jobFile = mock(Path.class);
-    JobConf jobConf = new JobConf();
     TaskAttemptImpl taImpl =
         new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
             taskSplitMetaInfo, jobConf, taListener, null,
@@ -346,6 +424,20 @@ public class TestTaskAttempt{
     return taImpl;
   }
 
+  private TaskAttemptImpl createReduceTaskAttemptImplForTest(
+      EventHandler eventHandler, Clock clock, JobConf jobConf) {
+    ApplicationId appId = ApplicationId.newInstance(1, 1);
+    JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+    TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+    TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+    Path jobFile = mock(Path.class);
+    TaskAttemptImpl taImpl =
+        new ReduceTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+            1, jobConf, taListener, null,
+            null, clock, null);
+    return taImpl;
+  }
+
   private void testMRAppHistory(MRApp app) throws Exception {
     Configuration conf = new Configuration();
     Job job = app.submit(conf);
@@ -1412,6 +1504,259 @@ public class TestTaskAttempt{
     assertFalse("InternalError occurred", eventHandler.internalError);
   }
 
+  @Test
+  public void testMapperCustomResourceTypes() {
+    initResourceTypes();
+    EventHandler eventHandler = mock(EventHandler.class);
+    TaskSplitMetaInfo taskSplitMetaInfo = new TaskSplitMetaInfo();
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.setLong(MRJobConfig.MAP_RESOURCE_TYPE_PREFIX
+        + CUSTOM_RESOURCE_NAME, 7L);
+    TaskAttemptImpl taImpl = createMapTaskAttemptImplForTest(eventHandler,
+        taskSplitMetaInfo, clock, jobConf);
+    ResourceInformation resourceInfo =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getResourceInformation(CUSTOM_RESOURCE_NAME);
+    assertEquals("Expecting the default unit (G)",
+        "G", resourceInfo.getUnits());
+    assertEquals(7L, resourceInfo.getValue());
+  }
+
+  @Test
+  public void testReducerCustomResourceTypes() {
+    initResourceTypes();
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX
+        + CUSTOM_RESOURCE_NAME, "3m");
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+    ResourceInformation resourceInfo =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getResourceInformation(CUSTOM_RESOURCE_NAME);
+    assertEquals("Expecting the specified unit (m)",
+        "m", resourceInfo.getUnits());
+    assertEquals(3L, resourceInfo.getValue());
+  }
+
+  @Test
+  public void testReducerMemoryRequestViaMapreduceReduceMemoryMb() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048);
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+    long memorySize =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getMemorySize();
+    assertEquals(2048, memorySize);
+  }
+
+  @Test
+  public void testReducerMemoryRequestViaMapreduceReduceResourceMemory() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX +
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY, "2 Gi");
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+    long memorySize =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getMemorySize();
+    assertEquals(2048, memorySize);
+  }
+
+  @Test
+  public void testReducerMemoryRequestDefaultMemory() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, new JobConf());
+    long memorySize =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getMemorySize();
+    assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, memorySize);
+  }
+
+  @Test
+  public void testReducerMemoryRequestWithoutUnits() {
+    Clock clock = SystemClock.getInstance();
+    for (String memoryResourceName : ImmutableList.of(
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
+        MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
+      EventHandler eventHandler = mock(EventHandler.class);
+      JobConf jobConf = new JobConf();
+      jobConf.setInt(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX +
+          memoryResourceName, 2048);
+      TaskAttemptImpl taImpl =
+          createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+      long memorySize =
+          getResourceInfoFromContainerRequest(taImpl, eventHandler).
+          getMemorySize();
+      assertEquals(2048, memorySize);
+    }
+  }
+
+  @Test
+  public void testReducerMemoryRequestOverriding() {
+    for (String memoryName : ImmutableList.of(
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
+        MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
+      TestAppender testAppender = new TestAppender();
+      final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
+      try {
+        logger.addAppender(testAppender);
+        EventHandler eventHandler = mock(EventHandler.class);
+        Clock clock = SystemClock.getInstance();
+        JobConf jobConf = new JobConf();
+        jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX + memoryName,
+            "3Gi");
+        jobConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048);
+        TaskAttemptImpl taImpl =
+            createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+        long memorySize =
+            getResourceInfoFromContainerRequest(taImpl, eventHandler).
+            getMemorySize();
+        assertEquals(3072, memorySize);
+        assertTrue(testAppender.getLogEvents().stream()
+            .anyMatch(e -> e.getLevel() == Level.WARN && ("Configuration " +
+                "mapreduce.reduce.resource." + memoryName + "=3Gi is " +
+                "overriding the mapreduce.reduce.memory.mb=2048 configuration")
+                    .equals(e.getMessage())));
+      } finally {
+        logger.removeAppender(testAppender);
+      }
+    }
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testReducerMemoryRequestMultipleName() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    for (String memoryName : ImmutableList.of(
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
+        MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
+      jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX + memoryName,
+          "3Gi");
+    }
+    createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+  }
+
+  @Test
+  public void testReducerCpuRequestViaMapreduceReduceCpuVcores() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 3);
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+    int vCores =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getVirtualCores();
+    assertEquals(3, vCores);
+  }
+
+  @Test
+  public void testReducerCpuRequestViaMapreduceReduceResourceVcores() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX +
+        MRJobConfig.RESOURCE_TYPE_NAME_VCORE, "5");
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+    int vCores =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getVirtualCores();
+    assertEquals(5, vCores);
+  }
+
+  @Test
+  public void testReducerCpuRequestDefaultMemory() {
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    TaskAttemptImpl taImpl =
+        createReduceTaskAttemptImplForTest(eventHandler, clock, new JobConf());
+    int vCores =
+        getResourceInfoFromContainerRequest(taImpl, eventHandler).
+        getVirtualCores();
+    assertEquals(MRJobConfig.DEFAULT_REDUCE_CPU_VCORES, vCores);
+  }
+
+  @Test
+  public void testReducerCpuRequestOverriding() {
+    TestAppender testAppender = new TestAppender();
+    final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
+    try {
+      logger.addAppender(testAppender);
+      EventHandler eventHandler = mock(EventHandler.class);
+      Clock clock = SystemClock.getInstance();
+      JobConf jobConf = new JobConf();
+      jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX +
+          MRJobConfig.RESOURCE_TYPE_NAME_VCORE, "7");
+      jobConf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 9);
+      TaskAttemptImpl taImpl =
+          createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+      long vCores =
+          getResourceInfoFromContainerRequest(taImpl, eventHandler).
+          getVirtualCores();
+      assertEquals(7, vCores);
+      assertTrue(testAppender.getLogEvents().stream().anyMatch(
+          e -> e.getLevel() == Level.WARN && ("Configuration " +
+              "mapreduce.reduce.resource.vcores=7 is overriding the " +
+              "mapreduce.reduce.cpu.vcores=9 configuration").equals(
+                  e.getMessage())));
+    } finally {
+      logger.removeAppender(testAppender);
+    }
+  }
+
+  private Resource getResourceInfoFromContainerRequest(
+      TaskAttemptImpl taImpl, EventHandler eventHandler) {
+    taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+        TaskAttemptEventType.TA_SCHEDULE));
+
+    assertEquals("Task attempt is not in STARTING state", taImpl.getState(),
+        TaskAttemptState.STARTING);
+
+    ArgumentCaptor<Event> captor = ArgumentCaptor.forClass(Event.class);
+    verify(eventHandler, times(2)).handle(captor.capture());
+
+    List<ContainerRequestEvent> containerRequestEvents = new ArrayList<>();
+    for (Event e : captor.getAllValues()) {
+      if (e instanceof ContainerRequestEvent) {
+        containerRequestEvents.add((ContainerRequestEvent) e);
+      }
+    }
+    assertEquals("Expected one ContainerRequestEvent after scheduling "
+        + "task attempt", 1, containerRequestEvents.size());
+
+    return containerRequestEvents.get(0).getCapability();
+  }
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testReducerCustomResourceTypeWithInvalidUnit() {
+    initResourceTypes();
+    EventHandler eventHandler = mock(EventHandler.class);
+    Clock clock = SystemClock.getInstance();
+    JobConf jobConf = new JobConf();
+    jobConf.set(MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX
+        + CUSTOM_RESOURCE_NAME, "3z");
+    createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
+  }
+
+  private void initResourceTypes() {
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+        CustomResourceTypesConfigurationProvider.class.getName());
+    ResourceUtils.resetResourceTypes(conf);
+  }
+
   private void setupTaskAttemptFinishingMonitor(
       EventHandler eventHandler, JobConf jobConf, AppContext appCtx) {
     TaskAttemptFinishingMonitor taskAttemptFinishingMonitor =

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -363,12 +363,47 @@ public interface MRJobConfig {
 
   public static final String MAP_INPUT_START = "mapreduce.map.input.start";
 
+  /**
+   * Configuration key for specifying memory requirement for the mapper.
+   * Kept for backward-compatibility, mapreduce.map.resource.memory
+   * is the new preferred way to specify this.
+   */
   public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb";
   public static final int DEFAULT_MAP_MEMORY_MB = 1024;
 
+  /**
+   * Configuration key for specifying CPU requirement for the mapper.
+   * Kept for backward-compatibility, mapreduce.map.resource.vcores
+   * is the new preferred way to specify this.
+   */
   public static final String MAP_CPU_VCORES = "mapreduce.map.cpu.vcores";
   public static final int DEFAULT_MAP_CPU_VCORES = 1;
 
+  /**
+   * Custom resource names required by the mapper should be
+   * appended to this prefix, the value's format is {amount}[ ][{unit}].
+   * If no unit is defined, the default unit will be used.
+   * Standard resource names: memory (default unit: Mi), vcores
+   */
+  public static final String MAP_RESOURCE_TYPE_PREFIX =
+      "mapreduce.map.resource.";
+
+  /**
+   * Resource type name for CPU vcores.
+   */
+  public static final String RESOURCE_TYPE_NAME_VCORE = "vcores";
+
+  /**
+   * Resource type name for memory.
+   */
+  public static final String RESOURCE_TYPE_NAME_MEMORY = "memory";
+
+  /**
+   * Alternative resource type name for memory.
+   */
+  public static final String RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY =
+      "memory-mb";
+
   public static final String MAP_ENV = "mapreduce.map.env";
 
   public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts";
@@ -417,12 +452,31 @@ public interface MRJobConfig {
 
   public static final String REDUCE_MARKRESET_BUFFER_SIZE = "mapreduce.reduce.markreset.buffer.size";
 
+  /**
+   * Configuration key for specifying memory requirement for the reducer.
+   * Kept for backward-compatibility, mapreduce.reduce.resource.memory
+   * is the new preferred way to specify this.
+   */
   public static final String REDUCE_MEMORY_MB = "mapreduce.reduce.memory.mb";
   public static final int DEFAULT_REDUCE_MEMORY_MB = 1024;
 
+  /**
+   * Configuration key for specifying CPU requirement for the reducer.
+   * Kept for backward-compatibility, mapreduce.reduce.resource.vcores
+   * is the new preferred way to specify this.
+   */
   public static final String REDUCE_CPU_VCORES = "mapreduce.reduce.cpu.vcores";
   public static final int DEFAULT_REDUCE_CPU_VCORES = 1;
 
+  /**
+   * Resource names required by the reducer should be
+   * appended to this prefix, the value's format is {amount}[ ][{unit}].
+   * If no unit is defined, the default unit will be used.
+   * Standard resource names: memory (default unit: Mi), vcores
+   */
+  public static final String REDUCE_RESOURCE_TYPE_PREFIX =
+      "mapreduce.reduce.resource.";
+
   public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes";
 
   public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent";
@@ -608,7 +662,10 @@ public interface MRJobConfig {
   public static final String DEFAULT_MR_AM_STAGING_DIR = 
     "/tmp/hadoop-yarn/staging";
 
-  /** The amount of memory the MR app master needs.*/
+  /** The amount of memory the MR app master needs.
+   * Kept for backward-compatibility, yarn.app.mapreduce.am.resource.memory is
+   * the new preferred way to specify this
+   */
   public static final String MR_AM_VMEM_MB =
     MR_AM_PREFIX+"resource.mb";
   public static final int DEFAULT_MR_AM_VMEM_MB = 1536;
@@ -618,6 +675,15 @@ public interface MRJobConfig {
     MR_AM_PREFIX+"resource.cpu-vcores";
   public static final int DEFAULT_MR_AM_CPU_VCORES = 1;
 
+  /**
+   * Resource names required by the MR AM should be
+   * appended to this prefix, the value's format is {amount}[ ][{unit}].
+   * If no unit is defined, the default unit will be used
+   * Standard resource names: memory (default unit: Mi), vcores
+   */
+  public static final String MR_AM_RESOURCE_PREFIX =
+      MR_AM_PREFIX + "resource.";
+
   /** Command line arguments passed to the MR app master.*/
   public static final String MR_AM_COMMAND_OPTS =
     MR_AM_PREFIX+"command-opts";

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static org.apache.hadoop.mapreduce.MRJobConfig.MR_AM_RESOURCE_PREFIX;
+
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -84,6 +87,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -93,6 +97,8 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -659,16 +665,76 @@ public class YARNRunner implements ClientProtocol {
 
   private List<ResourceRequest> generateResourceRequests() throws IOException {
     Resource capability = recordFactory.newRecordInstance(Resource.class);
-    capability.setMemorySize(
-        conf.getInt(
-            MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
-        )
-    );
-    capability.setVirtualCores(
-        conf.getInt(
-            MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
-        )
-    );
+    boolean memorySet = false;
+    boolean cpuVcoresSet = false;
+    List<ResourceInformation> resourceRequests = ResourceUtils
+        .getRequestedResourcesFromConfig(conf, MR_AM_RESOURCE_PREFIX);
+    for (ResourceInformation resourceReq : resourceRequests) {
+      String resourceName = resourceReq.getName();
+      if (MRJobConfig.RESOURCE_TYPE_NAME_MEMORY.equals(resourceName) ||
+          MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY.equals(
+              resourceName)) {
+        if (memorySet) {
+          throw new IllegalArgumentException(
+              "Only one of the following keys " +
+                  "can be specified for a single job: " +
+                  MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY + ", " +
+                  MRJobConfig.RESOURCE_TYPE_NAME_MEMORY);
+        }
+        String units = isEmpty(resourceReq.getUnits()) ?
+            ResourceUtils.getDefaultUnit(ResourceInformation.MEMORY_URI) :
+              resourceReq.getUnits();
+        capability.setMemorySize(
+            UnitsConversionUtil.convert(units, "Mi", resourceReq.getValue()));
+        memorySet = true;
+        if (conf.get(MRJobConfig.MR_AM_VMEM_MB) != null) {
+          LOG.warn("Configuration " + MR_AM_RESOURCE_PREFIX +
+              resourceName + "=" + resourceReq.getValue() +
+              resourceReq.getUnits() + " is overriding the " +
+              MRJobConfig.MR_AM_VMEM_MB + "=" +
+              conf.get(MRJobConfig.MR_AM_VMEM_MB) + " configuration");
+        }
+      } else if (MRJobConfig.RESOURCE_TYPE_NAME_VCORE.equals(resourceName)) {
+        capability.setVirtualCores(
+            (int) UnitsConversionUtil.convert(resourceReq.getUnits(), "",
+                resourceReq.getValue()));
+        cpuVcoresSet = true;
+        if (conf.get(MRJobConfig.MR_AM_CPU_VCORES) != null) {
+          LOG.warn("Configuration " + MR_AM_RESOURCE_PREFIX +
+              resourceName + "=" + resourceReq.getValue() +
+              resourceReq.getUnits() + " is overriding the " +
+              MRJobConfig.MR_AM_CPU_VCORES + "=" +
+              conf.get(MRJobConfig.MR_AM_CPU_VCORES) + " configuration");
+        }
+      } else if (!MRJobConfig.MR_AM_VMEM_MB.equals(
+          MR_AM_RESOURCE_PREFIX + resourceName) &&
+          !MRJobConfig.MR_AM_CPU_VCORES.equals(
+              MR_AM_RESOURCE_PREFIX + resourceName)) {
+        // the "mb", "cpu-vcores" resource types are not processed here
+        // since the yarn.app.mapreduce.am.resource.mb,
+        // yarn.app.mapreduce.am.resource.cpu-vcores keys are used for
+        // backward-compatibility - which is handled after this loop
+        ResourceInformation resourceInformation = capability
+            .getResourceInformation(resourceName);
+        resourceInformation.setUnits(resourceReq.getUnits());
+        resourceInformation.setValue(resourceReq.getValue());
+        capability.setResourceInformation(resourceName, resourceInformation);
+      }
+    }
+    if (!memorySet) {
+      capability.setMemorySize(
+          conf.getInt(
+              MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
+          )
+      );
+    }
+    if (!cpuVcoresSet) {
+      capability.setVirtualCores(
+          conf.getInt(
+              MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
+          )
+      );
+    }
     if (LOG.isDebugEnabled()) {
       LOG.debug("AppMaster capability = " + capability);
     }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java

@@ -186,6 +186,82 @@ public class TestTextInputFormat {
     verifyPartitions(473608, 110, file, codec, conf);
   }
 
+  // Test a corner case when position of stream is right after BZip2 marker
+  @Test (timeout=900000)
+  public void testSplitableCodecs2() throws IOException {
+    JobConf conf = new JobConf(defaultConf);
+    // Create the codec
+    CompressionCodec codec = null;
+    try {
+      codec = (CompressionCodec)
+      ReflectionUtils.newInstance(conf.getClassByName("org.apache.hadoop.io.compress.BZip2Codec"), conf);
+    } catch (ClassNotFoundException cnfe) {
+      throw new IOException("Illegal codec!");
+    }
+    Path file = new Path(workDir, "test"+codec.getDefaultExtension());
+
+    FileSystem localFs = FileSystem.getLocal(conf);
+    localFs.delete(workDir, true);
+    FileInputFormat.setInputPaths(conf, workDir);
+
+    int length = 250000;
+    LOG.info("creating; entries = " + length);
+    // create a file with length entries
+    Writer writer =
+        new OutputStreamWriter(codec.createOutputStream(localFs.create(file)));
+    try {
+      for (int i = 0; i < length; i++) {
+        writer.write(Integer.toString(i));
+        writer.write("\n");
+      }
+    } finally {
+      writer.close();
+    }
+
+    // Test split positions around a block boundary where the block does
+    // not start on a byte boundary.
+    for (long splitpos = 203418; splitpos < 203430; ++splitpos) {
+      TextInputFormat format = new TextInputFormat();
+      format.configure(conf);
+      LOG.info("setting block size of the input file to " + splitpos);
+      conf.setLong("mapreduce.input.fileinputformat.split.minsize", splitpos);
+      LongWritable key = new LongWritable();
+      Text value = new Text();
+      InputSplit[] splits = format.getSplits(conf, 2);
+      LOG.info("splitting: got =        " + splits.length);
+
+      // check each split
+      BitSet bits = new BitSet(length);
+      for (int j = 0; j < splits.length; j++) {
+        LOG.debug("split[" + j + "]= " + splits[j]);
+        RecordReader<LongWritable, Text> reader =
+            format.getRecordReader(splits[j], conf, Reporter.NULL);
+        try {
+          int counter = 0;
+          while (reader.next(key, value)) {
+            int v = Integer.parseInt(value.toString());
+            LOG.debug("read " + v);
+            if (bits.get(v)) {
+              LOG.warn("conflict with " + v + " in split " + j +
+                  " at position " + reader.getPos());
+            }
+            assertFalse("Key in multiple partitions.", bits.get(v));
+            bits.set(v);
+            counter++;
+          }
+          if (counter > 0) {
+            LOG.info("splits[" + j + "]=" + splits[j] + " count=" + counter);
+          } else {
+            LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + counter);
+          }
+        } finally {
+          reader.close();
+        }
+      }
+      assertEquals("Some keys in no partition.", length, bits.cardinality());
+    }
+  }
+
   private void verifyPartitions(int length, int numSplits, Path file,
       CompressionCodec codec, JobConf conf) throws IOException {
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java

@@ -33,10 +33,12 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
@@ -44,6 +46,7 @@ import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -69,6 +72,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -96,28 +100,37 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.log4j.Appender;
+import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Layout;
+import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.log4j.SimpleLayout;
 import org.apache.log4j.WriterAppender;
+import org.apache.log4j.spi.LoggingEvent;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
+import com.google.common.collect.ImmutableList;
+
 /**
  * Test YarnRunner and make sure the client side plugin works
  * fine
@@ -131,6 +144,53 @@ public class TestYARNRunner {
       MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.substring(0,
           MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.lastIndexOf("%"));
 
+  private static class CustomResourceTypesConfigurationProvider
+      extends LocalConfigurationProvider {
+
+    @Override
+    public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+        String name) throws YarnException, IOException {
+      if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+        return new ByteArrayInputStream(
+            ("<configuration>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types</name>\n" +
+            "   <value>a-custom-resource</value>\n" +
+            " </property>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types.a-custom-resource.units</name>\n" +
+            "   <value>G</value>\n" +
+            " </property>\n" +
+            "</configuration>\n").getBytes());
+      } else {
+        return super.getConfigurationInputStream(bootstrapConf, name);
+      }
+    }
+  }
+
+  private static class TestAppender extends AppenderSkeleton {
+
+    private final List<LoggingEvent> logEvents = new CopyOnWriteArrayList<>();
+
+    @Override
+    public boolean requiresLayout() {
+      return false;
+    }
+
+    @Override
+    public void close() {
+    }
+
+    @Override
+    protected void append(LoggingEvent arg0) {
+      logEvents.add(arg0);
+    }
+
+    private List<LoggingEvent> getLogEvents() {
+      return logEvents;
+    }
+  }
+
   private YARNRunner yarnRunner;
   private ResourceMgrDelegate resourceMgrDelegate;
   private YarnConfiguration conf;
@@ -143,6 +203,11 @@ public class TestYARNRunner {
   private  ClientServiceDelegate clientDelegate;
   private static final String failString = "Rejected job";
 
+  @BeforeClass
+  public static void setupBeforeClass() {
+    ResourceUtils.resetResourceTypes(new Configuration());
+  }
+
   @Before
   public void setUp() throws Exception {
     resourceMgrDelegate = mock(ResourceMgrDelegate.class);
@@ -175,6 +240,7 @@ public class TestYARNRunner {
   @After
   public void cleanup() {
     FileUtil.fullyDelete(testWorkDir);
+    ResourceUtils.resetResourceTypes(new Configuration());
   }
 
   @Test(timeout=20000)
@@ -881,4 +947,99 @@ public class TestYARNRunner {
         .get("hadoop.tmp.dir").equals("testconfdir"));
     UserGroupInformation.reset();
   }
+
+  @Test
+  public void testCustomAMRMResourceType() throws Exception {
+    initResourceTypes();
+    String customResourceName = "a-custom-resource";
+
+    JobConf jobConf = new JobConf();
+
+    jobConf.setInt(MRJobConfig.MR_AM_RESOURCE_PREFIX +
+        customResourceName, 5);
+    jobConf.setInt(MRJobConfig.MR_AM_CPU_VCORES, 3);
+
+    yarnRunner = new YARNRunner(jobConf);
+
+    submissionContext = buildSubmitContext(yarnRunner, jobConf);
+
+    List<ResourceRequest> resourceRequests =
+        submissionContext.getAMContainerResourceRequests();
+
+    Assert.assertEquals(1, resourceRequests.size());
+    ResourceRequest resourceRequest = resourceRequests.get(0);
+
+    ResourceInformation resourceInformation = resourceRequest.getCapability()
+        .getResourceInformation(customResourceName);
+    Assert.assertEquals("Expecting the default unit (G)",
+        "G", resourceInformation.getUnits());
+    Assert.assertEquals(5L, resourceInformation.getValue());
+    Assert.assertEquals(3, resourceRequest.getCapability().getVirtualCores());
+  }
+
+  @Test
+  public void testAMRMemoryRequest() throws Exception {
+    for (String memoryName : ImmutableList.of(
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
+        MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
+      JobConf jobConf = new JobConf();
+      jobConf.set(MRJobConfig.MR_AM_RESOURCE_PREFIX + memoryName, "3 Gi");
+
+      yarnRunner = new YARNRunner(jobConf);
+
+      submissionContext = buildSubmitContext(yarnRunner, jobConf);
+
+      List<ResourceRequest> resourceRequests =
+          submissionContext.getAMContainerResourceRequests();
+
+      Assert.assertEquals(1, resourceRequests.size());
+      ResourceRequest resourceRequest = resourceRequests.get(0);
+
+      long memorySize = resourceRequest.getCapability().getMemorySize();
+      Assert.assertEquals(3072, memorySize);
+    }
+  }
+
+  @Test
+  public void testAMRMemoryRequestOverriding() throws Exception {
+    for (String memoryName : ImmutableList.of(
+        MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
+        MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
+      TestAppender testAppender = new TestAppender();
+      Logger logger = Logger.getLogger(YARNRunner.class);
+      logger.addAppender(testAppender);
+      try {
+        JobConf jobConf = new JobConf();
+        jobConf.set(MRJobConfig.MR_AM_RESOURCE_PREFIX + memoryName, "3 Gi");
+        jobConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 2048);
+
+        yarnRunner = new YARNRunner(jobConf);
+
+        submissionContext = buildSubmitContext(yarnRunner, jobConf);
+
+        List<ResourceRequest> resourceRequests =
+            submissionContext.getAMContainerResourceRequests();
+
+        Assert.assertEquals(1, resourceRequests.size());
+        ResourceRequest resourceRequest = resourceRequests.get(0);
+
+        long memorySize = resourceRequest.getCapability().getMemorySize();
+        Assert.assertEquals(3072, memorySize);
+        assertTrue(testAppender.getLogEvents().stream().anyMatch(
+            e -> e.getLevel() == Level.WARN && ("Configuration " +
+                "yarn.app.mapreduce.am.resource." + memoryName + "=3Gi is " +
+                "overriding the yarn.app.mapreduce.am.resource.mb=2048 " +
+                "configuration").equals(e.getMessage())));
+      } finally {
+        logger.removeAppender(testAppender);
+      }
+    }
+  }
+
+  private void initResourceTypes() {
+    Configuration configuration = new Configuration();
+    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+        CustomResourceTypesConfigurationProvider.class.getName());
+    ResourceUtils.resetResourceTypes(configuration);
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java

@@ -57,7 +57,6 @@ import javax.crypto.SecretKey;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -84,7 +83,6 @@ import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
 import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
 import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
@@ -855,8 +853,6 @@ public class ShuffleHandler extends AuxiliaryService {
     private static final int ALLOWED_CONCURRENCY = 16;
     private final Configuration conf;
     private final IndexCache indexCache;
-    private final LocalDirAllocator lDirAlloc =
-      new LocalDirAllocator(YarnConfiguration.NM_LOCAL_DIRS);
     private int port;
     private final LoadingCache<AttemptPathIdentifier, AttemptPathInfo> pathCache =
       CacheBuilder.newBuilder().expireAfterAccess(EXPIRE_AFTER_ACCESS_MINUTES,
@@ -889,10 +885,10 @@ public class ShuffleHandler extends AuxiliaryService {
             Exception {
           String base = getBaseLocation(key.jobId, key.user);
           String attemptBase = base + key.attemptId;
-          Path indexFileName = lDirAlloc.getLocalPathToRead(
-              attemptBase + "/" + INDEX_FILE_NAME, conf);
-          Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
-              attemptBase + "/" + DATA_FILE_NAME, conf);
+          Path indexFileName = getAuxiliaryLocalPathHandler()
+              .getLocalPathForRead(attemptBase + "/" + INDEX_FILE_NAME);
+          Path mapOutputFileName = getAuxiliaryLocalPathHandler()
+              .getLocalPathForRead(attemptBase + "/" + DATA_FILE_NAME);
 
           if (LOG.isDebugEnabled()) {
             LOG.debug("Loaded : " + key + " via loader");

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.mapred;
 
+import org.apache.hadoop.test.GenericTestUtils;
 import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
 import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@@ -26,6 +27,7 @@ import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
 import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeTrue;
+import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -69,12 +71,14 @@ import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.PureJavaCrc32;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
 import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
+import org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.jboss.netty.channel.Channel;
@@ -99,8 +103,12 @@ import org.eclipse.jetty.http.HttpHeader;
 public class TestShuffleHandler {
   static final long MiB = 1024 * 1024; 
   private static final Log LOG = LogFactory.getLog(TestShuffleHandler.class);
+  private static final File ABS_LOG_DIR = GenericTestUtils.getTestDir(
+      TestShuffleHandler.class.getSimpleName() + "LocDir");
 
   class MockShuffleHandler extends org.apache.hadoop.mapred.ShuffleHandler {
+    private AuxiliaryLocalPathHandler pathHandler =
+        new TestAuxiliaryLocalPathHandler();
     @Override
     protected Shuffle getShuffle(final Configuration conf) {
       return new Shuffle(conf) {
@@ -140,11 +148,35 @@ public class TestShuffleHandler {
         }
       };
     }
+
+    @Override
+    public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
+      return pathHandler;
+    }
   }
 
-  private static class MockShuffleHandler2 extends org.apache.hadoop.mapred.ShuffleHandler {
-    boolean socketKeepAlive = false;
+  private class TestAuxiliaryLocalPathHandler
+      implements AuxiliaryLocalPathHandler {
+    @Override
+    public Path getLocalPathForRead(String path) throws IOException {
+      return new Path(ABS_LOG_DIR.getAbsolutePath(), path);
+    }
+
+    @Override
+    public Path getLocalPathForWrite(String path) throws IOException {
+      return new Path(ABS_LOG_DIR.getAbsolutePath());
+    }
 
+    @Override
+    public Path getLocalPathForWrite(String path, long size)
+        throws IOException {
+      return new Path(ABS_LOG_DIR.getAbsolutePath());
+    }
+  }
+
+  private static class MockShuffleHandler2 extends
+      org.apache.hadoop.mapred.ShuffleHandler {
+    boolean socketKeepAlive = false;
     @Override
     protected Shuffle getShuffle(final Configuration conf) {
       return new Shuffle(conf) {
@@ -479,6 +511,11 @@ public class TestShuffleHandler {
     conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100);
     HttpURLConnection conn = null;
     MockShuffleHandler2 shuffleHandler = new MockShuffleHandler2();
+    AuxiliaryLocalPathHandler pathHandler =
+        mock(AuxiliaryLocalPathHandler.class);
+    when(pathHandler.getLocalPathForRead(anyString())).thenThrow(
+        new DiskChecker.DiskErrorException("Test"));
+    shuffleHandler.setAuxiliaryLocalPathHandler(pathHandler);
     try {
       shuffleHandler.init(conf);
       shuffleHandler.start();
@@ -668,19 +705,16 @@ public class TestShuffleHandler {
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
     UserGroupInformation.setConfiguration(conf);
-    File absLogDir = new File("target",
-        TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile();
-    conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, ABS_LOG_DIR.getAbsolutePath());
     ApplicationId appId = ApplicationId.newInstance(12345, 1);
     LOG.info(appId.toString());
     String appAttemptId = "attempt_12345_1_m_1_0";
     String user = "randomUser";
     String reducerId = "0";
     List<File> fileMap = new ArrayList<File>();
-    createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId,
+    createShuffleHandlerFiles(ABS_LOG_DIR, user, appId.toString(), appAttemptId,
         conf, fileMap);
     ShuffleHandler shuffleHandler = new ShuffleHandler() {
-
       @Override
       protected Shuffle getShuffle(Configuration conf) {
         // replace the shuffle handler with one stubbed for testing
@@ -696,6 +730,8 @@ public class TestShuffleHandler {
         };
       }
     };
+    AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
+    shuffleHandler.setAuxiliaryLocalPathHandler(pathHandler);
     shuffleHandler.init(conf);
     try {
       shuffleHandler.start();
@@ -740,7 +776,7 @@ public class TestShuffleHandler {
       Assert.assertTrue((new String(byteArr)).contains(message));
     } finally {
       shuffleHandler.stop();
-      FileUtil.fullyDelete(absLogDir);
+      FileUtil.fullyDelete(ABS_LOG_DIR);
     }
   }
 
@@ -801,10 +837,14 @@ public class TestShuffleHandler {
     final File tmpDir = new File(System.getProperty("test.build.data",
         System.getProperty("java.io.tmpdir")),
         TestShuffleHandler.class.getName());
+    ShuffleHandler shuffle = new ShuffleHandler();
+    AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
+    shuffle.setAuxiliaryLocalPathHandler(pathHandler);
     Configuration conf = new Configuration();
     conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
-    ShuffleHandler shuffle = new ShuffleHandler();
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS,
+        ABS_LOG_DIR.getAbsolutePath());
     // emulate aux services startup with recovery enabled
     shuffle.setRecoveryPath(new Path(tmpDir.toString()));
     tmpDir.mkdirs();
@@ -830,6 +870,7 @@ public class TestShuffleHandler {
       // emulate shuffle handler restart
       shuffle.close();
       shuffle = new ShuffleHandler();
+      shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
       shuffle.start();
@@ -872,6 +913,9 @@ public class TestShuffleHandler {
     conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
     ShuffleHandler shuffle = new ShuffleHandler();
+    AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
+    shuffle.setAuxiliaryLocalPathHandler(pathHandler);
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, ABS_LOG_DIR.getAbsolutePath());
     // emulate aux services startup with recovery enabled
     shuffle.setRecoveryPath(new Path(tmpDir.toString()));
     tmpDir.mkdirs();
@@ -897,6 +941,7 @@ public class TestShuffleHandler {
       // emulate shuffle handler restart
       shuffle.close();
       shuffle = new ShuffleHandler();
+      shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
       shuffle.start();
@@ -914,6 +959,7 @@ public class TestShuffleHandler {
       Assert.assertEquals(version11, shuffle.loadVersion());
       shuffle.close();
       shuffle = new ShuffleHandler();
+      shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
       shuffle.start();
@@ -930,6 +976,7 @@ public class TestShuffleHandler {
       Assert.assertEquals(version21, shuffle.loadVersion());
       shuffle.close();
       shuffle = new ShuffleHandler();
+      shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
     
@@ -979,16 +1026,15 @@ public class TestShuffleHandler {
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "simple");
     UserGroupInformation.setConfiguration(conf);
-    File absLogDir = new File("target", TestShuffleHandler.class.
-        getSimpleName() + "LocDir").getAbsoluteFile();
-    conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_LOCAL_DIRS, ABS_LOG_DIR.getAbsolutePath());
     ApplicationId appId = ApplicationId.newInstance(12345, 1);
     String appAttemptId = "attempt_12345_1_m_1_0";
     String user = "randomUser";
     String reducerId = "0";
     List<File> fileMap = new ArrayList<File>();
-    createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId,
+    createShuffleHandlerFiles(ABS_LOG_DIR, user, appId.toString(), appAttemptId,
         conf, fileMap);
+    AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
     ShuffleHandler shuffleHandler = new ShuffleHandler() {
       @Override
       protected Shuffle getShuffle(Configuration conf) {
@@ -1032,6 +1078,7 @@ public class TestShuffleHandler {
         };
       }
     };
+    shuffleHandler.setAuxiliaryLocalPathHandler(pathHandler);
     shuffleHandler.init(conf);
     try {
       shuffleHandler.start();
@@ -1070,7 +1117,7 @@ public class TestShuffleHandler {
           0, failures.size());
     } finally {
       shuffleHandler.stop();
-      FileUtil.fullyDelete(absLogDir);
+      FileUtil.fullyDelete(ABS_LOG_DIR);
     }
   }
 

hadoop-project/pom.xml

@@ -65,7 +65,7 @@
     <java.security.egd>file:///dev/urandom</java.security.egd>
 
     <!-- avro version -->
-    <avro.version>1.7.4</avro.version>
+    <avro.version>1.7.7</avro.version>
 
     <!-- jersey version -->
     <jersey.version>1.19</jersey.version>
@@ -1552,6 +1552,32 @@
             </goals>
           </execution>
         </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>animal-sniffer-maven-plugin</artifactId>
+        <version>1.16</version>
+        <executions>
+          <execution>
+            <id>signature-check</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>check</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <signature>
+            <groupId>org.codehaus.mojo.signature</groupId>
+            <artifactId>java18</artifactId>
+            <version>1.0</version>
+          </signature>
+          <ignores>
+            <ignore>sun.misc.*</ignore>
+            <ignore>sun.net.*</ignore>
+            <ignore>sun.nio.ch.*</ignore>
+          </ignores>
+        </configuration>
       </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>

hadoop-tools/hadoop-resourceestimator/pom.xml

@@ -29,7 +29,7 @@
         <relativePath>../../hadoop-project</relativePath>
     </parent>
     <artifactId>hadoop-resourceestimator</artifactId>
-    <name>Apache Resource Estimator Service</name>
+    <name>Apache Hadoop Resource Estimator Service</name>
     <packaging>jar</packaging>
     <dependencies>
         <dependency>

hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java

@@ -173,8 +173,8 @@ public class SLSRunner extends Configured implements Tool {
     // <AMType, Class> map
     for (Map.Entry e : tempConf) {
       String key = e.getKey().toString();
-      if (key.startsWith(SLSConfiguration.AM_TYPE)) {
-        String amType = key.substring(SLSConfiguration.AM_TYPE.length());
+      if (key.startsWith(SLSConfiguration.AM_TYPE_PREFIX)) {
+        String amType = key.substring(SLSConfiguration.AM_TYPE_PREFIX.length());
         amClassMap.put(amType, Class.forName(tempConf.get(key)));
       }
     }
@@ -384,33 +384,36 @@ public class SLSRunner extends Configured implements Tool {
   }
 
   private void createAMForJob(Map jsonJob) throws YarnException {
-    long jobStartTime = Long.parseLong(jsonJob.get("job.start.ms").toString());
+    long jobStartTime = Long.parseLong(
+        jsonJob.get(SLSConfiguration.JOB_START_MS).toString());
 
     long jobFinishTime = 0;
-    if (jsonJob.containsKey("job.end.ms")) {
-      jobFinishTime = Long.parseLong(jsonJob.get("job.end.ms").toString());
+    if (jsonJob.containsKey(SLSConfiguration.JOB_END_MS)) {
+      jobFinishTime = Long.parseLong(
+          jsonJob.get(SLSConfiguration.JOB_END_MS).toString());
     }
 
-    String user = (String) jsonJob.get("job.user");
+    String user = (String) jsonJob.get(SLSConfiguration.JOB_USER);
     if (user == null) {
       user = "default";
     }
 
-    String queue = jsonJob.get("job.queue.name").toString();
+    String queue = jsonJob.get(SLSConfiguration.JOB_QUEUE_NAME).toString();
     increaseQueueAppNum(queue);
 
-    String amType = (String)jsonJob.get("am.type");
+    String amType = (String)jsonJob.get(SLSConfiguration.AM_TYPE);
     if (amType == null) {
       amType = SLSUtils.DEFAULT_JOB_TYPE;
     }
 
     int jobCount = 1;
-    if (jsonJob.containsKey("job.count")) {
-      jobCount = Integer.parseInt(jsonJob.get("job.count").toString());
+    if (jsonJob.containsKey(SLSConfiguration.JOB_COUNT)) {
+      jobCount = Integer.parseInt(
+          jsonJob.get(SLSConfiguration.JOB_COUNT).toString());
     }
     jobCount = Math.max(jobCount, 1);
 
-    String oldAppId = (String)jsonJob.get("job.id");
+    String oldAppId = (String)jsonJob.get(SLSConfiguration.JOB_ID);
     // Job id is generated automatically if this job configuration allows
     // multiple job instances
     if(jobCount > 1) {
@@ -426,7 +429,7 @@ public class SLSRunner extends Configured implements Tool {
   private List<ContainerSimulator> getTaskContainers(Map jsonJob)
       throws YarnException {
     List<ContainerSimulator> containers = new ArrayList<>();
-    List tasks = (List) jsonJob.get("job.tasks");
+    List tasks = (List) jsonJob.get(SLSConfiguration.JOB_TASKS);
     if (tasks == null || tasks.size() == 0) {
       throw new YarnException("No task for the job!");
     }
@@ -434,17 +437,22 @@ public class SLSRunner extends Configured implements Tool {
     for (Object o : tasks) {
       Map jsonTask = (Map) o;
 
-      String hostname = (String) jsonTask.get("container.host");
+      String hostname = (String) jsonTask.get(SLSConfiguration.TASK_HOST);
 
       long duration = 0;
-      if (jsonTask.containsKey("duration.ms")) {
-        duration = Integer.parseInt(jsonTask.get("duration.ms").toString());
-      } else if (jsonTask.containsKey("container.start.ms") &&
-          jsonTask.containsKey("container.end.ms")) {
-        long taskStart = Long.parseLong(jsonTask.get("container.start.ms")
-            .toString());
-        long taskFinish = Long.parseLong(jsonTask.get("container.end.ms")
-            .toString());
+      if (jsonTask.containsKey(SLSConfiguration.TASK_DURATION_MS)) {
+        duration = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.TASK_DURATION_MS).toString());
+      } else if (jsonTask.containsKey(SLSConfiguration.DURATION_MS)) {
+        // Also support "duration.ms" for backward compatibility
+        duration = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.DURATION_MS).toString());
+      } else if (jsonTask.containsKey(SLSConfiguration.TASK_START_MS) &&
+          jsonTask.containsKey(SLSConfiguration.TASK_END_MS)) {
+        long taskStart = Long.parseLong(
+            jsonTask.get(SLSConfiguration.TASK_START_MS).toString());
+        long taskFinish = Long.parseLong(
+            jsonTask.get(SLSConfiguration.TASK_END_MS).toString());
         duration = taskFinish - taskStart;
       }
       if (duration <= 0) {
@@ -453,32 +461,33 @@ public class SLSRunner extends Configured implements Tool {
       }
 
       Resource res = getDefaultContainerResource();
-      if (jsonTask.containsKey("container.memory")) {
-        int containerMemory =
-            Integer.parseInt(jsonTask.get("container.memory").toString());
+      if (jsonTask.containsKey(SLSConfiguration.TASK_MEMORY)) {
+        int containerMemory = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.TASK_MEMORY).toString());
         res.setMemorySize(containerMemory);
       }
 
-      if (jsonTask.containsKey("container.vcores")) {
-        int containerVCores =
-            Integer.parseInt(jsonTask.get("container.vcores").toString());
+      if (jsonTask.containsKey(SLSConfiguration.CONTAINER_VCORES)) {
+        int containerVCores = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.CONTAINER_VCORES).toString());
         res.setVirtualCores(containerVCores);
       }
 
       int priority = DEFAULT_MAPPER_PRIORITY;
-      if (jsonTask.containsKey("container.priority")) {
-        priority = Integer.parseInt(jsonTask.get("container.priority")
-            .toString());
+      if (jsonTask.containsKey(SLSConfiguration.TASK_PRIORITY)) {
+        priority = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.TASK_PRIORITY).toString());
       }
 
       String type = "map";
-      if (jsonTask.containsKey("container.type")) {
-        type = jsonTask.get("container.type").toString();
+      if (jsonTask.containsKey(SLSConfiguration.TASK_TYPE)) {
+        type = jsonTask.get(SLSConfiguration.TASK_TYPE).toString();
       }
 
       int count = 1;
-      if (jsonTask.containsKey("count")) {
-        count = Integer.parseInt(jsonTask.get("count").toString());
+      if (jsonTask.containsKey(SLSConfiguration.COUNT)) {
+        count = Integer.parseInt(
+            jsonTask.get(SLSConfiguration.COUNT).toString());
       }
       count = Math.max(count, 1);
 
@@ -708,14 +717,14 @@ public class SLSRunner extends Configured implements Tool {
       return amContainerResource;
     }
 
-    if (jsonJob.containsKey("am.memory")) {
+    if (jsonJob.containsKey(SLSConfiguration.AM_MEMORY)) {
       amContainerResource.setMemorySize(
-          Long.parseLong(jsonJob.get("am.memory").toString()));
+          Long.parseLong(jsonJob.get(SLSConfiguration.AM_MEMORY).toString()));
     }
 
-    if (jsonJob.containsKey("am.vcores")) {
+    if (jsonJob.containsKey(SLSConfiguration.AM_VCORES)) {
       amContainerResource.setVirtualCores(
-          Integer.parseInt(jsonJob.get("am.vcores").toString()));
+          Integer.parseInt(jsonJob.get(SLSConfiguration.AM_VCORES).toString()));
     }
     return amContainerResource;
   }

hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java

@@ -62,12 +62,15 @@ public class SLSConfiguration {
   public static final String AM_HEARTBEAT_INTERVAL_MS = AM_PREFIX
                                                   + "heartbeat.interval.ms";
   public static final int AM_HEARTBEAT_INTERVAL_MS_DEFAULT = 1000;
-  public static final String AM_TYPE = AM_PREFIX + "type.";
+  public static final String AM_TYPE = AM_PREFIX + "type";
+  public static final String AM_TYPE_PREFIX = AM_TYPE + ".";
 
+  public static final String AM_MEMORY = AM_PREFIX + "memory";
   public static final String AM_CONTAINER_MEMORY = AM_PREFIX +
       "container.memory";
   public static final int AM_CONTAINER_MEMORY_DEFAULT = 1024;
 
+  public static final String AM_VCORES = AM_PREFIX + "vcores";
   public static final String AM_CONTAINER_VCORES = AM_PREFIX +
       "container.vcores";
   public static final int AM_CONTAINER_VCORES_DEFAULT = 1;
@@ -85,4 +88,33 @@ public class SLSConfiguration {
         conf.getLong(AM_CONTAINER_MEMORY, AM_CONTAINER_MEMORY_DEFAULT),
         conf.getInt(AM_CONTAINER_VCORES, AM_CONTAINER_VCORES_DEFAULT));
   }
+
+  // input file
+
+  // nodes
+  public static final String NUM_NODES = "num.nodes";
+  public static final String NUM_RACKS = "num.racks";
+
+  // job
+  public static final String JOB_PREFIX = "job.";
+  public static final String JOB_ID = JOB_PREFIX + "id";
+  public static final String JOB_START_MS = JOB_PREFIX + "start.ms";
+  public static final String JOB_END_MS = JOB_PREFIX + "end.ms";
+  public static final String JOB_QUEUE_NAME = JOB_PREFIX + "queue.name";
+  public static final String JOB_USER = JOB_PREFIX + "user";
+  public static final String JOB_COUNT = JOB_PREFIX + "count";
+  public static final String JOB_TASKS = JOB_PREFIX + "tasks";
+
+  // task
+  public static final String COUNT = "count";
+  public static final String TASK_CONTAINER = "container.";
+  public static final String TASK_HOST = TASK_CONTAINER + "host";
+  public static final String TASK_START_MS = TASK_CONTAINER + "start.ms";
+  public static final String TASK_END_MS = TASK_CONTAINER + "end.ms";
+  public static final String DURATION_MS = "duration.ms";
+  public static final String TASK_DURATION_MS = TASK_CONTAINER + DURATION_MS;
+  public static final String TASK_PRIORITY = TASK_CONTAINER + "priority";
+  public static final String TASK_TYPE = TASK_CONTAINER + "type";
+  public static final String TASK_MEMORY = TASK_CONTAINER + "memory";
+
 }

hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java

@@ -39,6 +39,7 @@ import org.apache.hadoop.tools.rumen.JobTraceReader;
 import org.apache.hadoop.tools.rumen.LoggedJob;
 import org.apache.hadoop.tools.rumen.LoggedTask;
 import org.apache.hadoop.tools.rumen.LoggedTaskAttempt;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
 
 @Private
 @Unstable
@@ -118,21 +119,22 @@ public class SLSUtils {
   }
 
   private static void addNodes(Set<String> nodeSet, Map jsonEntry) {
-    if (jsonEntry.containsKey("num.nodes")) {
-      int numNodes = Integer.parseInt(jsonEntry.get("num.nodes").toString());
+    if (jsonEntry.containsKey(SLSConfiguration.NUM_NODES)) {
+      int numNodes = Integer.parseInt(
+          jsonEntry.get(SLSConfiguration.NUM_NODES).toString());
       int numRacks = 1;
-      if (jsonEntry.containsKey("num.racks")) {
+      if (jsonEntry.containsKey(SLSConfiguration.NUM_RACKS)) {
         numRacks = Integer.parseInt(
-            jsonEntry.get("num.racks").toString());
+            jsonEntry.get(SLSConfiguration.NUM_RACKS).toString());
       }
       nodeSet.addAll(generateNodes(numNodes, numRacks));
     }
 
-    if (jsonEntry.containsKey("job.tasks")) {
-      List tasks = (List) jsonEntry.get("job.tasks");
+    if (jsonEntry.containsKey(SLSConfiguration.JOB_TASKS)) {
+      List tasks = (List) jsonEntry.get(SLSConfiguration.JOB_TASKS);
       for (Object o : tasks) {
         Map jsonTask = (Map) o;
-        String hostname = (String) jsonTask.get("container.host");
+        String hostname = (String) jsonTask.get(SLSConfiguration.TASK_HOST);
         if (hostname != null) {
           nodeSet.add(hostname);
         }

hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md

@@ -344,7 +344,7 @@ Here we provide an example format of the sls json file, which contains 2 jobs. T
         "container.host" : "/default-rack/node1",  // host the container asks for
         "container.start.ms" : 6664,  // container start time, optional
         "container.end.ms" : 23707,   // container finish time, optional
-        "duration.ms":  50000,        // duration of the container, optional if start and end time is specified
+        "container.duration.ms":  50000, // duration of the container, optional if start and end time is specified
         "container.priority" : 20,    // priority of the container, optional, the default value is 20
         "container.type" : "map"      // type of the container, could be "map" or "reduce", optional, the default value is "map"
       }, {

hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg

@@ -14,3 +14,4 @@ feature.tc.enabled=0
 #  docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
 #  docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
 #  docker.privileged-containers.enabled=0
+#  docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java

@@ -22,6 +22,7 @@ import java.util.Arrays;
 
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -66,8 +67,10 @@ public abstract class Resource implements Comparable<Resource> {
   // copy array, etc.
   protected static final int NUM_MANDATORY_RESOURCES = 2;
 
-  protected static final int MEMORY_INDEX = 0;
-  protected static final int VCORES_INDEX = 1;
+  @Private
+  public static final int MEMORY_INDEX = 0;
+  @Private
+  public static final int VCORES_INDEX = 1;
 
   @Public
   @Stable
@@ -460,11 +463,11 @@ public abstract class Resource implements Comparable<Resource> {
   @Override
   public int hashCode() {
     final int prime = 47;
-    long result = 0;
+    int result = 0;
     for (ResourceInformation entry : resources) {
       result = prime * result + entry.hashCode();
     }
-    return (int) result;
+    return result;
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.api.records;
 
 import com.google.common.collect.ImmutableMap;
-import org.apache.curator.shaded.com.google.common.reflect.ClassPath;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java

@@ -155,9 +155,6 @@ public class LightWeightResource extends Resource {
   @Override
   public int hashCode() {
     final int prime = 47;
-    long result = prime + getMemorySize();
-    result = prime * result + getVirtualCores();
-
-    return (int) result;
+    return prime * (prime + Long.hashCode(getMemorySize())) + getVirtualCores();
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1483,6 +1483,35 @@ public class YarnConfiguration extends Configuration {
   @Private
   public static final String DEFAULT_NM_GPU_PATH_TO_EXEC = "";
 
+  /**
+   * Settings to control which implementation of docker plugin for GPU will be
+   * used.
+   *
+   * By default uses NVIDIA docker v1.
+   */
+  @Private
+  public static final String NM_GPU_DOCKER_PLUGIN_IMPL =
+      NM_GPU_RESOURCE_PREFIX + "docker-plugin";
+
+  @Private
+  public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
+
+  @Private
+  public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
+      NVIDIA_DOCKER_V1;
+
+  /**
+   * This setting controls end point of nvidia-docker-v1 plugin
+   */
+  @Private
+  public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =
+      NM_GPU_RESOURCE_PREFIX + "docker-plugin." + NVIDIA_DOCKER_V1
+          + ".endpoint";
+
+  @Private
+  public static final String DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =
+      "http://localhost:3476/v1.0/docker/cli";
+
 
   /** NM Webapp address.**/
   public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/** An Interface that can retrieve local directories to read from or write to.
+ *  Components can implement this interface to link it to
+ *  their own Directory Handler Service
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface AuxiliaryLocalPathHandler {
+  /**
+   * Get a path from the local FS for reading for a given Auxiliary Service.
+   * @param path the requested path
+   * @return the complete path to the file on a local disk
+   * @throws IOException if the file read encounters a problem
+   */
+  Path getLocalPathForRead(String path) throws IOException;
+
+  /**
+   * Get a path from the local FS for writing for a given Auxiliary Service.
+   * @param path the requested path
+   * @return the complete path to the file on a local disk
+   * @throws IOException if the path creations fails
+   */
+  Path getLocalPathForWrite(String path) throws IOException;
+
+  /**
+   * Get a path from the local FS for writing a file of an estimated size
+   * for a given Auxiliary Service.
+   * @param path the requested path
+   * @param size the size of the file that is going to be written
+   * @return the complete path to the file on a local disk
+   * @throws IOException if the path creations fails
+   */
+  Path getLocalPathForWrite(String path, long size) throws IOException;
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java

@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 public abstract class AuxiliaryService extends AbstractService {
 
   private Path recoveryPath = null;
+  private AuxiliaryLocalPathHandler auxiliaryLocalPathHandler;
 
   protected AuxiliaryService(String name) {
     super(name);
@@ -123,4 +124,24 @@ public abstract class AuxiliaryService extends AbstractService {
   public void setRecoveryPath(Path recoveryPath) {
     this.recoveryPath = recoveryPath;
   }
+
+  /**
+   * Method that gets the local dirs path handler for this Auxiliary Service.
+   *
+   * @return auxiliaryPathHandler object that is used to read from and write to
+   * valid local Dirs.
+   */
+  public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
+    return this.auxiliaryLocalPathHandler;
+  }
+
+  /**
+   * Method that sets the local dirs path handler for this Auxiliary Service.
+   *
+   * @param auxiliaryLocalPathHandler the pathHandler for this auxiliary service
+   */
+  public void setAuxiliaryLocalPathHandler(
+      AuxiliaryLocalPathHandler auxiliaryLocalPathHandler) {
+    this.auxiliaryLocalPathHandler = auxiliaryLocalPathHandler;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -42,7 +42,10 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Helper class to read the resource-types to be supported by the system.
@@ -56,6 +59,8 @@ public class ResourceUtils {
 
   private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
   private static final String VCORES = ResourceInformation.VCORES.getName();
+  private static final Pattern RESOURCE_REQUEST_VALUE_PATTERN =
+      Pattern.compile("^([0-9]+) ?([a-zA-Z]*)$");
 
   private static volatile boolean initializedResources = false;
   private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
@@ -600,4 +605,43 @@ public class ResourceUtils {
     ResourceUtils
         .initializeResourcesFromResourceInformationMap(resourceInformationMap);
   }
+
+  /**
+   * From a given configuration get all entries representing requested
+   * resources: entries that match the {prefix}{resourceName}={value}[{units}]
+   * pattern.
+   * @param configuration The configuration
+   * @param prefix Keys with this prefix are considered from the configuration
+   * @return The list of requested resources as described by the configuration
+   */
+  public static List<ResourceInformation> getRequestedResourcesFromConfig(
+      Configuration configuration, String prefix) {
+    List<ResourceInformation> result = new ArrayList<>();
+    Map<String, String> customResourcesMap = configuration
+        .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$");
+    for (Entry<String, String> resource : customResourcesMap.entrySet()) {
+      String resourceName = resource.getKey().substring(prefix.length());
+      Matcher matcher =
+          RESOURCE_REQUEST_VALUE_PATTERN.matcher(resource.getValue());
+      if (!matcher.matches()) {
+        String errorMsg = "Invalid resource request specified for property "
+            + resource.getKey() + ": \"" + resource.getValue()
+            + "\", expected format is: value[ ][units]";
+        LOG.error(errorMsg);
+        throw new IllegalArgumentException(errorMsg);
+      }
+      long value = Long.parseLong(matcher.group(1));
+      String unit = matcher.group(2);
+      if (unit.isEmpty()) {
+        unit = ResourceUtils.getDefaultUnit(resourceName);
+      }
+      ResourceInformation resourceInformation = new ResourceInformation();
+      resourceInformation.setName(resourceName);
+      resourceInformation.setValue(value);
+      resourceInformation.setUnits(unit);
+      result.add(resourceInformation);
+    }
+    return result;
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java

@@ -47,8 +47,6 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -113,6 +111,8 @@ import org.apache.log4j.LogManager;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.sun.jersey.api.client.ClientHandlerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * An ApplicationMaster for executing shell commands on a set of launched
@@ -179,7 +179,8 @@ import com.sun.jersey.api.client.ClientHandlerException;
 @InterfaceStability.Unstable
 public class ApplicationMaster {
 
-  private static final Log LOG = LogFactory.getLog(ApplicationMaster.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(ApplicationMaster.class);
 
   @VisibleForTesting
   @Private
@@ -349,7 +350,7 @@ public class ApplicationMaster {
       appMaster.run();
       result = appMaster.finish();
     } catch (Throwable t) {
-      LOG.fatal("Error running ApplicationMaster", t);
+      LOG.error("Error running ApplicationMaster", t);
       LogManager.shutdown();
       ExitUtil.terminate(1, t);
     }
@@ -388,7 +389,7 @@ public class ApplicationMaster {
     } catch (IOException e) {
       e.printStackTrace();
     } finally {
-      IOUtils.cleanup(LOG, buf);
+      IOUtils.cleanupWithLogger(LOG, buf);
     }
   }
 
@@ -630,7 +631,7 @@ public class ApplicationMaster {
     LOG.info("Executing with tokens:");
     while (iter.hasNext()) {
       Token<?> token = iter.next();
-      LOG.info(token);
+      LOG.info(token.toString());
       if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
         iter.remove();
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java

@@ -36,8 +36,6 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -85,6 +83,8 @@ import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Client for Distributed Shell application submission to YARN.
@@ -121,7 +121,8 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 @InterfaceStability.Unstable
 public class Client {
 
-  private static final Log LOG = LogFactory.getLog(Client.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(Client.class);
 
   private static final int DEFAULT_AM_MEMORY = 100;
   private static final int DEFAULT_AM_VCORES = 1;
@@ -237,7 +238,7 @@ public class Client {
       }
       result = client.run();
     } catch (Throwable t) {
-      LOG.fatal("Error running Client", t);
+      LOG.error("Error running Client", t);
       System.exit(1);
     }
     if (result) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/ContainerLaunchFailAppMaster.java

@@ -21,14 +21,14 @@ package org.apache.hadoop.yarn.applications.distributedshell;
 import java.nio.ByteBuffer;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ContainerLaunchFailAppMaster extends ApplicationMaster {
 
-  private static final Log LOG =
-    LogFactory.getLog(ContainerLaunchFailAppMaster.class);
+  private static final Logger LOG =
+    LoggerFactory.getLogger(ContainerLaunchFailAppMaster.class);
 
   public ContainerLaunchFailAppMaster() {
     super();
@@ -69,7 +69,7 @@ public class ContainerLaunchFailAppMaster extends ApplicationMaster {
       appMaster.run();
       result = appMaster.finish();
     } catch (Throwable t) {
-      LOG.fatal("Error running ApplicationMaster", t);
+      LOG.error("Error running ApplicationMaster", t);
       System.exit(1);
     }
     if (result) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java

@@ -20,13 +20,14 @@ package org.apache.hadoop.yarn.applications.distributedshell;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDSFailedAppMaster extends ApplicationMaster {
 
-  private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestDSFailedAppMaster.class);
 
   @Override
   public void run() throws YarnException, IOException, InterruptedException {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSSleepingAppMaster.java

@@ -18,12 +18,14 @@
 
 package org.apache.hadoop.yarn.applications.distributedshell;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDSSleepingAppMaster extends ApplicationMaster{
 
-  private static final Log LOG = LogFactory.getLog(TestDSSleepingAppMaster.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestDSSleepingAppMaster.class);
   private static final long SLEEP_TIME = 5000;
 
   public static void main(String[] args) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java

@@ -39,8 +39,6 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileContext;
@@ -96,11 +94,13 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDistributedShell {
 
-  private static final Log LOG =
-      LogFactory.getLog(TestDistributedShell.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestDistributedShell.class);
 
   protected MiniYARNCluster yarnCluster = null;
   protected MiniDFSCluster hdfsCluster = null;
@@ -892,11 +892,11 @@ public class TestDistributedShell {
     };
 
     //Before run the DS, the default the log level is INFO
-    final Log LOG_Client =
-        LogFactory.getLog(Client.class);
+    final Logger LOG_Client =
+        LoggerFactory.getLogger(Client.class);
     Assert.assertTrue(LOG_Client.isInfoEnabled());
     Assert.assertFalse(LOG_Client.isDebugEnabled());
-    final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class);
+    final Logger LOG_AM = LoggerFactory.getLogger(ApplicationMaster.class);
     Assert.assertTrue(LOG_AM.isInfoEnabled());
     Assert.assertFalse(LOG_AM.isDebugEnabled());
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java

@@ -21,8 +21,6 @@ import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -34,10 +32,12 @@ import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestDistributedShellWithNodeLabels {
-  private static final Log LOG =
-      LogFactory.getLog(TestDistributedShellWithNodeLabels.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestDistributedShellWithNodeLabels.class);
   
   static final int NUM_NMS = 2;
   TestDistributedShell distShellTest;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java

@@ -36,8 +36,6 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.security.Credentials;
@@ -60,6 +58,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The UnmanagedLauncher is a simple client that launches and unmanaged AM. An
@@ -75,7 +75,8 @@ import org.apache.hadoop.yarn.util.Records;
  * report app completion.
  */
 public class UnmanagedAMLauncher {
-  private static final Log LOG = LogFactory.getLog(UnmanagedAMLauncher.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(UnmanagedAMLauncher.class);
 
   private Configuration conf;
 
@@ -110,7 +111,7 @@ public class UnmanagedAMLauncher {
       }
       client.run();
     } catch (Throwable t) {
-      LOG.fatal("Error running Client", t);
+      LOG.error("Error running Client", t);
       System.exit(1);
     }
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java

@@ -28,8 +28,6 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URL;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -47,10 +45,12 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestUnmanagedAMLauncher {
-  private static final Log LOG = LogFactory
-      .getLog(TestUnmanagedAMLauncher.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(TestUnmanagedAMLauncher.class);
 
   protected static MiniYARNCluster yarnCluster = null;
   protected static Configuration conf = new YarnConfiguration();
@@ -128,7 +128,7 @@ public class TestUnmanagedAMLauncher {
     String classpath = getTestRuntimeClasspath();
     String javaHome = System.getenv("JAVA_HOME");
     if (javaHome == null) {
-      LOG.fatal("JAVA_HOME not defined. Test not running.");
+      LOG.error("JAVA_HOME not defined. Test not running.");
       return;
     }
     String[] args = {
@@ -170,7 +170,7 @@ public class TestUnmanagedAMLauncher {
     String classpath = getTestRuntimeClasspath();
     String javaHome = System.getenv("JAVA_HOME");
     if (javaHome == null) {
-      LOG.fatal("JAVA_HOME not defined. Test not running.");
+      LOG.error("JAVA_HOME not defined. Test not running.");
       return;
     }
     String[] args = {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java

@@ -23,8 +23,6 @@ import java.util.Collection;
 import java.util.function.Supplier;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -48,12 +46,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
     AbstractService {
-  private static final Log LOG = LogFactory.getLog(AMRMClient.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(AMRMClient.class);
 
   private TimelineV2Client timelineV2Client;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java

@@ -24,8 +24,6 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
@@ -53,6 +51,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * <code>AMRMClientAsync</code> handles communication with the ResourceManager
@@ -107,7 +107,8 @@ import com.google.common.base.Preconditions;
 @Stable
 public abstract class AMRMClientAsync<T extends ContainerRequest> 
 extends AbstractService {
-  private static final Log LOG = LogFactory.getLog(AMRMClientAsync.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(AMRMClientAsync.class);
   
   protected final AMRMClient<T> client;
   protected final CallbackHandler handler;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java

@@ -25,8 +25,6 @@ import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -51,13 +49,16 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
 public class AMRMClientAsyncImpl<T extends ContainerRequest> 
 extends AMRMClientAsync<T> {
   
-  private static final Log LOG = LogFactory.getLog(AMRMClientAsyncImpl.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(AMRMClientAsyncImpl.class);
   
   private final HeartbeatThread heartbeatThread;
   private final CallbackHandlerThread handlerThread;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/NMClientAsyncImpl.java

@@ -36,8 +36,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -63,12 +61,15 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
 public class NMClientAsyncImpl extends NMClientAsync {
 
-  private static final Log LOG = LogFactory.getLog(NMClientAsyncImpl.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(NMClientAsyncImpl.class);
 
   protected static final int INITIAL_THREAD_POOL_SIZE = 10;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java

@@ -34,8 +34,6 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.AbstractMap.SimpleEntry;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -80,12 +78,15 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
 public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
 
-  private static final Log LOG = LogFactory.getLog(AMRMClientImpl.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(AMRMClientImpl.class);
   private static final List<String> ANY_LIST =
       Collections.singletonList(ResourceRequest.ANY);
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java

@@ -25,8 +25,6 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
@@ -46,6 +44,8 @@ import org.apache.hadoop.yarn.security.NMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -53,7 +53,8 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @LimitedPrivate({ "MapReduce", "YARN" })
 public class ContainerManagementProtocolProxy {
-  static final Log LOG = LogFactory.getLog(ContainerManagementProtocolProxy.class);
+  static final Logger LOG =
+          LoggerFactory.getLogger(ContainerManagementProtocolProxy.class);
 
   private final int maxConnectedNMs;
   private final Map<String, ContainerManagementProtocolProxyData> cmProxy;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java

@@ -27,8 +27,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -56,6 +54,8 @@ import org.apache.hadoop.yarn.client.api.NMClient;
 import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * <p>
@@ -85,7 +85,8 @@ import org.apache.hadoop.yarn.ipc.RPCUtil;
 @Unstable
 public class NMClientImpl extends NMClient {
 
-  private static final Log LOG = LogFactory.getLog(NMClientImpl.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(NMClientImpl.class);
 
   // The logically coherent operations on startedContainers is synchronized to
   // ensure they are atomic

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/RemoteRequestsTable.java

@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.Priority;
@@ -36,10 +34,13 @@ import java.util.TreeMap;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.ResourceRequestInfo;
 import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.ProfileCapabilityComparator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class RemoteRequestsTable<T> implements Iterable<ResourceRequestInfo>{
 
-  private static final Log LOG = LogFactory.getLog(RemoteRequestsTable.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(RemoteRequestsTable.class);
 
   private ProfileCapabilityComparator resourceComparator;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java

@@ -22,8 +22,6 @@ package org.apache.hadoop.yarn.client.api.impl;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -46,6 +44,8 @@ import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory;
 import org.apache.hadoop.yarn.util.Records;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * An implementation of the SharedCacheClient API.
@@ -53,8 +53,8 @@ import com.google.common.annotations.VisibleForTesting;
 @Private
 @Unstable
 public class SharedCacheClientImpl extends SharedCacheClient {
-  private static final Log LOG = LogFactory
-      .getLog(SharedCacheClientImpl.class);
+  private static final Logger LOG = LoggerFactory
+      .getLogger(SharedCacheClientImpl.class);
 
   private ClientSCMProtocol scmClient;
   private InetSocketAddress scmAddress;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -29,8 +29,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -131,12 +129,15 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
 public class YarnClientImpl extends YarnClient {
 
-  private static final Log LOG = LogFactory.getLog(YarnClientImpl.class);
+  private static final Logger LOG = LoggerFactory
+          .getLogger(YarnClientImpl.class);
 
   protected ApplicationClientProtocol rmClient;
   protected long submitPollIntervalMillis;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java

@@ -55,8 +55,6 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.time.DateFormatUtils;
 import org.apache.commons.lang.time.DurationFormatUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -78,12 +76,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TopCLI extends YarnCLI {
 
   private static final String CLUSTER_INFO_URL = "/ws/v1/cluster/info";
 
-  private static final Log LOG = LogFactory.getLog(TopCLI.class);
+  private static final Logger LOG = LoggerFactory
+          .getLogger(TopCLI.class);
   private String CLEAR = "\u001b[2J";
   private String CLEAR_LINE = "\u001b[2K";
   private String SET_CURSOR_HOME = "\u001b[H";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestGetGroups.java

@@ -23,8 +23,6 @@ import java.io.PrintStream;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
@@ -37,10 +35,13 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestGetGroups extends GetGroupsTestBase {
   
-  private static final Log LOG = LogFactory.getLog(TestGetGroups.class);
+  private static final Logger LOG =
+          LoggerFactory.getLogger(TestGetGroups.class);
   
   private static ResourceManager resourceManager;
   

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java

@@ -34,8 +34,6 @@ import java.util.concurrent.TimeoutException;
 
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.ClientBaseWithFixes;
 import org.apache.hadoop.ha.HAServiceProtocol;
@@ -64,10 +62,12 @@ import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.base.Supplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestRMFailover extends ClientBaseWithFixes {
-  private static final Log LOG =
-      LogFactory.getLog(TestRMFailover.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRMFailover.class.getName());
   private static final HAServiceProtocol.StateChangeRequestInfo req =
       new HAServiceProtocol.StateChangeRequestInfo(
           HAServiceProtocol.RequestSource.REQUEST_BY_USER);
@@ -114,7 +114,7 @@ public class TestRMFailover extends ClientBaseWithFixes {
         client.getApplications();
         return;
       } catch (Exception e) {
-        LOG.error(e);
+        LOG.error(e.toString());
       } finally {
         client.stop();
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceManagerAdministrationProtocolPBClientImpl.java

@@ -22,8 +22,6 @@ import java.net.InetSocketAddress;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
@@ -53,6 +51,8 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.*;
 
@@ -61,8 +61,8 @@ import static org.junit.Assert.*;
  */
 public class TestResourceManagerAdministrationProtocolPBClientImpl {
   private static ResourceManager resourceManager;
-  private static final Log LOG = LogFactory
-          .getLog(TestResourceManagerAdministrationProtocolPBClientImpl.class);
+  private static final Logger LOG = LoggerFactory
+          .getLogger(TestResourceManagerAdministrationProtocolPBClientImpl.class);
   private final RecordFactory recordFactory = RecordFactoryProvider
           .getRecordFactory(null);