Merge branch 'trunk' into HDFS-7240

.gitignore

@@ -44,3 +44,4 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/dist
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
 yarnregistry.pdf
 patchprocess/
+.history/

LICENSE.txt

@@ -766,6 +766,31 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
 Apache HBase - Server which contains JQuery minified javascript library version 1.8.3
 Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7
+--------------------------------------------------------------------------------
+
+MIT License
+
+Copyright (c) 2003-2017 Optimatika
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+For:
 oj! Algorithms - version 43.0
 --------------------------------------------------------------------------------
 
@@ -1845,6 +1870,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 The binary distribution of this product bundles these dependencies under the
 following license:
 FindBugs-jsr305 3.0.0
+dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved.
 --------------------------------------------------------------------------------
 (2-clause BSD)
 Redistribution and use in source and binary forms, with or without

NOTICE.txt

@@ -581,3 +581,13 @@ The binary distribution of this product bundles binaries of
 Ehcache 3.3.1,
 which has the following notices:
  * Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software

dev-support/bin/create-release

@@ -489,9 +489,9 @@ function dockermode
     echo "RUN mkdir -p /maven"
     echo "RUN chown -R ${user_name} /maven"
 
-    # we always force build with the Oracle JDK
+    # we always force build with the OpenJDK JDK
     # but with the correct version
-    echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-oracle"
+    echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
     echo "USER ${user_name}"
     printf "\n\n"
   ) | docker build -t "${imgname}" -

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -86,6 +86,31 @@
         <include>*-sources.jar</include>
       </includes>
     </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+      <includes>
+        <include>*-sources.jar</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
+      <outputDirectory>etc/hadoop</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+      <includes>
+        <include>*-sources.jar</include>
+      </includes>
+    </fileSet>
     <fileSet>
       <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>

hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

@@ -103,6 +103,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aliyun</artifactId>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-aws</artifactId>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.conf;
 
+import com.ctc.wstx.api.ReaderConfig;
 import com.ctc.wstx.io.StreamBootstrapper;
 import com.ctc.wstx.io.SystemId;
 import com.ctc.wstx.stax.WstxInputFactory;
@@ -70,6 +71,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLStreamConstants;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
@@ -91,6 +93,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -206,19 +209,31 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private static final String DEFAULT_STRING_CHECK =
     "testingforemptydefaultvalue";
 
+  private static boolean restrictSystemPropsDefault = false;
+  private boolean restrictSystemProps = restrictSystemPropsDefault;
   private boolean allowNullValueProperties = false;
 
   private static class Resource {
     private final Object resource;
     private final String name;
+    private final boolean restrictParser;
     
     public Resource(Object resource) {
       this(resource, resource.toString());
     }
-    
+
+    public Resource(Object resource, boolean useRestrictedParser) {
+      this(resource, resource.toString(), useRestrictedParser);
+    }
+
     public Resource(Object resource, String name) {
+      this(resource, name, getRestrictParserDefault(resource));
+    }
+
+    public Resource(Object resource, String name, boolean restrictParser) {
       this.resource = resource;
       this.name = name;
+      this.restrictParser = restrictParser;
     }
     
     public String getName(){
@@ -228,11 +243,28 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     public Object getResource() {
       return resource;
     }
-    
+
+    public boolean isParserRestricted() {
+      return restrictParser;
+    }
+
     @Override
     public String toString() {
       return name;
     }
+
+    private static boolean getRestrictParserDefault(Object resource) {
+      if (resource instanceof String) {
+        return false;
+      }
+      UserGroupInformation user;
+      try {
+        user = UserGroupInformation.getCurrentUser();
+      } catch (IOException e) {
+        throw new RuntimeException("Unable to determine current user", e);
+      }
+      return user.getRealUser() != null;
+    }
   }
   
   /**
@@ -254,7 +286,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       new ConcurrentHashMap<String, Boolean>());
   
   private boolean loadDefaults = true;
-  
+
   /**
    * Configuration objects
    */
@@ -778,6 +810,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         this.overlay = (Properties)other.overlay.clone();
       }
 
+      this.restrictSystemProps = other.restrictSystemProps;
       if (other.updatingResource != null) {
         this.updatingResource = new ConcurrentHashMap<String, String[]>(
            other.updatingResource);
@@ -826,6 +859,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     }
   }
 
+  public static void setRestrictSystemPropertiesDefault(boolean val) {
+    restrictSystemPropsDefault = val;
+  }
+
+  public void setRestrictSystemProperties(boolean val) {
+    this.restrictSystemProps = val;
+  }
+
   /**
    * Add a configuration resource. 
    * 
@@ -839,6 +880,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     addResourceObject(new Resource(name));
   }
 
+  public void addResource(String name, boolean restrictedParser) {
+    addResourceObject(new Resource(name, restrictedParser));
+  }
+
   /**
    * Add a configuration resource. 
    * 
@@ -853,6 +898,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     addResourceObject(new Resource(url));
   }
 
+  public void addResource(URL url, boolean restrictedParser) {
+    addResourceObject(new Resource(url, restrictedParser));
+  }
+
   /**
    * Add a configuration resource. 
    * 
@@ -867,6 +916,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     addResourceObject(new Resource(file));
   }
 
+  public void addResource(Path file, boolean restrictedParser) {
+    addResourceObject(new Resource(file, restrictedParser));
+  }
+
   /**
    * Add a configuration resource. 
    * 
@@ -884,6 +937,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     addResourceObject(new Resource(in));
   }
 
+  public void addResource(InputStream in, boolean restrictedParser) {
+    addResourceObject(new Resource(in, restrictedParser));
+  }
+
   /**
    * Add a configuration resource. 
    * 
@@ -897,7 +954,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   public void addResource(InputStream in, String name) {
     addResourceObject(new Resource(in, name));
   }
-  
+
+  public void addResource(InputStream in, String name,
+      boolean restrictedParser) {
+    addResourceObject(new Resource(in, name, restrictedParser));
+  }
+
   /**
    * Add a configuration resource.
    *
@@ -907,7 +969,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
    * @param conf Configuration object from which to load properties
    */
   public void addResource(Configuration conf) {
-    addResourceObject(new Resource(conf.getProps()));
+    addResourceObject(new Resource(conf.getProps(), conf.restrictSystemProps));
   }
 
   
@@ -927,6 +989,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   
   private synchronized void addResourceObject(Resource resource) {
     resources.add(resource);                      // add to resources
+    restrictSystemProps |= resource.isParserRestricted();
     reloadConfiguration();
   }
 
@@ -1035,34 +1098,36 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       final String var = eval.substring(varBounds[SUB_START_IDX],
           varBounds[SUB_END_IDX]);
       String val = null;
-      try {
-        if (var.startsWith("env.") && 4 < var.length()) {
-          String v = var.substring(4);
-          int i = 0;
-          for (; i < v.length(); i++) {
-            char c = v.charAt(i);
-            if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
-              val = getenv(v.substring(0, i));
-              if (val == null || val.length() == 0) {
-                val = v.substring(i + 2);
-              }
-              break;
-            } else if (c == '-') {
-              val = getenv(v.substring(0, i));
-              if (val == null) {
-                val = v.substring(i + 1);
+      if (!restrictSystemProps) {
+        try {
+          if (var.startsWith("env.") && 4 < var.length()) {
+            String v = var.substring(4);
+            int i = 0;
+            for (; i < v.length(); i++) {
+              char c = v.charAt(i);
+              if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
+                val = getenv(v.substring(0, i));
+                if (val == null || val.length() == 0) {
+                  val = v.substring(i + 2);
+                }
+                break;
+              } else if (c == '-') {
+                val = getenv(v.substring(0, i));
+                if (val == null) {
+                  val = v.substring(i + 1);
+                }
+                break;
               }
-              break;
             }
+            if (i == v.length()) {
+              val = getenv(v);
+            }
+          } else {
+            val = getProperty(var);
           }
-          if (i == v.length()) {
-            val = getenv(v);
-          }
-        } else {
-          val = getProperty(var);
+        } catch (SecurityException se) {
+          LOG.warn("Unexpected SecurityException in Configuration", se);
         }
-      } catch(SecurityException se) {
-        LOG.warn("Unexpected SecurityException in Configuration", se);
       }
       if (val == null) {
         val = getRaw(var);
@@ -1129,6 +1194,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     this.allowNullValueProperties = val;
   }
 
+  public void setRestrictSystemProps(boolean val) {
+    this.restrictSystemProps = val;
+  }
+
   /**
    * Return existence of the <code>name</code> property, but only for
    * names which have no valid value, usually non-existent or commented
@@ -2719,7 +2788,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     return configMap;
   }
 
-  private XMLStreamReader parse(URL url)
+  private XMLStreamReader parse(URL url, boolean restricted)
       throws IOException, XMLStreamException {
     if (!quietmode) {
       if (LOG.isDebugEnabled()) {
@@ -2736,11 +2805,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       // with other users.
       connection.setUseCaches(false);
     }
-    return parse(connection.getInputStream(), url.toString());
+    return parse(connection.getInputStream(), url.toString(), restricted);
   }
 
-  private XMLStreamReader parse(InputStream is, String systemIdStr)
-      throws IOException, XMLStreamException {
+  private XMLStreamReader parse(InputStream is, String systemIdStr,
+      boolean restricted) throws IOException, XMLStreamException {
     if (!quietmode) {
       LOG.debug("parsing input stream " + is);
     }
@@ -2748,9 +2817,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       return null;
     }
     SystemId systemId = SystemId.construct(systemIdStr);
-    return XML_INPUT_FACTORY.createSR(XML_INPUT_FACTORY.createPrivateConfig(),
-        systemId, StreamBootstrapper.getInstance(null, systemId, is), false,
-        true);
+    ReaderConfig readerConfig = XML_INPUT_FACTORY.createPrivateConfig();
+    if (restricted) {
+      readerConfig.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+    }
+    return XML_INPUT_FACTORY.createSR(readerConfig, systemId,
+        StreamBootstrapper.getInstance(null, systemId, is), false, true);
   }
 
   private void loadResources(Properties properties,
@@ -2758,7 +2830,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
                              boolean quiet) {
     if(loadDefaults) {
       for (String resource : defaultResources) {
-        loadResource(properties, new Resource(resource), quiet);
+        loadResource(properties, new Resource(resource, false), quiet);
       }
     }
     
@@ -2778,12 +2850,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       name = wrapper.getName();
       XMLStreamReader2 reader = null;
       boolean returnCachedProperties = false;
+      boolean isRestricted = wrapper.isParserRestricted();
 
       if (resource instanceof URL) {                  // an URL resource
-        reader = (XMLStreamReader2)parse((URL)resource);
+        reader = (XMLStreamReader2)parse((URL)resource, isRestricted);
       } else if (resource instanceof String) {        // a CLASSPATH resource
         URL url = getResource((String)resource);
-        reader = (XMLStreamReader2)parse(url);
+        reader = (XMLStreamReader2)parse(url, isRestricted);
       } else if (resource instanceof Path) {          // a file resource
         // Can't use FileSystem API or we get an infinite loop
         // since FileSystem uses Configuration API.  Use java.io.File instead.
@@ -2794,10 +2867,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
             LOG.debug("parsing File " + file);
           }
           reader = (XMLStreamReader2)parse(new BufferedInputStream(
-              new FileInputStream(file)), ((Path)resource).toString());
+              new FileInputStream(file)), ((Path)resource).toString(),
+              isRestricted);
         }
       } else if (resource instanceof InputStream) {
-        reader = (XMLStreamReader2)parse((InputStream)resource, null);
+        reader = (XMLStreamReader2)parse((InputStream)resource, null,
+            isRestricted);
         returnCachedProperties = true;
       } else if (resource instanceof Properties) {
         overlay(properties, (Properties)resource);
@@ -2879,12 +2954,17 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
             if (confInclude == null) {
               break;
             }
+            if (isRestricted) {
+              throw new RuntimeException("Error parsing resource " + wrapper
+                  + ": XInclude is not supported for restricted resources");
+            }
             // Determine if the included resource is a classpath resource
             // otherwise fallback to a file resource
             // xi:include are treated as inline and retain current source
             URL include = getResource(confInclude);
             if (include != null) {
-              Resource classpathResource = new Resource(include, name);
+              Resource classpathResource = new Resource(include, name,
+                  wrapper.isParserRestricted());
               loadResource(properties, classpathResource, quiet);
             } else {
               URL url;
@@ -2905,7 +2985,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
                 }
                 url = href.toURI().toURL();
               }
-              Resource uriResource = new Resource(url, name);
+              Resource uriResource = new Resource(url, name,
+                  wrapper.isParserRestricted());
               loadResource(properties, uriResource, quiet);
             }
             break;
@@ -2993,7 +3074,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
 
       if (returnCachedProperties) {
         overlay(properties, toAddTo);
-        return new Resource(toAddTo, name);
+        return new Resource(toAddTo, name, wrapper.isParserRestricted());
       }
       return null;
     } catch (IOException e) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java

@@ -30,20 +30,23 @@ import java.util.EnumSet;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.ByteBufferReadable;
 import org.apache.hadoop.fs.CanSetDropBehind;
 import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.CanUnbuffer;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
 import org.apache.hadoop.fs.HasFileDescriptor;
 import org.apache.hadoop.fs.PositionedReadable;
 import org.apache.hadoop.fs.ReadOption;
 import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.StreamCapabilitiesPolicy;
 import org.apache.hadoop.io.ByteBufferPool;
-
-import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * CryptoInputStream decrypts data. It is not thread-safe. AES CTR mode is
@@ -61,7 +64,7 @@ import com.google.common.base.Preconditions;
 public class CryptoInputStream extends FilterInputStream implements 
     Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor, 
     CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess, 
-    ReadableByteChannel {
+    ReadableByteChannel, CanUnbuffer, StreamCapabilities {
   private final byte[] oneByteBuf = new byte[1];
   private final CryptoCodec codec;
   private final Decryptor decryptor;
@@ -719,4 +722,27 @@ public class CryptoInputStream extends FilterInputStream implements
   public boolean isOpen() {
     return !closed;
   }
+
+  private void cleanDecryptorPool() {
+    decryptorPool.clear();
+  }
+
+  @Override
+  public void unbuffer() {
+    cleanBufferPool();
+    cleanDecryptorPool();
+    StreamCapabilitiesPolicy.unbuffer(in);
+  }
+
+  @Override
+  public boolean hasCapability(String capability) {
+    switch (StringUtils.toLowerCase(capability)) {
+    case StreamCapabilities.READAHEAD:
+    case StreamCapabilities.DROPBEHIND:
+    case StreamCapabilities.UNBUFFER:
+      return true;
+    default:
+      return false;
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java

@@ -133,6 +133,13 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   private static final ObjectWriter WRITER =
       new ObjectMapper().writerWithDefaultPrettyPrinter();
 
+  private final Text dtService;
+
+  // Allow fallback to default kms server port 9600 for certain tests that do
+  // not specify the port explicitly in the kms provider url.
+  @VisibleForTesting
+  public static volatile boolean fallbackDefaultPortForTesting = false;
+
   private class EncryptedQueueRefiller implements
     ValueQueue.QueueRefiller<EncryptedKeyVersion> {
 
@@ -297,7 +304,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     }
   }
 
-  private String kmsUrl;
+  private URL kmsUrl;
   private SSLFactory sslFactory;
   private ConnectionConfigurator configurator;
   private DelegationTokenAuthenticatedURL.Token authToken;
@@ -349,7 +356,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   public KMSClientProvider(URI uri, Configuration conf) throws IOException {
     super(conf);
     kmsUrl = createServiceURL(extractKMSPath(uri));
-    if ("https".equalsIgnoreCase(new URL(kmsUrl).getProtocol())) {
+    int kmsPort = kmsUrl.getPort();
+    if ((kmsPort == -1) && fallbackDefaultPortForTesting) {
+      kmsPort = 9600;
+    }
+
+    InetSocketAddress addr = new InetSocketAddress(kmsUrl.getHost(), kmsPort);
+    dtService = SecurityUtil.buildTokenService(addr);
+
+    if ("https".equalsIgnoreCase(kmsUrl.getProtocol())) {
       sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
       try {
         sslFactory.init();
@@ -385,19 +400,20 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
                     KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
             new EncryptedQueueRefiller());
     authToken = new DelegationTokenAuthenticatedURL.Token();
+    LOG.info("KMSClientProvider for KMS url: {} delegation token service: {}" +
+        " created.", kmsUrl, dtService);
   }
 
   private static Path extractKMSPath(URI uri) throws MalformedURLException, IOException {
     return ProviderUtils.unnestUri(uri);
   }
 
-  private static String createServiceURL(Path path) throws IOException {
+  private static URL createServiceURL(Path path) throws IOException {
     String str = new URL(path.toString()).toExternalForm();
     if (str.endsWith("/")) {
       str = str.substring(0, str.length() - 1);
     }
-    return new URL(str + KMSRESTConstants.SERVICE_VERSION + "/").
-        toExternalForm();
+    return new URL(str + KMSRESTConstants.SERVICE_VERSION + "/");
   }
 
   private URL createURL(String collection, String resource, String subResource,
@@ -996,7 +1012,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   public Token<?>[] addDelegationTokens(final String renewer,
       Credentials credentials) throws IOException {
     Token<?>[] tokens = null;
-    Text dtService = getDelegationTokenService();
     Token<?> token = credentials.getToken(dtService);
     if (token == null) {
       final URL url = createURL(null, null, null, null);
@@ -1033,21 +1048,14 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     }
     return tokens;
   }
-  
-  private Text getDelegationTokenService() throws IOException {
-    URL url = new URL(kmsUrl);
-    InetSocketAddress addr = new InetSocketAddress(url.getHost(),
-        url.getPort());
-    Text dtService = SecurityUtil.buildTokenService(addr);
-    return dtService;
-  }
 
   private boolean containsKmsDt(UserGroupInformation ugi) throws IOException {
     // Add existing credentials from the UGI, since provider is cached.
     Credentials creds = ugi.getCredentials();
     if (!creds.getAllTokens().isEmpty()) {
+      LOG.debug("Searching for token that matches service: {}", dtService);
       org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
-          dToken = creds.getToken(getDelegationTokenService());
+          dToken = creds.getToken(dtService);
       if (dToken != null) {
         return true;
       }
@@ -1058,9 +1066,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   private UserGroupInformation getActualUgi() throws IOException {
     final UserGroupInformation currentUgi = UserGroupInformation
         .getCurrentUser();
-    if (LOG.isDebugEnabled()) {
-      UserGroupInformation.logAllUserInfo(currentUgi);
-    }
+
+    UserGroupInformation.logAllUserInfo(LOG, currentUgi);
+
     // Use current user by default
     UserGroupInformation actualUgi = currentUgi;
     if (currentUgi.getRealUser() != null) {
@@ -1099,6 +1107,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 
   @VisibleForTesting
   String getKMSUrl() {
-    return kmsUrl;
+    return kmsUrl.toString();
   }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java

@@ -38,7 +38,7 @@ import org.apache.hadoop.util.IdentityHashStore;
 public class FSDataInputStream extends DataInputStream
     implements Seekable, PositionedReadable, 
       ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
-      HasEnhancedByteBufferAccess, CanUnbuffer {
+      HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities {
   /**
    * Map ByteBuffers that we have handed out to readers to ByteBufferPool 
    * objects
@@ -227,12 +227,15 @@ public class FSDataInputStream extends DataInputStream
 
   @Override
   public void unbuffer() {
-    try {
-      ((CanUnbuffer)in).unbuffer();
-    } catch (ClassCastException e) {
-      throw new UnsupportedOperationException("this stream " +
-          in.getClass().getName() + " does not " + "support unbuffering.");
+    StreamCapabilitiesPolicy.unbuffer(in);
+  }
+
+  @Override
+  public boolean hasCapability(String capability) {
+    if (in instanceof StreamCapabilities) {
+      return ((StreamCapabilities) in).hasCapability(capability);
     }
+    return false;
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -973,8 +973,7 @@ public abstract class FileSystem extends Configured implements Closeable {
    * @param opt If absent, assume {@link HandleOpt#path()}.
    * @throws IllegalArgumentException If the FileStatus does not belong to
    *         this FileSystem
-   * @throws UnsupportedOperationException If
-   *         {@link #createPathHandle(FileStatus, HandleOpt[])}
+   * @throws UnsupportedOperationException If {@link #createPathHandle}
    *         not overridden by subclass.
    * @throws UnsupportedOperationException If this FileSystem cannot enforce
    *         the specified constraints.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java

@@ -135,7 +135,16 @@ public class LocatedFileStatus extends FileStatus {
   public BlockLocation[] getBlockLocations() {
     return locations;
   }
-  
+
+  /**
+   * Hook for subclasses to lazily set block locations. The {@link #locations}
+   * field should be null before this is called.
+   * @param locations Block locations for this instance.
+   */
+  protected void setBlockLocations(BlockLocation[] locations) {
+    this.locations = locations;
+  }
+
   /**
    * Compare this FileStatus to another FileStatus
    * @param   o the FileStatus to be compared.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java

@@ -338,8 +338,7 @@ public final class Options {
     }
 
     /**
-     * Utility function for mapping
-     * {@link FileSystem#getPathHandle(FileStatus, HandleOpt[])} to a
+     * Utility function for mapping {@link FileSystem#getPathHandle} to a
      * fixed set of handle options.
      * @param fs Target filesystem
      * @param opt Options to bind in partially evaluated function

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java

@@ -23,27 +23,49 @@ import org.apache.hadoop.classification.InterfaceStability;
 
 /**
  * Interface to query streams for supported capabilities.
+ *
+ * Capability strings must be in lower case.
+ *
+ * Constant strings are chosen over enums in order to allow other file systems
+ * to define their own capabilities.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface StreamCapabilities {
+  /**
+   * Stream hflush capability implemented by {@link Syncable#hflush()}.
+   */
+  String HFLUSH = "hflush";
+
+  /**
+   * Stream hsync capability implemented by {@link Syncable#hsync()}.
+   */
+  String HSYNC = "hsync";
+
+  /**
+   * Stream setReadahead capability implemented by
+   * {@link CanSetReadahead#setReadahead(Long)}.
+   */
+  String READAHEAD = "in:readahead";
+
+  /**
+   * Stream setDropBehind capability implemented by
+   * {@link CanSetDropBehind#setDropBehind(Boolean)}.
+   */
+  String DROPBEHIND = "dropbehind";
+
+  /**
+   * Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.
+   */
+  String UNBUFFER = "in:unbuffer";
+
   /**
    * Capabilities that a stream can support and be queried for.
    */
+  @Deprecated
   enum StreamCapability {
-    /**
-     * Stream hflush capability to flush out the data in client's buffer.
-     * Streams with this capability implement {@link Syncable} and support
-     * {@link Syncable#hflush()}.
-     */
-    HFLUSH("hflush"),
-
-    /**
-     * Stream hsync capability to flush out the data in client's buffer and
-     * the disk device. Streams with this capability implement {@link Syncable}
-     * and support {@link Syncable#hsync()}.
-     */
-    HSYNC("hsync");
+    HFLUSH(StreamCapabilities.HFLUSH),
+    HSYNC(StreamCapabilities.HSYNC);
 
     private final String capability;
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java

@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.InputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Static methods to implement policies for {@link StreamCapabilities}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class StreamCapabilitiesPolicy {
+  /**
+   * Implement the policy for {@link CanUnbuffer#unbuffer()}.
+   *
+   * @param in the input stream
+   */
+  public static void unbuffer(InputStream in) {
+    try {
+      if (in instanceof StreamCapabilities
+          && ((StreamCapabilities) in).hasCapability(
+          StreamCapabilities.UNBUFFER)) {
+        ((CanUnbuffer) in).unbuffer();
+      }
+    } catch (ClassCastException e) {
+      throw new UnsupportedOperationException("this stream " +
+          in.getClass().getName() +
+          " claims to unbuffer but forgets to implement CanUnbuffer");
+    }
+  }
+}
+

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java

@@ -41,27 +41,54 @@ import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 @InterfaceStability.Evolving
 public class GzipCodec extends DefaultCodec {
   /**
-   * A bridge that wraps around a DeflaterOutputStream to make it 
+   * A bridge that wraps around a DeflaterOutputStream to make it
    * a CompressionOutputStream.
    */
   @InterfaceStability.Evolving
   protected static class GzipOutputStream extends CompressorStream {
 
     private static class ResetableGZIPOutputStream extends GZIPOutputStream {
+      /**
+       * Fixed ten-byte gzip header. See {@link GZIPOutputStream}'s source for
+       * details.
+       */
+      private static final byte[] GZIP_HEADER = new byte[] {
+          0x1f, (byte) 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+      private boolean reset = false;
 
       public ResetableGZIPOutputStream(OutputStream out) throws IOException {
         super(out);
       }
 
-      public void resetState() throws IOException {
-        def.reset();
+      public synchronized void resetState() throws IOException {
+        reset = true;
+      }
+
+      @Override
+      public synchronized void write(byte[] buf, int off, int len)
+          throws IOException {
+        if (reset) {
+          def.reset();
+          crc.reset();
+          out.write(GZIP_HEADER);
+          reset = false;
+        }
+        super.write(buf, off, len);
+      }
+
+      @Override
+      public synchronized void close() throws IOException {
+        reset = false;
+        super.close();
       }
+
     }
 
     public GzipOutputStream(OutputStream out) throws IOException {
       super(new ResetableGZIPOutputStream(out));
     }
-    
+
     /**
      * Allow children types to put a different type in here.
      * @param out the Deflater stream to use
@@ -69,7 +96,7 @@ public class GzipCodec extends DefaultCodec {
     protected GzipOutputStream(CompressorStream out) {
       super(out);
     }
-    
+
     @Override
     public void close() throws IOException {
       out.close();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSinkAdapter.java

@@ -51,7 +51,7 @@ class MetricsSinkAdapter implements SinkQueue.Consumer<MetricsBuffer> {
   private final Thread sinkThread;
   private volatile boolean stopping = false;
   private volatile boolean inError = false;
-  private final int period, firstRetryDelay, retryCount;
+  private final int periodMs, firstRetryDelay, retryCount;
   private final long oobPutTimeout;
   private final float retryBackoff;
   private final MetricsRegistry registry = new MetricsRegistry("sinkadapter");
@@ -62,7 +62,7 @@ class MetricsSinkAdapter implements SinkQueue.Consumer<MetricsBuffer> {
   MetricsSinkAdapter(String name, String description, MetricsSink sink,
                      String context, MetricsFilter sourceFilter,
                      MetricsFilter recordFilter, MetricsFilter metricFilter,
-                     int period, int queueCapacity, int retryDelay,
+                     int periodMs, int queueCapacity, int retryDelay,
                      float retryBackoff, int retryCount) {
     this.name = checkNotNull(name, "name");
     this.description = description;
@@ -71,7 +71,7 @@ class MetricsSinkAdapter implements SinkQueue.Consumer<MetricsBuffer> {
     this.sourceFilter = sourceFilter;
     this.recordFilter = recordFilter;
     this.metricFilter = metricFilter;
-    this.period = checkArg(period, period > 0, "period");
+    this.periodMs = checkArg(periodMs, periodMs > 0, "period");
     firstRetryDelay = checkArg(retryDelay, retryDelay > 0, "retry delay");
     this.retryBackoff = checkArg(retryBackoff, retryBackoff>1, "retry backoff");
     oobPutTimeout = (long)
@@ -93,9 +93,9 @@ class MetricsSinkAdapter implements SinkQueue.Consumer<MetricsBuffer> {
     sinkThread.setDaemon(true);
   }
 
-  boolean putMetrics(MetricsBuffer buffer, long logicalTime) {
-    if (logicalTime % period == 0) {
-      LOG.debug("enqueue, logicalTime="+ logicalTime);
+  boolean putMetrics(MetricsBuffer buffer, long logicalTimeMs) {
+    if (logicalTimeMs % periodMs == 0) {
+      LOG.debug("enqueue, logicalTime="+ logicalTimeMs);
       if (queue.enqueue(buffer)) {
         refreshQueueSizeGauge();
         return true;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java

@@ -519,7 +519,7 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
         conf.getFilter(SOURCE_FILTER_KEY),
         conf.getFilter(RECORD_FILTER_KEY),
         conf.getFilter(METRIC_FILTER_KEY),
-        conf.getInt(PERIOD_KEY, PERIOD_DEFAULT),
+        conf.getInt(PERIOD_KEY, PERIOD_DEFAULT) * 1000,
         conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT),
         conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT),
         conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT),
@@ -618,6 +618,11 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
     return sources.get(name);
   }
 
+  @VisibleForTesting
+  public MetricsSinkAdapter getSinkAdapter(String name) {
+    return sinks.get(name);
+  }
+
   private InitMode initMode() {
     LOG.debug("from system property: "+ System.getProperty(MS_INIT_MODE_KEY));
     LOG.debug("from environment variable: "+ System.getenv(MS_INIT_MODE_KEY));

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java

@@ -28,6 +28,8 @@ import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.log.metrics.EventCounter;
 import org.apache.hadoop.metrics2.MetricsCollector;
@@ -39,6 +41,8 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.Interns;
 import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
 import static org.apache.hadoop.metrics2.impl.MsInfo.*;
+
+import org.apache.hadoop.util.GcTimeMonitor;
 import org.apache.hadoop.util.JvmPauseMonitor;
 
 /**
@@ -85,6 +89,7 @@ public class JvmMetrics implements MetricsSource {
   private JvmPauseMonitor pauseMonitor = null;
   final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
       new ConcurrentHashMap<String, MetricsInfo[]>();
+  private GcTimeMonitor gcTimeMonitor = null;
 
   @VisibleForTesting
   JvmMetrics(String processName, String sessionId) {
@@ -96,6 +101,11 @@ public class JvmMetrics implements MetricsSource {
     this.pauseMonitor = pauseMonitor;
   }
 
+  public void setGcTimeMonitor(GcTimeMonitor gcTimeMonitor) {
+    Preconditions.checkNotNull(gcTimeMonitor);
+    this.gcTimeMonitor = gcTimeMonitor;
+  }
+
   public static JvmMetrics create(String processName, String sessionId,
                                   MetricsSystem ms) {
     return ms.register(JvmMetrics.name(), JvmMetrics.description(),
@@ -176,6 +186,11 @@ public class JvmMetrics implements MetricsSource {
       rb.addCounter(GcTotalExtraSleepTime,
           pauseMonitor.getTotalGcExtraSleepTime());
     }
+
+    if (gcTimeMonitor != null) {
+      rb.addCounter(GcTimePercentage,
+          gcTimeMonitor.getLatestGcData().getGcTimePercentage());
+    }
   }
 
   private MetricsInfo[] getGcInfo(String gcName) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java

@@ -51,7 +51,8 @@ public enum JvmMetricsInfo implements MetricsInfo {
   LogInfo("Total number of info log events"),
   GcNumWarnThresholdExceeded("Number of times that the GC warn threshold is exceeded"),
   GcNumInfoThresholdExceeded("Number of times that the GC info threshold is exceeded"),
-  GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds");
+  GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds"),
+  GcTimePercentage("Percentage of time the JVM was paused in GC");
 
   private final String desc;
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -1990,20 +1990,51 @@ public class UserGroupInformation {
     }
   }
 
-  public static void logAllUserInfo(UserGroupInformation ugi) throws
+  /**
+   * Log current UGI and token information into specified log.
+   * @param ugi - UGI
+   * @throws IOException
+   */
+  @InterfaceAudience.LimitedPrivate({"HDFS", "KMS"})
+  @InterfaceStability.Unstable
+  public static void logUserInfo(Logger log, String caption,
+      UserGroupInformation ugi) throws IOException {
+    if (log.isDebugEnabled()) {
+      log.debug(caption + " UGI: " + ugi);
+      for (Token<?> token : ugi.getTokens()) {
+        log.debug("+token:" + token);
+      }
+    }
+  }
+
+  /**
+   * Log all (current, real, login) UGI and token info into specified log.
+   * @param ugi - UGI
+   * @throws IOException
+   */
+  @InterfaceAudience.LimitedPrivate({"HDFS", "KMS"})
+  @InterfaceStability.Unstable
+  public static void logAllUserInfo(Logger log, UserGroupInformation ugi) throws
       IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("UGI: " + ugi);
+    if (log.isDebugEnabled()) {
+      logUserInfo(log, "Current", ugi.getCurrentUser());
       if (ugi.getRealUser() != null) {
-        LOG.debug("+RealUGI: " + ugi.getRealUser());
-      }
-      LOG.debug("+LoginUGI: " + ugi.getLoginUser());
-      for (Token<?> token : ugi.getTokens()) {
-        LOG.debug("+UGI token:" + token);
+        logUserInfo(log, "Real", ugi.getRealUser());
       }
+      logUserInfo(log, "Login", ugi.getLoginUser());
     }
   }
 
+  /**
+   * Log all (current, real, login) UGI and token info into UGI debug log.
+   * @param ugi - UGI
+   * @throws IOException
+   */
+  public static void logAllUserInfo(UserGroupInformation ugi) throws
+      IOException {
+    logAllUserInfo(LOG, ugi);
+  }
+
   private void print() throws IOException {
     System.out.println("User: " + getUserName());
     System.out.print("Group Ids: ");

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java

@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import com.google.common.base.Preconditions;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+/**
+ * This class monitors the percentage of time the JVM is paused in GC within
+ * the specified observation window, say 1 minute. The user can provide a
+ * hook which will be called whenever this percentage exceeds the specified
+ * threshold.
+ */
+public class GcTimeMonitor extends Thread {
+
+  private final long maxGcTimePercentage;
+  private final long observationWindowMs, sleepIntervalMs;
+  private final GcTimeAlertHandler alertHandler;
+
+  private final List<GarbageCollectorMXBean> gcBeans =
+      ManagementFactory.getGarbageCollectorMXBeans();
+  // Ring buffers containing GC timings and timestamps when timings were taken
+  private final TsAndData[] gcDataBuf;
+  private int bufSize, startIdx, endIdx;
+
+  private long startTime;
+  private final GcData curData = new GcData();
+  private volatile boolean shouldRun = true;
+
+  /**
+   * Create an instance of GCTimeMonitor. Once it's started, it will stay alive
+   * and monitor GC time percentage until shutdown() is called. If you don't
+   * put a limit on the number of GCTimeMonitor instances that you create, and
+   * alertHandler != null, you should necessarily call shutdown() once the given
+   * instance is not needed. Otherwise, you may create a memory leak, because
+   * each running GCTimeMonitor will keep its alertHandler object in memory,
+   * which in turn may reference and keep in memory many more other objects.
+   *
+   * @param observationWindowMs the interval over which the percentage
+   *   of GC time should be calculated. A practical value would be somewhere
+   *   between 30 sec and several minutes.
+   * @param sleepIntervalMs how frequently this thread should wake up to check
+   *   GC timings. This is also a frequency with which alertHandler will be
+   *   invoked if GC time percentage exceeds the specified limit. A practical
+   *   value would likely be 500..1000 ms.
+   * @param maxGcTimePercentage A GC time percentage limit (0..100) within
+   *   observationWindowMs. Once this is exceeded, alertHandler will be
+   *   invoked every sleepIntervalMs milliseconds until GC time percentage
+   *   falls below this limit.
+   * @param alertHandler a single method in this interface is invoked when GC
+   *   time percentage exceeds the specified limit.
+   */
+  public GcTimeMonitor(long observationWindowMs, long sleepIntervalMs,
+      int maxGcTimePercentage, GcTimeAlertHandler alertHandler) {
+    Preconditions.checkArgument(observationWindowMs > 0);
+    Preconditions.checkArgument(
+        sleepIntervalMs > 0 && sleepIntervalMs < observationWindowMs);
+    Preconditions.checkArgument(
+        maxGcTimePercentage >= 0 && maxGcTimePercentage <= 100);
+
+    this.observationWindowMs = observationWindowMs;
+    this.sleepIntervalMs = sleepIntervalMs;
+    this.maxGcTimePercentage = maxGcTimePercentage;
+    this.alertHandler = alertHandler;
+
+    bufSize = (int) (observationWindowMs / sleepIntervalMs + 2);
+    // Prevent the user from accidentally creating an abnormally big buffer,
+    // which will result in slow calculations and likely inaccuracy.
+    Preconditions.checkArgument(bufSize <= 128 * 1024);
+    gcDataBuf = new TsAndData[bufSize];
+    for (int i = 0; i < bufSize; i++) {
+      gcDataBuf[i] = new TsAndData();
+    }
+
+    this.setDaemon(true);
+    this.setName("GcTimeMonitor obsWindow = " + observationWindowMs +
+        ", sleepInterval = " + sleepIntervalMs +
+        ", maxGcTimePerc = " + maxGcTimePercentage);
+  }
+
+  @Override
+  public void run() {
+    startTime = System.currentTimeMillis();
+    curData.timestamp = startTime;
+    gcDataBuf[startIdx].setValues(startTime, 0);
+
+    while (shouldRun) {
+      try {
+        Thread.sleep(sleepIntervalMs);
+      } catch (InterruptedException ie) {
+        return;
+      }
+
+      calculateGCTimePercentageWithinObservedInterval();
+      if (alertHandler != null &&
+          curData.gcTimePercentage > maxGcTimePercentage) {
+        alertHandler.alert(curData.clone());
+      }
+    }
+  }
+
+  public void shutdown() {
+    shouldRun = false;
+  }
+
+  /** Returns a copy of the most recent data measured by this monitor. */
+  public GcData getLatestGcData() {
+    return curData.clone();
+  }
+
+  private void calculateGCTimePercentageWithinObservedInterval() {
+    long prevTotalGcTime = curData.totalGcTime;
+    long totalGcTime = 0;
+    long totalGcCount = 0;
+    for (GarbageCollectorMXBean gcBean : gcBeans) {
+      totalGcTime += gcBean.getCollectionTime();
+      totalGcCount += gcBean.getCollectionCount();
+    }
+    long gcTimeWithinSleepInterval = totalGcTime - prevTotalGcTime;
+
+    long ts = System.currentTimeMillis();
+    long gcMonitorRunTime = ts - startTime;
+
+    endIdx = (endIdx + 1) % bufSize;
+    gcDataBuf[endIdx].setValues(ts, gcTimeWithinSleepInterval);
+
+    // Move startIdx forward until we reach the first buffer entry with
+    // timestamp within the observation window.
+    long startObsWindowTs = ts - observationWindowMs;
+    while (gcDataBuf[startIdx].ts < startObsWindowTs && startIdx != endIdx) {
+      startIdx = (startIdx + 1) % bufSize;
+    }
+
+    // Calculate total GC time within observationWindowMs.
+    // We should be careful about GC time that passed before the first timestamp
+    // in our observation window.
+    long gcTimeWithinObservationWindow = Math.min(
+        gcDataBuf[startIdx].gcPause, gcDataBuf[startIdx].ts - startObsWindowTs);
+    if (startIdx != endIdx) {
+      for (int i = (startIdx + 1) % bufSize; i != endIdx;
+           i = (i + 1) % bufSize) {
+        gcTimeWithinObservationWindow += gcDataBuf[i].gcPause;
+      }
+    }
+
+    curData.update(ts, gcMonitorRunTime, totalGcTime, totalGcCount,
+        (int) (gcTimeWithinObservationWindow * 100 /
+          Math.min(observationWindowMs, gcMonitorRunTime)));
+  }
+
+  /**
+   * The user can provide an instance of a class implementing this interface
+   * when initializing a GcTimeMonitor to receive alerts when GC time
+   * percentage exceeds the specified threshold.
+   */
+  public interface GcTimeAlertHandler {
+    void alert(GcData gcData);
+  }
+
+  /** Encapsulates data about GC pauses measured at the specific timestamp. */
+  public static class GcData implements Cloneable {
+    private long timestamp;
+    private long gcMonitorRunTime, totalGcTime, totalGcCount;
+    private int gcTimePercentage;
+
+    /** Returns the absolute timestamp when this measurement was taken. */
+    public long getTimestamp() {
+      return timestamp;
+    }
+
+    /** Returns the time since the start of the associated GcTimeMonitor. */
+    public long getGcMonitorRunTime() {
+      return gcMonitorRunTime;
+    }
+
+    /** Returns accumulated GC time since this JVM started. */
+    public long getAccumulatedGcTime() {
+      return totalGcTime;
+    }
+
+    /** Returns the accumulated number of GC pauses since this JVM started. */
+    public long getAccumulatedGcCount() {
+      return totalGcCount;
+    }
+
+    /**
+     * Returns the percentage (0..100) of time that the JVM spent in GC pauses
+     * within the observation window of the associated GcTimeMonitor.
+     */
+    public int getGcTimePercentage() {
+      return gcTimePercentage;
+    }
+
+    private synchronized void update(long inTimestamp, long inGcMonitorRunTime,
+        long inTotalGcTime, long inTotalGcCount, int inGcTimePercentage) {
+      this.timestamp = inTimestamp;
+      this.gcMonitorRunTime = inGcMonitorRunTime;
+      this.totalGcTime = inTotalGcTime;
+      this.totalGcCount = inTotalGcCount;
+      this.gcTimePercentage = inGcTimePercentage;
+    }
+
+    @Override
+    public synchronized GcData clone() {
+      try {
+        return (GcData) super.clone();
+      } catch (CloneNotSupportedException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  private static class TsAndData {
+    private long ts;      // Timestamp when this measurement was taken
+    private long gcPause; // Total GC pause time within the interval between ts
+                          // and the timestamp of the previous measurement.
+
+    void setValues(long inTs, long inGcPause) {
+      this.ts = inTs;
+      this.gcPause = inGcPause;
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md

@@ -63,6 +63,14 @@ when the various labels are appropriate. As a general rule, all new interfaces
 and APIs should have the most limited labels (e.g. Private Unstable) that will
 not inhibit the intent of the interface or API.
 
+### Structure
+
+This document is arranged in sections according to the various compatibility
+concerns. Within each section an introductory text explains what compatibility
+means in that section, why it's important, and what the intent to support
+compatibility is. The subsequent "Policy" section then sets forth in specific
+terms what the governing policy is.
+
 ### Notational Conventions
 
 The key words "MUST" "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
@@ -77,12 +85,18 @@ flagged for removal. The standard meaning of the annotation is that the
 API element should not be used and may be removed in a later version.
 
 In all cases removing an element from an API is an incompatible
-change. In the case of [Stable](./InterfaceClassification.html#Stable) APIs,
-the change cannot be made between minor releases within the same major
-version. In addition, to allow consumers of the API time to adapt to the change,
-the API element to be removed should be marked as deprecated for a full major
-release before it is removed. For example, if a method is marked as deprecated
-in Hadoop 2.8, it cannot be removed until Hadoop 4.0.
+change. The stability of the element SHALL determine when such a change is
+permissible. A [Stable](./InterfaceClassification.html#Stable) element MUST
+be marked as deprecated for a full major release before it can be removed and
+SHALL NOT be removed in a minor or maintenance release. An
+[Evolving](./InterfaceClassification.html#Evolving) element MUST be marked as
+deprecated for a full minor release before it can be removed and SHALL NOT be
+removed during a maintenance release. An
+[Unstable](./InterfaceClassification.html#Unstable) element MAY be removed at
+any time. When possible an [Unstable](./InterfaceClassification.html#Unstable)
+element SHOULD be marked as deprecated for at least one release before being
+removed. For example, if a method is marked as deprecated in Hadoop 2.8, it
+cannot be removed until Hadoop 4.0.
 
 ### Policy
 
@@ -141,7 +155,7 @@ in hand.
 #### Semantic compatibility
 
 Apache Hadoop strives to ensure that the behavior of APIs remains consistent
-over versions, though changes for correctness may result in changes in
+across releases, though changes for correctness may result in changes in
 behavior. API behavior SHALL be specified by the JavaDoc API documentation
 where present and complete. When JavaDoc API documentation is not available,
 behavior SHALL be specified by the behavior expected by the related unit tests.
@@ -229,8 +243,8 @@ transports, such as SSL. Upgrading a service from SSLv2 to SSLv3 may break
 existing SSLv2 clients. The minimum supported major version of any transports
 MUST not increase across minor releases within a major version.
 
-Service ports are considered as part of the transport mechanism. Fixed
-service port numbers MUST be kept consistent to prevent breaking clients.
+Service ports are considered as part of the transport mechanism. Default
+service port numbers must be kept consistent to prevent breaking clients.
 
 #### Policy
 
@@ -281,9 +295,8 @@ according to the following:
 * Client-Server compatibility MUST be maintained so as to allow upgrading individual components without upgrading others. For example, upgrade HDFS from version 2.1.0 to 2.2.0 without upgrading MapReduce.
 * Server-Server compatibility MUST be maintained so as to allow mixed versions within an active cluster so the cluster may be upgraded without downtime in a rolling fashion.
 
-New transport mechanisms MUST only be introduced with minor or major version
-changes. Existing transport mechanisms MUST continue to be supported across
-minor versions within a major version. Service port numbers MUST remain
+Existing transport mechanisms MUST continue to be supported across
+minor versions within a major version. Default service port numbers MUST remain
 consistent across minor version numbers within a major version.
 
 ### REST APIs
@@ -312,7 +325,8 @@ The Hadoop REST APIs SHALL be considered
 numbers, the Hadoop REST APIs SHALL be considered
 [Public](./InterfaceClassification.html#Public) and
 [Stable](./InterfaceClassification.html#Stable), i.e. no incompatible changes
-are allowed to within an API version number.
+are allowed to within an API version number. A REST API version must be labeled
+as deprecated for a full major release before it can be removed.
 
 ### Log Output
 
@@ -325,7 +339,9 @@ use cases are also supported.
 
 All log output SHALL be considered
 [Public](./InterfaceClassification.html#Public) and
-[Evolving](./InterfaceClassification.html#Evolving).
+[Unstable](./InterfaceClassification.html#Unstable). For log output, an
+incompatible change is one that renders a parser unable to find or recognize
+a line of log output.
 
 ### Audit Log Output
 
@@ -400,37 +416,42 @@ server's jhist file format, SHALL be considered
 
 HDFS persists metadata (the image and edit logs) in a private file format.
 Incompatible changes to either the format or the metadata prevent subsequent
-releases from reading older metadata. Incompatible changes MUST include a
-process by which existing metadata may be upgraded. Changes SHALL be
-allowed to require more than one upgrade. Incompatible changes MUST result in
-the metadata version number being incremented.
+releases from reading older metadata. Incompatible changes must include a
+process by which existing metadata may be upgraded.
 
 Depending on the degree of incompatibility in the changes, the following
 potential scenarios can arise:
 
 * Automatic: The image upgrades automatically, no need for an explicit "upgrade".
-* Direct: The image is upgradable, but might require one explicit release "upgrade".
-* Indirect: The image is upgradable, but might require upgrading to intermediate release(s) first.
+* Direct: The image is upgradeable, but might require one explicit release "upgrade".
+* Indirect: The image is upgradeable, but might require upgrading to intermediate release(s) first.
 * Not upgradeable: The image is not upgradeable.
 
-HDFS data nodes store data in a private directory structure. The schema of that
-directory structure must remain stable to retain compatibility.
+HDFS data nodes store data in a private directory structure. Incompatible
+changes to the directory structure may prevent older releases from accessing
+stored data. Incompatible changes must include a process by which existing
+data directories may be upgraded.
 
 ###### Policy
 
 The HDFS metadata format SHALL be considered
 [Private](./InterfaceClassification.html#Private) and
 [Evolving](./InterfaceClassification.html#Evolving). Incompatible
-changes MUST include a process by which existing metada may be upgraded. The
-upgrade process MUST allow the cluster metadata to be rolled back to the older
-version and its older disk format. The rollback MUST restore the original data
-but is not REQUIRED to restore the updated data. Any incompatible change
-to the format MUST result in the major version number of the schema being
-incremented.
+changes MUST include a process by which existing metadata may be upgraded.
+The upgrade process SHALL be allowed to require more than one upgrade.
+The upgrade process MUST allow the cluster metadata to be
+rolled back to the older version and its older disk format. The rollback
+MUST restore the original data but is not REQUIRED to restore the updated
+data. Any incompatible change to the format MUST result in the major version
+number of the schema being incremented.
 
 The data node directory format SHALL be considered
 [Private](./InterfaceClassification.html#Private) and
-[Evolving](./InterfaceClassification.html#Evolving).
+[Evolving](./InterfaceClassification.html#Evolving). Incompatible
+changes MUST include a process by which existing data directories may be
+upgraded. The upgrade process SHALL be allowed to require more than one upgrade.
+The upgrade process MUST allow the data directories to be
+rolled back to the older layout.
 
 ##### AWS S3A Guard Metadata
 
@@ -533,15 +554,42 @@ The Hadoop Web UI SHALL be considered
 [Public](./InterfaceClassification.html#Public) and
 [Unstable](./InterfaceClassification.html#Unstable).
 
+### Functional Compatibility
+
+Users depend on the behavior of a Hadoop cluster remaining consistent across
+releases. Changes which cause unexpectedly different behaviors from the cluster
+can lead to frustration and long adoption cycles. No new configuration should
+be added which changes the behavior of an existing
+cluster, assuming the cluster's configuration files remain unchanged. For any
+new settings that are defined, care should be taken to ensure that the new
+setting does not change the behavior of existing clusters.
+
+#### Policy
+
+Changes to existing functionality MUST NOT change the default behavior or the
+meaning of existing configuration settings between maintenance releases within
+the same minor version, regardless of whether the changes arise from changes
+to the system or logic or to internal or external default configuration values.
+
+Changes to existing functionality SHOULD NOT change the default behavior or the
+meaning of existing configuration settings between minor releases within
+the same major version, though changes, such as to fix correctness or
+security issues, may require incompatible behavioral changes. Where possible
+such behavioral changes SHOULD be off by default.
+
 ### Hadoop Configuration Files
 
 Users use Hadoop-defined properties to configure and provide hints to Hadoop and
 custom properties to pass information to jobs. Users are encouraged to avoid
 using custom configuration property names that conflict with the namespace of
 Hadoop-defined properties and should avoid using any prefixes used by Hadoop,
-e.g. hadoop, io, ipc, fs, net, file, ftp, s3, kfs, ha, file, dfs, mapred,
+e.g. hadoop, io, ipc, fs, net, file, ftp, kfs, ha, file, dfs, mapred,
 mapreduce, and yarn.
 
+In addition to properties files, Hadoop uses other configuration files to
+set system behavior, such as the fair scheduler configuration file or the
+resource profiles configuration file.
+
 #### Policy
 
 Hadoop-defined properties (names and meanings) SHALL be considered
@@ -552,6 +600,14 @@ across major versions. Default values of Hadoop-defined properties SHALL be
 considered [Public](./InterfaceClassification.html#Public) and
 [Evolving](./InterfaceClassification.html#Evolving).
 
+Hadoop configuration files that are not governed by the above rules about
+Hadoop-defined properties SHALL be considered
+[Public](./InterfaceClassification.html#Public) and
+[Stable](./InterfaceClassification.html#Stable). The definition of an
+incompatible change depends on the particular configuration file format, but
+the general rule is that a compatible change will allow a configuration
+file that was valid before the change to remain valid after the change.
+
 ### Log4j Configuration Files
 
 The log output produced by Hadoop daemons and CLIs is governed by a set of
@@ -568,11 +624,11 @@ All Log4j configurations SHALL be considered
 ### Directory Structure
 
 Source code, artifacts (source and tests), user logs, configuration files,
-output, and job history are all stored on disk either local file system or HDFS.
-Changing the directory structure of these user-accessible files can break
-compatibility, even in cases where the original path is preserved via symbolic
-links (such as when the path is accessed by a servlet that is configured to
-not follow symbolic links).
+output, and job history are all stored on disk on either the local file system
+or HDFS. Changing the directory structure of these user-accessible files can
+break compatibility, even in cases where the original path is preserved via
+symbolic links (such as when the path is accessed by a servlet that is
+configured to not follow symbolic links).
 
 #### Policy
 
@@ -680,11 +736,11 @@ upgrading other dependent software components.
 #### Policies
 
 * Hardware
-    * Architecture: The community has no plans to restrict Hadoop to specific architectures, but can have family-specific optimizations.
+    * Architecture: Intel and AMD are the processor architectures currently supported by the community. The community has no plans to restrict Hadoop to specific architectures, but MAY have family-specific optimizations. Support for any processor architecture SHOULD NOT be dropped without first being documented as deprecated for a full major release and MUST NOT be dropped without first being deprecated for at least a full minor release.
     * Minimum resources: While there are no guarantees on the minimum resources required by Hadoop daemons, the developer community SHOULD avoid increasing requirements within a minor release.
-* Operating Systems: The community SHOULD maintain the same minimum OS requirements (OS kernel versions) within a minor release. Currently GNU/Linux and Microsoft Windows are the OSes officially supported by the community, while Apache Hadoop is known to work reasonably well on other OSes such as Apple MacOSX, Solaris, etc.
+* Operating Systems: The community SHOULD maintain the same minimum OS requirements (OS kernel versions) within a minor release. Currently GNU/Linux and Microsoft Windows are the OSes officially supported by the community, while Apache Hadoop is known to work reasonably well on other OSes such as Apple MacOSX, Solaris, etc. Support for any OS SHOULD NOT be dropped without first being documented as deprecated for a full major release and MUST NOT be dropped without first being deprecated for at least a full minor release.
 * The JVM requirements SHALL NOT change across minor releases within the same major release unless the JVM version in question becomes unsupported. The JVM version requirement MAY be different for different operating systems or even operating system releases.
-* File systems supported by Hadoop, e.g. through the HDFS FileSystem API, SHOULD not become unsupported between minor releases within a major version unless a migration path to an alternate client implementation is available.
+* File systems supported by Hadoop, e.g. through the FileSystem API, SHOULD not become unsupported between minor releases within a major version unless a migration path to an alternate client implementation is available.
 
 References
 ----------

hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md

@@ -0,0 +1,432 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+Apache Hadoop Downstream Developer's Guide
+==========================================
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+Purpose
+-------
+
+The point of this document is to provide downstream developers with a clear
+reference for what to expect when building applications against the Hadoop
+source base. This document is primarily a distillation of the
+[Hadoop Compatibility Guidelines](./Compatibility.html) and hence focuses on
+what the compatibility guarantees are for the various Hadoop interfaces
+across releases.
+
+### Target Audience
+
+The target audience for this document is any developer working on a project
+or application that builds or depends on Apache Hadoop, whether the
+dependency is on the source code itself, a build artifact, or interacting
+with a running system.
+
+Hadoop Releases
+---------------
+
+The Hadoop development community periodically produces new Hadoop releases to
+introduce new functionality and fix existing issues. Realeses fall into three
+categories:
+
+* Major: a major release will typically include significant new functionality and generally represents the largest upgrade compatibility risk. A major release increments the first number of the release version, e.g. going from 2.8.2 to 3.0.0.
+* Minor: a minor release will typically include some new functionality as well as fixes for some notable issues. A minor release should not pose much upgrade risk in most cases. A minor release increments the middle number of release version, e.g. going from 2.8.2 to 2.9.0.
+* Maintenance: a maintenance release should not include any new functionality. The purpose of a maintenance release is to resolve a set of issues that are deemed by the developer community to be significant enough to be worth pushing a new release to address them. Maintenance releases should pose very little upgrade risk. A maintenance release increments the final number in the release version, e.g. going from 2.8.2 to 2.8.3.
+
+Consuming Hadoop APIs
+---------------------
+
+When writing software that calls methods or uses classes that belong to Apache
+Hadoop, developers should adhere to the following guidelines. Failure to adhere
+to the guidelines may result in problems transitioning from one Hadoop release
+to another.
+
+### Privacy
+
+Packages, classes, and methods may be annotated with an audience annotation.
+The three privacy levels are:
+[Public](./InterfaceClassification.html#Public),
+[Limited-Private](./InterfaceClassification.html#Limited-Private), and
+[Private](./InterfaceClassification.html#Private). Downstream developers should
+only use packages, classes, methods, and fields that are marked as
+[Public](./InterfaceClassification.html#Public). Packages, classes, and methods
+that are not marked as [Public](./InterfaceClassification.html#Public) are
+considered internal to Hadoop and are intended only for consumption by other
+components of Hadoop.
+
+If an element has an annotation that conflicts with it's containing element's
+annotation, then the most restrictive annotation takes precedence. For example,
+If a [Private](./InterfaceClassification.html#Private) method is contained
+in a [Public](./InterfaceClassification.html#Public) class, then the method
+should be treated as [Private](./InterfaceClassification.html#Private). If a
+[Public](./InterfaceClassification.html#Public) method is contained in a
+[Private](./InterfaceClassification.html#Private) class, the method should be
+treated as [Private](./InterfaceClassification.html#Private).
+
+If a method has no privacy annotation, then it inherits its privacy from its
+class. If a class has no privacy, it inherits its privacy from its package.
+If a package has no privacy, it should be assumed to be
+[Private](./InterfaceClassification.html#Private).
+
+### Stability
+
+Packages, classes, and methods may be annotated with a stability annotation.
+There are three classes of stability:
+[Stable](./InterfaceClassification.html#Stable),
+[Evolving](./InterfaceClassification.html#Evolving), and
+[Unstable](./InterfaceClassification.html#Unstable). The stability annotations
+determine when
+[incompatible changes](./InterfaceClassification.html#Change-Compatibility)
+are allowed to be made. [Stable](./InterfaceClassification.html#Stable) means
+that no incompatible changes are allowed between major releases.
+[Evolving](./InterfaceClassification.html#Evolving) means no incompatible
+changes are allowed between minor releases.
+[Unstable](./InterfaceClassification.html#Unstable) means that incompatible
+changes are allowed at any time. As a downstream developer, it is best to
+avoid [Unstable](./InterfaceClassification.html#Unstable) APIs and where
+possible to prefer [Stable](./InterfaceClassification.html#Stable) ones.
+
+If a method has no stability annotation, then it inherits its stability from its
+class. If a class has no stability, it inherits its stability from its package.
+If a package has no stability, it should be assumed to be
+[Unstable](./InterfaceClassification.html#Unstable).
+
+
+#### Releases and Stability
+
+Per the above rules on API stability, new releases are allowed to change APIs
+as follows:
+
+| Release Type | Stable API Changes | Evolving API Changes | Unstable API Changes |
+|:---- |:---- |:---- |:---- |
+| Major | Allowed | Allowed | Allowed |
+| Minor | Not Allowed | Allowed | Allowed |
+| Maintenance | Not Allowed | Not Allowed | Allowed |
+
+Note that a major release is *allowed* to break compatibility of any API, even
+though the Hadoop developer community strives to maintain compatibility as much
+as possible, even across major releases. Note also that an
+[Unstable](./InterfaceClassification.html#Unstable) API may change at any time
+without notice.
+
+### Deprecation
+
+Classes or methods that are annotated as @Deprecated are no longer safe to use.
+The deprecated element should continue to function, but may and likely will be
+removed in a subsequent release. The stability annotation
+will determine the earliest release when the deprecated element can be removed.
+A [Stable](./InterfaceClassification.html#Stable) element cannot be removed
+until the next major release. An
+[Evolving](./InterfaceClassification.html#Evolving) element cannot be removed
+until the next minor release. An
+[Unstable](./InterfaceClassification.html#Unstable) element may be removed at
+any time and will typically not be marked as deprecated before it is removed.
+[Stable](./InterfaceClassification.html#Stable) and
+[Evolving](./InterfaceClassification.html#Evolving) elements must be marked as
+deprecated for a full major or minor release (respectively) before they can be
+removed. For example, if a [Stable](./InterfaceClassification.html#Stable)
+is marked as deprecated in Hadoop 3.1, it cannot be removed until Hadoop 5.0.
+
+### Semantic Compatibility
+
+The Apache Hadoop developer community strives to ensure that the behavior of
+APIs remains consistent across releases, though changes for correctness may
+result in changes in behavior. The API JavaDocs are considered the primary
+authority for the expected behavior of an API. In cases where the
+JavaDocs are insufficient or missing, the unit tests are considered the
+fallback authority for expected behavior. Where unit tests are not present,
+the intended behavior should be inferred from the naming. As much as possible
+downstream developers should avoid looking at the source code for the API
+itself to determine expected behavior as that approach can create dependencies
+on implementation details that are not expressly held as expected behavior by
+the Hadoop development community.
+
+In cases where the JavaDocs are insufficient to infer expected behavior,
+downstream developers are strongly encouraged to file a Hadoop JIRA to request
+the JavaDocs be added or improved.
+
+Be aware that fixes done for correctness reasons may cause changes to the
+expected behavior of an API, though such changes are expected to be
+accompanied by documentation that clarifies the new behavior.
+
+The Apache Hadoop developer community tries to maintain binary compatibility
+for end user applications across releases. Ideally no updates to applications
+will be required when upgrading to a new Hadoop release, assuming the
+application does not use [Private](./InterfaceClassification.html#Private),
+[Limited-Private](./InterfaceClassification.html#Limited-Private), or
+[Unstable](./InterfaceClassification.html#Unstable) APIs. MapReduce applications
+in particular are guaranteed binary compatibility across releases.
+
+### Compatibility Issues
+
+The [Hadoop Compatibility Specification](./Compatibility.html) states the
+standards that the Hadoop development community is expected to uphold, but for
+various reasons, the source code may not live up to the ideals of the
+[compatibility specification](./Compatibility.html).
+
+Two common issues that a downstream developer will encounter are:
+
+1. APIs that are needed for application development aren't
+[Public](./InterfaceClassification.html#Public)
+2. A [Public](./InterfaceClassification.html#Public) API on which a downstream
+application depends is changed unexpectedly and incompatibly.
+
+In both of these cases, downstream developers are strongly encouraged to raise
+the issues with the Hadoop developer community either by sending an email to
+the appropriate
+[developer mailing list](https://hadoop.apache.org/mailing_lists.html) or
+[filing a JIRA](https://hadoop.apache.org/issue_tracking.html) or both. The
+developer community appreciates the feedback.
+
+Downstream developers are encouraged to reach out to the Hadoop development
+community in any case when they encounter an issue while developing an
+application against Hadoop. Odds are good that if it's an issue for one
+developer, it's an issue that numerous developers have or will encounter.
+
+Using the FileSystem API
+------------------------
+
+In the specific case of working with streams in Hadoop, e.g.
+`FSDataOutputStream`, an application can programmatically query for the
+capabilities of the stream using the methods of the
+[StreamCapabilities](http://hadoop.apache.org/docs/current/api/org/apache/hadoop/fs/StreamCapabilities.html)
+class. Dynamically adjusting to stream capabilities can make an applcation
+more robust in the face of changing implementations and environments.
+
+Consuming Hadoop REST APIs
+--------------------------
+
+The Hadoop REST APIs are a primary interface for a variety of downstream
+and internal applications and services. To support REST clients, the Hadoop
+REST APIs are versioned and will not change incompatibly within a version.
+Both the endpoint itself along with the list of supported parameters and the
+output from the endpoint are prohibited from changing incompatibly within a
+REST endpoint version. Note, however, that introducing new fields and other
+additive changes are considered compatible changes, so any consumer of the
+REST API should be flexible enough to ignore unknown fields.
+
+The REST API version is a single number and has no relationship with the Hadoop
+version number. The version number is encoded in the endpoint URL prefixed
+with a 'v', for example 'v1'. A new REST endpoint version may only be
+introduced with a minor or major release. A REST endpoint version may only be
+removed after being labeled as deprecated for a full major release.
+
+Consuming Hadoop Output
+-----------------------
+
+Hadoop produces a variety of outputs that could conceivably be consumed by
+application clients or downstream libraries. When consuming output from Hadoop,
+please consider the following:
+
+* Hadoop log output is not expected to change with a maintenance release unless it resolves a correctness issue. While log output can be consumed by software directly, it is intended primarily for a human reader.
+* Hadoop produces audit logs for a variety of operations. The audit logs are intended to be machine readable, though the addition of new records and fields are considered to be compatible changes. Any consumer of the audit logs should allow for unexpected records and fields. The audit log format must not change incompatibly between major releases.
+* Metrics data produced by Hadoop is mostly intended for automated consumption. The metrics format may not change in an incompatible way between major releases, but new records and fields can be compatibly added at any time. Consumers of the metrics data should allow for unknown records and fields.
+
+Consuming Hadoop Data
+---------------------
+
+Binary file formats used by Hadoop to store data, such as sequence files, HAR
+files, etc, are guaranteed to remain compatible between minor releases. In
+addition, in cases where changes are made between major releases,
+both backward and forward compatibility must be maintained. Note that only the
+sequence file format is guaranteed not to change incompatibly, not the
+serialized classes that are contained therein.
+
+In addition to the data produced by operations, Hadoop maintains its state
+information in a variety of data stores in various formats, such as the HDFS
+metadata store, the YARN resource manager state store, and the YARN federation
+state store. All Hadoop internal data stores are considered internal and
+[Private](./InterfaceClassification.html#Private) to Hadoop. Downstream
+developers should not attempt to consume data from the Hadoop state store as
+the data and/or data format may change unpredictably.
+
+Automating Operations with the Hadoop CLI
+-----------------------------------------
+
+The set of tools that make up the Hadoop command-line interface are intended
+both for consumption by end users and by downstream developers who are
+creating tools that execute the CLI tools and parse the output. For this reason
+the Hadoop CLI tools are treated like an interface and held stable between
+major releases. Between major releases, no CLI tool options will be removed or
+change semantically. The output from CLI tools will likewise remain the same
+within a major version number. Note that any change to CLI tool output is
+considered an incompatible change, so between major version, the CLI output
+will not change. Note that the CLI tool output is distinct from
+the log output produced by the CLI tools. Log output is not intended for
+automated consumption and may change at any time.
+
+Consuming the Hadoop Web UI
+---------------------------
+
+The web UIs that are exposed by Hadoop are for human consumption only. Scraping
+the UIs for data is not a supported use case. No effort is made to ensure any
+kind of compatibility between the data displayed in any of the web UIs
+across releases.
+
+Working with Hadoop configurations
+----------------------------------
+
+Hadoop uses two primary forms of configuration files: XML configuration files
+and logging configuration files.
+
+### XML Configuration Files
+
+The XML configuration files contain a set of properties as name-value pairs.
+The names and meanings of the properties are defined by Hadoop and are
+guaranteed to be stable across minor releases. A property can only be removed
+in a major release and only if it has been marked as deprecated for at least a
+full major release. Most properties have a default value that will be used if
+the property is not explicitly set in the XML configuration files. The default
+property values will not be changed during a maintenance release.  For details
+about the properties supported by the various Hadoop components,
+see the component documentation.
+
+Downstream developers and users can add their own properties into the XML
+configuration files for use by their tools and applications. While Hadoop
+makes no formal restrictions about defining new properties, a new property
+that conflicts with a property defined by Hadoop can lead to unexpected and
+undesirable results. Users are encouraged to avoid using custom configuration
+property names that conflict with the namespace of Hadoop-defined properties
+and thus should avoid using any prefixes used by Hadoop,
+e.g. hadoop, io, ipc, fs, net, file, ftp, kfs, ha, file, dfs, mapred,
+mapreduce, and yarn.
+
+### Logging Configuration Files
+
+The log output produced by Hadoop daemons and CLIs is governed by a set of
+configuration files. These files control the minimum level of log message that
+will be output by the various components of Hadoop, as well as where and how
+those messages are stored. Between minor releases no changes will be made to
+the log configuration that reduce, eliminate, or redirect the log messages.
+
+### Other Configuration Files
+
+Hadoop makes use of a number of other types of configuration files in a variety
+of formats, such as the JSON resource profiles configuration or the XML fair
+scheduler configuration. No incompatible changes will be introduced to the
+configuration file formats within a minor release. Even between minor releases
+incompatible configuration file format changes will be avoided if possible.
+
+Using and Consuming Hadoop Artifacts
+------------------------------------
+
+### Source and Configuration Files
+
+As a downstream developer or consumer of Hadoop, it's possible to access all
+elements of the Hadoop platform, including source code, configuration files,
+build artifacts, etc. While the open nature of the platform allows it,
+developers should not create dependencies on these internal details of Hadoop
+as they may change at any time. The Hadoop development community will attempt,
+however, to keep the existing structure stable within a major version.
+
+The location and general structure of the Hadoop configuration files, job
+history information (as consumed by the job history server), and logs files
+generated by Hadoop will be maintained across maintenance releases.
+
+### Build Artifacts
+
+The build artifacts produced by the Hadoop build process, e.g. JAR files, are
+subject to change at any time and should not be treated as reliable, except
+for the client artifacts. Client artifacts and their contents will remain
+compatible within a major release. It is the goal of the Hadoop development
+community to allow application code to continue to function unchanged across
+minor releases and, whenever possible, across major releases. The current list
+of client artifacts is as follows:
+
+* hadoop-client
+* hadoop-client-api
+* hadoop-client-minicluster
+* hadoop-client-runtime
+* hadoop-hdfs-client
+* hadoop-hdfs-native-client
+* hadoop-mapreduce-client-app
+* hadoop-mapreduce-client-common
+* hadoop-mapreduce-client-core
+* hadoop-mapreduce-client-jobclient
+* hadoop-mapreduce-client-nativetask
+* hadoop-yarn-client
+
+### Environment Variables
+
+Some Hadoop components receive information through environment variables. For
+example, the ```HADOOP_OPTS``` environment variable is interpreted by most
+Hadoop processes as a string of additional JVM arguments to be used when
+starting a new JVM. Between minor releases the way Hadoop interprets environment
+variables will not change in an incompatible way. In other words, the same value
+placed into the same variable should produce the same result for all Hadoop
+releases within the same major version.
+
+### Library Dependencies
+
+Hadoop relies on a large number of third-party libraries for its operation. As
+much as possible the Hadoop developer community works to hide these dependencies
+from downstream developers. Some common libraries, such as Guava, could cause
+significant compatibility issues between Hadoop and downstream applications if
+those dependencies were exposed downstream. Nonetheless Hadoop does expose some
+of its dependencies, especially prior to Hadoop 3. No new dependency
+will be exposed by Hadoop via the client artifacts between major releases.
+
+A common downstream anti-pattern is to use the output of ```hadoop classpath```
+to set the downstream application's classpath or add all third-party JARs
+included with Hadoop to the downstream application's classpath. This practice
+creates a tight coupling between the downstream application and Hadoop's
+third-party dependencies, which leads to a fragile application that is hard to
+maintain as Hadoop's dependencies change. This practice is strongly discouraged.
+
+Hadoop depends on the Java virtual machine for its operation, which can
+impact downstream applications. To minimize disruption, the minimum supported
+version of the JVM will not change between major releases of Hadoop. In the
+event that the current minimum supported JVM version becomes unsupported
+between major releases, the minimum supported JVM version may be changed in a
+minor release.
+
+Hadoop also includes several native components, including compression, the
+container executor binary, and various native integrations. These native
+components introduce a set of native dependencies for Hadoop. The set of
+native dependencies can change in a minor release, but the Hadoop developer
+community will try to limit any dependency version changes to minor version
+changes as much as possible.
+
+### Hardware and OS Dependencies
+
+Hadoop is currently supported by the Hadoop developer community on Linux and
+Windows running on x86 and AMD processors. These OSes and processors are likely
+to remain supported for the foreseeable future. In the event that support plans
+change, the OS or processor to be dropped will be documented as deprecated
+for at least a full minor release, but ideally a full major release, before
+actually being dropped. Hadoop may function on other OSes and processor
+architectures, but the community may not be able to provide assistance in the
+event of issues.
+
+There are no guarantees on how the minimum resources required by Hadoop daemons
+will change between releases, even maintenance releases. Nonetheless, the
+Hadoop developer community will try to avoid increasing the requirements within
+a minor release.
+
+Any file systems supported Hadoop, such as through the FileSystem API, will
+in most cases continue to be supported throughout a major release. The only
+case where support for a file system can be dropped within a major version is
+if a clean migration path to an alternate client implementation is provided.
+
+Questions
+---------
+
+For question about developing applications and projects against Apache Hadoop,
+please contact the developer mailing list for the relevant component(s):
+
+* [dev-common](mailto:dev-common@apache.org)
+* [dev-hdfs](mailto:dev-hdfs@apache.org)
+* [dev-mapreduce](mailto:dev-mapreduce@apache.org)
+* [dev-yarn](mailto:dev-yarn@apache.org)

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -1359,7 +1359,8 @@ problems were not considered during the implementation of these loops.
 ## <a name="StreamCapability"></a> interface `StreamCapabilities`
 
 The `StreamCapabilities` provides a way to programmatically query the
-capabilities that an `OutputStream` supports.
+capabilities that `OutputStream`, `InputStream`, or other FileSystem class
+supports.
 
 ```java
 public interface StreamCapabilities {
@@ -1369,12 +1370,16 @@ public interface StreamCapabilities {
 
 ### `boolean hasCapability(capability)`
 
-Return true if the `OutputStream` has the desired capability.
+Return true if the `OutputStream`, `InputStream`, or other FileSystem class
+has the desired capability.
 
 The caller can query the capabilities of a stream using a string value.
-It currently supports to query:
-
- * `StreamCapabilties.HFLUSH` ("*hflush*"): the capability to flush out the data
- in client's buffer.
- * `StreamCapabilities.HSYNC` ("*hsync*"): capability to flush out the data in
- client's buffer and the disk device.
+Here is a table of possible string values:
+
+String       | Constant   | Implements       | Description
+-------------|------------|------------------|-------------------------------
+hflush       | HFLUSH     | Syncable         | Flush out the data in client's user buffer. After the return of this call, new readers will see the data.
+hsync        | HSYNC      | Syncable         | Flush out the data in client's user buffer all the way to the disk device (but the disk may have it in its cache). Similar to POSIX fsync.
+in:readahead | READAHEAD  | CanSetReadahead  | Set the readahead on the input stream.
+dropbehind   | DROPBEHIND | CanSetDropBehind | Drop the cache.
+in:unbuffer  | UNBUFFER   | CanUnbuffer      | Reduce the buffering on the input stream.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java

@@ -25,15 +25,23 @@ import org.junit.Test;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static org.junit.Assert.assertTrue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 
 /**
  * Base class for comparing fields in one or more Configuration classes
@@ -74,6 +82,15 @@ import static org.junit.Assert.assertTrue;
 @Ignore
 public abstract class TestConfigurationFieldsBase {
 
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestConfigurationFieldsBase.class);
+
+  private static final Logger LOG_CONFIG = LoggerFactory.getLogger(
+      "org.apache.hadoop.conf.TestConfigurationFieldsBase.config");
+
+  private static final Logger LOG_XML = LoggerFactory.getLogger(
+      "org.apache.hadoop.conf.TestConfigurationFieldsBase.xml");
+
   /**
    * Member variable for storing xml filename.
    */
@@ -98,27 +115,27 @@ public abstract class TestConfigurationFieldsBase {
 
   /**
    * Set of properties to skip extracting (and thus comparing later) in 
-   * extractMemberVariablesFromConfigurationFields.
+   * {@link #extractMemberVariablesFromConfigurationFields(Field[])}.
    */
-  protected Set<String> configurationPropsToSkipCompare = null;
+  protected Set<String> configurationPropsToSkipCompare = new HashSet<>();
 
   /**
    * Set of property prefixes to skip extracting (and thus comparing later)
    * in * extractMemberVariablesFromConfigurationFields.
    */
-  protected Set<String> configurationPrefixToSkipCompare = null;
+  protected Set<String> configurationPrefixToSkipCompare = new HashSet<>();
 
   /**
    * Set of properties to skip extracting (and thus comparing later) in 
    * extractPropertiesFromXml.
    */
-  protected Set<String> xmlPropsToSkipCompare = null;
+  protected Set<String> xmlPropsToSkipCompare = new HashSet<>();
 
   /**
    * Set of property prefixes to skip extracting (and thus comparing later)
    * in extractPropertiesFromXml.
    */
-  protected Set<String> xmlPrefixToSkipCompare = null;
+  protected Set<String> xmlPrefixToSkipCompare = new HashSet<>();
 
   /**
    * Member variable to store Configuration variables for later comparison.
@@ -155,13 +172,6 @@ public abstract class TestConfigurationFieldsBase {
   @SuppressWarnings("checkstyle:visibilitymodifier")
   protected Set<String> filtersForDefaultValueCollisionCheck = new HashSet<>();
 
-  /**
-   * Member variable for debugging base class operation
-   */
-  protected boolean configDebug = false;
-  protected boolean xmlDebug = false;
-  protected boolean defaultDebug = false;
-
   /**
    * Abstract method to be used by subclasses for initializing base
    * members.
@@ -173,27 +183,24 @@ public abstract class TestConfigurationFieldsBase {
    * variables from a Configuration type class.
    *
    * @param fields The class member variables
-   * @return HashMap containing <StringValue,MemberVariableName> entries
+   * @return HashMap containing (StringValue, MemberVariableName) entries
    */
   private HashMap<String,String>
       extractMemberVariablesFromConfigurationFields(Field[] fields) {
     // Sanity Check
-    if (fields==null)
+    if (fields == null)
       return null;
 
-    HashMap<String,String> retVal = new HashMap<String,String>();
+    HashMap<String,String> retVal = new HashMap<>();
 
     // Setup regexp for valid properties
     String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
     Pattern p = Pattern.compile(propRegex);
 
     // Iterate through class member variables
-    int totalFields = 0;
     String value;
     for (Field f : fields) {
-      if (configDebug) {
-        System.out.println("Field: " + f);
-      }
+      LOG_CONFIG.debug("Field: {}", f);
       // Filter out anything that isn't "public static final"
       if (!Modifier.isStatic(f.getModifiers()) ||
           !Modifier.isPublic(f.getModifiers()) ||
@@ -217,9 +224,7 @@ public abstract class TestConfigurationFieldsBase {
       } catch (IllegalAccessException iaException) {
         continue;
       }
-      if (configDebug) {
-        System.out.println("  Value: " + value);
-      }
+      LOG_CONFIG.debug("  Value: {}", value);
       // Special Case: Detect and ignore partial properties (ending in x)
       //               or file properties (ending in .xml)
       if (value.endsWith(".xml") ||
@@ -227,20 +232,16 @@ public abstract class TestConfigurationFieldsBase {
           value.endsWith("-"))
         continue;
       // Ignore known configuration props
-      if (configurationPropsToSkipCompare != null) {
-        if (configurationPropsToSkipCompare.contains(value)) {
-          continue;
-        }
+      if (configurationPropsToSkipCompare.contains(value)) {
+        continue;
       }
       // Ignore known configuration prefixes
       boolean skipPrefix = false;
-      if (configurationPrefixToSkipCompare != null) {
-        for (String cfgPrefix : configurationPrefixToSkipCompare) {
-	  if (value.startsWith(cfgPrefix)) {
-            skipPrefix = true;
-            break;
-	  }
-	}
+      for (String cfgPrefix : configurationPrefixToSkipCompare) {
+        if (value.startsWith(cfgPrefix)) {
+          skipPrefix = true;
+          break;
+        }
       }
       if (skipPrefix) {
         continue;
@@ -249,22 +250,16 @@ public abstract class TestConfigurationFieldsBase {
       //                  something like: blah.blah2(.blah3.blah4...)
       Matcher m = p.matcher(value);
       if (!m.find()) {
-        if (configDebug) {
-          System.out.println("  Passes Regex: false");
-        }
+        LOG_CONFIG.debug("  Passes Regex: false");
         continue;
       }
-      if (configDebug) {
-        System.out.println("  Passes Regex: true");
-      }
+      LOG_CONFIG.debug("  Passes Regex: true");
 
       // Save member variable/value as hash
       if (!retVal.containsKey(value)) {
         retVal.put(value,f.getName());
       } else {
-        if (configDebug) {
-          System.out.println("ERROR: Already found key for property " + value);
-        }
+        LOG_CONFIG.debug("ERROR: Already found key for property " + value);
       }
     }
 
@@ -275,11 +270,10 @@ public abstract class TestConfigurationFieldsBase {
    * Pull properties and values from filename.
    *
    * @param filename XML filename
-   * @return HashMap containing <Property,Value> entries from XML file
+   * @return HashMap containing &lt;Property,Value&gt; entries from XML file
    */
-  private HashMap<String,String> extractPropertiesFromXml
-      (String filename) {
-    if (filename==null) {
+  private HashMap<String,String> extractPropertiesFromXml(String filename) {
+    if (filename == null) {
       return null;
     }
 
@@ -288,48 +282,28 @@ public abstract class TestConfigurationFieldsBase {
     conf.setAllowNullValueProperties(true);
     conf.addResource(filename);
 
-    HashMap<String,String> retVal = new HashMap<String,String>();
+    HashMap<String,String> retVal = new HashMap<>();
     Iterator<Map.Entry<String,String>> kvItr = conf.iterator();
     while (kvItr.hasNext()) {
       Map.Entry<String,String> entry = kvItr.next();
       String key = entry.getKey();
       // Ignore known xml props
-      if (xmlPropsToSkipCompare != null) {
-        if (xmlPropsToSkipCompare.contains(key)) {
-          if (xmlDebug) {
-            System.out.println("  Skipping Full Key: " + key);
-          }
-          continue;
-        }
+      if (xmlPropsToSkipCompare.contains(key)) {
+        LOG_XML.debug("  Skipping Full Key: {}", key);
+        continue;
       }
       // Ignore known xml prefixes
-      boolean skipPrefix = false;
-      if (xmlPrefixToSkipCompare != null) {
-        for (String xmlPrefix : xmlPrefixToSkipCompare) {
-	  if (key.startsWith(xmlPrefix)) {
-	    skipPrefix = true;
-            break;
-	  }
-	}
-      }
-      if (skipPrefix) {
-        if (xmlDebug) {
-          System.out.println("  Skipping Prefix Key: " + key);
-        }
+      if (xmlPrefixToSkipCompare.stream().anyMatch(key::startsWith)) {
+        LOG_XML.debug("  Skipping Prefix Key: " + key);
         continue;
       }
       if (conf.onlyKeyExists(key)) {
-        retVal.put(key,null);
-        if (xmlDebug) {
-          System.out.println("  XML Key,Null Value: " + key);
-        }
+        retVal.put(key, null);
+        LOG_XML.debug("  XML Key,Null Value: " + key);
       } else {
-        String value = conf.get(key);
-        if (value!=null) {
-          retVal.put(key,entry.getValue());
-          if (xmlDebug) {
-            System.out.println("  XML Key,Valid Value: " + key);
-          }
+        if (conf.get(key) != null) {
+          retVal.put(key, entry.getValue());
+          LOG_XML.debug("  XML Key,Valid Value: " + key);
         }
       }
       kvItr.remove();
@@ -353,12 +327,12 @@ public abstract class TestConfigurationFieldsBase {
    * member variables from a Configuration type class.
    *
    * @param fields The class member variables
-   * @return HashMap containing <DefaultVariableName,DefaultValue> entries
+   * @return HashMap containing (DefaultVariableName, DefaultValue) entries
    */
   private HashMap<String,String>
       extractDefaultVariablesFromConfigurationFields(Field[] fields) {
     // Sanity Check
-    if (fields==null) {
+    if (fields == null) {
       return null;
     }
 
@@ -407,12 +381,11 @@ public abstract class TestConfigurationFieldsBase {
             boolean bValue = (boolean) f.get(null);
             retVal.put(f.getName(),Boolean.toString(bValue));
           } else {
-            if (defaultDebug) {
-              System.out.println("Config variable " + f.getName() + " has unknown type " + f.getType().getName());
-            }
+            LOG.debug("Config variable {} has unknown type {}",
+                f.getName(), f.getType().getName());
           }
         } catch (IllegalAccessException iaException) {
-          iaException.printStackTrace();
+          LOG.error("{}", f, iaException);
         }
       }
     }
@@ -427,8 +400,9 @@ public abstract class TestConfigurationFieldsBase {
    * @param keyMap2 The set to subtract
    * @return Returns set operation keyMap1-keyMap2
    */
-  private static Set<String> compareConfigurationToXmlFields(Map<String,String> keyMap1, Map<String,String> keyMap2) {
-    Set<String> retVal = new HashSet<String>(keyMap1.keySet());
+  private static Set<String> compareConfigurationToXmlFields(
+      Map<String,String> keyMap1, Map<String,String> keyMap2) {
+    Set<String> retVal = new HashSet<>(keyMap1.keySet());
     retVal.removeAll(keyMap2.keySet());
 
     return retVal;
@@ -443,60 +417,36 @@ public abstract class TestConfigurationFieldsBase {
     initializeMemberVariables();
 
     // Error if subclass hasn't set class members
-    assertTrue(xmlFilename!=null);
-    assertTrue(configurationClasses!=null);
+    assertNotNull(xmlFilename);
+    assertNotNull(configurationClasses);
 
     // Create class member/value map
-    configurationMemberVariables = new HashMap<String,String>();
-    if (configDebug) {
-      System.out.println("Reading configuration classes");
-      System.out.println("");
-    }
+    configurationMemberVariables = new HashMap<>();
+    LOG_CONFIG.debug("Reading configuration classes\n");
     for (Class c : configurationClasses) {
       Field[] fields = c.getDeclaredFields();
       Map<String,String> memberMap =
           extractMemberVariablesFromConfigurationFields(fields);
-      if (memberMap!=null) {
+      if (memberMap != null) {
         configurationMemberVariables.putAll(memberMap);
       }
     }
-    if (configDebug) {
-      System.out.println("");
-      System.out.println("=====");
-      System.out.println("");
-    }
+    LOG_CONFIG.debug("\n=====\n");
 
     // Create XML key/value map
-    if (xmlDebug) {
-      System.out.println("Reading XML property files");
-      System.out.println("");
-    }
+    LOG_XML.debug("Reading XML property files\n");
     xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
-    if (xmlDebug) {
-      System.out.println("");
-      System.out.println("=====");
-      System.out.println("");
-    }
+    LOG_XML.debug("\n=====\n");
 
     // Create default configuration variable key/value map
-    if (defaultDebug) {
-      System.out.println("Reading Config property files for defaults");
-      System.out.println("");
-    }
-    configurationDefaultVariables = new HashMap<String,String>();
-    for (Class c : configurationClasses) {
-      Field[] fields = c.getDeclaredFields();
-      Map<String,String> defaultMap =
-          extractDefaultVariablesFromConfigurationFields(fields);
-      if (defaultMap!=null) {
-        configurationDefaultVariables.putAll(defaultMap);
-      }
-    }
-    if (defaultDebug) {
-      System.out.println("");
-      System.out.println("=====");
-      System.out.println("");
-    }
+    LOG.debug("Reading Config property files for defaults\n");
+    configurationDefaultVariables = new HashMap<>();
+    Arrays.stream(configurationClasses)
+        .map(Class::getDeclaredFields)
+        .map(this::extractDefaultVariablesFromConfigurationFields)
+        .filter(Objects::nonNull)
+        .forEach(map -> configurationDefaultVariables.putAll(map));
+    LOG.debug("\n=====\n");
 
     // Find class members not in the XML file
     configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
@@ -514,17 +464,16 @@ public abstract class TestConfigurationFieldsBase {
   @Test
   public void testCompareConfigurationClassAgainstXml() {
     // Error if subclass hasn't set class members
-    assertTrue(xmlFilename!=null);
-    assertTrue(configurationClasses!=null);
+    assertNotNull(xmlFilename);
+    assertNotNull(configurationClasses);
 
     final int missingXmlSize = configurationFieldsMissingInXmlFile.size();
 
     for (Class c : configurationClasses) {
-      System.out.println(c);
+      LOG.info(c.toString());
     }
-    System.out.println("  (" + configurationMemberVariables.size() + " member variables)");
-    System.out.println();
-    StringBuffer xmlErrorMsg = new StringBuffer();
+    LOG.info("({} member variables)\n", configurationMemberVariables.size());
+    StringBuilder xmlErrorMsg = new StringBuilder();
     for (Class c : configurationClasses) {
       xmlErrorMsg.append(c);
       xmlErrorMsg.append(" ");
@@ -533,23 +482,33 @@ public abstract class TestConfigurationFieldsBase {
     xmlErrorMsg.append(missingXmlSize);
     xmlErrorMsg.append(" variables missing in ");
     xmlErrorMsg.append(xmlFilename);
-    System.out.println(xmlErrorMsg.toString());
-    System.out.println();
-    if (missingXmlSize==0) {
-      System.out.println("  (None)");
+    LOG.error(xmlErrorMsg.toString());
+    if (missingXmlSize == 0) {
+      LOG.info("  (None)");
     } else {
-      for (String missingField : configurationFieldsMissingInXmlFile) {
-        System.out.println("  " + missingField);
-      }
+      appendMissingEntries(xmlErrorMsg, configurationFieldsMissingInXmlFile);
     }
-    System.out.println();
-    System.out.println("=====");
-    System.out.println();
+    LOG.info("\n=====\n");
     if (errorIfMissingXmlProps) {
-      assertTrue(xmlErrorMsg.toString(), missingXmlSize==0);
+      assertEquals(xmlErrorMsg.toString(), 0, missingXmlSize);
     }
   }
 
+  /**
+   * Take a set of missing entries, sort, append to the string builder
+   * and also log at INFO.
+   * @param sb string builder
+   * @param missing set of missing entries
+   */
+  private void appendMissingEntries(StringBuilder sb, Set<String> missing) {
+    sb.append(" Entries: ");
+    new TreeSet<>(missing).forEach(
+        (s) -> {
+          LOG.info("  {}", s);
+          sb.append("  ").append(s);
+        });
+  }
+
   /**
    * Compares the properties that are in the XML properties file, but not
    * in the Configuration class.
@@ -557,35 +516,28 @@ public abstract class TestConfigurationFieldsBase {
   @Test
   public void testCompareXmlAgainstConfigurationClass() {
     // Error if subclass hasn't set class members
-    assertTrue(xmlFilename!=null);
-    assertTrue(configurationClasses!=null);
+    assertNotNull(xmlFilename);
+    assertNotNull(configurationClasses);
 
     final int missingConfigSize = xmlFieldsMissingInConfiguration.size();
 
-    System.out.println("File " + xmlFilename + " (" + xmlKeyValueMap.size() + " properties)");
-    System.out.println();
-    StringBuffer configErrorMsg = new StringBuffer();
+    LOG.info("File {} ({} properties)", xmlFilename, xmlKeyValueMap.size());
+    StringBuilder configErrorMsg = new StringBuilder();
     configErrorMsg.append(xmlFilename);
     configErrorMsg.append(" has ");
     configErrorMsg.append(missingConfigSize);
     configErrorMsg.append(" properties missing in");
-    for (Class c : configurationClasses) {
-      configErrorMsg.append("  " + c);
-    }
-    System.out.println(configErrorMsg.toString());
-    System.out.println();
-    if (missingConfigSize==0) {
-      System.out.println("  (None)");
+    Arrays.stream(configurationClasses)
+        .forEach(c -> configErrorMsg.append("  ").append(c));
+    LOG.info(configErrorMsg.toString());
+    if (missingConfigSize == 0) {
+      LOG.info("  (None)");
     } else {
-      for (String missingField : xmlFieldsMissingInConfiguration) {
-        System.out.println("  " + missingField);
-      }
+      appendMissingEntries(configErrorMsg, xmlFieldsMissingInConfiguration);
     }
-    System.out.println();
-    System.out.println("=====");
-    System.out.println();
-    if ( errorIfMissingConfigProps ) {
-      assertTrue(configErrorMsg.toString(), missingConfigSize==0);
+    LOG.info("\n=====\n");
+    if (errorIfMissingConfigProps) {
+      assertEquals(configErrorMsg.toString(), 0, missingConfigSize);
     }
   }
 
@@ -596,23 +548,23 @@ public abstract class TestConfigurationFieldsBase {
   @Test
   public void testXmlAgainstDefaultValuesInConfigurationClass() {
     // Error if subclass hasn't set class members
-    assertTrue(xmlFilename!=null);
-    assertTrue(configurationMemberVariables!=null);
-    assertTrue(configurationDefaultVariables!=null);
+    assertNotNull(xmlFilename);
+    assertNotNull(configurationMemberVariables);
+    assertNotNull(configurationDefaultVariables);
 
-    HashSet<String> xmlPropertiesWithEmptyValue = new HashSet<String>();
-    HashSet<String> configPropertiesWithNoDefaultConfig = new HashSet<String>();
+    TreeSet<String> xmlPropertiesWithEmptyValue = new TreeSet<>();
+    TreeSet<String> configPropertiesWithNoDefaultConfig = new TreeSet<>();
     HashMap<String,String> xmlPropertiesMatchingConfigDefault =
-        new HashMap<String,String>();
+        new HashMap<>();
     // Ugly solution.  Should have tuple-based solution.
-    HashMap<HashMap<String,String>,HashMap<String,String>> mismatchingXmlConfig =
-        new HashMap<HashMap<String,String>,HashMap<String,String>>();
+    HashMap<HashMap<String,String>, HashMap<String,String>> mismatchingXmlConfig
+        = new HashMap<>();
 
     for (Map.Entry<String,String> xEntry : xmlKeyValueMap.entrySet()) {
       String xmlProperty = xEntry.getKey();
       String xmlDefaultValue = xEntry.getValue();
       String configProperty = configurationMemberVariables.get(xmlProperty);
-      if (configProperty!=null) {
+      if (configProperty != null) {
         String defaultConfigName = null;
         String defaultConfigValue = null;
 
@@ -624,7 +576,7 @@ public abstract class TestConfigurationFieldsBase {
         String defaultNameCheck2 = null;
         if (configProperty.endsWith("_KEY")) {
           defaultNameCheck2 = configProperty
-              .substring(0,configProperty.length()-4) + "_DEFAULT";
+              .substring(0, configProperty.length() - 4) + "_DEFAULT";
         }
         String defaultValueCheck2 = configurationDefaultVariables
             .get(defaultNameCheck2);
@@ -634,115 +586,94 @@ public abstract class TestConfigurationFieldsBase {
             .get(defaultNameCheck3);
 
         // Pick the default value that exists
-        if (defaultValueCheck1!=null) {
+        if (defaultValueCheck1 != null) {
           defaultConfigName = defaultNameCheck1;
           defaultConfigValue = defaultValueCheck1;
-        } else if (defaultValueCheck2!=null) {
+        } else if (defaultValueCheck2 != null) {
           defaultConfigName = defaultNameCheck2;
           defaultConfigValue = defaultValueCheck2;
-        } else if (defaultValueCheck3!=null) {
+        } else if (defaultValueCheck3 != null) {
           defaultConfigName = defaultNameCheck3;
           defaultConfigValue = defaultValueCheck3;
         }
 
-        if (defaultConfigValue!=null) {
-          if (xmlDefaultValue==null) {
+        if (defaultConfigValue != null) {
+          if (xmlDefaultValue == null) {
             xmlPropertiesWithEmptyValue.add(xmlProperty);
           } else if (!xmlDefaultValue.equals(defaultConfigValue)) {
-            HashMap<String,String> xmlEntry =
-                new HashMap<String,String>();
-            xmlEntry.put(xmlProperty,xmlDefaultValue);
-            HashMap<String,String> configEntry =
-                new HashMap<String,String>();
-            configEntry.put(defaultConfigName,defaultConfigValue);
-            mismatchingXmlConfig.put(xmlEntry,configEntry);
-           } else {
+            HashMap<String, String> xmlEntry = new HashMap<>();
+            xmlEntry.put(xmlProperty, xmlDefaultValue);
+            HashMap<String, String> configEntry = new HashMap<>();
+            configEntry.put(defaultConfigName, defaultConfigValue);
+            mismatchingXmlConfig.put(xmlEntry, configEntry);
+          } else {
             xmlPropertiesMatchingConfigDefault
                 .put(xmlProperty, defaultConfigName);
           }
         } else {
           configPropertiesWithNoDefaultConfig.add(configProperty);
         }
-      } else {
       }
     }
 
     // Print out any unknown mismatching XML value/Config default value
-    System.out.println(this.xmlFilename + " has " +
-        mismatchingXmlConfig.size() +
-        " properties that do not match the default Config value");
-    if (mismatchingXmlConfig.size()==0) {
-      System.out.println("  (None)");
+    LOG.info("{} has {} properties that do not match the default Config value",
+        xmlFilename, mismatchingXmlConfig.size());
+    if (mismatchingXmlConfig.isEmpty()) {
+      LOG.info("  (None)");
     } else {
        for (Map.Entry<HashMap<String,String>,HashMap<String,String>> xcEntry :
            mismatchingXmlConfig.entrySet()) {
-         HashMap<String,String> xmlMap = xcEntry.getKey();
-         HashMap<String,String> configMap = xcEntry.getValue();
-         for (Map.Entry<String,String> xmlEntry : xmlMap.entrySet()) {
-           System.out.println("  XML Property: " + xmlEntry.getKey());
-           System.out.println("  XML Value:    " + xmlEntry.getValue());
-         }
-         for (Map.Entry<String,String> configEntry : configMap.entrySet()) {
-           System.out.println("  Config Name:  " + configEntry.getKey());
-           System.out.println("  Config Value: " + configEntry.getValue());
-         }
-         System.out.println("");
+         xcEntry.getKey().forEach((key, value) -> {
+           LOG.info("XML Property: {}", key);
+           LOG.info("XML Value:    {}", value);
+         });
+         xcEntry.getValue().forEach((key, value) -> {
+           LOG.info("Config Name:  {}", key);
+           LOG.info("Config Value: {}", value);
+         });
+         LOG.info("");
        }
     }
-    System.out.println();
+    LOG.info("\n");
 
     // Print out Config properties that have no corresponding DEFAULT_*
     // variable and cannot do any XML comparison (i.e. probably needs to
     // be checked by hand)
-    System.out.println("Configuration(s) have " +
-        configPropertiesWithNoDefaultConfig.size() +
+    LOG.info("Configuration(s) have {} " +
         " properties with no corresponding default member variable.  These" +
-        " will need to be verified manually.");
-    if (configPropertiesWithNoDefaultConfig.size()==0) {
-      System.out.println("  (None)");
+        " will need to be verified manually.",
+        configPropertiesWithNoDefaultConfig.size());
+    if (configPropertiesWithNoDefaultConfig.isEmpty()) {
+      LOG.info("  (None)");
     } else {
-      Iterator<String> cItr = configPropertiesWithNoDefaultConfig.iterator();
-      while (cItr.hasNext()) {
-        System.out.println("  " + cItr.next());
-      }
+      configPropertiesWithNoDefaultConfig.forEach(c -> LOG.info(" {}", c));
     }
-    System.out.println();
+    LOG.info("\n");
 
     // MAYBE TODO Print out any known mismatching XML value/Config default
 
     // Print out XML properties that have empty values (i.e. should result
     // in code-based default)
-    System.out.println(this.xmlFilename + " has " +
-        xmlPropertiesWithEmptyValue.size() + " properties with empty values");
-    if (xmlPropertiesWithEmptyValue.size()==0) {
-      System.out.println("  (None)");
+    LOG.info("{} has {} properties with empty values",
+        xmlFilename, xmlPropertiesWithEmptyValue.size());
+    if (xmlPropertiesWithEmptyValue.isEmpty()) {
+      LOG.info("  (None)");
     } else {
-      Iterator<String> xItr = xmlPropertiesWithEmptyValue.iterator();
-      while (xItr.hasNext()) {
-        System.out.println("  " + xItr.next());
-      }
+      xmlPropertiesWithEmptyValue.forEach(p -> LOG.info("  {}", p));
     }
-    System.out.println();
+    LOG.info("\n");
 
     // Print out any matching XML value/Config default value
-    System.out.println(this.xmlFilename + " has " +
-        xmlPropertiesMatchingConfigDefault.size() +
-        " properties which match a corresponding Config variable");
-    if (xmlPropertiesMatchingConfigDefault.size()==0) {
-      System.out.println("  (None)");
+    LOG.info("{} has {} properties which match a corresponding Config variable",
+        xmlFilename, xmlPropertiesMatchingConfigDefault.size());
+    if (xmlPropertiesMatchingConfigDefault.isEmpty()) {
+      LOG.info("  (None)");
     } else {
-      for (Map.Entry<String,String> xcEntry :
-          xmlPropertiesMatchingConfigDefault.entrySet()) {
-        System.out.println("  " + xcEntry.getKey() + " / " +
-            xcEntry.getValue());
-      }
+      xmlPropertiesMatchingConfigDefault.forEach(
+          (key, value) -> LOG.info("  {} / {}", key, value));
     }
-    System.out.println();
-
-    // Test separator
-    System.out.println();
-    System.out.println("=====");
-    System.out.println();
+    LOG.info("\n=====\n");
   }
 
   /**
@@ -754,8 +685,8 @@ public abstract class TestConfigurationFieldsBase {
   @Test
   public void testDefaultValueCollision() {
     for (String filter : filtersForDefaultValueCollisionCheck) {
-      System.out.println("Checking if any of the default values whose name " +
-          "contains string \"" + filter + "\" collide.");
+      LOG.info("Checking if any of the default values whose name " +
+          "contains string \"{}\" collide.", filter);
 
       // Map from filtered default value to name of the corresponding parameter.
       Map<String, String> filteredValues = new HashMap<>();
@@ -769,15 +700,14 @@ public abstract class TestConfigurationFieldsBase {
           if (StringUtils.isNumeric(ent.getValue())) {
             String crtValue =
                 filteredValues.putIfAbsent(ent.getValue(), ent.getKey());
-            assertTrue("Parameters " + ent.getKey() + " and " + crtValue +
-                " are using the same default value!", crtValue == null);
+            assertNull("Parameters " + ent.getKey() + " and " + crtValue +
+                " are using the same default value!", crtValue);
           }
           valuesChecked++;
         }
-      }
 
-      System.out.println(
-          "Checked " + valuesChecked + " default values for collision.");
+      }
+      LOG.info("Checked {} default values for collision.", valuesChecked);
     }
 
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java

@@ -27,6 +27,7 @@ import java.util.EnumSet;
 import java.util.Random;
 
 import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.CanUnbuffer;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
@@ -102,7 +103,32 @@ public abstract class CryptoStreamsTestBase {
     
     return total;
   }
-  
+
+  private int preadAll(PositionedReadable in, byte[] b, int off, int len)
+      throws IOException {
+    int n = 0;
+    int total = 0;
+    while (n != -1) {
+      total += n;
+      if (total >= len) {
+        break;
+      }
+      n = in.read(total, b, off + total, len - total);
+    }
+
+    return total;
+  }
+
+  private void preadCheck(PositionedReadable in) throws Exception {
+    byte[] result = new byte[dataLen];
+    int n = preadAll(in, result, 0, dataLen);
+
+    Assert.assertEquals(dataLen, n);
+    byte[] expectedData = new byte[n];
+    System.arraycopy(data, 0, expectedData, 0, n);
+    Assert.assertArrayEquals(result, expectedData);
+  }
+
   protected OutputStream getOutputStream(int bufferSize) throws IOException {
     return getOutputStream(bufferSize, key, iv);
   }
@@ -146,7 +172,6 @@ public abstract class CryptoStreamsTestBase {
     // EOF
     n = in.read(result, 0, dataLen);
     Assert.assertEquals(n, -1);
-    in.close();
   }
   
   /** Test crypto writing with different buffer size. */
@@ -730,4 +755,47 @@ public abstract class CryptoStreamsTestBase {
     
     in.close();
   }
+
+  /** Test unbuffer. */
+  @Test(timeout=120000)
+  public void testUnbuffer() throws Exception {
+    OutputStream out = getOutputStream(smallBufferSize);
+    writeData(out);
+
+    // Test buffered read
+    try (InputStream in = getInputStream(smallBufferSize)) {
+      // Test unbuffer after buffered read
+      readCheck(in);
+      ((CanUnbuffer) in).unbuffer();
+
+      if (in instanceof Seekable) {
+        // Test buffered read again after unbuffer
+        // Must seek to the beginning first
+        ((Seekable) in).seek(0);
+        readCheck(in);
+      }
+
+      // Test close after unbuffer
+      ((CanUnbuffer) in).unbuffer();
+      // The close will be called when exiting this try-with-resource block
+    }
+
+    // Test pread
+    try (InputStream in = getInputStream(smallBufferSize)) {
+      if (in instanceof PositionedReadable) {
+        PositionedReadable pin = (PositionedReadable) in;
+
+        // Test unbuffer after pread
+        preadCheck(pin);
+        ((CanUnbuffer) in).unbuffer();
+
+        // Test pread again after unbuffer
+        preadCheck(pin);
+
+        // Test close after unbuffer
+        ((CanUnbuffer) in).unbuffer();
+        // The close will be called when exiting this try-with-resource block
+      }
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java

@@ -29,11 +29,13 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ByteBufferReadable;
 import org.apache.hadoop.fs.CanSetDropBehind;
 import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.CanUnbuffer;
 import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
 import org.apache.hadoop.fs.HasFileDescriptor;
 import org.apache.hadoop.fs.PositionedReadable;
 import org.apache.hadoop.fs.ReadOption;
 import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Syncable;
 import org.apache.hadoop.io.ByteBufferPool;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -159,16 +161,18 @@ public class TestCryptoStreams extends CryptoStreamsTestBase {
     }
   }
   
-  public static class FakeInputStream extends InputStream implements 
-      Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor, 
-      CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess {
+  static class FakeInputStream extends InputStream
+      implements Seekable, PositionedReadable, ByteBufferReadable,
+                 HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
+                 HasEnhancedByteBufferAccess, CanUnbuffer,
+                 StreamCapabilities {
     private final byte[] oneByteBuf = new byte[1];
     private int pos = 0;
     private final byte[] data;
     private final int length;
     private boolean closed = false;
 
-    public FakeInputStream(DataInputBuffer in) {
+    FakeInputStream(DataInputBuffer in) {
       data = in.getData();
       length = in.getLength();
     }
@@ -349,6 +353,22 @@ public class TestCryptoStreams extends CryptoStreamsTestBase {
         UnsupportedOperationException {
     }
 
+    @Override
+    public void unbuffer() {
+    }
+
+    @Override
+    public boolean hasCapability(String capability) {
+      switch (capability.toLowerCase()) {
+      case StreamCapabilities.READAHEAD:
+      case StreamCapabilities.DROPBEHIND:
+      case StreamCapabilities.UNBUFFER:
+        return true;
+      default:
+        return false;
+      }
+    }
+
     @Override
     public FileDescriptor getFileDescriptor() throws IOException {
       return null;

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java

@@ -112,4 +112,9 @@ public class TestCryptoStreamsForLocalFS extends CryptoStreamsTestBase {
   @Test(timeout=10000)
   public void testSeekToNewSource() throws Exception {
   }
+
+  @Ignore("Local file input stream does not support unbuffer")
+  @Override
+  @Test
+  public void testUnbuffer() throws Exception {}
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java

@@ -120,4 +120,9 @@ public class TestCryptoStreamsNormal extends CryptoStreamsTestBase {
   @Override
   @Test(timeout=10000)
   public void testHasEnhancedByteBufferAccess() throws IOException {}
+
+  @Ignore("ByteArrayInputStream does not support unbuffer")
+  @Override
+  @Test
+  public void testUnbuffer() throws Exception {}
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java

@@ -39,8 +39,11 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.net.ConnectTimeoutException;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.junit.After;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -48,9 +51,20 @@ import com.google.common.collect.Sets;
 
 public class TestLoadBalancingKMSClientProvider {
 
+  @BeforeClass
+  public static void setup() throws IOException {
+    SecurityUtil.setTokenServiceUseIp(false);
+  }
+
+  @After
+  public void teardown() throws IOException {
+    KMSClientProvider.fallbackDefaultPortForTesting = false;
+  }
+
   @Test
   public void testCreation() throws Exception {
     Configuration conf = new Configuration();
+    KMSClientProvider.fallbackDefaultPortForTesting = true;
     KeyProvider kp = new KMSClientProvider.Factory().createProvider(new URI(
         "kms://http@host1/kms/foo"), conf);
     assertTrue(kp instanceof LoadBalancingKMSClientProvider);
@@ -231,6 +245,7 @@ public class TestLoadBalancingKMSClientProvider {
   @Test
   public void testClassCastException() throws Exception {
     Configuration conf = new Configuration();
+    KMSClientProvider.fallbackDefaultPortForTesting = true;
     KMSClientProvider p1 = new MyKMSClientProvider(
         new URI("kms://http@host1/kms/foo"), conf);
     LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java

@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.compress;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Verify resettable compressor.
+ */
+public class TestGzipCodec {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestGzipCodec.class);
+
+  private static final String DATA1 = "Dogs don't know it's not bacon!\n";
+  private static final String DATA2 = "It's baconnnn!!\n";
+  private GzipCodec codec = new GzipCodec();
+
+  @Before
+  public void setUp() {
+    codec.setConf(new Configuration(false));
+  }
+
+  // Test simple compression.
+  @Test
+  public void testSingleCompress() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    CompressionOutputStream cmpOut = codec.createOutputStream(baos);
+    cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+    cmpOut.finish();
+    cmpOut.close();
+
+    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+    GZIPInputStream cmpIn = new GZIPInputStream(bais);
+    byte[] buf = new byte[1024];
+    int len = cmpIn.read(buf);
+    String result = new String(buf, 0, len, StandardCharsets.UTF_8);
+    assertEquals("Input must match output", DATA1, result);
+  }
+
+  // Test multi-member gzip file created via finish(), resetState().
+  @Test
+  public void testResetCompress() throws IOException {
+    DataOutputBuffer dob = new DataOutputBuffer();
+    CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+    cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+    cmpOut.finish();
+    cmpOut.resetState();
+    cmpOut.write(DATA2.getBytes(StandardCharsets.UTF_8));
+    cmpOut.finish();
+    cmpOut.close();
+    dob.close();
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), 0, dob.getLength());
+    CompressionInputStream cmpIn = codec.createInputStream(dib);
+    byte[] buf = new byte[1024];
+    StringBuilder result = new StringBuilder();
+    int len = 0;
+    while (true) {
+      len = cmpIn.read(buf);
+      if (len < 0) {
+        break;
+      }
+      result.append(new String(buf, 0, len, StandardCharsets.UTF_8));
+    }
+    assertEquals("Output must match input", DATA1 + DATA2, result.toString());
+  }
+
+  // ensure all necessary methods are overwritten
+  @Test
+  public void testWriteOverride() throws IOException {
+    Random r = new Random();
+    long seed = r.nextLong();
+    LOG.info("seed: " + seed);
+    r.setSeed(seed);
+    byte[] buf = new byte[128];
+    r.nextBytes(buf);
+    DataOutputBuffer dob = new DataOutputBuffer();
+    CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+    cmpOut.write(buf);
+    int i = r.nextInt(128 - 10);
+    int l = r.nextInt(128 - i);
+    cmpOut.write(buf, i, l);
+    cmpOut.write((byte)(r.nextInt() & 0xFF));
+    cmpOut.close();
+
+    r.setSeed(seed);
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), 0, dob.getLength());
+    CompressionInputStream cmpIn = codec.createInputStream(dib);
+    byte[] vbuf = new byte[128];
+    assertEquals(128, cmpIn.read(vbuf));
+    assertArrayEquals(buf, vbuf);
+    r.nextBytes(vbuf);
+    int vi = r.nextInt(128 - 10);
+    int vl = r.nextInt(128 - vi);
+    assertEquals(vl, cmpIn.read(vbuf, 0, vl));
+    assertArrayEquals(Arrays.copyOfRange(buf, i,  i + l),
+        Arrays.copyOf(vbuf, vl));
+    assertEquals(r.nextInt() & 0xFF, cmpIn.read());
+    assertEquals(-1, cmpIn.read());
+  }
+
+  // don't write a new header if no data are written after reset
+  @Test
+  public void testIdempotentResetState() throws IOException {
+    DataOutputBuffer dob = new DataOutputBuffer();
+    CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+    cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+    cmpOut.finish();
+    cmpOut.finish();
+    cmpOut.finish();
+    cmpOut.resetState();
+    cmpOut.resetState();
+    cmpOut.finish();
+    cmpOut.resetState();
+    cmpOut.close();
+    dob.close();
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), 0, dob.getLength());
+    CompressionInputStream cmpIn = codec.createInputStream(dib);
+    byte[] buf = new byte[1024];
+    StringBuilder result = new StringBuilder();
+    int len = 0;
+    while (true) {
+      len = cmpIn.read(buf);
+      if (len < 0) {
+        break;
+      }
+      result.append(new String(buf, 0, len, StandardCharsets.UTF_8));
+    }
+    assertEquals("Output must match input", DATA1, result.toString());
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java

@@ -39,10 +39,12 @@ import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
 
 import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
 import com.google.common.collect.Iterables;
 
 import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.test.GenericTestUtils;
 import static org.apache.hadoop.test.MoreAsserts.*;
 
 import org.apache.hadoop.metrics2.AbstractMetric;
@@ -78,8 +80,11 @@ public class TestMetricsSystemImpl {
 
   public static class TestSink implements MetricsSink {
 
+    private List<Iterable<AbstractMetric>> metricValues = new ArrayList<>();
+
     @Override public void putMetrics(MetricsRecord record) {
       LOG.debug(record.toString());
+      metricValues.add(record.metrics());
     }
 
     @Override public void flush() {}
@@ -87,6 +92,10 @@ public class TestMetricsSystemImpl {
     @Override public void init(SubsetConfiguration conf) {
       LOG.debug(MetricsConfig.toString(conf));
     }
+
+    List<Iterable<AbstractMetric>> getMetricValues() {
+      return metricValues;
+    }
   }
 
   @Test public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
@@ -559,6 +568,46 @@ public class TestMetricsSystemImpl {
     ms.shutdown();
   }
 
+  @Test
+  public void testRegisterSinksMultiplePeriods() throws Exception {
+    new ConfigBuilder().add("test.sink.test1.period", 100000)
+        .add("test.sink.test1.class", TestSink.class.getName())
+        .add("test.sink.test2.period", 200000)
+        .add("test.sink.test2.class", TestSink.class.getName())
+        .save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
+    MetricsSystemImpl ms = new MetricsSystemImpl();
+    try {
+      ms.init("test");
+      TestSink sink1 = (TestSink) ms.getSinkAdapter("test1").sink();
+      TestSink sink2 = (TestSink) ms.getSinkAdapter("test2").sink();
+      assertEquals(0, sink1.getMetricValues().size());
+      assertEquals(0, sink2.getMetricValues().size());
+      ms.onTimerEvent();
+      // Give some time for the publish event to go through
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override
+        public Boolean get() {
+          return sink1.getMetricValues().size() > 0;
+        }
+      }, 10, 10000);
+      assertEquals(1, sink1.getMetricValues().size());
+      assertEquals(0, sink2.getMetricValues().size());
+      ms.onTimerEvent();
+      // Give some time for the publish event to go through
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override
+        public Boolean get() {
+          return sink1.getMetricValues().size() > 1 &&
+              sink2.getMetricValues().size() > 0;
+        }
+      }, 10, 10000);
+      assertEquals(2, sink1.getMetricValues().size());
+      assertEquals(1, sink2.getMetricValues().size());
+    } finally {
+      ms.shutdown();
+    }
+  }
+
   @Metrics(context="test")
   private static class TestSource {
     @Metric("C1 desc") MutableCounterLong c1;

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.metrics2.source;
 
+import org.apache.hadoop.util.GcTimeMonitor;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -35,6 +36,9 @@ import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.JvmPauseMonitor;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
 import static org.apache.hadoop.metrics2.impl.MsInfo.*;
 
@@ -43,17 +47,21 @@ public class TestJvmMetrics {
   @Rule
   public Timeout timeout = new Timeout(30000);
   private JvmPauseMonitor pauseMonitor;
+  private GcTimeMonitor gcTimeMonitor;
 
   /**
-   * Robust shutdown of the pause monitor if it hasn't been stopped already.
+   * Robust shutdown of the monitors if they haven't been stopped already.
    */
   @After
   public void teardown() {
     ServiceOperations.stop(pauseMonitor);
+    if (gcTimeMonitor != null) {
+      gcTimeMonitor.shutdown();
+    }
   }
 
   @Test
-  public void testPresence() {
+  public void testJvmPauseMonitorPresence() {
     pauseMonitor = new JvmPauseMonitor();
     pauseMonitor.init(new Configuration());
     pauseMonitor.start();
@@ -66,14 +74,35 @@ public class TestJvmMetrics {
     verify(rb).tag(ProcessName, "test");
     verify(rb).tag(SessionId, "test");
     for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
-      if (info.name().startsWith("Mem"))
+      if (info.name().startsWith("Mem")) {
         verify(rb).addGauge(eq(info), anyFloat());
-      else if (info.name().startsWith("Gc"))
+      } else if (info.name().startsWith("Gc") &&
+          !info.name().equals("GcTimePercentage")) {
         verify(rb).addCounter(eq(info), anyLong());
-      else if (info.name().startsWith("Threads"))
+      } else if (info.name().startsWith("Threads")) {
         verify(rb).addGauge(eq(info), anyInt());
-      else if (info.name().startsWith("Log"))
+      } else if (info.name().startsWith("Log")) {
         verify(rb).addCounter(eq(info), anyLong());
+      }
+    }
+  }
+
+  @Test
+  public void testGcTimeMonitorPresence() {
+    gcTimeMonitor = new GcTimeMonitor(60000, 1000, 70, null);
+    gcTimeMonitor.start();
+    JvmMetrics jvmMetrics = new JvmMetrics("test", "test");
+    jvmMetrics.setGcTimeMonitor(gcTimeMonitor);
+    MetricsRecordBuilder rb = getMetrics(jvmMetrics);
+    MetricsCollector mc = rb.parent();
+
+    verify(mc).addRecord(JvmMetrics);
+    verify(rb).tag(ProcessName, "test");
+    verify(rb).tag(SessionId, "test");
+    for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
+      if (info.name().equals("GcTimePercentage")) {
+        verify(rb).addCounter(eq(info), anyInt());
+      }
     }
   }
 
@@ -120,4 +149,55 @@ public class TestJvmMetrics {
     }
   }
 
+  @Test
+  public void testGcTimeMonitor() {
+    class Alerter implements GcTimeMonitor.GcTimeAlertHandler {
+      private volatile int numAlerts;
+      private volatile int maxGcTimePercentage;
+      @Override
+      public void alert(GcTimeMonitor.GcData gcData) {
+        numAlerts++;
+        if (gcData.getGcTimePercentage() > maxGcTimePercentage) {
+          maxGcTimePercentage = gcData.getGcTimePercentage();
+        }
+      }
+    }
+    Alerter alerter = new Alerter();
+
+    int alertGcPerc = 10;  // Alerter should be called if GC takes >= 10%
+    gcTimeMonitor = new GcTimeMonitor(60*1000, 100, alertGcPerc, alerter);
+    gcTimeMonitor.start();
+
+    int maxGcTimePercentage = 0;
+    long gcCount = 0;
+
+    // Generate a lot of garbage for some time and verify that the monitor
+    // reports at least some percentage of time in GC pauses, and that the
+    // alerter is invoked at least once.
+
+    List<String> garbageStrings = new ArrayList<>();
+
+    long startTime = System.currentTimeMillis();
+    // Run this for at least 1 sec for our monitor to collect enough data
+    while (System.currentTimeMillis() - startTime < 1000) {
+      for (int j = 0; j < 100000; j++) {
+        garbageStrings.add(
+            "Long string prefix just to fill memory with garbage " + j);
+      }
+      garbageStrings.clear();
+      System.gc();
+
+      GcTimeMonitor.GcData gcData = gcTimeMonitor.getLatestGcData();
+      int gcTimePercentage = gcData.getGcTimePercentage();
+      if (gcTimePercentage > maxGcTimePercentage) {
+        maxGcTimePercentage = gcTimePercentage;
+      }
+      gcCount = gcData.getAccumulatedGcCount();
+    }
+
+    Assert.assertTrue(maxGcTimePercentage > 0);
+    Assert.assertTrue(gcCount > 0);
+    Assert.assertTrue(alerter.numAlerts > 0);
+    Assert.assertTrue(alerter.maxGcTimePercentage >= alertGcPerc);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java

@@ -48,7 +48,6 @@ import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.AccessControlException;
@@ -188,8 +187,7 @@ public class Hdfs extends AbstractFileSystem {
 
       @Override
       public LocatedFileStatus next() throws IOException {
-        return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated(
-            getUri(), p);
+        return getNext().makeQualifiedLocated(getUri(), p);
       }
     };
   }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java

@@ -119,6 +119,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -2793,7 +2794,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     }
   }
 
-  public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+      throws IOException {
     checkOpen();
     try (TraceScope ignored = tracer.newScope("getErasureCodingPolicies")) {
       return namenode.getErasureCodingPolicies();

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java

@@ -57,6 +57,7 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
 import org.apache.hadoop.fs.ReadOption;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
 import org.apache.hadoop.hdfs.client.impl.BlockReaderFactory;
 import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
@@ -81,6 +82,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.IdentityHashStore;
 import org.apache.hadoop.util.StopWatch;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.htrace.core.SpanId;
 import org.apache.htrace.core.TraceScope;
 import org.apache.htrace.core.Tracer;
@@ -96,7 +98,7 @@ import javax.annotation.Nonnull;
 @InterfaceAudience.Private
 public class DFSInputStream extends FSInputStream
     implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
-    HasEnhancedByteBufferAccess, CanUnbuffer {
+               HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities {
   @VisibleForTesting
   public static boolean tcpReadsDisabledForTesting = false;
   private long hedgedReadOpsLoopNumForTesting = 0;
@@ -1779,4 +1781,16 @@ public class DFSInputStream extends FSInputStream
   public synchronized void unbuffer() {
     closeCurrentBlockReaders();
   }
+
+  @Override
+  public boolean hasCapability(String capability) {
+    switch (StringUtils.toLowerCase(capability)) {
+    case StreamCapabilities.READAHEAD:
+    case StreamCapabilities.DROPBEHIND:
+    case StreamCapabilities.UNBUFFER:
+      return true;
+    default:
+      return false;
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java

@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.hdfs;
 
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HFLUSH;
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HSYNC;
-
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -69,6 +66,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.DataChecksum.Type;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.htrace.core.TraceScope;
 import org.slf4j.Logger;
@@ -552,11 +550,13 @@ public class DFSOutputStream extends FSOutputSummer
 
   @Override
   public boolean hasCapability(String capability) {
-    if (capability.equalsIgnoreCase(HSYNC.getValue()) ||
-        capability.equalsIgnoreCase((HFLUSH.getValue()))) {
+    switch (StringUtils.toLowerCase(capability)) {
+    case StreamCapabilities.HSYNC:
+    case StreamCapabilities.HFLUSH:
       return true;
+    default:
+      return false;
     }
-    return false;
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -77,6 +77,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
@@ -84,7 +85,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
 import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -1211,8 +1211,7 @@ public class DistributedFileSystem extends FileSystem {
         T next;
         HdfsFileStatus fileStat = thisListing.getPartialListing()[i++];
         if (needLocation) {
-          next = (T)((HdfsLocatedFileStatus)fileStat)
-              .makeQualifiedLocated(getUri(), p);
+          next = (T)fileStat.makeQualifiedLocated(getUri(), p);
         } else {
           next = (T)fileStat.makeQualified(getUri(), p);
         }
@@ -2676,7 +2675,7 @@ public class DistributedFileSystem extends FileSystem {
    * @return all erasure coding policies supported by this file system.
    * @throws IOException
    */
-  public Collection<ErasureCodingPolicy> getAllErasureCodingPolicies()
+  public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
       throws IOException {
     return Arrays.asList(dfs.getErasureCodingPolicies());
   }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java

@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
@@ -537,7 +538,8 @@ public class HdfsAdmin {
    *
    * @throws IOException
    */
-  public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+      throws IOException {
     return dfs.getClient().getErasureCodingPolicies();
   }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -1619,7 +1619,7 @@ public interface ClientProtocol {
    * @throws IOException
    */
   @Idempotent
-  ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException;
+  ErasureCodingPolicyInfo[] getErasureCodingPolicies() throws IOException;
 
   /**
    * Get the erasure coding codecs loaded in Namenode.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java

@@ -21,7 +21,6 @@ import com.google.common.base.Preconditions;
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 
@@ -29,22 +28,23 @@ import java.io.Serializable;
 
 /**
  * A policy about how to write/read/code an erasure coding file.
+ * <p>
+ * Note this class should be lightweight and immutable, because it's cached
+ * by {@link SystemErasureCodingPolicies}, to be returned as a part of
+ * {@link HdfsFileStatus}.
  */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
 public final class ErasureCodingPolicy implements Serializable {
 
   private static final long serialVersionUID = 0x0079fe4e;
 
-  private String name;
+  private final String name;
   private final ECSchema schema;
   private final int cellSize;
-  private byte id;
-  private ErasureCodingPolicyState state;
-
+  private final byte id;
 
   public ErasureCodingPolicy(String name, ECSchema schema,
-      int cellSize, byte id, ErasureCodingPolicyState state) {
+      int cellSize, byte id) {
     Preconditions.checkNotNull(name);
     Preconditions.checkNotNull(schema);
     Preconditions.checkArgument(cellSize > 0, "cellSize must be positive");
@@ -54,22 +54,14 @@ public final class ErasureCodingPolicy implements Serializable {
     this.schema = schema;
     this.cellSize = cellSize;
     this.id = id;
-    this.state = state;
-  }
-
-  public ErasureCodingPolicy(String name, ECSchema schema, int cellSize,
-      byte id) {
-    this(name, schema, cellSize, id, ErasureCodingPolicyState.DISABLED);
   }
 
   public ErasureCodingPolicy(ECSchema schema, int cellSize, byte id) {
-    this(composePolicyName(schema, cellSize), schema, cellSize, id,
-        ErasureCodingPolicyState.DISABLED);
+    this(composePolicyName(schema, cellSize), schema, cellSize, id);
   }
 
   public ErasureCodingPolicy(ECSchema schema, int cellSize) {
-    this(composePolicyName(schema, cellSize), schema, cellSize, (byte) -1,
-        ErasureCodingPolicyState.DISABLED);
+    this(composePolicyName(schema, cellSize), schema, cellSize, (byte) -1);
   }
 
   public static String composePolicyName(ECSchema schema, int cellSize) {
@@ -86,10 +78,6 @@ public final class ErasureCodingPolicy implements Serializable {
     return name;
   }
 
-  public void setName(String name) {
-    this.name = name;
-  }
-
   public ECSchema getSchema() {
     return schema;
   }
@@ -114,39 +102,14 @@ public final class ErasureCodingPolicy implements Serializable {
     return id;
   }
 
-  public void setId(byte id) {
-    this.id = id;
-  }
-
-
   public boolean isReplicationPolicy() {
     return (id == ErasureCodeConstants.REPLICATION_POLICY_ID);
   }
 
-  public ErasureCodingPolicyState getState() {
-    return state;
-  }
-
-  public void setState(ErasureCodingPolicyState state) {
-    this.state = state;
-  }
-
   public boolean isSystemPolicy() {
     return (this.id < ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
   }
 
-  public boolean isEnabled() {
-    return (this.state == ErasureCodingPolicyState.ENABLED);
-  }
-
-  public boolean isDisabled() {
-    return (this.state == ErasureCodingPolicyState.DISABLED);
-  }
-
-  public boolean isRemoved() {
-    return (this.state == ErasureCodingPolicyState.REMOVED);
-  }
-
   @Override
   public boolean equals(Object o) {
     if (o == null) {
@@ -164,7 +127,6 @@ public final class ErasureCodingPolicy implements Serializable {
         .append(schema, rhs.schema)
         .append(cellSize, rhs.cellSize)
         .append(id, rhs.id)
-        .append(state, rhs.state)
         .isEquals();
   }
 
@@ -175,7 +137,6 @@ public final class ErasureCodingPolicy implements Serializable {
         .append(schema)
         .append(cellSize)
         .append(id)
-        .append(state)
         .toHashCode();
   }
 
@@ -184,8 +145,7 @@ public final class ErasureCodingPolicy implements Serializable {
     return "ErasureCodingPolicy=[" + "Name=" + name + ", "
         + "Schema=[" + schema.toString() + "], "
         + "CellSize=" + cellSize + ", "
-        + "Id=" + id + ", "
-        + "State=" + state.toString()
+        + "Id=" + id
         + "]";
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java

@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.Serializable;
+
+/**
+ * HDFS internal presentation of a {@link ErasureCodingPolicy}. Also contains
+ * additional information such as {@link ErasureCodingPolicyState}.
+ */
+@InterfaceAudience.Private
+public class ErasureCodingPolicyInfo implements Serializable {
+
+  private static final long serialVersionUID = 0x31;
+
+  private final ErasureCodingPolicy policy;
+  private ErasureCodingPolicyState state;
+
+  public ErasureCodingPolicyInfo(final ErasureCodingPolicy thePolicy,
+      final ErasureCodingPolicyState theState) {
+    Preconditions.checkNotNull(thePolicy);
+    Preconditions.checkNotNull(theState);
+    policy = thePolicy;
+    state = theState;
+  }
+
+  public ErasureCodingPolicyInfo(final ErasureCodingPolicy thePolicy) {
+    this(thePolicy, ErasureCodingPolicyState.DISABLED);
+  }
+
+  public ErasureCodingPolicy getPolicy() {
+    return policy;
+  }
+
+  public ErasureCodingPolicyState getState() {
+    return state;
+  }
+
+  public void setState(final ErasureCodingPolicyState newState) {
+    Preconditions.checkNotNull(newState, "New state should not be null.");
+    state = newState;
+  }
+
+  public boolean isEnabled() {
+    return (this.state == ErasureCodingPolicyState.ENABLED);
+  }
+
+  public boolean isDisabled() {
+    return (this.state == ErasureCodingPolicyState.DISABLED);
+  }
+
+  public boolean isRemoved() {
+    return (this.state == ErasureCodingPolicyState.REMOVED);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == null) {
+      return false;
+    }
+    if (o == this) {
+      return true;
+    }
+    if (o.getClass() != getClass()) {
+      return false;
+    }
+    ErasureCodingPolicyInfo rhs = (ErasureCodingPolicyInfo) o;
+    return new EqualsBuilder()
+        .append(policy, rhs.policy)
+        .append(state, rhs.state)
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(303855623, 582626729)
+        .append(policy)
+        .append(state)
+        .toHashCode();
+  }
+
+  @Override
+  public String toString() {
+    return policy.toString() + ", State=" + state.toString();
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.protocol;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -27,8 +26,7 @@ import java.io.IOException;
 /**
  * Value denotes the possible states of an ErasureCodingPolicy.
  */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
+@InterfaceAudience.Private
 public enum ErasureCodingPolicyState {
 
   /** Policy is disabled. It's policy default state. */

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java

@@ -26,15 +26,17 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 
-/** Interface that represents the over the wire information for a file.
+/**
+ * HDFS metadata for an entity in the filesystem.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class HdfsFileStatus extends FileStatus {
+public final class HdfsFileStatus extends LocatedFileStatus {
 
   private static final long serialVersionUID = 0x126eb82a;
 
@@ -49,11 +51,12 @@ public class HdfsFileStatus extends FileStatus {
   private final int childrenNum;
   private final byte storagePolicy;
 
+  // BlockLocations[] is the user-facing type
+  private transient LocatedBlocks hdfsloc;
+
   public static final byte[] EMPTY_NAME = new byte[0];
 
-  /**
-   * Set of features potentially active on an instance.
-   */
+  /** Set of features potentially active on an instance. */
   public enum Flags {
     HAS_ACL,
     HAS_CRYPT,
@@ -81,18 +84,19 @@ public class HdfsFileStatus extends FileStatus {
    * @param storagePolicy ID which specifies storage policy
    * @param ecPolicy the erasure coding policy
    */
-  protected HdfsFileStatus(long length, boolean isdir, int replication,
+  private HdfsFileStatus(long length, boolean isdir, int replication,
                          long blocksize, long mtime, long atime,
                          FsPermission permission, EnumSet<Flags> flags,
                          String owner, String group,
                          byte[] symlink, byte[] path, long fileId,
                          int childrenNum, FileEncryptionInfo feInfo,
-                         byte storagePolicy, ErasureCodingPolicy ecPolicy) {
+                         byte storagePolicy, ErasureCodingPolicy ecPolicy,
+                         LocatedBlocks hdfsloc) {
     super(length, isdir, replication, blocksize, mtime,
         atime, convert(isdir, symlink != null, permission, flags),
         owner, group, null, null,
         flags.contains(Flags.HAS_ACL), flags.contains(Flags.HAS_CRYPT),
-        flags.contains(Flags.HAS_EC));
+        flags.contains(Flags.HAS_EC), null);
     this.flags = flags;
     this.uSymlink = symlink;
     this.uPath = path;
@@ -101,6 +105,7 @@ public class HdfsFileStatus extends FileStatus {
     this.feInfo = feInfo;
     this.storagePolicy = storagePolicy;
     this.ecPolicy = ecPolicy;
+    this.hdfsloc = hdfsloc;
   }
 
   /**
@@ -152,7 +157,7 @@ public class HdfsFileStatus extends FileStatus {
    * Check if the local name is empty.
    * @return true if the name is empty
    */
-  public final boolean isEmptyLocalName() {
+  public boolean isEmptyLocalName() {
     return uPath.length == 0;
   }
 
@@ -160,7 +165,7 @@ public class HdfsFileStatus extends FileStatus {
    * Get the string representation of the local name.
    * @return the local name in string
    */
-  public final String getLocalName() {
+  public String getLocalName() {
     return DFSUtilClient.bytes2String(uPath);
   }
 
@@ -168,7 +173,7 @@ public class HdfsFileStatus extends FileStatus {
    * Get the Java UTF8 representation of the local name.
    * @return the local name in java UTF8
    */
-  public final byte[] getLocalNameInBytes() {
+  public byte[] getLocalNameInBytes() {
     return uPath;
   }
 
@@ -177,7 +182,7 @@ public class HdfsFileStatus extends FileStatus {
    * @param parent the parent path
    * @return the full path in string
    */
-  public final String getFullName(final String parent) {
+  public String getFullName(String parent) {
     if (isEmptyLocalName()) {
       return parent;
     }
@@ -195,7 +200,7 @@ public class HdfsFileStatus extends FileStatus {
    * @param parent the parent path
    * @return the full path
    */
-  public final Path getFullPath(final Path parent) {
+  public Path getFullPath(Path parent) {
     if (isEmptyLocalName()) {
       return parent;
     }
@@ -219,15 +224,15 @@ public class HdfsFileStatus extends FileStatus {
   /**
    * Opaque referant for the symlink, to be resolved at the client.
    */
-  public final byte[] getSymlinkInBytes() {
+  public byte[] getSymlinkInBytes() {
     return uSymlink;
   }
 
-  public final long getFileId() {
+  public long getFileId() {
     return fileId;
   }
 
-  public final FileEncryptionInfo getFileEncryptionInfo() {
+  public FileEncryptionInfo getFileEncryptionInfo() {
     return feInfo;
   }
 
@@ -239,12 +244,12 @@ public class HdfsFileStatus extends FileStatus {
     return ecPolicy;
   }
 
-  public final int getChildrenNum() {
+  public int getChildrenNum() {
     return childrenNum;
   }
 
   /** @return the storage policy id */
-  public final byte getStoragePolicy() {
+  public byte getStoragePolicy() {
     return storagePolicy;
   }
 
@@ -257,6 +262,10 @@ public class HdfsFileStatus extends FileStatus {
     return flags.contains(Flags.SNAPSHOT_ENABLED);
   }
 
+  public LocatedBlocks getLocatedBlocks() {
+    return hdfsloc;
+  }
+
   @Override
   public boolean equals(Object o) {
     // satisfy findbugs
@@ -277,11 +286,30 @@ public class HdfsFileStatus extends FileStatus {
    * @param parent Parent path of this element.
    * @return Reference to this instance.
    */
-  public final FileStatus makeQualified(URI defaultUri, Path parent) {
+  public FileStatus makeQualified(URI defaultUri, Path parent) {
     // fully-qualify path
     setPath(getFullPath(parent).makeQualified(defaultUri, null));
     return this; // API compatibility
+  }
 
+  /**
+   * This function is used to transform the underlying HDFS LocatedBlocks to
+   * BlockLocations. This method must be invoked before
+   * {@link #getBlockLocations()}.
+   *
+   * The returned BlockLocation will have different formats for replicated
+   * and erasure coded file.
+   * Please refer to
+   * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations
+   * (FileStatus, long, long)}
+   * for examples.
+   */
+  public LocatedFileStatus makeQualifiedLocated(URI defaultUri,
+                                                      Path path) {
+    makeQualified(defaultUri, path);
+    setBlockLocations(
+        DFSUtilClient.locatedBlocks2Locations(getLocatedBlocks()));
+    return this;
   }
 
   /**
@@ -311,6 +339,7 @@ public class HdfsFileStatus extends FileStatus {
     private byte storagePolicy             =
         HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
     private ErasureCodingPolicy ecPolicy   = null;
+    private LocatedBlocks locations        = null;
 
     /**
      * Set the length of the entity (default = 0).
@@ -489,13 +518,24 @@ public class HdfsFileStatus extends FileStatus {
       return this;
     }
 
+    /**
+     * Set the block locations for this entity (default = null).
+     * @param locations HDFS locations
+     *                  (see {@link #makeQualifiedLocated(URI, Path)})
+     * @return This Builder instance
+     */
+    public Builder locations(LocatedBlocks locations) {
+      this.locations = locations;
+      return this;
+    }
+
     /**
      * @return An {@link HdfsFileStatus} instance from these parameters.
      */
     public HdfsFileStatus build() {
       return new HdfsFileStatus(length, isdir, replication, blocksize,
           mtime, atime, permission, flags, owner, group, symlink, path, fileId,
-          childrenNum, feInfo, storagePolicy, ecPolicy);
+          childrenNum, feInfo, storagePolicy, ecPolicy, locations);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java

@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import java.net.URI;
-import java.util.EnumSet;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileEncryptionInfo;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-
-/**
- * Interface that represents the over the wire information
- * including block locations for a file.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class HdfsLocatedFileStatus extends HdfsFileStatus {
-
-  private static final long serialVersionUID = 0x23c73328;
-
-  /**
-   * Left transient, because {@link #makeQualifiedLocated(URI,Path)}
-   * is the user-facing type.
-   */
-  private transient LocatedBlocks locations;
-
-  /**
-   * Constructor
-   *
-   * @param length size
-   * @param isdir if this is directory
-   * @param block_replication the file's replication factor
-   * @param blocksize the file's block size
-   * @param modification_time most recent modification time
-   * @param access_time most recent access time
-   * @param permission permission
-   * @param owner owner
-   * @param group group
-   * @param symlink symbolic link
-   * @param path local path name in java UTF8 format
-   * @param fileId the file id
-   * @param locations block locations
-   * @param feInfo file encryption info
-   */
-  public HdfsLocatedFileStatus(long length, boolean isdir,
-      int block_replication, long blocksize, long modification_time,
-      long access_time, FsPermission permission, EnumSet<Flags> flags,
-      String owner, String group, byte[] symlink, byte[] path, long fileId,
-      LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo,
-      byte storagePolicy, ErasureCodingPolicy ecPolicy) {
-    super(length, isdir, block_replication, blocksize, modification_time,
-        access_time, permission, flags, owner, group, symlink, path, fileId,
-        childrenNum, feInfo, storagePolicy, ecPolicy);
-    this.locations = locations;
-  }
-
-  public LocatedBlocks getBlockLocations() {
-    return locations;
-  }
-
-  /**
-   * This function is used to transform the underlying HDFS LocatedBlocks to
-   * BlockLocations.
-   *
-   * The returned BlockLocation will have different formats for replicated
-   * and erasure coded file.
-   * Please refer to
-   * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations
-   * (FileStatus, long, long)}
-   * for examples.
-   */
-  public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
-      Path path) {
-    makeQualified(defaultUri, path);
-    return new LocatedFileStatus(this,
-        DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    // satisfy findbugs
-    return super.equals(o);
-  }
-
-  @Override
-  public int hashCode() {
-    // satisfy findbugs
-    return super.hashCode();
-  }
-}

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java

@@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
@@ -1782,17 +1783,17 @@ public class ClientNamenodeProtocolTranslatorPB implements
   }
 
   @Override
-  public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+      throws IOException {
     try {
       GetErasureCodingPoliciesResponseProto response = rpcProxy
           .getErasureCodingPolicies(null, VOID_GET_EC_POLICIES_REQUEST);
-      ErasureCodingPolicy[] ecPolicies =
-          new ErasureCodingPolicy[response.getEcPoliciesCount()];
+      ErasureCodingPolicyInfo[] ecPolicies =
+          new ErasureCodingPolicyInfo[response.getEcPoliciesCount()];
       int i = 0;
-      for (ErasureCodingPolicyProto ecPolicyProto :
-          response.getEcPoliciesList()) {
+      for (ErasureCodingPolicyProto proto : response.getEcPoliciesList()) {
         ecPolicies[i++] =
-            PBHelperClient.convertErasureCodingPolicy(ecPolicyProto);
+            PBHelperClient.convertErasureCodingPolicyInfo(proto);
       }
       return ecPolicies;
     } catch (ServiceException e) {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -55,6 +55,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.inotify.Event;
 import org.apache.hadoop.hdfs.inotify.EventBatch;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
@@ -79,6 +80,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
@@ -88,7 +90,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -1583,23 +1584,36 @@ public class PBHelperClient {
     EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags()
         ? convertFlags(fs.getFlags())
         : convertFlags(fs.getPermission());
-    return new HdfsLocatedFileStatus(
-        fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
-        fs.getBlockReplication(), fs.getBlocksize(),
-        fs.getModificationTime(), fs.getAccessTime(),
-        convert(fs.getPermission()),
-        flags,
-        fs.getOwner(), fs.getGroup(),
-        fs.getFileType().equals(FileType.IS_SYMLINK) ?
-            fs.getSymlink().toByteArray() : null,
-        fs.getPath().toByteArray(),
-        fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID,
-        fs.hasLocations() ? convert(fs.getLocations()) : null,
-        fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
-        fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
-        fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
-            : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
-        fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null);
+    return new HdfsFileStatus.Builder()
+        .length(fs.getLength())
+        .isdir(fs.getFileType().equals(FileType.IS_DIR))
+        .replication(fs.getBlockReplication())
+        .blocksize(fs.getBlocksize())
+        .mtime(fs.getModificationTime())
+        .atime(fs.getAccessTime())
+        .perm(convert(fs.getPermission()))
+        .flags(flags)
+        .owner(fs.getOwner())
+        .group(fs.getGroup())
+        .symlink(FileType.IS_SYMLINK.equals(fs.getFileType())
+            ? fs.getSymlink().toByteArray()
+            : null)
+        .path(fs.getPath().toByteArray())
+        .fileId(fs.hasFileId()
+            ? fs.getFileId()
+            : HdfsConstants.GRANDFATHER_INODE_ID)
+        .locations(fs.hasLocations() ? convert(fs.getLocations()) : null)
+        .children(fs.hasChildrenNum() ? fs.getChildrenNum() : -1)
+        .feInfo(fs.hasFileEncryptionInfo()
+            ? convert(fs.getFileEncryptionInfo())
+            : null)
+        .storagePolicy(fs.hasStoragePolicy()
+            ? (byte) fs.getStoragePolicy()
+            : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED)
+        .ecPolicy(fs.hasEcPolicy()
+            ? convertErasureCodingPolicy(fs.getEcPolicy())
+            : null)
+        .build();
   }
 
   private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) {
@@ -1862,10 +1876,10 @@ public class PBHelperClient {
     if (dl == null)
       return null;
     List<HdfsFileStatusProto> partList =  dl.getPartialListingList();
-    return new DirectoryListing(partList.isEmpty() ?
-        new HdfsLocatedFileStatus[0] :
-        convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
-        dl.getRemainingEntries());
+    return new DirectoryListing(partList.isEmpty()
+        ? new HdfsFileStatus[0]
+        : convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
+                  dl.getRemainingEntries());
   }
 
   public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) {
@@ -2159,12 +2173,9 @@ public class PBHelperClient {
     if (fs.getFileEncryptionInfo() != null) {
       builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
     }
-    if (fs instanceof HdfsLocatedFileStatus) {
-      final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
-      LocatedBlocks locations = lfs.getBlockLocations();
-      if (locations != null) {
-        builder.setLocations(convert(locations));
-      }
+    LocatedBlocks locations = fs.getLocatedBlocks();
+    if (locations != null) {
+      builder.setLocations(convert(locations));
     }
     if(fs.getErasureCodingPolicy() != null) {
       builder.setEcPolicy(convertErasureCodingPolicy(
@@ -2956,6 +2967,9 @@ public class PBHelperClient {
     return HdfsProtos.ErasureCodingPolicyState.valueOf(state.getValue());
   }
 
+  /**
+   * Convert the protobuf to a {@link ErasureCodingPolicy}.
+   */
   public static ErasureCodingPolicy convertErasureCodingPolicy(
       ErasureCodingPolicyProto proto) {
     final byte id = (byte) (proto.getId() & 0xFF);
@@ -2969,28 +2983,62 @@ public class PBHelperClient {
           "Missing schema field in ErasureCodingPolicy proto");
       Preconditions.checkArgument(proto.hasCellSize(),
           "Missing cellsize field in ErasureCodingPolicy proto");
-      Preconditions.checkArgument(proto.hasState(),
-          "Missing state field in ErasureCodingPolicy proto");
 
       return new ErasureCodingPolicy(proto.getName(),
           convertECSchema(proto.getSchema()),
-          proto.getCellSize(), id, convertECState(proto.getState()));
+          proto.getCellSize(), id);
     }
     return policy;
   }
 
-  public static ErasureCodingPolicyProto convertErasureCodingPolicy(
+  /**
+   * Convert the protobuf to a {@link ErasureCodingPolicyInfo}. This should only
+   * be needed when the caller is interested in the state of the policy.
+   */
+  public static ErasureCodingPolicyInfo convertErasureCodingPolicyInfo(
+      ErasureCodingPolicyProto proto) {
+    ErasureCodingPolicy policy = convertErasureCodingPolicy(proto);
+    ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
+    Preconditions.checkArgument(proto.hasState(),
+        "Missing state field in ErasureCodingPolicy proto");
+    info.setState(convertECState(proto.getState()));
+    return info;
+  }
+
+  private static ErasureCodingPolicyProto.Builder createECPolicyProtoBuilder(
       ErasureCodingPolicy policy) {
-    ErasureCodingPolicyProto.Builder builder = ErasureCodingPolicyProto
-        .newBuilder()
-        .setId(policy.getId());
+    final ErasureCodingPolicyProto.Builder builder =
+        ErasureCodingPolicyProto.newBuilder().setId(policy.getId());
     // If it's not a built-in policy, need to set the optional fields.
     if (SystemErasureCodingPolicies.getByID(policy.getId()) == null) {
       builder.setName(policy.getName())
           .setSchema(convertECSchema(policy.getSchema()))
-          .setCellSize(policy.getCellSize())
-          .setState(convertECState(policy.getState()));
+          .setCellSize(policy.getCellSize());
     }
+    return builder;
+  }
+
+  /**
+   * Convert a {@link ErasureCodingPolicy} to protobuf.
+   * This means no state of the policy will be set on the protobuf.
+   */
+  public static ErasureCodingPolicyProto convertErasureCodingPolicy(
+      ErasureCodingPolicy policy) {
+    return createECPolicyProtoBuilder(policy).build();
+  }
+
+  /**
+   * Convert a {@link ErasureCodingPolicyInfo} to protobuf.
+   * The protobuf will have the policy, and state. State is relevant when:
+   * 1. Persisting a policy to fsimage
+   * 2. Returning the policy to the RPC call
+   * {@link DistributedFileSystem#getAllErasureCodingPolicies()}
+   */
+  public static ErasureCodingPolicyProto convertErasureCodingPolicy(
+      ErasureCodingPolicyInfo info) {
+    final ErasureCodingPolicyProto.Builder builder =
+        createECPolicyProtoBuilder(info.getPolicy());
+    builder.setState(convertECState(info.getState()));
     return builder.build();
   }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -780,6 +780,7 @@ public class WebHdfsFileSystem extends FileSystem
           try {
             IOException newIoe = ioe.getClass().getConstructor(String.class)
                 .newInstance(node + ": " + ioe.getMessage());
+            newIoe.initCause(ioe.getCause());
             newIoe.setStackTrace(ioe.getStackTrace());
             ioe = newIoe;
           } catch (NoSuchMethodException | SecurityException 

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicyInfo.java

@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.junit.Test;
+
+import static org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies.RS_6_3_POLICY_ID;
+import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.DISABLED;
+import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.ENABLED;
+import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.REMOVED;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test {@link ErasureCodingPolicyInfo}.
+ */
+public class TestErasureCodingPolicyInfo {
+
+  @Test
+  public void testPolicyAndStateCantBeNull() {
+    try {
+      new ErasureCodingPolicyInfo(null);
+      fail("Null policy should fail");
+    } catch (NullPointerException expected) {
+    }
+
+    try {
+      new ErasureCodingPolicyInfo(SystemErasureCodingPolicies
+          .getByID(RS_6_3_POLICY_ID), null);
+      fail("Null policy should fail");
+    } catch (NullPointerException expected) {
+    }
+  }
+
+  @Test
+  public void testStates() {
+    ErasureCodingPolicyInfo info =
+        new ErasureCodingPolicyInfo(SystemErasureCodingPolicies
+          .getByID(RS_6_3_POLICY_ID));
+
+    info.setState(ENABLED);
+    assertFalse(info.isDisabled());
+    assertTrue(info.isEnabled());
+    assertFalse(info.isRemoved());
+
+    info.setState(REMOVED);
+    assertFalse(info.isDisabled());
+    assertFalse(info.isEnabled());
+    assertTrue(info.isRemoved());
+
+    info.setState(DISABLED);
+    assertTrue(info.isDisabled());
+    assertFalse(info.isEnabled());
+    assertFalse(info.isRemoved());
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
+import java.nio.file.FileSystemException;
 
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -33,9 +34,14 @@ import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
 import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
 import org.apache.hadoop.hdfs.nfs.mount.Mountd;
 import org.apache.hadoop.hdfs.nfs.mount.RpcProgramMountd;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 public class TestExportsTable {
+
+  @Rule
+  public ExpectedException exception = ExpectedException.none();
  
   @Test
   public void testHdfsExportPoint() throws IOException {
@@ -70,7 +76,7 @@ public class TestExportsTable {
   }
 
   @Test
-  public void testViewFsExportPoint() throws IOException {
+  public void testViewFsMultipleExportPoint() throws IOException {
     NfsConfiguration config = new NfsConfiguration();
     MiniDFSCluster cluster = null;
     String clusterName = RandomStringUtils.randomAlphabetic(10);
@@ -182,6 +188,56 @@ public class TestExportsTable {
     }
   }
 
+  @Test
+  public void testViewFsRootExportPoint() throws IOException {
+    NfsConfiguration config = new NfsConfiguration();
+    MiniDFSCluster cluster = null;
+    String clusterName = RandomStringUtils.randomAlphabetic(10);
+
+    String exportPoint = "/";
+    config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY, exportPoint);
+    config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+        FsConstants.VIEWFS_SCHEME + "://" + clusterName);
+    // Use emphral port in case tests are running in parallel
+    config.setInt("nfs3.mountd.port", 0);
+    config.setInt("nfs3.server.port", 0);
+    config.set("nfs.http.address", "0.0.0.0:0");
+
+    try {
+      cluster =
+          new MiniDFSCluster.Builder(config).nnTopology(
+              MiniDFSNNTopology.simpleFederatedTopology(2))
+              .numDataNodes(2)
+              .build();
+      cluster.waitActive();
+      DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+      DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+      cluster.waitActive();
+      Path base1 = new Path("/user1");
+      Path base2 = new Path("/user2");
+      hdfs1.delete(base1, true);
+      hdfs2.delete(base2, true);
+      hdfs1.mkdirs(base1);
+      hdfs2.mkdirs(base2);
+      ConfigUtil.addLink(config, clusterName, "/hdfs1",
+          hdfs1.makeQualified(base1).toUri());
+      ConfigUtil.addLink(config, clusterName, "/hdfs2",
+          hdfs2.makeQualified(base2).toUri());
+
+      exception.expect(FileSystemException.class);
+      exception.
+          expectMessage("Only HDFS is supported as underlyingFileSystem, "
+              + "fs scheme:viewfs");
+      // Start nfs
+      final Nfs3 nfsServer = new Nfs3(config);
+      nfsServer.startServiceInternal(false);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
   @Test
   public void testHdfsInternalExportPoint() throws IOException {
     NfsConfiguration config = new NfsConfiguration();
@@ -219,4 +275,34 @@ public class TestExportsTable {
       }
     }
   }
+
+  @Test
+  public void testInvalidFsExport() throws IOException {
+    NfsConfiguration config = new NfsConfiguration();
+    MiniDFSCluster cluster = null;
+
+    // Use emphral port in case tests are running in parallel
+    config.setInt("nfs3.mountd.port", 0);
+    config.setInt("nfs3.server.port", 0);
+    config.set("nfs.http.address", "0.0.0.0:0");
+
+    try {
+      cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+      cluster.waitActive();
+      config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+          FsConstants.LOCAL_FS_URI.toString());
+
+      exception.expect(FileSystemException.class);
+      exception.
+          expectMessage("Only HDFS is supported as underlyingFileSystem, "
+              + "fs scheme:file");
+      // Start nfs
+      final Nfs3 nfsServer = new Nfs3(config);
+      nfsServer.startServiceInternal(false);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml

@@ -269,11 +269,10 @@
         <Method name="visitFile" />
         <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
     </Match>
-    <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not.
-         Defensible compatibility choices over time create odd corners. -->
+    <!-- BlockLocations are user-facing, but LocatedBlocks are not. -->
     <Match>
-        <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" />
-        <Field name="locations" />
+        <Class name="org.apache.hadoop.hdfs.protocol.HdfsFileStatus" />
+        <Field name="hdfsloc" />
         <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" />
     </Match>
     <Match>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java

@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
@@ -1683,11 +1684,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   public GetErasureCodingPoliciesResponseProto getErasureCodingPolicies(RpcController controller,
       GetErasureCodingPoliciesRequestProto request) throws ServiceException {
     try {
-      ErasureCodingPolicy[] ecPolicies = server.getErasureCodingPolicies();
+      ErasureCodingPolicyInfo[] ecpInfos = server.getErasureCodingPolicies();
       GetErasureCodingPoliciesResponseProto.Builder resBuilder = GetErasureCodingPoliciesResponseProto
           .newBuilder();
-      for (ErasureCodingPolicy ecPolicy : ecPolicies) {
-        resBuilder.addEcPolicies(PBHelperClient.convertErasureCodingPolicy(ecPolicy));
+      for (ErasureCodingPolicyInfo info : ecpInfos) {
+        resBuilder.addEcPolicies(
+            PBHelperClient.convertErasureCodingPolicy(info));
       }
       return resBuilder.build();
     } catch (IOException e) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 
+import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos
@@ -51,6 +52,8 @@ import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 
 /**
@@ -263,25 +266,63 @@ public class JournalNodeSyncer {
   }
 
   private List<InetSocketAddress> getOtherJournalNodeAddrs() {
-    URI uri = null;
+    String uriStr = "";
     try {
-      String uriStr = conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+      uriStr = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+
+      if (uriStr == null || uriStr.isEmpty()) {
+        if (nameServiceId != null) {
+          uriStr = conf.getTrimmed(DFSConfigKeys
+              .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "." + nameServiceId);
+        }
+      }
+
       if (uriStr == null || uriStr.isEmpty()) {
-        LOG.warn("Could not construct Shared Edits Uri");
+        HashSet<String> sharedEditsUri = Sets.newHashSet();
+        if (nameServiceId != null) {
+          Collection<String> nnIds = DFSUtilClient.getNameNodeIds(
+              conf, nameServiceId);
+          for (String nnId : nnIds) {
+            String suffix = nameServiceId + "." + nnId;
+            uriStr = conf.getTrimmed(DFSConfigKeys
+                .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "." + suffix);
+            sharedEditsUri.add(uriStr);
+          }
+          if (sharedEditsUri.size() > 1) {
+            uriStr = null;
+            LOG.error("The conf property " + DFSConfigKeys
+                .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " not set properly, " +
+                "it has been configured with different journalnode values " +
+                sharedEditsUri.toString() + " for a" +
+                " single nameserviceId" + nameServiceId);
+          }
+        }
+      }
+
+      if (uriStr == null || uriStr.isEmpty()) {
+        LOG.error("Could not construct Shared Edits Uri");
         return null;
+      } else {
+        return getJournalAddrList(uriStr);
       }
-      uri = new URI(uriStr);
-      return Util.getLoggerAddresses(uri,
-          Sets.newHashSet(jn.getBoundIpcAddress()));
+
     } catch (URISyntaxException e) {
       LOG.error("The conf property " + DFSConfigKeys
           .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " not set properly.");
     } catch (IOException e) {
-      LOG.error("Could not parse JournalNode addresses: " + uri);
+      LOG.error("Could not parse JournalNode addresses: " + uriStr);
     }
     return null;
   }
 
+  private List<InetSocketAddress> getJournalAddrList(String uriStr) throws
+      URISyntaxException,
+      IOException {
+    URI uri = new URI(uriStr);
+    return Util.getLoggerAddresses(uri,
+        Sets.newHashSet(jn.getBoundIpcAddress()));
+  }
+
   private JournalIdProto convertJournalId(String journalId) {
     return QJournalProtocolProtos.JournalIdProto.newBuilder()
       .setIdentifier(journalId)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -775,10 +775,11 @@ public class BlockManager implements BlockStatsMXBean {
           String storageId = storage.getStorageID();
           DatanodeStorageInfo storageInfo = node.getStorageInfo(storageId);
           State state = (storageInfo == null) ? null : storageInfo.getState();
-          out.println("Block=" + block.getBlockId() + "\tNode=" + node.getName()
-              + "\tStorageID=" + storageId + "\tStorageState=" + state
-              + "\tTotalReplicas=" +
-              blocksMap.numNodes(block)
+          out.println("Block=" + block.toString()
+              + "\tSize=" + block.getNumBytes()
+              + "\tNode=" + node.getName() + "\tStorageID=" + storageId
+              + "\tStorageState=" + state
+              + "\tTotalReplicas=" + blocksMap.numNodes(block)
               + "\tReason=" + corruptReplicas.getCorruptReason(block, node));
           numNodesToFind--;
           if (numNodesToFind == 0) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java

@@ -614,7 +614,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
 
       if (LOG.isDebugEnabled()) {
         LOG.debug("Failed to choose from local rack (location = " + localRack
-            + "); the second replica is not found, retry choosing ramdomly", e);
+            + "); the second replica is not found, retry choosing randomly", e);
       }
       //the second replica is not found, randomly choose one from the network
       return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
@@ -636,7 +636,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
     } catch(NotEnoughReplicasException e) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Failed to choose from the next rack (location = " + nextRack
-            + "), retry choosing ramdomly", e);
+            + "), retry choosing randomly", e);
       }
       //otherwise randomly choose one from the network
       return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java

@@ -62,10 +62,17 @@ public class BlockPlacementPolicyRackFaultTolerant extends BlockPlacementPolicyD
    * randomly.
    * 2. If total replica expected is bigger than numOfRacks, it choose:
    *  2a. Fill each rack exactly (maxNodesPerRack-1) replicas.
-   *  2b. For some random racks, place one more replica to each one of them, until
-   *  numOfReplicas have been chosen. <br>
-   * In the end, the difference of the numbers of replicas for each two racks
-   * is no more than 1.
+   *  2b. For some random racks, place one more replica to each one of them,
+   *  until numOfReplicas have been chosen. <br>
+   * 3. If after step 2, there are still replicas not placed (due to some
+   * racks have fewer datanodes than maxNodesPerRack), the rest of the replicas
+   * is placed evenly on the rest of the racks who have Datanodes that have
+   * not been placed a replica.
+   * 4. If after step 3, there are still replicas not placed. A
+   * {@link NotEnoughReplicasException} is thrown.
+   * <p>
+   * For normal setups, step 2 would suffice. So in the end, the difference
+   * of the numbers of replicas for each two racks is no more than 1.
    * Either way it always prefer local storage.
    * @return local node of writer
    */
@@ -132,24 +139,63 @@ public class BlockPlacementPolicyRackFaultTolerant extends BlockPlacementPolicyD
       chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
           maxNodesPerRack, results, avoidStaleNodes, storageTypes);
     } catch (NotEnoughReplicasException e) {
-      LOG.debug("Only able to place {} of {} (maxNodesPerRack={}) nodes " +
-              "evenly across racks, falling back to uneven placement.",
-          results.size(), numOfReplicas, maxNodesPerRack);
+      LOG.warn("Only able to place {} of total expected {}"
+              + " (maxNodesPerRack={}, numOfReplicas={}) nodes "
+              + "evenly across racks, falling back to evenly place on the "
+              + "remaining racks. This may not guarantee rack-level fault "
+              + "tolerance. Please check if the racks are configured properly.",
+          results.size(), totalReplicaExpected, maxNodesPerRack, numOfReplicas);
       LOG.debug("Caught exception was:", e);
+      chooseEvenlyFromRemainingRacks(writer, excludedNodes, blocksize,
+          maxNodesPerRack, results, avoidStaleNodes, storageTypes,
+          totalReplicaExpected, e);
+
+    }
+
+    return writer;
+  }
+
+  /**
+   * Choose as evenly as possible from the racks which have available datanodes.
+   */
+  private void chooseEvenlyFromRemainingRacks(Node writer,
+      Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
+      List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
+      EnumMap<StorageType, Integer> storageTypes, int totalReplicaExpected,
+      NotEnoughReplicasException e) throws NotEnoughReplicasException {
+    int numResultsOflastChoose = 0;
+    NotEnoughReplicasException lastException = e;
+    int bestEffortMaxNodesPerRack = maxNodesPerRack;
+    while (results.size() != totalReplicaExpected &&
+        numResultsOflastChoose != results.size()) {
       // Exclude the chosen nodes
+      final Set<Node> newExcludeNodes = new HashSet<>();
       for (DatanodeStorageInfo resultStorage : results) {
         addToExcludedNodes(resultStorage.getDatanodeDescriptor(),
-            excludedNodes);
+            newExcludeNodes);
       }
 
       LOG.trace("Chosen nodes: {}", results);
       LOG.trace("Excluded nodes: {}", excludedNodes);
-      numOfReplicas = totalReplicaExpected - results.size();
-      chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
-          totalReplicaExpected, results, avoidStaleNodes, storageTypes);
+      LOG.trace("New Excluded nodes: {}", newExcludeNodes);
+      final int numOfReplicas = totalReplicaExpected - results.size();
+      numResultsOflastChoose = results.size();
+      try {
+        chooseOnce(numOfReplicas, writer, newExcludeNodes, blocksize,
+            ++bestEffortMaxNodesPerRack, results, avoidStaleNodes,
+            storageTypes);
+      } catch (NotEnoughReplicasException nere) {
+        lastException = nere;
+      } finally {
+        excludedNodes.addAll(newExcludeNodes);
+      }
     }
 
-    return writer;
+    if (numResultsOflastChoose != totalReplicaExpected) {
+      LOG.debug("Best effort placement failed: expecting {} replicas, only "
+          + "chose {}.", totalReplicaExpected, numResultsOflastChoose);
+      throw lastException;
+    }
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java

@@ -153,22 +153,22 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
   public StorageReport[] getStorageReports(String bpid)
       throws IOException {
     List<StorageReport> reports;
-    synchronized (statsLock) {
-      List<FsVolumeImpl> curVolumes = volumes.getVolumes();
-      reports = new ArrayList<>(curVolumes.size());
-      for (FsVolumeImpl volume : curVolumes) {
-        try (FsVolumeReference ref = volume.obtainReference()) {
-          StorageReport sr = new StorageReport(volume.toDatanodeStorage(),
-              false,
-              volume.getCapacity(),
-              volume.getDfsUsed(),
-              volume.getAvailable(),
-              volume.getBlockPoolUsed(bpid),
-              volume.getNonDfsUsed());
-          reports.add(sr);
-        } catch (ClosedChannelException e) {
-          continue;
-        }
+    // Volumes are the references from a copy-on-write snapshot, so the
+    // access on the volume metrics doesn't require an additional lock.
+    List<FsVolumeImpl> curVolumes = volumes.getVolumes();
+    reports = new ArrayList<>(curVolumes.size());
+    for (FsVolumeImpl volume : curVolumes) {
+      try (FsVolumeReference ref = volume.obtainReference()) {
+        StorageReport sr = new StorageReport(volume.toDatanodeStorage(),
+            false,
+            volume.getCapacity(),
+            volume.getDfsUsed(),
+            volume.getAvailable(),
+            volume.getBlockPoolUsed(bpid),
+            volume.getNonDfsUsed());
+        reports.add(sr);
+      } catch (ClosedChannelException e) {
+        continue;
       }
     }
 
@@ -247,9 +247,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
   private final int smallBufferSize;
 
-  // Used for synchronizing access to usage stats
-  private final Object statsLock = new Object();
-
   final LocalFileSystem localFS;
 
   private boolean blockPinningEnabled;
@@ -583,9 +580,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
    */
   @Override // FSDatasetMBean
   public long getDfsUsed() throws IOException {
-    synchronized(statsLock) {
-      return volumes.getDfsUsed();
-    }
+    return volumes.getDfsUsed();
   }
 
   /**
@@ -593,9 +588,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
    */
   @Override // FSDatasetMBean
   public long getBlockPoolUsed(String bpid) throws IOException {
-    synchronized(statsLock) {
-      return volumes.getBlockPoolUsed(bpid);
-    }
+    return volumes.getBlockPoolUsed(bpid);
   }
   
   /**
@@ -611,9 +604,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
    */
   @Override // FSDatasetMBean
   public long getCapacity() throws IOException {
-    synchronized(statsLock) {
-      return volumes.getCapacity();
-    }
+    return volumes.getCapacity();
   }
 
   /**
@@ -621,9 +612,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
    */
   @Override // FSDatasetMBean
   public long getRemaining() throws IOException {
-    synchronized(statsLock) {
-      return volumes.getRemaining();
-    }
+    return volumes.getRemaining();
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java

@@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -342,43 +341,38 @@ public class FsVolumeImpl implements FsVolumeSpi {
 
   private void decDfsUsedAndNumBlocks(String bpid, long value,
                                       boolean blockFileDeleted) {
-    try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
-      BlockPoolSlice bp = bpSlices.get(bpid);
-      if (bp != null) {
-        bp.decDfsUsed(value);
-        if (blockFileDeleted) {
-          bp.decrNumBlocks();
-        }
+    // BlockPoolSlice map is thread safe, and update the space used or
+    // number of blocks are atomic operations, so it doesn't require to
+    // hold the dataset lock.
+    BlockPoolSlice bp = bpSlices.get(bpid);
+    if (bp != null) {
+      bp.decDfsUsed(value);
+      if (blockFileDeleted) {
+        bp.decrNumBlocks();
       }
     }
   }
 
   void incDfsUsedAndNumBlocks(String bpid, long value) {
-    try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
-      BlockPoolSlice bp = bpSlices.get(bpid);
-      if (bp != null) {
-        bp.incDfsUsed(value);
-        bp.incrNumBlocks();
-      }
+    BlockPoolSlice bp = bpSlices.get(bpid);
+    if (bp != null) {
+      bp.incDfsUsed(value);
+      bp.incrNumBlocks();
     }
   }
 
   void incDfsUsed(String bpid, long value) {
-    try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
-      BlockPoolSlice bp = bpSlices.get(bpid);
-      if (bp != null) {
-        bp.incDfsUsed(value);
-      }
+    BlockPoolSlice bp = bpSlices.get(bpid);
+    if (bp != null) {
+      bp.incDfsUsed(value);
     }
   }
 
   @VisibleForTesting
   public long getDfsUsed() throws IOException {
     long dfsUsed = 0;
-    try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
-      for(BlockPoolSlice s : bpSlices.values()) {
-        dfsUsed += s.getDfsUsed();
-      }
+    for(BlockPoolSlice s : bpSlices.values()) {
+      dfsUsed += s.getDfsUsed();
     }
     return dfsUsed;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java

@@ -44,7 +44,6 @@ import org.apache.hadoop.security.token.Token;
 
 import java.io.IOException;
 import java.net.URI;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.EnumSet;
 import java.util.List;
@@ -143,9 +142,13 @@ class ParameterParser {
   }
 
   public EnumSet<CreateFlag> createFlag() {
-    String cf =
-        decodeComponent(param(CreateFlagParam.NAME), StandardCharsets.UTF_8);
-
+    String cf = "";
+    if (param(CreateFlagParam.NAME) != null) {
+      QueryStringDecoder decoder = new QueryStringDecoder(
+          param(CreateFlagParam.NAME),
+          StandardCharsets.UTF_8);
+      cf = decoder.path();
+    }
     return new CreateFlagParam(cf).getValue();
   }
 
@@ -158,61 +161,6 @@ class ParameterParser {
     return p == null ? null : p.get(0);
   }
 
-  /**
-   * The following function behaves exactly the same as netty's
-   * <code>QueryStringDecoder#decodeComponent</code> except that it
-   * does not decode the '+' character as space. WebHDFS takes this scheme
-   * to maintain the backward-compatibility for pre-2.7 releases.
-   */
-  private static String decodeComponent(final String s, final Charset charset) {
-    if (s == null) {
-      return "";
-    }
-    final int size = s.length();
-    boolean modified = false;
-    for (int i = 0; i < size; i++) {
-      final char c = s.charAt(i);
-      if (c == '%' || c == '+') {
-        modified = true;
-        break;
-      }
-    }
-    if (!modified) {
-      return s;
-    }
-    final byte[] buf = new byte[size];
-    int pos = 0;  // position in `buf'.
-    for (int i = 0; i < size; i++) {
-      char c = s.charAt(i);
-      if (c == '%') {
-        if (i == size - 1) {
-          throw new IllegalArgumentException("unterminated escape sequence at" +
-                                                 " end of string: " + s);
-        }
-        c = s.charAt(++i);
-        if (c == '%') {
-          buf[pos++] = '%';  // "%%" -> "%"
-          break;
-        }
-        if (i == size - 1) {
-          throw new IllegalArgumentException("partial escape sequence at end " +
-                                                 "of string: " + s);
-        }
-        c = decodeHexNibble(c);
-        final char c2 = decodeHexNibble(s.charAt(++i));
-        if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) {
-          throw new IllegalArgumentException(
-              "invalid escape sequence `%" + s.charAt(i - 1) + s.charAt(
-                  i) + "' at index " + (i - 2) + " of: " + s);
-        }
-        c = (char) (c * 16 + c2);
-        // Fall through.
-      }
-      buf[pos++] = (byte) c;
-    }
-    return new String(buf, 0, pos, charset);
-  }
-
   /**
    * Helper to decode half of a hexadecimal number from a string.
    * @param c The ASCII character of the hexadecimal number to decode.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java

@@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
@@ -1785,7 +1786,8 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
   }
 
   @Override
-  public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+      throws IOException {
     checkOperation(OperationCategory.READ, false);
     return null;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java

@@ -364,7 +364,7 @@ public class Mover {
           if (!isSnapshotPathInCurrent(fullPath)) {
             // the full path is a snapshot path but it is also included in the
             // current directory tree, thus ignore it.
-            processFile(fullPath, (HdfsLocatedFileStatus) status, result);
+            processFile(fullPath, status, result);
           }
         } catch (IOException e) {
           LOG.warn("Failed to check the status of " + parent
@@ -374,7 +374,7 @@ public class Mover {
     }
 
     /** @return true if it is necessary to run another round of migration */
-    private void processFile(String fullPath, HdfsLocatedFileStatus status,
+    private void processFile(String fullPath, HdfsFileStatus status,
         Result result) {
       byte policyId = status.getStoragePolicy();
       if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
@@ -395,7 +395,7 @@ public class Mover {
           status.getReplication());
 
       final ErasureCodingPolicy ecPolicy = status.getErasureCodingPolicy();
-      final LocatedBlocks locatedBlocks = status.getBlockLocations();
+      final LocatedBlocks locatedBlocks = status.getLocatedBlocks();
       final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete();
       List<LocatedBlock> lbs = locatedBlocks.getLocatedBlocks();
       for (int i = 0; i < lbs.size(); i++) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -65,18 +66,18 @@ public final class ErasureCodingPolicyManager {
    * All policies sorted by name for fast querying, include built-in policy,
    * user defined policy, removed policy.
    */
-  private Map<String, ErasureCodingPolicy> policiesByName;
+  private Map<String, ErasureCodingPolicyInfo> policiesByName;
 
   /**
    * All policies sorted by ID for fast querying, including built-in policy,
    * user defined policy, removed policy.
    */
-  private Map<Byte, ErasureCodingPolicy> policiesByID;
+  private Map<Byte, ErasureCodingPolicyInfo> policiesByID;
 
   /**
    * For better performance when query all Policies.
    */
-  private ErasureCodingPolicy[] allPolicies;
+  private ErasureCodingPolicyInfo[] allPolicies;
 
   /**
    * All enabled policies sorted by name for fast querying, including built-in
@@ -120,15 +121,17 @@ public final class ErasureCodingPolicyManager {
      */
     for (ErasureCodingPolicy policy :
         SystemErasureCodingPolicies.getPolicies()) {
-      policiesByName.put(policy.getName(), policy);
-      policiesByID.put(policy.getId(), policy);
+      final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
+      policiesByName.put(policy.getName(), info);
+      policiesByID.put(policy.getId(), info);
     }
 
-    if (!defaultPolicyName.trim().isEmpty()) {
-      ErasureCodingPolicy ecPolicy = policiesByName.get(defaultPolicyName);
-      if (ecPolicy == null) {
+    if (!defaultPolicyName.isEmpty()) {
+      final ErasureCodingPolicyInfo info =
+          policiesByName.get(defaultPolicyName);
+      if (info == null) {
         String names = policiesByName.values()
-            .stream().map(ErasureCodingPolicy::getName)
+            .stream().map((pi) -> pi.getPolicy().getName())
             .collect(Collectors.joining(", "));
         String msg = String.format("EC policy '%s' specified at %s is not a "
                 + "valid policy. Please choose from list of available "
@@ -138,11 +141,13 @@ public final class ErasureCodingPolicyManager {
             names);
         throw new HadoopIllegalArgumentException(msg);
       }
-      enabledPoliciesByName.put(ecPolicy.getName(), ecPolicy);
+      info.setState(ErasureCodingPolicyState.ENABLED);
+      enabledPoliciesByName.put(info.getPolicy().getName(), info.getPolicy());
     }
     enabledPolicies =
         enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
-    allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]);
+    allPolicies =
+        policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
 
     maxCellSize = conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY,
@@ -190,23 +195,50 @@ public final class ErasureCodingPolicyManager {
    * Get all system defined policies and user defined policies.
    * @return all policies
    */
-  public ErasureCodingPolicy[] getPolicies() {
+  public ErasureCodingPolicyInfo[] getPolicies() {
     return allPolicies;
   }
 
   /**
-   * Get a policy by policy ID, including system policy and user defined policy.
+   * Get a {@link ErasureCodingPolicy} by policy ID, including system policy
+   * and user defined policy.
    * @return ecPolicy, or null if not found
    */
   public ErasureCodingPolicy getByID(byte id) {
+    final ErasureCodingPolicyInfo ecpi = getPolicyInfoByID(id);
+    if (ecpi == null) {
+      return null;
+    }
+    return ecpi.getPolicy();
+  }
+
+  /**
+   * Get a {@link ErasureCodingPolicyInfo} by policy ID, including system policy
+   * and user defined policy.
+   */
+  private ErasureCodingPolicyInfo getPolicyInfoByID(final byte id) {
     return this.policiesByID.get(id);
   }
 
   /**
-   * Get a policy by policy ID, including system policy and user defined policy.
+   * Get a {@link ErasureCodingPolicy} by policy name, including system
+   * policy and user defined policy.
    * @return ecPolicy, or null if not found
    */
   public ErasureCodingPolicy getByName(String name) {
+    final ErasureCodingPolicyInfo ecpi = getPolicyInfoByName(name);
+    if (ecpi == null) {
+      return null;
+    }
+    return ecpi.getPolicy();
+  }
+
+  /**
+   * Get a {@link ErasureCodingPolicyInfo} by policy name, including system
+   * policy and user defined policy.
+   * @return ecPolicy, or null if not found
+   */
+  private ErasureCodingPolicyInfo getPolicyInfoByName(final String name) {
     return this.policiesByName.get(name);
   }
 
@@ -224,9 +256,6 @@ public final class ErasureCodingPolicyManager {
    */
   public synchronized ErasureCodingPolicy addPolicy(
       ErasureCodingPolicy policy) {
-    // Set policy state into DISABLED when adding into Hadoop.
-    policy.setState(ErasureCodingPolicyState.DISABLED);
-
     if (!CodecUtil.hasCodec(policy.getCodecName())) {
       throw new HadoopIllegalArgumentException("Codec name "
           + policy.getCodecName() + " is not supported");
@@ -240,7 +269,8 @@ public final class ErasureCodingPolicyManager {
 
     String assignedNewName = ErasureCodingPolicy.composePolicyName(
         policy.getSchema(), policy.getCellSize());
-    for (ErasureCodingPolicy p : getPolicies()) {
+    for (ErasureCodingPolicyInfo info : getPolicies()) {
+      final ErasureCodingPolicy p = info.getPolicy();
       if (p.getName().equals(assignedNewName)) {
         LOG.info("The policy name " + assignedNewName + " already exists");
         return p;
@@ -261,11 +291,13 @@ public final class ErasureCodingPolicyManager {
           ErasureCodeConstants.MAX_POLICY_ID);
     }
 
-    policy.setName(assignedNewName);
-    policy.setId(getNextAvailablePolicyID());
-    this.policiesByName.put(policy.getName(), policy);
-    this.policiesByID.put(policy.getId(), policy);
-    allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]);
+    policy = new ErasureCodingPolicy(assignedNewName, policy.getSchema(),
+        policy.getCellSize(), getNextAvailablePolicyID());
+    final ErasureCodingPolicyInfo pi = new ErasureCodingPolicyInfo(policy);
+    this.policiesByName.put(policy.getName(), pi);
+    this.policiesByID.put(policy.getId(), pi);
+    allPolicies =
+        policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
     return policy;
   }
 
@@ -283,12 +315,13 @@ public final class ErasureCodingPolicyManager {
    * Remove an User erasure coding policy by policyName.
    */
   public synchronized void removePolicy(String name) {
-    ErasureCodingPolicy ecPolicy = policiesByName.get(name);
-    if (ecPolicy == null) {
+    final ErasureCodingPolicyInfo info = policiesByName.get(name);
+    if (info == null) {
       throw new HadoopIllegalArgumentException("The policy name " +
           name + " does not exist");
     }
 
+    final ErasureCodingPolicy ecPolicy = info.getPolicy();
     if (ecPolicy.isSystemPolicy()) {
       throw new HadoopIllegalArgumentException("System erasure coding policy " +
           name + " cannot be removed");
@@ -299,7 +332,7 @@ public final class ErasureCodingPolicyManager {
       enabledPolicies =
           enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
     }
-    ecPolicy.setState(ErasureCodingPolicyState.REMOVED);
+    info.setState(ErasureCodingPolicyState.REMOVED);
     LOG.info("Remove erasure coding policy " + name);
 
     /*
@@ -310,10 +343,10 @@ public final class ErasureCodingPolicyManager {
 
   @VisibleForTesting
   public List<ErasureCodingPolicy> getRemovedPolicies() {
-    ArrayList<ErasureCodingPolicy> removedPolicies =
-        new ArrayList<ErasureCodingPolicy>();
-    for (ErasureCodingPolicy ecPolicy : policiesByName.values()) {
-      if (ecPolicy.isRemoved()) {
+    ArrayList<ErasureCodingPolicy> removedPolicies = new ArrayList<>();
+    for (ErasureCodingPolicyInfo info : policiesByName.values()) {
+      final ErasureCodingPolicy ecPolicy = info.getPolicy();
+      if (info.isRemoved()) {
         removedPolicies.add(ecPolicy);
       }
     }
@@ -324,8 +357,8 @@ public final class ErasureCodingPolicyManager {
    * Disable an erasure coding policy by policyName.
    */
   public synchronized void disablePolicy(String name) {
-    ErasureCodingPolicy ecPolicy = policiesByName.get(name);
-    if (ecPolicy == null) {
+    ErasureCodingPolicyInfo info = policiesByName.get(name);
+    if (info == null) {
       throw new HadoopIllegalArgumentException("The policy name " +
           name + " does not exist");
     }
@@ -335,7 +368,7 @@ public final class ErasureCodingPolicyManager {
       enabledPolicies =
           enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
     }
-    ecPolicy.setState(ErasureCodingPolicyState.DISABLED);
+    info.setState(ErasureCodingPolicyState.DISABLED);
     LOG.info("Disable the erasure coding policy " + name);
   }
 
@@ -343,14 +376,15 @@ public final class ErasureCodingPolicyManager {
    * Enable an erasure coding policy by policyName.
    */
   public synchronized void enablePolicy(String name) {
-    ErasureCodingPolicy ecPolicy = policiesByName.get(name);
-    if (ecPolicy == null) {
+    final ErasureCodingPolicyInfo info = policiesByName.get(name);
+    if (info == null) {
       throw new HadoopIllegalArgumentException("The policy name " +
           name + " does not exist");
     }
 
+    final ErasureCodingPolicy ecPolicy = info.getPolicy();
     enabledPoliciesByName.put(name, ecPolicy);
-    ecPolicy.setState(ErasureCodingPolicyState.ENABLED);
+    info.setState(ErasureCodingPolicyState.ENABLED);
     enabledPolicies =
         enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]);
     LOG.info("Enable the erasure coding policy " + name);
@@ -359,17 +393,19 @@ public final class ErasureCodingPolicyManager {
   /**
    * Load an erasure coding policy into erasure coding manager.
    */
-  private void loadPolicy(ErasureCodingPolicy policy) {
+  private void loadPolicy(ErasureCodingPolicyInfo info) {
+    Preconditions.checkNotNull(info);
+    final ErasureCodingPolicy policy = info.getPolicy();
     if (!CodecUtil.hasCodec(policy.getCodecName()) ||
         policy.getCellSize() > maxCellSize) {
       // If policy is not supported in current system, set the policy state to
       // DISABLED;
-      policy.setState(ErasureCodingPolicyState.DISABLED);
+      info.setState(ErasureCodingPolicyState.DISABLED);
     }
 
-    this.policiesByName.put(policy.getName(), policy);
-    this.policiesByID.put(policy.getId(), policy);
-    if (policy.isEnabled()) {
+    this.policiesByName.put(policy.getName(), info);
+    this.policiesByID.put(policy.getId(), info);
+    if (info.isEnabled()) {
       enablePolicy(policy.getName());
     }
   }
@@ -380,11 +416,13 @@ public final class ErasureCodingPolicyManager {
    * @param ecPolicies contains ErasureCodingPolicy list
    *
    */
-  public synchronized void loadPolicies(List<ErasureCodingPolicy> ecPolicies) {
+  public synchronized void loadPolicies(
+      List<ErasureCodingPolicyInfo> ecPolicies) {
     Preconditions.checkNotNull(ecPolicies);
-    for (ErasureCodingPolicy p : ecPolicies) {
+    for (ErasureCodingPolicyInfo p : ecPolicies) {
       loadPolicy(p);
     }
-    allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]);
+    allPolicies =
+        policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.XAttrHelper;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.WritableUtils;
@@ -366,10 +367,10 @@ final class FSDirErasureCodingOp {
    * Get available erasure coding polices.
    *
    * @param fsn namespace
-   * @return {@link ErasureCodingPolicy} array
+   * @return {@link ErasureCodingPolicyInfo} array
    */
-  static ErasureCodingPolicy[] getErasureCodingPolicies(final FSNamesystem fsn)
-      throws IOException {
+  static ErasureCodingPolicyInfo[] getErasureCodingPolicies(
+      final FSNamesystem fsn) throws IOException {
     assert fsn.hasReadLock();
     return fsn.getErasureCodingPolicyManager().getPolicies();
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java

@@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -256,14 +255,13 @@ class FSDirStatAndListingOp {
         listing[i] =
             createFileStatus(fsd, iip, child, childStoragePolicy, needLocation);
         listingCnt++;
-        if (listing[i] instanceof HdfsLocatedFileStatus) {
-            // Once we  hit lsLimit locations, stop.
-            // This helps to prevent excessively large response payloads.
-            // Approximate #locations with locatedBlockCount() * repl_factor
-            LocatedBlocks blks =
-                ((HdfsLocatedFileStatus)listing[i]).getBlockLocations();
-            locationBudget -= (blks == null) ? 0 :
-               blks.locatedBlockCount() * listing[i].getReplication();
+        LocatedBlocks blks = listing[i].getLocatedBlocks();
+        if (blks != null) {
+          // Once we  hit lsLimit locations, stop.
+          // This helps to prevent excessively large response payloads.
+          // Approximate #locations with locatedBlockCount() * repl_factor
+          locationBudget -=
+              blks.locatedBlockCount() * listing[i].getReplication();
         }
       }
       // truncate return array if necessary
@@ -486,31 +484,26 @@ class FSDirStatAndListingOp {
       String owner, String group, byte[] symlink, byte[] path, long fileId,
       int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
       ErasureCodingPolicy ecPolicy, LocatedBlocks locations) {
-    if (locations == null) {
-      return new HdfsFileStatus.Builder()
-          .length(length)
-          .isdir(isdir)
-          .replication(replication)
-          .blocksize(blocksize)
-          .mtime(mtime)
-          .atime(atime)
-          .perm(permission)
-          .flags(flags)
-          .owner(owner)
-          .group(group)
-          .symlink(symlink)
-          .path(path)
-          .fileId(fileId)
-          .children(childrenNum)
-          .feInfo(feInfo)
-          .storagePolicy(storagePolicy)
-          .ecPolicy(ecPolicy)
-          .build();
-    } else {
-      return new HdfsLocatedFileStatus(length, isdir, replication, blocksize,
-          mtime, atime, permission, flags, owner, group, symlink, path,
-          fileId, locations, childrenNum, feInfo, storagePolicy, ecPolicy);
-    }
+    return new HdfsFileStatus.Builder()
+        .length(length)
+        .isdir(isdir)
+        .replication(replication)
+        .blocksize(blocksize)
+        .mtime(mtime)
+        .atime(atime)
+        .perm(permission)
+        .flags(flags)
+        .owner(owner)
+        .group(group)
+        .symlink(symlink)
+        .path(path)
+        .fileId(fileId)
+        .children(childrenNum)
+        .feInfo(feInfo)
+        .storagePolicy(storagePolicy)
+        .ecPolicy(ecPolicy)
+        .locations(locations)
+        .build();
   }
 
   private static ContentSummary getContentSummaryInt(FSDirectory fsd,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java

@@ -41,7 +41,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
 import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.slf4j.Logger;
@@ -380,10 +380,10 @@ public final class FSImageFormatProtobuf {
     private void loadErasureCodingSection(InputStream in)
         throws IOException {
       ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in);
-      List<ErasureCodingPolicy> ecPolicies = Lists
+      List<ErasureCodingPolicyInfo> ecPolicies = Lists
           .newArrayListWithCapacity(s.getPoliciesCount());
       for (int i = 0; i < s.getPoliciesCount(); ++i) {
-        ecPolicies.add(PBHelperClient.convertErasureCodingPolicy(
+        ecPolicies.add(PBHelperClient.convertErasureCodingPolicyInfo(
             s.getPolicies(i)));
       }
       fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies);
@@ -586,11 +586,11 @@ public final class FSImageFormatProtobuf {
     private void saveErasureCodingSection(
         FileSummary.Builder summary) throws IOException {
       final FSNamesystem fsn = context.getSourceNamesystem();
-      ErasureCodingPolicy[] ecPolicies =
+      ErasureCodingPolicyInfo[] ecPolicies =
           fsn.getErasureCodingPolicyManager().getPolicies();
       ArrayList<ErasureCodingPolicyProto> ecPolicyProtoes =
           new ArrayList<ErasureCodingPolicyProto>();
-      for (ErasureCodingPolicy p : ecPolicies) {
+      for (ErasureCodingPolicyInfo p : ecPolicies) {
         ecPolicyProtoes.add(PBHelperClient.convertErasureCodingPolicy(p));
       }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -90,6 +90,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
 
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
@@ -7418,16 +7419,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   }
 
   /**
-   * Get available erasure coding polices
+   * Get all erasure coding polices.
    */
-  ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  ErasureCodingPolicyInfo[] getErasureCodingPolicies() throws IOException {
     final String operationName = "getErasureCodingPolicies";
     boolean success = false;
     checkOperation(OperationCategory.READ);
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      final ErasureCodingPolicy[] ret =
+      final ErasureCodingPolicyInfo[] ret =
           FSDirErasureCodingOp.getErasureCodingPolicies(this);
       success = true;
       return ret;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java

@@ -101,6 +101,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.FSLimitException;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
@@ -2303,7 +2304,8 @@ public class NameNodeRpcServer implements NamenodeProtocols {
   }
 
   @Override // ClientProtocol
-  public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
+  public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
+      throws IOException {
     checkNNStartup();
     return namesystem.getErasureCodingPolicies();
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java

@@ -471,6 +471,13 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
   void check(String parent, HdfsFileStatus file, Result replRes, Result ecRes)
       throws IOException {
     String path = file.getFullName(parent);
+    if (showprogress &&
+        (totalDirs + totalSymlinks + replRes.totalFiles + ecRes.totalFiles)
+            % 100 == 0) {
+      out.println();
+      out.flush();
+    }
+
     if (file.isDirectory()) {
       checkDir(path, replRes, ecRes);
       return;
@@ -489,10 +496,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
 
     final Result r = file.getErasureCodingPolicy() != null ? ecRes: replRes;
     collectFileSummary(path, file, r, blocks);
-    if (showprogress && (replRes.totalFiles + ecRes.totalFiles) % 100 == 0) {
-      out.println();
-      out.flush();
-    }
     collectBlocksSummary(parent, file, r, blocks);
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java

@@ -439,6 +439,7 @@ public class StandbyCheckpointer {
               namesystem.setNeedRollbackFsImage(false);
             }
             lastCheckpointTime = now;
+            LOG.info("Checkpoint finished successfully.");
           }
         } catch (SaveNamespaceCancelledException ce) {
           LOG.info("Checkpoint was cancelled: " + ce.getMessage());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.util.ECPolicyLoader;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.tools.TableListing;
@@ -111,16 +112,16 @@ public class ECAdmin extends Configured implements Tool {
 
       final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
-        Collection<ErasureCodingPolicy> policies =
+        final Collection<ErasureCodingPolicyInfo> policies =
             dfs.getAllErasureCodingPolicies();
         if (policies.isEmpty()) {
           System.out.println("There is no erasure coding policies in the " +
               "cluster.");
         } else {
           System.out.println("Erasure Coding Policies:");
-          for (ErasureCodingPolicy policy : policies) {
+          for (ErasureCodingPolicyInfo policy : policies) {
             if (policy != null) {
-              System.out.println(policy.toString());
+              System.out.println(policy);
             }
           }
         }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
+import com.google.common.base.Preconditions;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET;
@@ -34,7 +35,6 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -61,6 +61,8 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
@@ -69,6 +71,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
@@ -147,6 +150,8 @@ class OfflineImageReconstructor {
     this.events = factory.createXMLEventReader(reader);
     this.sections = new HashMap<>();
     this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor());
+    this.sections.put(ErasureCodingSectionProcessor.NAME,
+        new ErasureCodingSectionProcessor());
     this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor());
     this.sections.put(SecretManagerSectionProcessor.NAME,
         new SecretManagerSectionProcessor());
@@ -490,6 +495,76 @@ class OfflineImageReconstructor {
     }
   }
 
+  private class ErasureCodingSectionProcessor implements SectionProcessor {
+    static final String NAME = "ErasureCodingSection";
+
+    @Override
+    public void process() throws IOException {
+      Node node = new Node();
+      loadNodeChildren(node, "ErasureCodingSection fields");
+      ErasureCodingSection.Builder builder = ErasureCodingSection.newBuilder();
+      while (true) {
+        ErasureCodingPolicyProto.Builder policyBuilder =
+            ErasureCodingPolicyProto.newBuilder();
+        Node ec = node.removeChild(ERASURE_CODING_SECTION_POLICY);
+        if (ec == null) {
+          break;
+        }
+        int policyId = ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_ID);
+        policyBuilder.setId(policyId);
+        String name = ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_NAME);
+        policyBuilder.setName(name);
+        Integer cellSize =
+            ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_CELL_SIZE);
+        policyBuilder.setCellSize(cellSize);
+        String policyState =
+            ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_STATE);
+        if (policyState != null) {
+          policyBuilder.setState(
+              HdfsProtos.ErasureCodingPolicyState.valueOf(policyState));
+        }
+
+        Node schema = ec.removeChild(ERASURE_CODING_SECTION_SCHEMA);
+        Preconditions.checkNotNull(schema);
+
+        ECSchemaProto.Builder schemaBuilder = ECSchemaProto.newBuilder();
+        String codecName =
+            schema.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME);
+        schemaBuilder.setCodecName(codecName);
+        Integer dataUnits =
+            schema.removeChildInt(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS);
+        schemaBuilder.setDataUnits(dataUnits);
+        Integer parityUnits = schema.
+            removeChildInt(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS);
+        schemaBuilder.setParityUnits(parityUnits);
+        Node options = schema
+            .removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTIONS);
+        if (options != null) {
+          while (true) {
+            Node option =
+                options.removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTION);
+            if (option == null) {
+              break;
+            }
+            String key = option
+                .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY);
+            String value = option
+                .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE);
+            schemaBuilder.addOptions(HdfsProtos.ECSchemaOptionEntryProto
+                .newBuilder().setKey(key).setValue(value).build());
+          }
+        }
+        policyBuilder.setSchema(schemaBuilder.build());
+
+        builder.addPolicies(policyBuilder.build());
+      }
+      ErasureCodingSection section = builder.build();
+      section.writeDelimitedTo(out);
+      node.verifyNoRemainingKeys("ErasureCodingSection");
+      recordSectionLength(SectionName.ERASURE_CODING.name());
+    }
+  }
+
   private class INodeSectionProcessor implements SectionProcessor {
     static final String NAME = "INodeSection";
 
@@ -548,7 +623,7 @@ class OfflineImageReconstructor {
     }
     switch (type) {
     case "FILE":
-      processFileXml(node, inodeBld );
+      processFileXml(node, inodeBld);
       break;
     case "DIRECTORY":
       processDirectoryXml(node, inodeBld);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java

@@ -28,6 +28,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.Map;
 import java.util.TimeZone;
 
 import com.google.protobuf.ByteString;
@@ -36,15 +37,20 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
@@ -60,6 +66,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.util.XMLUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.util.LimitInputStream;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -79,6 +86,8 @@ import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_
 @InterfaceAudience.Private
 public final class PBImageXmlWriter {
   public static final String NAME_SECTION_NAME = "NameSection";
+  public static final String ERASURE_CODING_SECTION_NAME =
+      "ErasureCodingSection";
   public static final String INODE_SECTION_NAME = "INodeSection";
   public static final String SECRET_MANAGER_SECTION_NAME =
       "SecretManagerSection";
@@ -109,6 +118,33 @@ public final class PBImageXmlWriter {
   public static final String NAME_SECTION_LAST_ALLOCATED_STRIPED_BLOCK_ID =
       "lastAllocatedStripedBlockId";
 
+  public static final String ERASURE_CODING_SECTION_POLICY =
+      "erasureCodingPolicy";
+  public static final String ERASURE_CODING_SECTION_POLICY_ID =
+      "policyId";
+  public static final String ERASURE_CODING_SECTION_POLICY_NAME =
+      "policyName";
+  public static final String ERASURE_CODING_SECTION_POLICY_CELL_SIZE =
+      "cellSize";
+  public static final String ERASURE_CODING_SECTION_POLICY_STATE =
+      "policyState";
+  public static final String ERASURE_CODING_SECTION_SCHEMA =
+      "ecSchema";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME =
+      "codecName";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS =
+      "dataUnits";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS =
+      "parityUnits";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_OPTIONS =
+      "extraOptions";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION =
+      "option";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY =
+      "key";
+  public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE =
+      "value";
+
   public static final String INODE_SECTION_LAST_INODE_ID = "lastInodeId";
   public static final String INODE_SECTION_NUM_INODES = "numInodes";
   public static final String INODE_SECTION_TYPE = "type";
@@ -297,6 +333,9 @@ public final class PBImageXmlWriter {
         case STRING_TABLE:
           loadStringTable(is);
           break;
+        case ERASURE_CODING:
+          dumpErasureCodingSection(is);
+          break;
         case INODE:
           dumpINodeSection(is);
           break;
@@ -527,6 +566,47 @@ public final class PBImageXmlWriter {
     }
   }
 
+  private void dumpErasureCodingSection(InputStream in) throws IOException {
+    ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in);
+    if (s.getPoliciesCount() > 0) {
+      out.println("<" + ERASURE_CODING_SECTION_NAME + ">");
+      for (int i = 0; i < s.getPoliciesCount(); ++i) {
+        HdfsProtos.ErasureCodingPolicyProto policy = s.getPolicies(i);
+        dumpErasureCodingPolicy(PBHelperClient
+            .convertErasureCodingPolicyInfo(policy));
+      }
+      out.println("</" + ERASURE_CODING_SECTION_NAME + ">\n");
+    }
+  }
+
+  private void dumpErasureCodingPolicy(ErasureCodingPolicyInfo ecPolicyInfo) {
+    ErasureCodingPolicy ecPolicy = ecPolicyInfo.getPolicy();
+    out.println("<" + ERASURE_CODING_SECTION_POLICY + ">");
+    o(ERASURE_CODING_SECTION_POLICY_ID, ecPolicy.getId());
+    o(ERASURE_CODING_SECTION_POLICY_NAME, ecPolicy.getName());
+    o(ERASURE_CODING_SECTION_POLICY_CELL_SIZE, ecPolicy.getCellSize());
+    o(ERASURE_CODING_SECTION_POLICY_STATE, ecPolicyInfo.getState());
+    out.println("<" + ERASURE_CODING_SECTION_SCHEMA + ">");
+    ECSchema schema = ecPolicy.getSchema();
+    o(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME, schema.getCodecName());
+    o(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS, schema.getNumDataUnits());
+    o(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS,
+        schema.getNumParityUnits());
+    if (schema.getExtraOptions().size() > 0) {
+      out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTIONS + ">");
+      for (Map.Entry<String, String> option :
+          schema.getExtraOptions().entrySet()) {
+        out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTION + ">");
+        o(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY, option.getKey());
+        o(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE, option.getValue());
+        out.println("</" + ERASURE_CODING_SECTION_SCHEMA_OPTION + ">");
+      }
+      out.println("</" + ERASURE_CODING_SECTION_SCHEMA_OPTIONS + ">");
+    }
+    out.println("</" + ERASURE_CODING_SECTION_SCHEMA + ">");
+    out.println("</" + ERASURE_CODING_SECTION_POLICY + ">\n");
+  }
+
   private void dumpINodeSection(InputStream in) throws IOException {
     INodeSection s = INodeSection.parseDelimitedFrom(in);
     out.print("<" + INODE_SECTION_NAME + ">");

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js

@@ -440,22 +440,29 @@
           }).complete(function(data) {
             numCompleted++;
             if(numCompleted == files.length) {
-              $('#modal-upload-file').modal('hide');
-              $('#modal-upload-file-button').button('reset');
+              reset_upload_button();
               browse_directory(current_directory);
             }
           }).error(function(jqXHR, textStatus, errorThrown) {
             numCompleted++;
+            reset_upload_button();
             show_err_msg("Couldn't upload the file " + file.file.name + ". "+ errorThrown);
           });
         }).error(function(jqXHR, textStatus, errorThrown) {
           numCompleted++;
+          reset_upload_button();
           show_err_msg("Couldn't find datanode to write file. " + errorThrown);
         });
       })();
     }
   });
 
+  //Reset the upload button
+  function reset_upload_button() {
+    $('#modal-upload-file').modal('hide');
+    $('#modal-upload-file-button').button('reset');
+  }
+
   //Store the list of files which have been checked into session storage
   function store_selected_files(current_directory) {
     sessionStorage.setItem("source_directory", current_directory);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java

@@ -118,6 +118,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
 import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -147,6 +149,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
 import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -300,6 +303,19 @@ public class DFSTestUtil {
     }
   }
 
+  public static ErasureCodingPolicyState getECPolicyState(
+      final ErasureCodingPolicy policy) {
+    final ErasureCodingPolicyInfo[] policyInfos =
+        ErasureCodingPolicyManager.getInstance().getPolicies();
+    for (ErasureCodingPolicyInfo pi : policyInfos) {
+      if (pi.getPolicy().equals(policy)) {
+        return pi.getState();
+      }
+    }
+    throw new IllegalArgumentException("ErasureCodingPolicy <" + policy
+        + "> doesn't exist in the policies:" + Arrays.toString(policyInfos));
+  }
+
   /** class MyFile contains enough information to recreate the contents of
    * a single file.
    */

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java

@@ -1092,11 +1092,11 @@ public class TestBlockStoragePolicy {
     return types;
   }
 
-  private void checkLocatedBlocks(HdfsLocatedFileStatus status, int blockNum,
+  private void checkLocatedBlocks(HdfsFileStatus status, int blockNum,
                                   int replicaNum, StorageType... types) {
     List<StorageType> typeList = Lists.newArrayList();
     Collections.addAll(typeList, types);
-    LocatedBlocks lbs = status.getBlockLocations();
+    LocatedBlocks lbs = status.getLocatedBlocks();
     Assert.assertEquals(blockNum, lbs.getLocatedBlocks().size());
     for (LocatedBlock lb : lbs.getLocatedBlocks()) {
       Assert.assertEquals(replicaNum, lb.getStorageTypes().length);
@@ -1127,7 +1127,7 @@ public class TestBlockStoragePolicy {
       HdfsFileStatus[] status = fs.getClient().listPaths(foo.toString(),
           HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
       checkDirectoryListing(status, policyId);
-      HdfsLocatedFileStatus fooStatus = (HdfsLocatedFileStatus) status[0];
+      HdfsFileStatus fooStatus = status[0];
       checkLocatedBlocks(fooStatus, 1, 3, before);
 
       // change the replication factor to 5
@@ -1140,7 +1140,7 @@ public class TestBlockStoragePolicy {
       status = fs.getClient().listPaths(foo.toString(),
           HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
       checkDirectoryListing(status, policyId);
-      fooStatus = (HdfsLocatedFileStatus) status[0];
+      fooStatus = status[0];
       checkLocatedBlocks(fooStatus, 1, numDataNodes, after);
 
       // change the replication factor back to 3
@@ -1157,7 +1157,7 @@ public class TestBlockStoragePolicy {
       status = fs.getClient().listPaths(foo.toString(),
           HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
       checkDirectoryListing(status, policyId);
-      fooStatus = (HdfsLocatedFileStatus) status[0];
+      fooStatus = status[0];
       checkLocatedBlocks(fooStatus, 1, REPLICATION, before);
     } finally {
       cluster.shutdown();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java

@@ -274,7 +274,7 @@ public class TestDFSOutputStream {
     when(client.getTracer()).thenReturn(FsTracer.get(new Configuration()));
     client.clientRunning = true;
     DataStreamer stream = new DataStreamer(
-        mock(HdfsFileStatus.class),
+        new HdfsFileStatus.Builder().build(),
         mock(ExtendedBlock.class),
         client,
         "foo", null, null, null, null, null, null);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
@@ -34,6 +35,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.event.Level;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 /**
  * Test erasure coding block placement with skewed # nodes per rack.
  */
@@ -42,10 +50,10 @@ public class TestErasureCodingMultipleRacks {
       LoggerFactory.getLogger(TestErasureCodingMultipleRacks.class);
 
   static {
-    GenericTestUtils.setLogLevel(BlockPlacementPolicy.LOG, Level.DEBUG);
-    GenericTestUtils.setLogLevel(BlockPlacementPolicyDefault.LOG, Level.DEBUG);
+    GenericTestUtils.setLogLevel(BlockPlacementPolicy.LOG, Level.TRACE);
+    GenericTestUtils.setLogLevel(BlockPlacementPolicyDefault.LOG, Level.TRACE);
     GenericTestUtils.setLogLevel(BlockPlacementPolicyRackFaultTolerant.LOG,
-        Level.DEBUG);
+        Level.TRACE);
     GenericTestUtils.setLogLevel(NetworkTopology.LOG, Level.DEBUG);
   }
 
@@ -62,20 +70,38 @@ public class TestErasureCodingMultipleRacks {
   private DistributedFileSystem dfs;
 
   @Before
-  public void setup() throws Exception {
+  public void setup() {
     ecPolicy = getPolicy();
-    final int dataUnits = ecPolicy.getNumDataUnits();
-    final int parityUnits = ecPolicy.getNumParityUnits();
-    final int numDatanodes = dataUnits + parityUnits;
-    final int numRacks = 2;
+    conf = new HdfsConfiguration();
+    // disable load consideration to test placement only.
+    conf.setBoolean(DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false);
+  }
+
+  /**
+   * Setup cluster with desired number of DN, racks, and specified number of
+   * rack that only has 1 DN. Other racks will be evenly setup with the number
+   * of DNs.
+   * <p>
+   * This is not done as a {@link Before}, so test cases can setup differently.
+   *
+   * @param numDatanodes number of total Datanodes.
+   * @param numRacks number of total racks
+   * @param numSingleDnRacks number of racks that only has 1 DN
+   * @throws Exception
+   */
+  public void setupCluster(final int numDatanodes, final int numRacks,
+      final int numSingleDnRacks) throws Exception {
+    assert numDatanodes > numRacks;
+    assert numRacks > numSingleDnRacks;
+    assert numSingleDnRacks >= 0;
     final String[] racks = new String[numDatanodes];
-    for (int i = 0; i < numRacks; i++) {
+    for (int i = 0; i < numSingleDnRacks; i++) {
       racks[i] = "/rack" + i;
     }
-    for (int i = numRacks; i < numDatanodes; i++) {
-      racks[i] = "/rack" + (numRacks - 1);
+    for (int i = numSingleDnRacks; i < numDatanodes; i++) {
+      racks[i] =
+          "/rack" + (numSingleDnRacks + (i % (numRacks - numSingleDnRacks)));
     }
-    conf = new HdfsConfiguration();
     cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(numDatanodes)
         .racks(racks)
@@ -92,16 +118,89 @@ public class TestErasureCodingMultipleRacks {
     }
   }
 
+  // Extreme case.
+  @Test
+  public void testSkewedRack1() throws Exception {
+    final int dataUnits = ecPolicy.getNumDataUnits();
+    final int parityUnits = ecPolicy.getNumParityUnits();
+    setupCluster(dataUnits + parityUnits, 2, 1);
+
+    final int filesize = ecPolicy.getNumDataUnits() * ecPolicy.getCellSize();
+    byte[] contents = new byte[filesize];
+
+    final Path path = new Path("/testfile");
+    LOG.info("Writing file " + path);
+    DFSTestUtil.writeFile(dfs, path, contents);
+    BlockLocation[] blocks = dfs.getFileBlockLocations(path, 0, Long.MAX_VALUE);
+    assertEquals(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(),
+        blocks[0].getHosts().length);
+  }
+
+  // 1 rack has many nodes, other racks have single node. Extreme case.
+  @Test
+  public void testSkewedRack2() throws Exception {
+    final int dataUnits = ecPolicy.getNumDataUnits();
+    final int parityUnits = ecPolicy.getNumParityUnits();
+    setupCluster(dataUnits + parityUnits * 2, dataUnits, dataUnits - 1);
+
+    final int filesize = ecPolicy.getNumDataUnits() * ecPolicy.getCellSize();
+    byte[] contents = new byte[filesize];
+
+    final Path path = new Path("/testfile");
+    LOG.info("Writing file " + path);
+    DFSTestUtil.writeFile(dfs, path, contents);
+    BlockLocation[] blocks = dfs.getFileBlockLocations(path, 0, Long.MAX_VALUE);
+    assertEquals(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(),
+        blocks[0].getHosts().length);
+  }
+
+  // 2 racks have sufficient nodes, other racks has 1. Should be able to
+  // tolerate 1 rack failure.
   @Test
-  public void testSkewedRack() throws Exception {
-    final int filesize = ecPolicy.getNumDataUnits() * ecPolicy
-        .getCellSize();
+  public void testSkewedRack3() throws Exception {
+    final int dataUnits = ecPolicy.getNumDataUnits();
+    final int parityUnits = ecPolicy.getNumParityUnits();
+    // Create enough extra DNs on the 2 racks to test even placement.
+    // Desired placement is parityUnits replicas on the 2 racks, and 1 replica
+    // on the rest of the racks (which only have 1 DN)
+    setupCluster(dataUnits + parityUnits * 4, dataUnits - parityUnits + 2,
+        dataUnits - parityUnits);
+
+    final int filesize = ecPolicy.getNumDataUnits() * ecPolicy.getCellSize();
     byte[] contents = new byte[filesize];
 
-    for (int i = 0; i < 10; i++) {
+    for (int i = 0; i < 10; ++i) {
       final Path path = new Path("/testfile" + i);
       LOG.info("Writing file " + path);
       DFSTestUtil.writeFile(dfs, path, contents);
+      BlockLocation[] blocks =
+          dfs.getFileBlockLocations(path, 0, Long.MAX_VALUE);
+      assertEquals(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(),
+          blocks[0].getHosts().length);
+      assertRackFailureTolerated(blocks[0].getTopologyPaths());
+    }
+  }
+
+  // Verifies that no more than numParityUnits is placed on a rack.
+  private void assertRackFailureTolerated(final String[] topologies) {
+    final Map<String, Integer> racksCount = new HashMap<>();
+    for (String t : topologies) {
+      final Integer count = racksCount.get(getRackName(t));
+      if (count == null) {
+        racksCount.put(getRackName(t), 1);
+      } else {
+        racksCount.put(getRackName(t), count + 1);
+      }
     }
+    LOG.info("Rack count map is: {}", racksCount);
+
+    for (Integer count : racksCount.values()) {
+      assertTrue(count <= ecPolicy.getNumParityUnits());
+    }
+  }
+
+  private String getRackName(final String topology) {
+    assert topology.indexOf('/', 1) > 0;
+    return topology.substring(0, topology.indexOf('/', 1));
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -49,6 +50,7 @@ import org.junit.rules.Timeout;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
@@ -375,10 +377,16 @@ public class TestErasureCodingPolicies {
 
   @Test
   public void testGetAllErasureCodingPolicies() throws Exception {
-    Collection<ErasureCodingPolicy> allECPolicies = fs
+    Collection<ErasureCodingPolicyInfo> allECPolicies = fs
         .getAllErasureCodingPolicies();
-    assertTrue("All system policies should be enabled",
-        allECPolicies.containsAll(SystemErasureCodingPolicies.getPolicies()));
+    final List<ErasureCodingPolicy> sysPolicies =
+        new ArrayList<>(SystemErasureCodingPolicies.getPolicies());
+    for (ErasureCodingPolicyInfo ecpi : allECPolicies) {
+      if (ecpi.isEnabled()) {
+        sysPolicies.remove(ecpi.getPolicy());
+      }
+    }
+    assertTrue("All system policies should be enabled", sysPolicies.isEmpty());
 
     // Query after add a new policy
     ECSchema toAddSchema = new ECSchema("rs", 5, 2);
@@ -609,11 +617,11 @@ public class TestErasureCodingPolicies {
     fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
 
     String ecPolicyName = null;
-    Collection<ErasureCodingPolicy> allPolicies =
+    final Collection<ErasureCodingPolicyInfo> allPoliciesInfo =
         fs.getAllErasureCodingPolicies();
-    for (ErasureCodingPolicy policy : allPolicies) {
-      if (!ecPolicy.equals(policy)) {
-        ecPolicyName = policy.getName();
+    for (ErasureCodingPolicyInfo info : allPoliciesInfo) {
+      if (!ecPolicy.equals(info.getPolicy())) {
+        ecPolicyName = info.getPolicy().getName();
         break;
       }
     }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java

@@ -106,6 +106,24 @@ public class TestJournalNode {
         "testJournalNodeSyncerNotStartWhenSyncEnabled")) {
       conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
           "qjournal://jn0:9900;jn1:9901");
+    } else if (testName.getMethodName().equals(
+        "testJournalNodeSyncwithFederationTypeConfigWithNameServiceId")) {
+      conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1",
+          "qjournal://journalnode0:9900;journalnode0:9901");
+    } else if (testName.getMethodName().equals(
+        "testJournalNodeSyncwithFederationTypeConfigWithNamenodeId")) {
+      conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".ns1", "nn1,nn2");
+      conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn1",
+          "qjournal://journalnode0:9900;journalnode1:9901");
+      conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn2",
+          "qjournal://journalnode0:9900;journalnode1:9901");
+    } else if (testName.getMethodName().equals(
+        "testJournalNodeSyncwithFederationTypeIncorrectConfigWithNamenodeId")) {
+      conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".ns1", "nn1,nn2");
+      conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn1",
+          "qjournal://journalnode0:9900;journalnode1:9901");
+      conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn2",
+          "qjournal://journalnode0:9902;journalnode1:9903");
     }
     jn = new JournalNode();
     jn.setConf(conf);
@@ -387,7 +405,7 @@ public class TestJournalNode {
 
   @Test
   public void testJournalNodeSyncerNotStartWhenSyncDisabled()
-      throws IOException{
+      throws IOException {
     //JournalSyncer will not be started, as journalsync is not enabled
     conf.setBoolean(DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_KEY, false);
     jn.getOrCreateJournal(journalId);
@@ -408,7 +426,7 @@ public class TestJournalNode {
 
   @Test
   public void testJournalNodeSyncerNotStartWhenSyncEnabledIncorrectURI()
-      throws IOException{
+      throws IOException {
     //JournalSyncer will not be started,
     // as shared edits hostnames are not resolved
     jn.getOrCreateJournal(journalId);
@@ -431,7 +449,7 @@ public class TestJournalNode {
 
   @Test
   public void testJournalNodeSyncerNotStartWhenSyncEnabled()
-      throws IOException{
+      throws IOException {
     //JournalSyncer will not be started,
     // as shared edits hostnames are not resolved
     jn.getOrCreateJournal(journalId);
@@ -452,9 +470,84 @@ public class TestJournalNode {
 
   }
 
-  private void setupStaticHostResolution(int nameServiceIdCount,
+
+  @Test
+  public void testJournalNodeSyncwithFederationTypeConfigWithNameServiceId()
+      throws IOException {
+    //JournalSyncer will not be started, as nameserviceId passed is null,
+    // but configured shared edits dir is appended with nameserviceId
+    setupStaticHostResolution(2, "journalnode");
+    jn.getOrCreateJournal(journalId);
+    Assert.assertEquals(false,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(false,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+    //Trying by passing nameserviceId and resolve hostnames
+    // now IstriedJournalSyncerStartWithnsId should be set
+    // and also journalnode syncer will also be started
+
+    jn.getOrCreateJournal(journalId, "ns1");
+    Assert.assertEquals(true,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(true,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+  }
+
+  @Test
+  public void testJournalNodeSyncwithFederationTypeConfigWithNamenodeId()
+      throws IOException {
+    //JournalSyncer will not be started, as nameserviceId passed is null,
+    // but configured shared edits dir is appended with nameserviceId +
+    // namenodeId
+    setupStaticHostResolution(2, "journalnode");
+    jn.getOrCreateJournal(journalId);
+    Assert.assertEquals(false,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(false,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+    //Trying by passing nameserviceId and resolve hostnames
+    // now IstriedJournalSyncerStartWithnsId should be set
+    // and also journalnode syncer will also be started
+
+    jn.getOrCreateJournal(journalId, "ns1");
+    Assert.assertEquals(true,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(true,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+  }
+
+  @Test
+  public void
+      testJournalNodeSyncwithFederationTypeIncorrectConfigWithNamenodeId()
+      throws IOException {
+    //JournalSyncer will not be started, as nameserviceId passed is null,
+    // but configured shared edits dir is appended with nameserviceId +
+    // namenodeId
+    setupStaticHostResolution(2, "journalnode");
+    jn.getOrCreateJournal(journalId);
+    Assert.assertEquals(false,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(false,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+    //Trying by passing nameserviceId and resolve hostnames
+    // now IstriedJournalSyncerStartWithnsId should  be set
+    // and  journalnode syncer will not  be started
+    // as for each nnId, different shared Edits dir value is configured
+
+    jn.getOrCreateJournal(journalId, "ns1");
+    Assert.assertEquals(false,
+        jn.getJournalSyncerStatus(journalId));
+    Assert.assertEquals(true,
+        jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+  }
+
+
+  private void setupStaticHostResolution(int journalNodeCount,
                                          String hostname) {
-    for (int i = 0; i < nameServiceIdCount; i++) {
+    for (int i = 0; i < journalNodeCount; i++) {
       NetUtils.addStaticResolution(hostname + i,
           "localhost");
     }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java

@@ -1318,8 +1318,9 @@ public class TestBlockManager {
       assertTrue("Unexpected text in metasave," +
               "was expecting corrupt blocks section!", foundIt);
       corruptBlocksLine = reader.readLine();
-      String regex = "Block=[0-9]+\\tNode=.*\\tStorageID=.*StorageState.*" +
-          "TotalReplicas=.*Reason=GENSTAMP_MISMATCH";
+      String regex = "Block=blk_[0-9]+_[0-9]+\\tSize=.*\\tNode=.*" +
+          "\\tStorageID=.*\\tStorageState.*" +
+          "\\tTotalReplicas=.*\\tReason=GENSTAMP_MISMATCH";
       assertTrue("Unexpected corrupt block section in metasave!",
           corruptBlocksLine.matches(regex));
       corruptBlocksLine = reader.readLine();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HAUtilClient;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -25,14 +26,17 @@ import org.apache.hadoop.hdfs.web.resources.DelegationParam;
 import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.junit.Assert.fail;
+
 import io.netty.handler.codec.http.QueryStringDecoder;
 
 import java.io.IOException;
+import java.util.EnumSet;
 
-import static org.mockito.Mockito.mock;
 
 public class TestParameterParser {
   private static final String LOGICAL_NAME = "minidfs";
@@ -63,6 +67,81 @@ public class TestParameterParser {
     Assert.assertEquals(EXPECTED_PATH, testParser.path());
   }
 
+  @Test
+  public void testCreateFlag() {
+    final String path = "/test1?createflag=append,sync_block";
+    Configuration conf = new Configuration();
+    QueryStringDecoder decoder = new QueryStringDecoder(
+        WebHdfsHandler.WEBHDFS_PREFIX + path);
+    ParameterParser testParser = new ParameterParser(decoder, conf);
+    EnumSet<CreateFlag> actual = testParser.createFlag();
+    EnumSet<CreateFlag> expected = EnumSet.of(CreateFlag.APPEND,
+        CreateFlag.SYNC_BLOCK);
+    Assert.assertEquals(expected.toString(), actual.toString());
+
+
+    final String path1 = "/test1?createflag=append";
+    decoder = new QueryStringDecoder(
+        WebHdfsHandler.WEBHDFS_PREFIX + path1);
+    testParser = new ParameterParser(decoder, conf);
+
+    actual = testParser.createFlag();
+    expected = EnumSet.of(CreateFlag.APPEND);
+    Assert.assertEquals(expected, actual);
+
+    final String path2 = "/test1";
+    decoder = new QueryStringDecoder(
+        WebHdfsHandler.WEBHDFS_PREFIX + path2);
+    testParser = new ParameterParser(decoder, conf);
+    actual = testParser.createFlag();
+    Assert.assertEquals(0, actual.size());
+
+    final String path3 = "/test1?createflag=create,overwrite";
+    decoder = new QueryStringDecoder(
+        WebHdfsHandler.WEBHDFS_PREFIX + path3);
+    testParser = new ParameterParser(decoder, conf);
+    actual = testParser.createFlag();
+    expected = EnumSet.of(CreateFlag.CREATE, CreateFlag
+        .OVERWRITE);
+    Assert.assertEquals(expected.toString(), actual.toString());
+
+
+    final String path4 = "/test1?createflag=";
+    decoder = new QueryStringDecoder(
+        WebHdfsHandler.WEBHDFS_PREFIX + path4);
+    testParser = new ParameterParser(decoder, conf);
+    actual = testParser.createFlag();
+    Assert.assertEquals(0, actual.size());
+
+    //Incorrect value passed to createflag
+    try {
+      final String path5 = "/test1?createflag=overwrite,";
+      decoder = new QueryStringDecoder(
+          WebHdfsHandler.WEBHDFS_PREFIX + path5);
+      testParser = new ParameterParser(decoder, conf);
+      actual = testParser.createFlag();
+      fail("It should throw Illegal Argument Exception");
+    } catch (Exception e) {
+      GenericTestUtils
+          .assertExceptionContains("No enum constant", e);
+    }
+
+    //Incorrect value passed to createflag
+    try {
+      final String path6 = "/test1?createflag=,";
+      decoder = new QueryStringDecoder(
+          WebHdfsHandler.WEBHDFS_PREFIX + path6);
+      testParser = new ParameterParser(decoder, conf);
+      actual = testParser.createFlag();
+      fail("It should throw Illegal Argument Exception");
+    } catch (Exception e) {
+      GenericTestUtils
+          .assertExceptionContains("No enum constant", e);
+    }
+
+
+  }
+
   @Test
   public void testOffset() throws IOException {
     final long X = 42;

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java

@@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
@@ -312,18 +311,17 @@ public class TestStorageMover {
 
     private void verifyFile(final Path parent, final HdfsFileStatus status,
         final Byte expectedPolicyId) throws Exception {
-      HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus) status;
-      byte policyId = fileStatus.getStoragePolicy();
+      byte policyId = status.getStoragePolicy();
       BlockStoragePolicy policy = policies.getPolicy(policyId);
       if (expectedPolicyId != null) {
         Assert.assertEquals((byte)expectedPolicyId, policy.getId());
       }
       final List<StorageType> types = policy.chooseStorageTypes(
           status.getReplication());
-      for(LocatedBlock lb : fileStatus.getBlockLocations().getLocatedBlocks()) {
+      for(LocatedBlock lb : status.getLocatedBlocks().getLocatedBlocks()) {
         final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types,
             lb.getStorageTypes());
-        Assert.assertTrue(fileStatus.getFullName(parent.toString())
+        Assert.assertTrue(status.getFullName(parent.toString())
             + " with policy " + policy + " has non-empty overlap: " + diff
             + ", the corresponding block is " + lb.getBlock().getLocalBlock(),
             diff.removeOverlap(true));

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java

@@ -748,7 +748,8 @@ public class TestFSEditLogLoader {
       // check if new policy is reapplied through edit log
       ErasureCodingPolicy ecPolicy =
           ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId());
-      assertEquals(ErasureCodingPolicyState.DISABLED, ecPolicy.getState());
+      assertEquals(ErasureCodingPolicyState.DISABLED,
+          DFSTestUtil.getECPolicyState(ecPolicy));
 
       // 2. enable policy
       fs.enableErasureCodingPolicy(newPolicy.getName());
@@ -756,7 +757,8 @@ public class TestFSEditLogLoader {
       cluster.waitActive();
       ecPolicy =
           ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId());
-      assertEquals(ErasureCodingPolicyState.ENABLED, ecPolicy.getState());
+      assertEquals(ErasureCodingPolicyState.ENABLED,
+          DFSTestUtil.getECPolicyState(ecPolicy));
 
       // create a new file, use the policy
       final Path dirPath = new Path("/striped");
@@ -773,7 +775,8 @@ public class TestFSEditLogLoader {
       cluster.waitActive();
       ecPolicy =
           ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId());
-      assertEquals(ErasureCodingPolicyState.DISABLED, ecPolicy.getState());
+      assertEquals(ErasureCodingPolicyState.DISABLED,
+          DFSTestUtil.getECPolicyState(ecPolicy));
       // read file
       DFSTestUtil.readFileAsBytes(fs, filePath);
 
@@ -783,7 +786,8 @@ public class TestFSEditLogLoader {
       cluster.waitActive();
       ecPolicy =
           ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId());
-      assertEquals(ErasureCodingPolicyState.REMOVED, ecPolicy.getState());
+      assertEquals(ErasureCodingPolicyState.REMOVED,
+          DFSTestUtil.getECPolicyState(ecPolicy));
       // read file
       DFSTestUtil.readFileAsBytes(fs, filePath);
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java

@@ -870,7 +870,8 @@ public class TestFSImage {
           newPolicy, ecPolicy);
       assertEquals(
           "Newly added erasure coding policy should be of disabled state",
-          ErasureCodingPolicyState.DISABLED, ecPolicy.getState());
+          ErasureCodingPolicyState.DISABLED,
+          DFSTestUtil.getECPolicyState(ecPolicy));
 
       // Test enable/disable/remove user customized erasure coding policy
       testChangeErasureCodingPolicyState(cluster, blockSize, newPolicy);
@@ -880,14 +881,12 @@ public class TestFSImage {
     }
   }
 
-
   private void testChangeErasureCodingPolicyState(MiniDFSCluster cluster,
       int blockSize, ErasureCodingPolicy targetPolicy) throws IOException {
     DistributedFileSystem fs = cluster.getFileSystem();
 
     // 1. Enable an erasure coding policy
     fs.enableErasureCodingPolicy(targetPolicy.getName());
-    targetPolicy.setState(ErasureCodingPolicyState.ENABLED);
     // Create file, using the new policy
     final Path dirPath = new Path("/striped");
     final Path filePath = new Path(dirPath, "file");
@@ -910,13 +909,13 @@ public class TestFSImage {
     assertEquals("The erasure coding policy is not found",
         targetPolicy, ecPolicy);
     assertEquals("The erasure coding policy should be of enabled state",
-        ErasureCodingPolicyState.ENABLED, ecPolicy.getState());
+        ErasureCodingPolicyState.ENABLED,
+        DFSTestUtil.getECPolicyState(ecPolicy));
     // Read file regardless of the erasure coding policy state
     DFSTestUtil.readFileAsBytes(fs, filePath);
 
     // 2. Disable an erasure coding policy
     fs.disableErasureCodingPolicy(ecPolicy.getName());
-    targetPolicy.setState(ErasureCodingPolicyState.DISABLED);
     // Save namespace and restart NameNode
     fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
     fs.saveNamespace();
@@ -929,7 +928,8 @@ public class TestFSImage {
     assertEquals("The erasure coding policy is not found",
         targetPolicy, ecPolicy);
     assertEquals("The erasure coding policy should be of disabled state",
-        ErasureCodingPolicyState.DISABLED, ecPolicy.getState());
+        ErasureCodingPolicyState.DISABLED,
+        DFSTestUtil.getECPolicyState(ecPolicy));
     // Read file regardless of the erasure coding policy state
     DFSTestUtil.readFileAsBytes(fs, filePath);
 
@@ -944,7 +944,7 @@ public class TestFSImage {
       return;
     }
 
-    targetPolicy.setState(ErasureCodingPolicyState.REMOVED);
+    fs.removeErasureCodingPolicy(ecPolicy.getName());
     // Save namespace and restart NameNode
     fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
     fs.saveNamespace();
@@ -957,7 +957,8 @@ public class TestFSImage {
     assertEquals("The erasure coding policy saved into and loaded from " +
         "fsImage is bad", targetPolicy, ecPolicy);
     assertEquals("The erasure coding policy should be of removed state",
-        ErasureCodingPolicyState.REMOVED, ecPolicy.getState());
+        ErasureCodingPolicyState.REMOVED,
+        DFSTestUtil.getECPolicyState(ecPolicy));
     // Read file regardless of the erasure coding policy state
     DFSTestUtil.readFileAsBytes(fs, filePath);
     fs.delete(dirPath, true);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
+import com.google.common.collect.ImmutableMap;
 import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
 import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
 import static org.apache.hadoop.fs.permission.AclEntryType.OTHER;
@@ -24,7 +25,19 @@ import static org.apache.hadoop.fs.permission.AclEntryType.USER;
 import static org.apache.hadoop.fs.permission.FsAction.ALL;
 import static org.apache.hadoop.fs.permission.FsAction.EXECUTE;
 import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
 import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_NAME;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_CELL_SIZE;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_NAME;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_STATE;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME;
+import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_OPTION;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -48,11 +61,14 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
@@ -91,6 +107,10 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 import org.xml.sax.Attributes;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
@@ -106,6 +126,7 @@ public class TestOfflineImageViewer {
   private static final String TEST_RENEWER = "JobTracker";
   private static File originalFsimage = null;
   private static int filesECCount = 0;
+  private static String addedErasureCodingPolicyName = null;
 
   // namespace as written to dfs, to be compared with viewer's output
   final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
@@ -142,6 +163,15 @@ public class TestOfflineImageViewer {
       DistributedFileSystem hdfs = cluster.getFileSystem();
       hdfs.enableErasureCodingPolicy(ecPolicy.getName());
 
+      Map<String, String> options = ImmutableMap.of("k1", "v1", "k2", "v2");
+      ECSchema schema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME,
+          10, 4, options);
+      ErasureCodingPolicy policy = new ErasureCodingPolicy(schema, 1024);
+      AddErasureCodingPolicyResponse[] responses =
+          hdfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{policy});
+      addedErasureCodingPolicyName = responses[0].getPolicy().getName();
+      hdfs.enableErasureCodingPolicy(addedErasureCodingPolicyName);
+
       // Create a reasonable namespace
       for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
         Path dir = new Path("/dir" + i);
@@ -272,8 +302,9 @@ public class TestOfflineImageViewer {
       }
       LOG.debug("original FS image file is " + originalFsimage);
     } finally {
-      if (cluster != null)
+      if (cluster != null) {
         cluster.shutdown();
+      }
     }
   }
 
@@ -585,6 +616,7 @@ public class TestOfflineImageViewer {
       IOUtils.closeStream(out);
     }
   }
+
   private void testPBDelimitedWriter(String db)
       throws IOException, InterruptedException {
     final String DELIMITER = "\t";
@@ -808,4 +840,71 @@ public class TestOfflineImageViewer {
       IOUtils.closeStream(out);
     }
   }
+
+  private static String getXmlString(Element element, String name) {
+    NodeList id = element.getElementsByTagName(name);
+    Element line = (Element) id.item(0);
+    if (line == null) {
+      return "";
+    }
+    Node first = line.getFirstChild();
+    // handle empty <key></key>
+    if (first == null) {
+      return "";
+    }
+    String val = first.getNodeValue();
+    if (val == null) {
+      return "";
+    }
+    return val;
+  }
+
+  @Test
+  public void testOfflineImageViewerForECPolicies() throws Exception {
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    PrintStream o = new PrintStream(output);
+    PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
+    v.visit(new RandomAccessFile(originalFsimage, "r"));
+    final String xml = output.toString();
+
+    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilder db = dbf.newDocumentBuilder();
+    InputSource is = new InputSource();
+    is.setCharacterStream(new StringReader(xml));
+    Document dom = db.parse(is);
+    NodeList ecSection = dom.getElementsByTagName(ERASURE_CODING_SECTION_NAME);
+    assertEquals(1, ecSection.getLength());
+    NodeList policies =
+        dom.getElementsByTagName(ERASURE_CODING_SECTION_POLICY);
+    assertEquals(1 + SystemErasureCodingPolicies.getPolicies().size(),
+        policies.getLength());
+    for (int i = 0; i < policies.getLength(); i++) {
+      Element policy = (Element) policies.item(i);
+      String name = getXmlString(policy, ERASURE_CODING_SECTION_POLICY_NAME);
+      if (name.equals(addedErasureCodingPolicyName)) {
+        String cellSize =
+            getXmlString(policy, ERASURE_CODING_SECTION_POLICY_CELL_SIZE);
+        assertEquals("1024", cellSize);
+        String state =
+            getXmlString(policy, ERASURE_CODING_SECTION_POLICY_STATE);
+        assertEquals(ErasureCodingPolicyState.ENABLED.toString(), state);
+
+        Element schema = (Element) policy
+            .getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA).item(0);
+        String codecName =
+            getXmlString(schema, ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME);
+        assertEquals(ErasureCodeConstants.RS_CODEC_NAME, codecName);
+
+        NodeList options =
+            schema.getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA_OPTION);
+        assertEquals(2, options.getLength());
+        Element option1 = (Element) options.item(0);
+        assertEquals("k1", getXmlString(option1, "key"));
+        assertEquals("v1", getXmlString(option1, "value"));
+        Element option2 = (Element) options.item(1);
+        assertEquals("k2", getXmlString(option2, "key"));
+        assertEquals("v2", getXmlString(option2, "value"));
+      }
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java

@@ -1452,4 +1452,63 @@ public class TestWebHDFS {
       }
     }
   }
+
+  /**
+   * Tests that {@link WebHdfsFileSystem.AbstractRunner} propagates original
+   * exception's stacktrace and cause during runWithRetry attempts.
+   * @throws Exception
+   */
+  @Test
+  public void testExceptionPropogationInAbstractRunner() throws Exception{
+    final Configuration conf = WebHdfsTestUtil.createConf();
+    final Path dir = new Path("/testExceptionPropogationInAbstractRunner");
+
+    conf.setBoolean(HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY, true);
+
+    final short numDatanodes = 1;
+    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(numDatanodes)
+        .build();
+    try {
+      cluster.waitActive();
+      final FileSystem fs = WebHdfsTestUtil
+          .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+      //create a file
+      final long length = 1L << 20;
+      final Path file1 = new Path(dir, "testFile");
+
+      DFSTestUtil.createFile(fs, file1, length, numDatanodes, 20120406L);
+
+      //get file status and check that it was written properly.
+      final FileStatus s1 = fs.getFileStatus(file1);
+      assertEquals("Write failed for file " + file1, length, s1.getLen());
+
+      FSDataInputStream in = fs.open(file1);
+      in.read(); // Connection is made only when the first read() occurs.
+      final WebHdfsInputStream webIn =
+          (WebHdfsInputStream)(in.getWrappedStream());
+
+      final String msg = "Throwing dummy exception";
+      IOException ioe = new IOException(msg, new DummyThrowable());
+
+      WebHdfsFileSystem.ReadRunner readRunner = spy(webIn.getReadRunner());
+      doThrow(ioe).when(readRunner).getResponse(any(HttpURLConnection.class));
+
+      webIn.setReadRunner(readRunner);
+
+      try {
+        webIn.read();
+        fail("Read should have thrown IOException.");
+      } catch (IOException e) {
+        assertTrue(e.getMessage().contains(msg));
+        assertTrue(e.getCause() instanceof DummyThrowable);
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  final static class DummyThrowable extends Throwable {
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java

@@ -26,8 +26,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.NoSuchElementException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -45,6 +43,8 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.CryptoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * <code>BackupStore</code> is an utility class that is used to support
@@ -60,7 +60,8 @@ import org.apache.hadoop.mapreduce.CryptoUtils;
 @InterfaceStability.Unstable
 public class BackupStore<K,V> {
 
-  private static final Log LOG = LogFactory.getLog(BackupStore.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(BackupStore.class.getName());
   private static final int MAX_VINT_SIZE = 9;
   private static final int EOF_MARKER_SIZE = 2 * MAX_VINT_SIZE;
   private final TaskAttemptID tid;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/CleanupQueue.java

@@ -21,16 +21,16 @@ package org.apache.hadoop.mapred;
 import java.io.IOException;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 class CleanupQueue {
 
-  public static final Log LOG =
-    LogFactory.getLog(CleanupQueue.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(CleanupQueue.class);
 
   private static PathCleanupThread cleanupThread;
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java

@@ -30,7 +30,6 @@ import java.util.HashMap;
 import java.util.Iterator;
 
 import org.apache.commons.collections.IteratorUtils;
-import org.apache.commons.logging.Log;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.mapreduce.FileSystemCounter;
@@ -44,6 +43,7 @@ import org.apache.hadoop.mapreduce.counters.GenericCounter;
 import org.apache.hadoop.mapreduce.counters.Limits;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
+import org.slf4j.Logger;
 
 import com.google.common.collect.Iterators;
 
@@ -596,7 +596,7 @@ public class Counters
    * Logs the current counter values.
    * @param log The log to use.
    */
-  public void log(Log log) {
+  public void log(Logger log) {
     log.info("Counters: " + size());
     for(Group group: this) {
       log.info("  " + group.getDisplayName());

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/DeprecatedQueueConfigurationParser.java

@@ -23,8 +23,8 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import static org.apache.hadoop.mapred.QueueManager.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.List;
 import java.util.Map;
@@ -37,8 +37,8 @@ import java.util.ArrayList;
  * 
  */
 class DeprecatedQueueConfigurationParser extends QueueConfigurationParser {
-  private static final Log LOG =
-    LogFactory.getLog(DeprecatedQueueConfigurationParser.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DeprecatedQueueConfigurationParser.class);
   static final String MAPRED_QUEUE_NAMES_KEY = "mapred.queue.names";
 
   DeprecatedQueueConfigurationParser(Configuration conf) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java

@@ -30,8 +30,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.BlockLocation;
@@ -50,6 +48,8 @@ import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** 
  * A base class for file-based {@link InputFormat}.
@@ -68,8 +68,8 @@ import com.google.common.collect.Iterables;
 @InterfaceStability.Stable
 public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
 
-  public static final Log LOG =
-    LogFactory.getLog(FileInputFormat.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(FileInputFormat.class);
   
   @Deprecated
   public enum Counter {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileOutputCommitter.java

@@ -20,12 +20,12 @@ package org.apache.hadoop.mapred;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** An {@link OutputCommitter} that commits files specified 
  * in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}. 
@@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path;
 @InterfaceStability.Stable
 public class FileOutputCommitter extends OutputCommitter {
 
-  public static final Log LOG = LogFactory.getLog(
+  public static final Logger LOG = LoggerFactory.getLogger(
       "org.apache.hadoop.mapred.FileOutputCommitter");
   
   /**

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java

@@ -43,8 +43,8 @@ import org.apache.hadoop.io.compress.Decompressor;
 import org.apache.hadoop.io.serializer.SerializationFactory;
 import org.apache.hadoop.io.serializer.Serializer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * <code>IFile</code> is the simple &lt;key-len, value-len, key, value&gt; format
@@ -56,7 +56,7 @@ import org.apache.commons.logging.LogFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class IFile {
-  private static final Log LOG = LogFactory.getLog(IFile.class);
+  private static final Logger LOG = LoggerFactory.getLogger(IFile.class);
   public static final int EOF_MARKER = -1; // End of File Marker
   
   /**

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java

@@ -25,8 +25,6 @@ import java.io.IOException;
 import java.io.InputStream;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.ChecksumException;
@@ -36,6 +34,8 @@ import org.apache.hadoop.io.ReadaheadPool;
 import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.util.DataChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 /**
  * A checksum input stream, used for IFiles.
  * Used to validate the checksum of files created by {@link IFileOutputStream}. 
@@ -59,7 +59,8 @@ public class IFileInputStream extends InputStream {
   private boolean readahead;
   private int readaheadLength;
 
-  public static final Log LOG = LogFactory.getLog(IFileInputStream.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(IFileInputStream.class);
 
   private boolean disableChecksumValidation = false;
   

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java

@@ -22,17 +22,17 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class IndexCache {
 
   private final JobConf conf;
   private final int totalMemoryAllowed;
   private AtomicInteger totalMemoryUsed = new AtomicInteger();
-  private static final Log LOG = LogFactory.getLog(IndexCache.class);
+  private static final Logger LOG = LoggerFactory.getLogger(IndexCache.class);
 
   private final ConcurrentHashMap<String,IndexInformation> cache =
     new ConcurrentHashMap<String,IndexInformation>();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java

@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.JobACL;
@@ -29,11 +27,13 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 public class JobACLsManager {
 
-  static final Log LOG = LogFactory.getLog(JobACLsManager.class);
+  static final Logger LOG = LoggerFactory.getLogger(JobACLsManager.class);
   Configuration conf;
   private final AccessControlList adminAcl;
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java

@@ -24,8 +24,6 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -53,6 +51,8 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** 
  * A map/reduce job configuration.
@@ -115,7 +115,7 @@ import org.apache.hadoop.util.Tool;
 @InterfaceStability.Stable
 public class JobConf extends Configuration {
 
-  private static final Log LOG = LogFactory.getLog(JobConf.class);
+  private static final Logger LOG = LoggerFactory.getLogger(JobConf.class);
   private static final Pattern JAVA_OPTS_XMX_PATTERN =
           Pattern.compile(".*(?:^|\\s)-Xmx(\\d+)([gGmMkK]?)(?:$|\\s).*");
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java

@@ -24,20 +24,20 @@ import java.net.URISyntaxException;
 import java.util.concurrent.Delayed;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.params.ClientPNames;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.params.CoreConnectionPNames;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class JobEndNotifier {
-  private static final Log LOG =
-    LogFactory.getLog(JobEndNotifier.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JobEndNotifier.class.getName());
 
  
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JvmContext.java

@@ -24,13 +24,13 @@ import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class JvmContext implements Writable {
 
-  public static final Log LOG =
-    LogFactory.getLog(JvmContext.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(JvmContext.class);
   
   JVMId jvmId;
   String pid;